<commit_before>// Copyright (c) 2012, Joyent, Inc. All rights reserved.
var assert = require('assert-plus');
var EventEmitter = require('events').EventEmitter;
var HeartbeatServer = require('./heartbeatServer');
var HeartbeatClient = require('./heartbeatClient');
var path = require('path');
var PostgresMgr = require('./postgresMgr');
var util = require('util');
var uuid = require('node-uuid');
var vasync = require('vasync');
var verror = require('verror');
var zkplus = require('zkplus');

/**
 * Represents a Postgres node in a shard. We want to be as aggressive as
 * possible on pruning errors, which means the shard will emit error events
 * which should indicate to the consumer to restart the shard.
 *
 * @constructor
 *
 * @param {string} shardPath The path under which znodes for this shard are
 * stored.
 * @param {object} zkCfg The ZK configs.
 * @param {object} postgresMgrCfg The PG configs.
 * @param {object} heartbeaterCfg The heartbeater configs.
 * @param {integer} postgresPort
 * @param {integer} backupPort The port used by the backup service.
 * @param {string} url The url of the current node.
 * @param {integer} heartbeatInterval The interval of the heartbeater.
 * @param {integer} ttl The ttl of the registrar node.
 * @param {object} log The bunyan log object.
 */
function Shard(options) {
        assert.object(options, 'options');
        assert.object(options.log, 'options.log');
        assert.string(options.shardPath, 'options.shardPath');
        assert.object(options.zkCfg, 'options.zkCfg');
        assert.object(options.postgresMgrCfg, 'options.postgresMgrCfg');
        assert.object(options.heartbeaterCfg, 'options.heartbeaterCfg');
        assert.number(options.backupPort, 'options.backupPort');
        assert.number(options.postgresPort, 'options.postgresPort');
        assert.string(options.url, 'options.url');
        assert.number(options.heartbeatPort, 'options.heartbeatPort');
        assert.number(options.heartbeatInterval, 'options.heartbeatInterval');
        assert.number(options.ttl, 'options.ttl');

        EventEmitter.call(this);
        var self = this;
        this._log = options.log;

        /**
        * The path under which znodes for this shard are stored
        */
        this._shardPath = options.shardPath;

        /**
         * The registrar path, since the shardPath is usually
         * /prefix/com.joyent.foo.bar.baz, we strip out the first /prefix
         * replace . with / and append /pg
         */
        this._registrarPathPrefix =
                '/' +
                (self._shardPath.substring(self._shardPath.indexOf('/', 1) + 1))
                .split('.').reverse().join('/');

        this._registrarPath = this._registrarPathPrefix + '/pg';

        this._ttl = options.ttl;

        /**
        * The zk cfg
        */
        this._zkCfg = options.zkCfg;

        /**
        * The url of this peer
        */
        this._url = options.url;

        /**
        * This object's uuid
        */
        this._uuid = uuid();

        /**
        * The postgres manager
        */
        this._postgresMgr = new PostgresMgr(options.postgresMgrCfg);
        this._postgresPort = options.postgresPort;

        this._heartbeatServer = new HeartbeatServer({
                log: self._log,
                port: options.heartbeatPort
        });

        this._heartbeatPort = options.heartbeatPort;

        this._heartbeatClient = null;

        this._backupPort = options.backupPort;

        this._heartbeatInterval = options.heartbeatInterval;

        this._currStandby = null;

        /**
         * The queue used to manage transition events such that they occur
         * serially and automically.
         */
        this._transitionQueue = new TransitionQueue({
                log: self._log
        });

        /**
        * The ZK session
        */
        this._zk = zkplus.createClient(self._zkCfg);
        this._election = null;

        self._zk.on('connect', function() {
                _cleanup(self, function(err) {
                        if (err) {
                                self._log.error({err: err},
                                'unable to clean up previous emphemeral nodes');
                                self.emit('error', err);
                        }
                        self._election = zkplus.createGenericElection({
                                client: self._zk,
                                path: self._shardPath,
                                pathPrefix: self._url,
                                object: {}
                        });
                        _init(self);
                });
        });

        self._zk.on('session_expired', function() {
                self._log.error('zk session expired');
                self.emit('error', 'zk session expired');
        });
}

module.exports = Shard;
util.inherits(Shard, EventEmitter);

// Transition functions

function _leader(ctx, cb) {
        var self = ctx.self;
        var postgresMgr = self._postgresMgr;
        var log = self._log;
        log.info({
                currStandby: self._currStandby
        },'going to primary mode');

        if (self._heartbeatClient) {
                log.info('stopping heartbeat');
                self._heartbeatClient.stop();
        }

        postgresMgr.primary(self._currStandby, function(err) {
                if (err) {
                        log.error({
                                err: err
                        }, 'unable to transition to primary');
                        throw new verror.VError(err);
                }

                _writeRegistrar(self, function(err2) {
                        if (err2) {
                                log.error({
                                        err: err2
                                }, 'unable to write primary registrar');
                                throw new verror.VError(err2);
                        }
                        return cb();
                });
        });
}

function _newLeader(ctx, cb) {
        var self = ctx.self;
        var log = self._log;
        var myLeader = ctx.leader;
        var pgUrl = _transformPgUrl(self, myLeader);
        var backupUrl = _transformBackupUrl(self, myLeader);
        var heartbeatUrl = _transformHeartbeatUrl(self, myLeader);
        var postgresMgr = self._postgresMgr;
        log.info({
                leader: myLeader,
                primaryUrl: pgUrl,
                backupUrl: backupUrl,
                heartbeatUrl: heartbeatUrl
        }, 'going to standby mode');

        postgresMgr.standby(pgUrl, backupUrl, function(err) {
                if (err) {
                        log.error({
                                err: err,
                                pgUrl: pgUrl,
                                backupUrl: backupUrl
                        }, 'unable to transition to standby');
                        throw new verror.VError(err);
                }

                if (self._heartbeatClient) {
                        log.info('stopping heartbeat');
                        self._heartbeatClient.stop();
                }

                // start heartbeating
                self._heartbeatClient = new HeartbeatClient({
                        log: log,
                        primaryUrl: heartbeatUrl,
                        url: self._url,
                        heartbeatInterval: self._heartbeatInterval
                });

                log.info('starting heartbeat');
                self._heartbeatClient.start(function() {
                        cb();
                });
        });
}

function _newStandby(ctx, cb) {
        var self = ctx.self;
        var log = self._log;
        var postgresMgr = self._postgresMgr;
        var standby = ctx.standby;
        var standbyUrl = _transformPgUrl(self, standby);

        if (standbyUrl === self._currStandby) {
                log.info({
                        prevStandby: self._currStandby,
                        standbyUrl: standbyUrl
                }, 'standby not changed');

                return cb();
        }

        self._currStandby = standbyUrl;
        if (!self._election.amLeader) {
                log.info({
                        currStandby: self._currStandby
                }, 'not primary, ignoring standby heartbeats,' +
                ' but updating currStandby');

                return cb();
        }

        log.info({
                standby: standbyUrl
        }, 'going to primary mode with sync standby');

        postgresMgr.updateStandby(standbyUrl, function(err) {
                if (err) {
                        //log.error({
                                //err: err,
                                //standby: standbyUrl
                        //}, 'unable to transition to primary');
                        throw new verror.VError(err, 'unable to transition to primary');
                }

                log.info({
                        standby: standbyUrl
                }, 'finished primary transition');
                return cb();
        });

        return true;
}

function _expired(ctx, cb) {
        var self = ctx.self;
        var postgresMgr = self._postgresMgr;
        var standby = ctx.standby;
        var log = self._log;
        var standbyUrl = _transformPgUrl(self, standby);

        if (standbyUrl !== self._currStandby) {
                log.info({
                        currStandby: self._currStandby,
                        expiredStandby: standbyUrl
                }, 'expired standby does not match currStandby,' +
                ' ignoring');
                return cb();
        }

        self._currStandby = null;

        // only remove standby if currently leader. If not leader, the async
        // standby doesn't require any action.
        if (self._election.amLeader) {
                log.info({
                        standby: standbyUrl
                }, 'standby expired, going to primary standalone mode');
                postgresMgr.updateStandby(null, function(err) {
                        if (err) {
                                log.error({
                                        err: err
                                }, 'unable to transition to primary');
                                throw new verror.VError(err);
                        }
                        return cb();
                });
        } else {
                log.info({
                        standby: standbyUrl
                }, 'async standby expired, I am not primary, ignoring');
                return cb();
        }

        return true;
}

// private functions

/**
 * Delete any previous emphemeral zk-nodes belonging to this pg node.
 */
function _cleanup(self, callback) {
        var log = self._log;
        var zk = self._zk;

        log.info({
                shardPath: self._shardPath,
                url: self._url
        },'entering cleanup');
        var tasks = [
                function mkdirp(_, cb) {
                        zk.mkdirp(self._shardPath, function(err) {
                                log.debug({
                                        path: self._shardPath,
                                        err: err
                                }, 'returned from mkdirp');
                                cb(err);
                        });
                },
                function readdir(_, cb) {
                        log.debug({
                                shardPath: self._shardPath
                        }, 'listing election dir');
                        zk.readdir(self._shardPath, function(err, nodes) {
                                log.debug({
                                        err: err,
                                        nodes: nodes
                                }, 'returned from readdir');
                                if (err) {
                                        return cb(err);
                                } else {
                                        _.nodes = nodes;
                                        return cb();
                                }
                        });
                },
                function determineNodes(_, cb) {
                        log.debug({
                                nodes: _.nodes
                        }, 'entering determine nodes');
                        var nodes = [];
                        for (var i = 0; i < _.nodes.length; i++) {
                                var node = _.nodes[i];
                                if (node.split('-')[0] === self._url) {
                                        nodes.push(node);
                                }
                        }

                        _.nodes = nodes;
                        cb();
                },
                function delPrevNodes(_, cb) {
                        log.debug({
                                nodes: _.nodes
                        },'entering delPrevNodes');
                        var done = 0;
                        for (var i = 0; i < _.nodes.length; i++) {
                                var spath = self._shardPath + '/' + _.nodes[i];
                                log.debug({
                                        node: spath
                                }, 'deleting prev node');
                                zk.rmr(spath, function(err) {
                                        if (err) {
                                                return cb(err);
                                        } else {
                                                done++;
                                                var nLength = _.nodes.length;
                                                if (done === (nLength - 1)) {
                                                        return cb();
                                                }
                                        }
                                        return true;
                                });

                        }

                        return cb();
                }
        ];

        vasync.pipeline({funcs: tasks, arg: {}}, function(err) {
                if (err) {
                        log.error('Unable to delete previous emphemeral nodes');
                        return callback(err);
                } else {
                        log.info('finished deleting previous emphemeral nodes');
                        return callback();
                }
        });
}

/**
 * initialize this node
 */
function _init(self) {
        var log = self._log;
        log.info('initializing shard');
        var queue = self._transitionQueue;

        self._election.on('leader', function() {
                log.info('leader event');
                var req = {
                        invoke: _leader,
                        ctx: {
                                self: self
                        },
                        transition: 'leader'
                };

                queue.push(req);
        });

        self._election.on('newLeader', function(myLeader) {
                log.info({
                        newLeader: myLeader
                }, 'newLeader event');
                var req = {
                        invoke: _newLeader,
                        ctx: {
                                self: self,
                                leader: myLeader
                        },
                        transition: 'newLeader'
                };

                queue.push(req);
        });

        self._heartbeatServer.on('newStandby', function(standby) {
                log.info({
                        standby: standby
                }, 'newStandby event');
                var opts = {
                        ctx: self,
                        standby: standby
                };
                var req = {
                        invoke: _newStandby,
                        ctx: {
                                self: self,
                                standby: standby
                        },
                        transition: 'newStandby'
                };

                queue.push(req);
        });

        self._heartbeatServer.on('expired', function(standby) {
                log.info({
                        standby: standby
                }, 'expired event');
                var req = {
                        invoke: _expired,
                        ctx: {
                                self: self,
                                standby: standby
                        },
                        transition: 'expired'
                };

                queue.push(req);
        });

        self._heartbeatServer.init();
        self._election.vote(function(err) {
                if (err) {
                        throw new verror.VError(err);
                }

                log.info('finished vote');
        });
}

function _transformPgUrl(self, url) {
        return 'tcp://postgres@' + url.split('-')[0] + ':' +
                self._postgresPort + '/postgres';
}

function _transformBackupUrl(self, url) {
        return  'http://' + url.split('-')[0] + ':' + self._backupPort;
}

function _transformHeartbeatUrl(self, url) {
        return  'http://' + url.split('-')[0]+ ':' + self._heartbeatPort;
}

function _writeRegistrar(self, callback) {
        var log = self._log;
        var zk = self._zk;

        var opts = {
                flags:  ['ephemeral'],
                object: {
                        type: 'database',
                        database: {
                                primary: _transformPgUrl(self, self._url),
                                ttl: self._ttl
                        }
                }
        };

        log.info({
                registrarPath: self._registrarPath,
                url: _transformPgUrl(self, self._url),
                opts: opts
        }, 'entering write registrar');

        var tasks = [
                function mkdirp(_, cb) {
                        zk.mkdirp(self._registrarPathPrefix, function(err) {
                                var err2;
                                if (err)  {
                                        err2 = verror.VError(err);
                                }
                                cb(err2);
                        });
                },
                function rm(_, cb) {
                        zk.rmr(self._registrarPath, function() {
                                // don't check for err on rmr, it could fail if
                                // the node DNE
                                cb();
                        });
                },
                function create(_, cb) {
                        zk.create(self._registrarPath, opts, function(err) {
                                var err2;
                                if (err)  {
                                        err2 = verror.VError(err);
                                }
                                cb(err2);
                        });
                }
        ];

        vasync.pipeline({funcs: tasks, arg: {}}, function(err) {
                if (err) {
                        log.error('Unable to write registrar');
                        return callback(err);
                } else {
                        log.info('finished writing registrar');
                        return callback();
                }
        });

        return true;
}

// private classes

/**
 * Queue used to serialize state transitions
 */
function TransitionQueue(options) {
        var self = this;
        this._queue = [];
        this._log = options.log;
        this._emitter = new EventEmitter();

        var log = self._log;
        var isPop = false;
        var req = null;
        self._emitter.on('push', function() {
                if (isPop) {
                        log.info({
                                queue_length: self._queue.length,
                                curr_req: req.transition,
                                new_req: self._queue[self._queue.length - 1].transition
                        }, 'waiting for current transition to finish before ' +
                        'executing pushed req');
                        return;
                } else {
                        exec();
                }
        });

        function exec() {
                isPop = true;
                req = self._queue.shift();
                req.invoke(req.ctx, function() {
                        log.info({
                                transition: req.transition
                        }, 'finished transition');
                        if (self._queue.length > 0) {
                                log.info({
                                        queue_length: self._queue.length
                                }, 'more elements in the queue,' +
                                ' continuing exec');
                                exec();
                        } else {
                                log.info({
                                        queue_length: self._queue.length
                                }, 'no more elements in queue');
                                isPop = false;
                        }
                });
        }
}

TransitionQueue.prototype.push = function push(req) {
        var self = this;
        var log = self._log;
        log.info({
                req: req.transition
        }, 'pushing request');
        self._queue.push(req);
        self._emitter.emit('push');
};
<commit_msg>don't log error when throwing error<commit_after>// Copyright (c) 2012, Joyent, Inc. All rights reserved.
var assert = require('assert-plus');
var EventEmitter = require('events').EventEmitter;
var HeartbeatServer = require('./heartbeatServer');
var HeartbeatClient = require('./heartbeatClient');
var path = require('path');
var PostgresMgr = require('./postgresMgr');
var util = require('util');
var uuid = require('node-uuid');
var vasync = require('vasync');
var verror = require('verror');
var zkplus = require('zkplus');

/**
 * Represents a Postgres node in a shard. We want to be as aggressive as
 * possible on pruning errors, which means the shard will emit error events
 * which should indicate to the consumer to restart the shard.
 *
 * @constructor
 *
 * @param {string} shardPath The path under which znodes for this shard are
 * stored.
 * @param {object} zkCfg The ZK configs.
 * @param {object} postgresMgrCfg The PG configs.
 * @param {object} heartbeaterCfg The heartbeater configs.
 * @param {integer} postgresPort
 * @param {integer} backupPort The port used by the backup service.
 * @param {string} url The url of the current node.
 * @param {integer} heartbeatInterval The interval of the heartbeater.
 * @param {integer} ttl The ttl of the registrar node.
 * @param {object} log The bunyan log object.
 */
function Shard(options) {
        assert.object(options, 'options');
        assert.object(options.log, 'options.log');
        assert.string(options.shardPath, 'options.shardPath');
        assert.object(options.zkCfg, 'options.zkCfg');
        assert.object(options.postgresMgrCfg, 'options.postgresMgrCfg');
        assert.object(options.heartbeaterCfg, 'options.heartbeaterCfg');
        assert.number(options.backupPort, 'options.backupPort');
        assert.number(options.postgresPort, 'options.postgresPort');
        assert.string(options.url, 'options.url');
        assert.number(options.heartbeatPort, 'options.heartbeatPort');
        assert.number(options.heartbeatInterval, 'options.heartbeatInterval');
        assert.number(options.ttl, 'options.ttl');

        EventEmitter.call(this);
        var self = this;
        this._log = options.log;

        /**
        * The path under which znodes for this shard are stored
        */
        this._shardPath = options.shardPath;

        /**
         * The registrar path, since the shardPath is usually
         * /prefix/com.joyent.foo.bar.baz, we strip out the first /prefix
         * replace . with / and append /pg
         */
        this._registrarPathPrefix =
                '/' +
                (self._shardPath.substring(self._shardPath.indexOf('/', 1) + 1))
                .split('.').reverse().join('/');

        this._registrarPath = this._registrarPathPrefix + '/pg';

        this._ttl = options.ttl;

        /**
        * The zk cfg
        */
        this._zkCfg = options.zkCfg;

        /**
        * The url of this peer
        */
        this._url = options.url;

        /**
        * This object's uuid
        */
        this._uuid = uuid();

        /**
        * The postgres manager
        */
        this._postgresMgr = new PostgresMgr(options.postgresMgrCfg);
        this._postgresPort = options.postgresPort;

        this._heartbeatServer = new HeartbeatServer({
                log: self._log,
                port: options.heartbeatPort
        });

        this._heartbeatPort = options.heartbeatPort;

        this._heartbeatClient = null;

        this._backupPort = options.backupPort;

        this._heartbeatInterval = options.heartbeatInterval;

        this._currStandby = null;

        /**
         * The queue used to manage transition events such that they occur
         * serially and automically.
         */
        this._transitionQueue = new TransitionQueue({
                log: self._log
        });

        /**
        * The ZK session
        */
        this._zk = zkplus.createClient(self._zkCfg);
        this._election = null;

        self._zk.on('connect', function() {
                _cleanup(self, function(err) {
                        if (err) {
                                self._log.error({err: err},
                                'unable to clean up previous emphemeral nodes');
                                self.emit('error', err);
                        }
                        self._election = zkplus.createGenericElection({
                                client: self._zk,
                                path: self._shardPath,
                                pathPrefix: self._url,
                                object: {}
                        });
                        _init(self);
                });
        });

        self._zk.on('session_expired', function() {
                self._log.error('zk session expired');
                self.emit('error', 'zk session expired');
        });
}

module.exports = Shard;
util.inherits(Shard, EventEmitter);

// Transition functions

function _leader(ctx, cb) {
        var self = ctx.self;
        var postgresMgr = self._postgresMgr;
        var log = self._log;
        log.info({
                currStandby: self._currStandby
        },'going to primary mode');

        if (self._heartbeatClient) {
                log.info('stopping heartbeat');
                self._heartbeatClient.stop();
        }

        postgresMgr.primary(self._currStandby, function(err) {
                if (err) {
                        log.error({
                                err: err
                        }, 'unable to transition to primary');
                        throw new verror.VError(err);
                }

                _writeRegistrar(self, function(err2) {
                        if (err2) {
                                log.error({
                                        err: err2
                                }, 'unable to write primary registrar');
                                throw new verror.VError(err2);
                        }
                        return cb();
                });
        });
}

function _newLeader(ctx, cb) {
        var self = ctx.self;
        var log = self._log;
        var myLeader = ctx.leader;
        var pgUrl = _transformPgUrl(self, myLeader);
        var backupUrl = _transformBackupUrl(self, myLeader);
        var heartbeatUrl = _transformHeartbeatUrl(self, myLeader);
        var postgresMgr = self._postgresMgr;
        log.info({
                leader: myLeader,
                primaryUrl: pgUrl,
                backupUrl: backupUrl,
                heartbeatUrl: heartbeatUrl
        }, 'going to standby mode');

        postgresMgr.standby(pgUrl, backupUrl, function(err) {
                if (err) {
                        log.error({
                                err: err,
                                pgUrl: pgUrl,
                                backupUrl: backupUrl
                        }, 'unable to transition to standby');
                        throw new verror.VError(err);
                }

                if (self._heartbeatClient) {
                        log.info('stopping heartbeat');
                        self._heartbeatClient.stop();
                }

                // start heartbeating
                self._heartbeatClient = new HeartbeatClient({
                        log: log,
                        primaryUrl: heartbeatUrl,
                        url: self._url,
                        heartbeatInterval: self._heartbeatInterval
                });

                log.info('starting heartbeat');
                self._heartbeatClient.start(function() {
                        cb();
                });
        });
}

function _newStandby(ctx, cb) {
        var self = ctx.self;
        var log = self._log;
        var postgresMgr = self._postgresMgr;
        var standby = ctx.standby;
        var standbyUrl = _transformPgUrl(self, standby);

        if (standbyUrl === self._currStandby) {
                log.info({
                        prevStandby: self._currStandby,
                        standbyUrl: standbyUrl
                }, 'standby not changed');

                return cb();
        }

        self._currStandby = standbyUrl;
        if (!self._election.amLeader) {
                log.info({
                        currStandby: self._currStandby
                }, 'not primary, ignoring standby heartbeats,' +
                ' but updating currStandby');

                return cb();
        }

        log.info({
                standby: standbyUrl
        }, 'going to primary mode with sync standby');

        postgresMgr.updateStandby(standbyUrl, function(err) {
                if (err) {
                        throw new verror.VError(err,
                                'unable to transition to primary');
                }

                log.info({
                        standby: standbyUrl
                }, 'finished primary transition');
                return cb();
        });

        return true;
}

function _expired(ctx, cb) {
        var self = ctx.self;
        var postgresMgr = self._postgresMgr;
        var standby = ctx.standby;
        var log = self._log;
        var standbyUrl = _transformPgUrl(self, standby);

        if (standbyUrl !== self._currStandby) {
                log.info({
                        currStandby: self._currStandby,
                        expiredStandby: standbyUrl
                }, 'expired standby does not match currStandby,' +
                ' ignoring');
                return cb();
        }

        self._currStandby = null;

        // only remove standby if currently leader. If not leader, the async
        // standby doesn't require any action.
        if (self._election.amLeader) {
                log.info({
                        standby: standbyUrl
                }, 'standby expired, going to primary standalone mode');
                postgresMgr.updateStandby(null, function(err) {
                        if (err) {
                                log.error({
                                        err: err
                                }, 'unable to transition to primary');
                                throw new verror.VError(err);
                        }
                        return cb();
                });
        } else {
                log.info({
                        standby: standbyUrl
                }, 'async standby expired, I am not primary, ignoring');
                return cb();
        }

        return true;
}

// private functions

/**
 * Delete any previous emphemeral zk-nodes belonging to this pg node.
 */
function _cleanup(self, callback) {
        var log = self._log;
        var zk = self._zk;

        log.info({
                shardPath: self._shardPath,
                url: self._url
        },'entering cleanup');
        var tasks = [
                function mkdirp(_, cb) {
                        zk.mkdirp(self._shardPath, function(err) {
                                log.debug({
                                        path: self._shardPath,
                                        err: err
                                }, 'returned from mkdirp');
                                cb(err);
                        });
                },
                function readdir(_, cb) {
                        log.debug({
                                shardPath: self._shardPath
                        }, 'listing election dir');
                        zk.readdir(self._shardPath, function(err, nodes) {
                                log.debug({
                                        err: err,
                                        nodes: nodes
                                }, 'returned from readdir');
                                if (err) {
                                        return cb(err);
                                } else {
                                        _.nodes = nodes;
                                        return cb();
                                }
                        });
                },
                function determineNodes(_, cb) {
                        log.debug({
                                nodes: _.nodes
                        }, 'entering determine nodes');
                        var nodes = [];
                        for (var i = 0; i < _.nodes.length; i++) {
                                var node = _.nodes[i];
                                if (node.split('-')[0] === self._url) {
                                        nodes.push(node);
                                }
                        }

                        _.nodes = nodes;
                        cb();
                },
                function delPrevNodes(_, cb) {
                        log.debug({
                                nodes: _.nodes
                        },'entering delPrevNodes');
                        var done = 0;
                        for (var i = 0; i < _.nodes.length; i++) {
                                var spath = self._shardPath + '/' + _.nodes[i];
                                log.debug({
                                        node: spath
                                }, 'deleting prev node');
                                zk.rmr(spath, function(err) {
                                        if (err) {
                                                return cb(err);
                                        } else {
                                                done++;
                                                var nLength = _.nodes.length;
                                                if (done === (nLength - 1)) {
                                                        return cb();
                                                }
                                        }
                                        return true;
                                });

                        }

                        return cb();
                }
        ];

        vasync.pipeline({funcs: tasks, arg: {}}, function(err) {
                if (err) {
                        log.error('Unable to delete previous emphemeral nodes');
                        return callback(err);
                } else {
                        log.info('finished deleting previous emphemeral nodes');
                        return callback();
                }
        });
}

/**
 * initialize this node
 */
function _init(self) {
        var log = self._log;
        log.info('initializing shard');
        var queue = self._transitionQueue;

        self._election.on('leader', function() {
                log.info('leader event');
                var req = {
                        invoke: _leader,
                        ctx: {
                                self: self
                        },
                        transition: 'leader'
                };

                queue.push(req);
        });

        self._election.on('newLeader', function(myLeader) {
                log.info({
                        newLeader: myLeader
                }, 'newLeader event');
                var req = {
                        invoke: _newLeader,
                        ctx: {
                                self: self,
                                leader: myLeader
                        },
                        transition: 'newLeader'
                };

                queue.push(req);
        });

        self._heartbeatServer.on('newStandby', function(standby) {
                log.info({
                        standby: standby
                }, 'newStandby event');
                var opts = {
                        ctx: self,
                        standby: standby
                };
                var req = {
                        invoke: _newStandby,
                        ctx: {
                                self: self,
                                standby: standby
                        },
                        transition: 'newStandby'
                };

                queue.push(req);
        });

        self._heartbeatServer.on('expired', function(standby) {
                log.info({
                        standby: standby
                }, 'expired event');
                var req = {
                        invoke: _expired,
                        ctx: {
                                self: self,
                                standby: standby
                        },
                        transition: 'expired'
                };

                queue.push(req);
        });

        self._heartbeatServer.init();
        self._election.vote(function(err) {
                if (err) {
                        throw new verror.VError(err);
                }

                log.info('finished vote');
        });
}

function _transformPgUrl(self, url) {
        return 'tcp://postgres@' + url.split('-')[0] + ':' +
                self._postgresPort + '/postgres';
}

function _transformBackupUrl(self, url) {
        return  'http://' + url.split('-')[0] + ':' + self._backupPort;
}

function _transformHeartbeatUrl(self, url) {
        return  'http://' + url.split('-')[0]+ ':' + self._heartbeatPort;
}

function _writeRegistrar(self, callback) {
        var log = self._log;
        var zk = self._zk;

        var opts = {
                flags:  ['ephemeral'],
                object: {
                        type: 'database',
                        database: {
                                primary: _transformPgUrl(self, self._url),
                                ttl: self._ttl
                        }
                }
        };

        log.info({
                registrarPath: self._registrarPath,
                url: _transformPgUrl(self, self._url),
                opts: opts
        }, 'entering write registrar');

        var tasks = [
                function mkdirp(_, cb) {
                        zk.mkdirp(self._registrarPathPrefix, function(err) {
                                var err2;
                                if (err)  {
                                        err2 = verror.VError(err);
                                }
                                cb(err2);
                        });
                },
                function rm(_, cb) {
                        zk.rmr(self._registrarPath, function() {
                                // don't check for err on rmr, it could fail if
                                // the node DNE
                                cb();
                        });
                },
                function create(_, cb) {
                        zk.create(self._registrarPath, opts, function(err) {
                                var err2;
                                if (err)  {
                                        err2 = verror.VError(err);
                                }
                                cb(err2);
                        });
                }
        ];

        vasync.pipeline({funcs: tasks, arg: {}}, function(err) {
                if (err) {
                        log.error('Unable to write registrar');
                        return callback(err);
                } else {
                        log.info('finished writing registrar');
                        return callback();
                }
        });

        return true;
}

// private classes

/**
 * Queue used to serialize state transitions
 */
function TransitionQueue(options) {
        var self = this;
        this._queue = [];
        this._log = options.log;
        this._emitter = new EventEmitter();

        var log = self._log;
        var isPop = false;
        var req = null;
        self._emitter.on('push', function() {
                if (isPop) {
                        log.info({
                                queue_length: self._queue.length,
                                curr_req: req.transition,
                                new_req: self._queue[self._queue.length - 1].transition
                        }, 'waiting for current transition to finish before ' +
                        'executing pushed req');
                        return;
                } else {
                        exec();
                }
        });

        function exec() {
                isPop = true;
                req = self._queue.shift();
                req.invoke(req.ctx, function() {
                        log.info({
                                transition: req.transition
                        }, 'finished transition');
                        if (self._queue.length > 0) {
                                log.info({
                                        queue_length: self._queue.length
                                }, 'more elements in the queue,' +
                                ' continuing exec');
                                exec();
                        } else {
                                log.info({
                                        queue_length: self._queue.length
                                }, 'no more elements in queue');
                                isPop = false;
                        }
                });
        }
}

TransitionQueue.prototype.push = function push(req) {
        var self = this;
        var log = self._log;
        log.info({
                req: req.transition
        }, 'pushing request');
        self._queue.push(req);
        self._emitter.emit('push');
};
<|endoftext|>


<commit_before>/**
 * @list dependencies
 */

var ID = require('short-id')
  , mongoose = require('mongoose')
  , Promise = require('node-promise').Promise
  , ShortURL = require('../models/ShortURL').ShortURL;

/**
 * @configure short-id
 */

ID.configure({
  length: 6,
  algorithm: 'sha1',
  salt: Math.random
});

/**
 * @method connect
 * @param {String} mongdb Mongo DB String to connect to
 */

exports.connect = function(mongodb) {
  mongoose.connect(mongodb);
  exports.connection = mongoose.connection;
};

/**
 * @method generate
 * @param {Object} options Must at least include a `URL` attribute
 */

exports.generate = function(document) {
  var generatePromise
    , promise = new Promise()
    , generatedHash = ID.store(document.URL)
    , query = { URL : document.URL };
  document['hash'] = generatedHash;
  document['data'] = (document.data) ? document.data : null;
  generatePromise = ShortURL.findOrCreate(query, document, {});
  generatePromise.then(function(ShortURLObject) {
    promise.resolve(ShortURLObject);
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method retrieve
 * @param {Object} options Must at least include a `hash` attribute
 */

exports.retrieve = function(hash) {
  var promise = new Promise();
  var query = { hash : hash }
    , update = { $inc: { hits: 1 } }
    , options = { multi: true };
  var retrievePromise = ShortURL.findOne(query);
  ShortURL.update( query, update , options , function (){ } );
  retrievePromise.then(function(ShortURLObject) {
    if (ShortURLObject && ShortURLObject !== null) {
      promise.resolve(ShortURLObject);
    } else {
      promise.reject(new Error('MongoDB - Cannot find Document'), true);
    };
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method hits
 * @param {Object} options Must at least include a `hash` attribute
 */

exports.hits = function(hash) {
  var promise = new Promise();
  var query = { hash : hash }
    , options = { multi: true };
  var retrievePromise = ShortURL.findOne(query);
  ShortURL.update(query, update, options, function(){ });
  retrievePromise.then(function(ShortURLObject) {
    if (ShortURLObject && ShortURLObject !== null) {
      promise.resolve(ShortURLObject.hits);
    } else {
      promise.reject(new Error('MongoDB - Cannot find Document'), true);
    };
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method list
 * @description List all Shortened URLs
 */

exports.list = function() {
  var promise = new Promise();
  var listPromise = ShortURL.find({});
  listPromise.then(function(ShortenedURLObjects) {
    promise.resolve(ShortenedURLObjects);
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};
<commit_msg>Removed an extra promise layer<commit_after>/**
 * @list dependencies
 */

var ID = require('short-id')
  , mongoose = require('mongoose')
  , Promise = require('node-promise').Promise
  , ShortURL = require('../models/ShortURL').ShortURL;

/**
 * @configure short-id
 */

ID.configure({
  length: 6,
  algorithm: 'sha1',
  salt: Math.random
});

/**
 * @method connect
 * @param {String} mongdb Mongo DB String to connect to
 */

exports.connect = function(mongodb) {
  mongoose.connect(mongodb);
  exports.connection = mongoose.connection;
};

/**
 * @method generate
 * @param {Object} options Must at least include a `URL` attribute
 */

exports.generate = function(document) {
  var generatePromise
    , promise = new Promise()
    , generatedHash = ID.store(document.URL)
    , query = { URL : document.URL };
  document['hash'] = generatedHash;
  document['data'] = (document.data) ? document.data : null;
  generatePromise = ShortURL.findOrCreate(query, document, {});
  generatePromise.then(function(ShortURLObject) {
    promise.resolve(ShortURLObject);
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method retrieve
 * @param {Object} options Must at least include a `hash` attribute
 */

exports.retrieve = function(hash) {
  var promise = new Promise();
  var query = { hash : hash }
    , update = { $inc: { hits: 1 } }
    , options = { multi: true };
  var retrievePromise = ShortURL.findOne(query);
  ShortURL.update( query, update , options , function (){ } );
  retrievePromise.then(function(ShortURLObject) {
    if (ShortURLObject && ShortURLObject !== null) {
      promise.resolve(ShortURLObject);
    } else {
      promise.reject(new Error('MongoDB - Cannot find Document'), true);
    };
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method hits
 * @param {Object} options Must at least include a `hash` attribute
 */

exports.hits = function(hash) {
  var promise = new Promise();
  var query = { hash : hash }
    , options = { multi: true };
  var retrievePromise = ShortURL.findOne(query);
  ShortURL.update(query, update, options, function(){ });
  retrievePromise.then(function(ShortURLObject) {
    if (ShortURLObject && ShortURLObject !== null) {
      promise.resolve(ShortURLObject.hits);
    } else {
      promise.reject(new Error('MongoDB - Cannot find Document'), true);
    };
  }, function(error) {
    promise.reject(error, true);
  });
  return promise;
};

/**
 * @method list
 * @description List all Shortened URLs
 */

exports.list = function() {
  return ShortURL.find({});
};
<|endoftext|>


<commit_before>const self = require("sdk/self");

const {Hotkey} = require("sdk/hotkeys");
const {Panel} = require("sdk/panel");
const {ToggleButton} = require("sdk/ui/button/toggle");

/**
 * Registers and returns the hotkey.
 */
exports.GroupsHotkey = function(onPress) {
  /**
   * NOTE
   *
   * Since we're removing Panorama in 1222490 it should be safe to reassign
   * Ctrl-Shift-E for our replacement
   */
  return Hotkey({
    combo: "accel-shift-e",
    onPress
  });
};

/**
 * Creates and returns a Panel displaying the Tab Groupes and their Tabs.
 */
exports.GroupsPanel = function() {
  return Panel({
    contentScriptFile: [
      self.data.url("vendor/react-0.14.2.js"),
      self.data.url("vendor/react-dom-0.14.2.min.js"),
      self.data.url("vendor/redux.min.js"),
      self.data.url("vendor/react-redux.min.js"),
      self.data.url("vendor/immutable.min.js"),

      self.data.url("action_creators.js"),
      self.data.url("reducer.js"),
      self.data.url("components/app.js"),
      self.data.url("components/group.js"),
      self.data.url("components/grouplist.js"),
      self.data.url("components/tab.js"),
      self.data.url("components/tablist.js"),
      self.data.url("groupspanel.js")
    ],
    contentScriptWhen: "ready",
    contentURL: self.data.url("groupspanel.html")
  });
};

/**
 * Creates and returns a ToggleButton for toggling the Panel.
 */
exports.PanelButton = function() {
  return ToggleButton({
    id: "tabgroups-show",
    icon: {
      "32": self.data.url("assets/images/icon-32.png"),
      "64": self.data.url("assets/images/icon-64.png")
    },
    label: "Show Tab Groups"
  });
};
<commit_msg>Load scripts immediately<commit_after>const self = require("sdk/self");

const {Hotkey} = require("sdk/hotkeys");
const {Panel} = require("sdk/panel");
const {ToggleButton} = require("sdk/ui/button/toggle");

/**
 * Registers and returns the hotkey.
 */
exports.GroupsHotkey = function(onPress) {
  /**
   * NOTE
   *
   * Since we're removing Panorama in 1222490 it should be safe to reassign
   * Ctrl-Shift-E for our replacement
   */
  return Hotkey({
    combo: "accel-shift-e",
    onPress
  });
};

/**
 * Creates and returns a Panel displaying the Tab Groupes and their Tabs.
 */
exports.GroupsPanel = function() {
  return Panel({
    contentScriptFile: [
      self.data.url("vendor/react-0.14.2.js"),
      self.data.url("vendor/react-dom-0.14.2.min.js"),
      self.data.url("vendor/redux.min.js"),
      self.data.url("vendor/react-redux.min.js"),
      self.data.url("vendor/immutable.min.js"),

      self.data.url("action_creators.js"),
      self.data.url("reducer.js"),
      self.data.url("components/app.js"),
      self.data.url("components/group.js"),
      self.data.url("components/grouplist.js"),
      self.data.url("components/tab.js"),
      self.data.url("components/tablist.js"),
      self.data.url("groupspanel.js")
    ],
    contentURL: self.data.url("groupspanel.html")
  });
};

/**
 * Creates and returns a ToggleButton for toggling the Panel.
 */
exports.PanelButton = function() {
  return ToggleButton({
    id: "tabgroups-show",
    icon: {
      "32": self.data.url("assets/images/icon-32.png"),
      "64": self.data.url("assets/images/icon-64.png")
    },
    label: "Show Tab Groups"
  });
};
<|endoftext|>


<commit_before>/**
 * Library methods for styles and css tasks
 *
 * @author Nikhil Modak
 */

'use strict';

var SKIN_FILES_PATTERN = '**/*(*.css|*.scss|*.sass|*.ftl|*.png|*.svg|*.jpg|*.jpeg|skin.properties)';
var lazyReq = require('lazy-req')(require);
var sass = lazyReq('gulp-sass');
var connect = lazyReq('gulp-connect');
var sourcemaps = lazyReq('gulp-sourcemaps');
var rename = lazyReq('gulp-rename');
var plumber = lazyReq('gulp-plumber');
var path = lazyReq('path');
var replace = lazyReq('gulp-replace');
var concat = lazyReq('gulp-concat');
var gulpif = lazyReq('gulp-if');
var newer = lazyReq('gulp-newer');
var fs = lazyReq('fs');

module.exports = function (gulp, gutil) {
  var server;

  function getServer() {
    return server ? server : server = require('./server.js')(gulp, gutil);
  }

  var cors = function (req, res, next) {
    res.setHeader('Access-Control-Allow-Origin', '*');
    res.setHeader('Access-Control-Allow-Headers', '*');
    next();
  };

  return {
    process: function (patterns, dest, files) {
      return gulp.src(patterns)
       .pipe(gutil.env.filterFiles(files))
       .pipe(gulp.dest(dest));
    },
    compile: function (lr) {
      var server = getServer();
      var version = server.localSkinCompileVersion();
      var feature = server.localSkinCompileFeature();
      var skin = server.localSkinCompileSkin();
      var dest = server.localSkinCompileDest();
      var skinPath = 'res/feature/' + feature + '/' + version + '/res/skins/' + skin + '/sass/skin.scss';
      var includePaths = [
        'res/feature/responsivepeak/' + version + '/res/skins/',
        'res/feature/responsivebase/' + version + '/res/skins/',
        'res/feature/responsivebase/common/res/skins/'
      ];

      return gulp.src(skinPath)
        .pipe(lr ? plumber()() : gutil.noop())
        .pipe(sourcemaps().init())
        .pipe(sass()({
          includePaths: includePaths
        }))
        .pipe(sourcemaps().write())
        .pipe(rename()(skin + '.css'))
        .pipe(gulp.dest(dest))
        .pipe(lr ? lr() : gutil.noop());
    },
    server: function () {
      connect().server({
        root: [
          getServer().localServerDir(),
          'web'
        ],
        port: getServer().localServerPort(),
        middleware: function () {
          return [cors];
        }
      });
    },
    vars: function () {
      var baseFeaturePath = 'res/feature/responsivebase';
      var peakFeaturePath = 'res/feature/responsivepeak';
      var baseSassPath = 'res/skins/bootstrap_base/sass';
      var peakSassPath = 'res/skins/responsive_peak/sass';
      var varsPath = 'codebook/_variables-*.scss';

      function getSkinVersions(dir) {
        return fs().readdirSync(dir)
          .filter(function(file) {
            return fs().statSync(path().join(dir, file)).isDirectory();
          });
      }

      var peakVars = getSkinVersions(peakFeaturePath).map(function(version) {
        return gulp.src([path().join(baseFeaturePath, version, '**', varsPath),
            path().join(peakFeaturePath, version, '**', varsPath)])
          .pipe(newer()(path().join(peakFeaturePath, version, peakSassPath, '_variables.scss')))
          .pipe(replace()('!default', ''))
          .pipe(replace()(/^\s*(\$.*)/gm, '//$1'))
          .pipe(replace()(/(\s*);/gm, ';'))
          .pipe(rename()('_variables.scss'))
          .pipe(gulpif()(function(file){ return file.path.indexOf(baseFeaturePath) >= 0; },
            gulp.dest(path().join(baseFeaturePath, version, baseSassPath))))
          .pipe(concat()('_variables.scss'))
          .pipe(gulp.dest(path().join(peakFeaturePath, version, peakSassPath)));
      });
      return peakVars;
    },
    FILES_PATTERN: SKIN_FILES_PATTERN
  };
};
<commit_msg>fix for <= 15.11 directory names<commit_after>/**
 * Library methods for styles and css tasks
 *
 * @author Nikhil Modak
 */

'use strict';

var SKIN_FILES_PATTERN = '**/*(*.css|*.scss|*.sass|*.ftl|*.png|*.svg|*.jpg|*.jpeg|skin.properties)';
var lazyReq = require('lazy-req')(require);
var sass = lazyReq('gulp-sass');
var connect = lazyReq('gulp-connect');
var sourcemaps = lazyReq('gulp-sourcemaps');
var rename = lazyReq('gulp-rename');
var plumber = lazyReq('gulp-plumber');
var path = lazyReq('path');
var replace = lazyReq('gulp-replace');
var concat = lazyReq('gulp-concat');
var gulpif = lazyReq('gulp-if');
var newer = lazyReq('gulp-newer');
var fs = lazyReq('fs');

module.exports = function (gulp, gutil) {
  var server;

  function getServer() {
    return server ? server : server = require('./server.js')(gulp, gutil);
  }

  var cors = function (req, res, next) {
    res.setHeader('Access-Control-Allow-Origin', '*');
    res.setHeader('Access-Control-Allow-Headers', '*');
    next();
  };

  return {
    process: function (patterns, dest, files) {
      return gulp.src(patterns)
       .pipe(gutil.env.filterFiles(files))
       .pipe(gulp.dest(dest));
    },
    compile: function (lr) {
      var server = getServer();
      var version = server.localSkinCompileVersion();
      var feature = server.localSkinCompileFeature();
      var skin = server.localSkinCompileSkin();
      var dest = server.localSkinCompileDest();
      var skinPath = 'res/feature/' + feature + '/' + version + '/res/skins/' + skin + '/sass/skin.scss';
      var includePaths = [
        'res/feature/responsivepeak/' + version + '/res/skins/',
        'res/feature/responsivebase/' + version + '/res/skins/',
        'res/feature/responsivebase/common/res/skins/'
      ];

      return gulp.src(skinPath)
        .pipe(lr ? plumber()() : gutil.noop())
        .pipe(sourcemaps().init())
        .pipe(sass()({
          includePaths: includePaths
        }))
        .pipe(sourcemaps().write())
        .pipe(rename()(skin + '.css'))
        .pipe(gulp.dest(dest))
        .pipe(lr ? lr() : gutil.noop());
    },
    server: function () {
      connect().server({
        root: [
          getServer().localServerDir(),
          'web'
        ],
        port: getServer().localServerPort(),
        middleware: function () {
          return [cors];
        }
      });
    },
    vars: function () {
      var baseFeaturePath = 'res/feature/responsivebase';
      var peakFeaturePath = 'res/feature/responsivepeak';
      var baseSassPath = 'res/skins/bootstrap_base/sass';
      var peakSassPath = 'res/skins/responsive_peak/sass';
      var varsPath = 'codebook/_variables-*.scss';

      function getSkinVersions(dir) {
        try {
          fs().accessSync(dir, fs().F_OK);
          return fs().readdirSync(dir)
            .filter(function(file) {
              return fs().statSync(path().join(dir, file)).isDirectory();
            });
        } catch(e) {
          gutil.log(gutil.colors.red('Warning: Skin feature path not found (this is expected for <= 15.11): ', dir));
          return [];
        }
      }

      var peakVars = getSkinVersions(peakFeaturePath).map(function(version) {
        return gulp.src([path().join(baseFeaturePath, version, '**', varsPath),
            path().join(peakFeaturePath, version, '**', varsPath)])
          .pipe(newer()(path().join(peakFeaturePath, version, peakSassPath, '_variables.scss')))
          .pipe(replace()('!default', ''))
          .pipe(replace()(/^\s*(\$.*)/gm, '//$1'))
          .pipe(replace()(/(\s*);/gm, ';'))
          .pipe(rename()('_variables.scss'))
          .pipe(gulpif()(function(file){ return file.path.indexOf(baseFeaturePath) >= 0; },
            gulp.dest(path().join(baseFeaturePath, version, baseSassPath))))
          .pipe(concat()('_variables.scss'))
          .pipe(gulp.dest(path().join(peakFeaturePath, version, peakSassPath)));
      });
      return peakVars;
    },
    FILES_PATTERN: SKIN_FILES_PATTERN
  };
};
<|endoftext|>


<commit_before>/**
 * Provides methods for working with the job queue
 **/
var generateJob = require('../util/utility').generateJob;
var config = require('../config');
var bull = require('bull');
var url = require('url');
var async = require('async');
// parse the url
var conn_info = url.parse(config.REDIS_URL, true /* parse query string */ );
if (conn_info.protocol !== 'redis:')
{
    throw new Error('connection string must use the redis: protocol');
}
var options = {
    port: conn_info.port || 6379,
    host: conn_info.hostname,
    options: conn_info.query
};
if (conn_info.auth)
{
    options.redis.auth = conn_info.auth.replace(/.*?:/, '');
}

function generateKey(type, state)
{
    return ["bull", type, state].join(":");
}

function getQueue(type)
{
    return bull(type, options.port, options.host);
}

function addToQueue(queue, payload, options, cb)
{
    var job = generateJob(queue.name, payload);
    options.attempts = options.attempts || 15;
    options.backoff = options.backoff ||
    {
        delay: 60 * 1000,
        type: 'exponential'
    };
    queue.add(job, options).then(function(queuejob)
    {
        //console.log("created %s jobId: %s", queue.name, queuejob.jobId);
        cb(null, queuejob);
    }).catch(cb);
}

function getCounts(redis, cb)
{
    var types = ["request", "mmr", "parse", "cache"];
    async.map(types, getQueueCounts, function(err, result)
    {
        var obj = {};
        result.forEach(function(r, i)
        {
            obj[types[i]] = r;
        });
        cb(err, obj);
    });

    function getQueueCounts(type, cb)
    {
        async.series(
        {
            "wait": function(cb)
            {
                redis.llen(generateKey(type, "wait"), cb);
            },
            "act": function(cb)
            {
                redis.llen(generateKey(type, "active"), cb);
            },
            "del": function(cb)
            {
                redis.zcard(generateKey(type, "delayed"), cb);
            },
            "comp": function(cb)
            {
                redis.scard(generateKey(type, "completed"), cb);
            },
            "fail": function(cb)
            {
                redis.scard(generateKey(type, "failed"), cb);
            }
        }, cb);
    }
}

function cleanup(redis, cb)
{
    var types = ["request", "mmr", "parse", "cache"];
    async.each(types, function(key, cb)
    {
        var queue = getQueue(key);
        async.each(['completed', 'failed', 'delayed'], function(type, cb)
        {
            queue.clean(24 * 60 * 60 * 1000, type);
            queue.once('cleaned', function(job, type)
            {
                console.log('cleaned %s %s jobs from queue %s', job.length, type, key);
                cb();
            });
        }, cb);
    }, cb);
}
module.exports = {
    getQueue: getQueue,
    addToQueue: addToQueue,
    getCounts: getCounts,
    cleanup: cleanup
};<commit_msg>refactor types array in queue.js<commit_after>/**
 * Provides methods for working with the job queue
 **/
var generateJob = require('../util/utility').generateJob;
var config = require('../config');
var bull = require('bull');
var url = require('url');
var async = require('async');
var types = ["request", "mmr", "parse", "cache", "fullhistory"];
// parse the url
var conn_info = url.parse(config.REDIS_URL, true /* parse query string */ );
if (conn_info.protocol !== 'redis:')
{
    throw new Error('connection string must use the redis: protocol');
}
var options = {
    port: conn_info.port || 6379,
    host: conn_info.hostname,
    options: conn_info.query
};
if (conn_info.auth)
{
    options.redis.auth = conn_info.auth.replace(/.*?:/, '');
}

function generateKey(type, state)
{
    return ["bull", type, state].join(":");
}

function getQueue(type)
{
    return bull(type, options.port, options.host);
}

function addToQueue(queue, payload, options, cb)
{
    var job = generateJob(queue.name, payload);
    options.attempts = options.attempts || 15;
    options.backoff = options.backoff ||
    {
        delay: 60 * 1000,
        type: 'exponential'
    };
    queue.add(job, options).then(function(queuejob)
    {
        //console.log("created %s jobId: %s", queue.name, queuejob.jobId);
        cb(null, queuejob);
    }).catch(cb);
}

function getCounts(redis, cb)
{
    async.map(types, getQueueCounts, function(err, result)
    {
        var obj = {};
        result.forEach(function(r, i)
        {
            obj[types[i]] = r;
        });
        cb(err, obj);
    });

    function getQueueCounts(type, cb)
    {
        async.series(
        {
            "wait": function(cb)
            {
                redis.llen(generateKey(type, "wait"), cb);
            },
            "act": function(cb)
            {
                redis.llen(generateKey(type, "active"), cb);
            },
            "del": function(cb)
            {
                redis.zcard(generateKey(type, "delayed"), cb);
            },
            "comp": function(cb)
            {
                redis.scard(generateKey(type, "completed"), cb);
            },
            "fail": function(cb)
            {
                redis.scard(generateKey(type, "failed"), cb);
            }
        }, cb);
    }
}

function cleanup(redis, cb)
{
    async.each(types, function(key, cb)
    {
        var queue = getQueue(key);
        async.each(['completed', 'failed', 'delayed'], function(type, cb)
        {
            queue.clean(24 * 60 * 60 * 1000, type);
            queue.once('cleaned', function(job, type)
            {
                console.log('cleaned %s %s jobs from queue %s', job.length, type, key);
                cb();
            });
        }, cb);
    }, cb);
}
module.exports = {
    getQueue: getQueue,
    addToQueue: addToQueue,
    getCounts: getCounts,
    cleanup: cleanup
};
<|endoftext|>


<commit_before>/*
 *   Copyright 2014-2016 Guy Bedford (http://guybedford.com)
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

var chalk = require('chalk');

var apiResolver;
var logging = true;

var logBuffer = '';
var errBuffer = '';

var logged = false;
var cli = module.exports;
var Promise = require('bluebird');

var ui = exports;

var readline = require('readline');

var Writable = require('stream').Writable;

function MutableStdout(opts) {
  Writable.call(this, opts);
}
MutableStdout.prototype = Object.create(Writable.prototype);
MutableStdout.prototype._write = function(chunk, encoding, callback) {
  if (!this.muted)
    process.stdout.write(chunk, encoding);
  callback();
};

var mutableStdout = new MutableStdout();

ui.logLevel = 3; // corresponds to 'info'
var logTypes = ['err', 'warn', 'ok', 'info', 'debug'];

// if storing any logs, dump them on exit
var inputRedact;
process.on('exit', function() {
  if (!apiResolver) {
    if (inputRedact) {
      console.log('');
      inputRedact();
    }
    if (logBuffer)
      process.stdout.write(logBuffer);
    if (errBuffer)
      process.stderr.write(errBuffer);
    console.log('');
  }
});

exports.setResolver = function(resolver) {
  apiResolver = resolver;
};

// use the default option in all prompts
// throws an error for prompts that don't take a default
var useDefaults;
exports.useDefaults = function(_useDefaults) {
  if (_useDefaults === undefined)
    _useDefaults = true;
  useDefaults = _useDefaults;
};

exports.setLogLevel = function(level) {
  if (!level)
    return;

  var levelIndex = logTypes.indexOf(level);

  if (levelIndex == -1)
    ui.log('warn', 'Unknown log level: ' + level);
  else
    ui.logLevel = levelIndex;
};

var format = exports.format = {
  q: function(msg, opt) {
    return moduleMsg(msg + (opt ? ' [' + opt + ']' : '') + ': ');
  },
  err: function(msg) {
    return '\n' + chalk.red.bold('err  ') + moduleMsg(msg, false, false);
  },
  info: function(msg) {
    return '     ' + moduleMsg(msg, true);
  },
  warn: function(msg) {
    return '\n' + chalk.yellow.bold('warn ') + moduleMsg(msg, true);
  },
  ok: function(msg) {
    return chalk.green.bold('ok   ') + moduleMsg(msg, true);
  }
};

exports.log = function(type, msg) {
  if (apiResolver)
    return apiResolver.emit('log', type, msg);

  logged = true;

  if (arguments.length === 1) {
    msg = type;
    type = null;
  }

  msg = msg || '';

  msg = msg.toString();

  if (type)
    msg = (format[type] || format.info)(msg.toString());

  var logLevel = logTypes.indexOf(type);
  if (logLevel == -1)
    logLevel = 3;
  if (logLevel <= ui.logLevel) {
    if (logging) {
      if (type != 'err')
        console.log(msg);
      else
        console.error(msg);
    }
    else {
      if (type != 'err')
        logBuffer += msg + '\n';
      else
        errBuffer += msg + '\n';
    }
  }
};

function highlight(text) {
  return text
    .replace(/([\s\`\(]|^)%([^%]+)%([\s\`\)\r\n\,\.]|$)/g, '$1' + chalk.bold('$2') + '$3')
    .replace(/([\s\`\(]|^)\`([^\`]+)\`([\s\`\)\r\n\,\.]|$)/g, '$1' + chalk.cyan('$2') + '$3')
    .replace(/([\s\`\%]|^)\_([^\_]+)\_([\s\`\%\r\n\,\.]|$)/g, '$1' + chalk.underline('$2') + '$3')
    .replace(/\t/g, '  ');
}

exports.wordWrap = wordWrap;
function wordWrap(text, columns, leftIndent, rightIndent, skipFirstLineIndent) {
  leftIndent = leftIndent || 0;
  rightIndent = rightIndent || 0;

  var leftSpaces = '';
  var i;

  for (i = 0; i < leftIndent; i++)
    leftSpaces += ' ';

  // simple word wrapping
  columns = columns - leftIndent - rightIndent;
  var output = [];
  var lastBreak = 0;
  var lastSpace = 0;
  var skipLength = 0; // skip control characters
  var controlStart = false;
  for (i = 0; i < text.length; i++) {
    if (text[i] == '`' || text[i] == '%' || text[i] == '_') {
      if (controlStart && (('\r\n `,.' + (text[i] == '_' ? '%' : ')')).indexOf(text[i + 1]) != -1 || !text[i + 1])) {
        controlStart = false;
        skipLength++;
        continue;
      }
      else if (lastBreak == lastSpace || !text[i - 1] || (' `' + (text[i] == '_' ? '%' : '(')).indexOf(text[i - 1]) != -1) {
        controlStart = true;
        skipLength++;
        continue;
      }
    }
    if (text[i] == ' ')
      lastSpace = i;

    // existing newline adds buffer
    if (text[i] == '\n') {
      lastSpace = i;
      output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substring(lastBreak, i) + '\n');
      lastSpace = i;
      lastBreak = i + 1;
      skipLength = 0;
    }

    // on expected breaking point
    else if ((i  - skipLength) != lastBreak &&
          (i - lastBreak - skipLength) % columns == 0 &&
          lastSpace > lastBreak) {
      output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substring(lastBreak, lastSpace) + '\n');
      lastBreak = lastSpace + 1;
      skipLength = 0;
    }
  }
  output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substr(lastBreak));
  return output.join('');
}

function moduleMsg(msg, tab, wrap) {
  if (tab)
    msg = wordWrap(msg, process.stdout.columns, tab ? 5 : 0).substr(5);
  else if (wrap)
    msg = wordWrap(msg, process.stdout.columns);

  // formatting
  return highlight(msg);
}

var inputQueue = [];
var confirm = exports.confirm = function(msg, def, options) {
  if (typeof def == 'object') {
    options = def;
    def = undefined;
  }
  options = options || {};

  if (apiResolver)
    return new Promise(function(resolve) {
      apiResolver.emit('prompt', {
        type: 'confirm',
        message: msg,
        default: def
      }, function(answer) {
        resolve(answer);
      });
    });

  var defText;
  if (def === true)
    defText = 'Yes';
  else if (def === false)
    defText = 'No';
  else
    def = undefined;

  if (useDefaults) {
    if (def === undefined)
      defText = 'Yes';
    process.stdout.write(format.q(msg) + defText + '\n');
    return Promise.resolve(!!def);
  }

  return cli.input(msg + (defText ? '' : ' [Yes/No]'), defText, {
    edit: options.edit,
    info: options.info,
    silent: options.silent,
    options: ['Yes', 'No']
  })
  .then(function(reply) {
    if (reply == 'Yes')
      return true;
    if (reply == 'No')
      return false;
    return confirm(msg, def, options);
  });
};

var inputFunc = function() {};
process.stdin.on('keypress', function(chunk, key) {
  inputFunc(key);
});

exports.input = function(msg, def, options, queue) {
  if (typeof def == 'object') {
    options = def;
    def = undefined;
  }
  options = options || {};
  if (typeof options == 'boolean')
    options = {
      silent: true,
      edit: false,
      clearOnType: false,
      info: undefined,
      completer: undefined,
      validate: undefined,
      validationError: undefined
    };

  options.completer = function(partialLine) {
    if (!partialLine.length || !options.options)
      return [[], partialLine];

    partialLine = partialLine.toLowerCase().trim();

    var hits = options.options.filter(function(c) {
      return c.toLowerCase().indexOf(partialLine) == 0;
    });

    // we only return one option, to avoid printing out completions
    return [hits.length ? [hits[0]] : [], partialLine];
  };

  options = options || {};

  if (useDefaults) {
    process.stdout.write(format.q(msg) + def + '\n');
    return Promise.resolve(def);
  }

  if (apiResolver)
    return new Promise(function(resolve) {
      apiResolver.emit('prompt', {
        type: 'input',
        message: msg,
        default: def
      }, function(answer) {
        resolve(answer.input);
      });
    });

  return new Promise(function(resolve, reject) {
    if (!logging && !queue)
      return inputQueue.push({
        args: [msg, def, options, true],
        resolve: resolve,
        reject: reject
      });

    // if there is info text for this question, first write out the hint text below
    // then restore the cursor and write out the question
    var infoLines = 1;
    if (options.info || options.validationError) {
      var infoText = '\n';
      if (options.info)
        infoText += highlight(wordWrap(options.info, process.stdout.columns, 5, 5)) + '\n';
      if (options.validationError)
        infoText += format.warn(wordWrap(options.validationError, process.stdout.columns, 0, 5)) + '\n';
      infoText += '\n';
      process.stdout.write(infoText);
      infoLines = infoText.split('\n').length;
    }

    if (logging && logged)
      process.stdout.write('\n');
    logging = false;

    var rl = readline.createInterface({
      input: process.stdin,
      output: mutableStdout,
      terminal: true,
      completer: options.completer
    });

    var questionMessage = format.q(msg, !options.edit && def || '');
    infoLines += questionMessage.split('\n').length - 1;

    if (options.clearOnType && options.edit)
      inputFunc = function(key) {
        if (key.name != 'return' && key.name != 'backspace' && key.name != 'left') {
          var len = (def || '').length;
          // this actually works, which was rather unexpected
          for (var i = 0; i < len; i++)
            process.stdin.emit('keypress', '\b', { name: 'backspace' });
        }
        inputFunc = function() {};
      };

    rl.question(questionMessage, function(inputVal) {
      inputFunc = function() {};
      if (mutableStdout.muted)
        console.log('');
      rl.close();

      // run completions by default on enter when provided
      if (options.options)
        inputVal = options.completer(inputVal)[0][0] || '';

      inputVal = inputVal.trim();

      if (!options.edit)
        inputVal = inputVal || def;

      var retryErr;
      if (options.validate)
        retryErr = options.validate(inputVal);

      inputRedact(inputVal, !!retryErr);
      inputRedact = null;

      if (retryErr)
        return cli.input(msg, def, {
          info: options.info,
          silent: options.silent,
          edit: options.edit,
          clearOnType: options.clearOnType,
          completer: options.completer,
          validate: options.validate,
          validationError: retryErr.toString()
        }, true).then(resolve, reject);

      // bump the input queue
      var next = inputQueue.shift();
      if (next)
        cli.input.apply(null, next.args).then(next.resolve, next.reject);
      else {
        process.stdout.write(logBuffer);
        process.stderr.write(errBuffer);
        logBuffer = '';
        errBuffer = '';
        logging = true;
        logged = false;
      }

      resolve(inputVal);
    });

    inputRedact = function(inputVal, retry) {
      inputVal = inputVal || '';
      var inputText = !retry ? format.q(msg, !options.edit && def || '') + inputVal + '\n' : '';
      // erase the help text, rewrite the question above, and begin the next question
      process.stdout.write('\033[' + infoLines + 'A\033[J' + inputText);
    };

    mutableStdout.muted = false;

    if (options.silent)
      mutableStdout.muted = true;

    if (def && options.edit)
      rl.write(def);
  });
};
<commit_msg>ui control character adjustemnts<commit_after>/*
 *   Copyright 2014-2016 Guy Bedford (http://guybedford.com)
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

var chalk = require('chalk');

var apiResolver;
var logging = true;

var logBuffer = '';
var errBuffer = '';

var logged = false;
var cli = module.exports;
var Promise = require('bluebird');

var ui = exports;

var readline = require('readline');

var Writable = require('stream').Writable;

function MutableStdout(opts) {
  Writable.call(this, opts);
}
MutableStdout.prototype = Object.create(Writable.prototype);
MutableStdout.prototype._write = function(chunk, encoding, callback) {
  if (!this.muted)
    process.stdout.write(chunk, encoding);
  callback();
};

var mutableStdout = new MutableStdout();

ui.logLevel = 3; // corresponds to 'info'
var logTypes = ['err', 'warn', 'ok', 'info', 'debug'];

// if storing any logs, dump them on exit
var inputRedact;
process.on('exit', function() {
  if (!apiResolver) {
    if (inputRedact) {
      console.log('');
      inputRedact();
    }
    if (logBuffer)
      process.stdout.write(logBuffer);
    if (errBuffer)
      process.stderr.write(errBuffer);
    console.log('');
  }
});

exports.setResolver = function(resolver) {
  apiResolver = resolver;
};

// use the default option in all prompts
// throws an error for prompts that don't take a default
var useDefaults;
exports.useDefaults = function(_useDefaults) {
  if (_useDefaults === undefined)
    _useDefaults = true;
  useDefaults = _useDefaults;
};

exports.setLogLevel = function(level) {
  if (!level)
    return;

  var levelIndex = logTypes.indexOf(level);

  if (levelIndex == -1)
    ui.log('warn', 'Unknown log level: ' + level);
  else
    ui.logLevel = levelIndex;
};

var format = exports.format = {
  q: function(msg, opt) {
    return moduleMsg(msg + (opt ? ' [' + opt + ']' : '') + ': ');
  },
  err: function(msg) {
    return '\n' + chalk.red.bold('err  ') + moduleMsg(msg, false, false);
  },
  info: function(msg) {
    return '     ' + moduleMsg(msg, true);
  },
  warn: function(msg) {
    return '\n' + chalk.yellow.bold('warn ') + moduleMsg(msg, true);
  },
  ok: function(msg) {
    return chalk.green.bold('ok   ') + moduleMsg(msg, true);
  }
};

exports.log = function(type, msg) {
  if (apiResolver)
    return apiResolver.emit('log', type, msg);

  logged = true;

  if (arguments.length === 1) {
    msg = type;
    type = null;
  }

  msg = msg || '';

  msg = msg.toString();

  if (type)
    msg = (format[type] || format.info)(msg.toString());

  var logLevel = logTypes.indexOf(type);
  if (logLevel == -1)
    logLevel = 3;
  if (logLevel <= ui.logLevel) {
    if (logging) {
      if (type != 'err')
        console.log(msg);
      else
        console.error(msg);
    }
    else {
      if (type != 'err')
        logBuffer += msg + '\n';
      else
        errBuffer += msg + '\n';
    }
  }
};

function isControlTerminateChar(c, control) {
  return !c || ('\r\n ,.' + (control == '_' ? '%' : '')).indexOf(c) != -1;
}

function highlight(text) {
  return text
    .replace(/([\s\`\(]|^)%([^%]+)%/g, function(match, index, capture) {
      if (isControlTerminateChar(text[index + 1], '%'))
        return match.replace('%' + capture + '%', chalk.bold(capture));
      else
        return match;
    })
    .replace(/([\s\`\(]|^)\`([^\`]+)\`/g, function(match, index, capture) {
      if (isControlTerminateChar(text[index + 1], '`'))
        return match.replace('`' + capture + '`', chalk.cyan(capture));
      else
        return match;
    })
    .replace(/([\s\`\%]|^)\_([^\_]+)\_/g, function(match, index, capture) {
      if (isControlTerminateChar(text[index + 1], '_'))
        return match.replace('_' + capture + '_', chalk.underline(capture));
      else
        return match;
    })
    .replace(/\t/g, '  ');
}

exports.wordWrap = wordWrap;
function wordWrap(text, columns, leftIndent, rightIndent, skipFirstLineIndent) {
  leftIndent = leftIndent || 0;
  rightIndent = rightIndent || 0;

  var leftSpaces = '';
  var i;

  for (i = 0; i < leftIndent; i++)
    leftSpaces += ' ';

  // simple word wrapping
  columns = columns - leftIndent - rightIndent;
  var output = [];
  var lastBreak = 0;
  var lastSpace = 0;
  var skipLength = 0; // skip control characters
  var controlStart = false;
  for (i = 0; i < text.length; i++) {
    if (text[i] == '`' || text[i] == '%' || text[i] == '_') {
      if (controlStart && isControlTerminateChar(text[i + 1], text[i])) {
        controlStart = false;
        skipLength++;
        continue;
      }
      else if (lastBreak == lastSpace || !text[i - 1] || (' `' + (text[i] == '_' ? '%' : '(')).indexOf(text[i - 1]) != -1) {
        controlStart = true;
        skipLength++;
        continue;
      }
    }
    if (text[i] == ' ')
      lastSpace = i;

    // existing newline adds buffer
    if (text[i] == '\n') {
      lastSpace = i;
      output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substring(lastBreak, i) + '\n');
      lastSpace = i;
      lastBreak = i + 1;
      skipLength = 0;
    }

    // on expected breaking point
    else if ((i  - skipLength) != lastBreak &&
          (i - lastBreak - skipLength) % columns == 0 &&
          lastSpace > lastBreak) {
      output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substring(lastBreak, lastSpace) + '\n');
      lastBreak = lastSpace + 1;
      skipLength = 0;
    }
  }
  output.push((output.length == 0 && skipFirstLineIndent ? '' : leftSpaces) + text.substr(lastBreak));
  return output.join('');
}

function moduleMsg(msg, tab, wrap) {
  if (tab)
    msg = wordWrap(msg, process.stdout.columns, tab ? 5 : 0).substr(5);
  else if (wrap)
    msg = wordWrap(msg, process.stdout.columns);

  // formatting
  return highlight(msg);
}

var inputQueue = [];
var confirm = exports.confirm = function(msg, def, options) {
  if (typeof def == 'object') {
    options = def;
    def = undefined;
  }
  options = options || {};

  if (apiResolver)
    return new Promise(function(resolve) {
      apiResolver.emit('prompt', {
        type: 'confirm',
        message: msg,
        default: def
      }, function(answer) {
        resolve(answer);
      });
    });

  var defText;
  if (def === true)
    defText = 'Yes';
  else if (def === false)
    defText = 'No';
  else
    def = undefined;

  if (useDefaults) {
    if (def === undefined)
      defText = 'Yes';
    process.stdout.write(format.q(msg) + defText + '\n');
    return Promise.resolve(!!def);
  }

  return cli.input(msg + (defText ? '' : ' [Yes/No]'), defText, {
    edit: options.edit,
    info: options.info,
    silent: options.silent,
    options: ['Yes', 'No']
  })
  .then(function(reply) {
    if (reply == 'Yes')
      return true;
    if (reply == 'No')
      return false;
    return confirm(msg, def, options);
  });
};

var inputFunc = function() {};
process.stdin.on('keypress', function(chunk, key) {
  inputFunc(key);
});

exports.input = function(msg, def, options, queue) {
  if (typeof def == 'object') {
    options = def;
    def = undefined;
  }
  options = options || {};
  if (typeof options == 'boolean')
    options = {
      silent: true,
      edit: false,
      clearOnType: false,
      info: undefined,
      completer: undefined,
      validate: undefined,
      validationError: undefined
    };

  options.completer = function(partialLine) {
    if (!partialLine.length || !options.options)
      return [[], partialLine];

    partialLine = partialLine.toLowerCase().trim();

    var hits = options.options.filter(function(c) {
      return c.toLowerCase().indexOf(partialLine) == 0;
    });

    // we only return one option, to avoid printing out completions
    return [hits.length ? [hits[0]] : [], partialLine];
  };

  options = options || {};

  if (useDefaults) {
    process.stdout.write(format.q(msg) + def + '\n');
    return Promise.resolve(def);
  }

  if (apiResolver)
    return new Promise(function(resolve) {
      apiResolver.emit('prompt', {
        type: 'input',
        message: msg,
        default: def
      }, function(answer) {
        resolve(answer.input);
      });
    });

  return new Promise(function(resolve, reject) {
    if (!logging && !queue)
      return inputQueue.push({
        args: [msg, def, options, true],
        resolve: resolve,
        reject: reject
      });

    // if there is info text for this question, first write out the hint text below
    // then restore the cursor and write out the question
    var infoLines = 1;
    if (options.info || options.validationError) {
      var infoText = '\n';
      if (options.info)
        infoText += highlight(wordWrap(options.info, process.stdout.columns, 5, 5)) + '\n';
      if (options.validationError)
        infoText += format.warn(wordWrap(options.validationError, process.stdout.columns, 0, 5)) + '\n';
      infoText += '\n';
      process.stdout.write(infoText);
      infoLines = infoText.split('\n').length;
    }

    if (logging && logged)
      process.stdout.write('\n');
    logging = false;

    var rl = readline.createInterface({
      input: process.stdin,
      output: mutableStdout,
      terminal: true,
      completer: options.completer
    });

    var questionMessage = format.q(msg, !options.edit && def || '');
    infoLines += questionMessage.split('\n').length - 1;

    if (options.clearOnType && options.edit)
      inputFunc = function(key) {
        if (key.name != 'return' && key.name != 'backspace' && key.name != 'left') {
          var len = (def || '').length;
          // this actually works, which was rather unexpected
          for (var i = 0; i < len; i++)
            process.stdin.emit('keypress', '\b', { name: 'backspace' });
        }
        inputFunc = function() {};
      };

    rl.question(questionMessage, function(inputVal) {
      inputFunc = function() {};
      if (mutableStdout.muted)
        console.log('');
      rl.close();

      // run completions by default on enter when provided
      if (options.options)
        inputVal = options.completer(inputVal)[0][0] || '';

      inputVal = inputVal.trim();

      if (!options.edit)
        inputVal = inputVal || def;

      var retryErr;
      if (options.validate)
        retryErr = options.validate(inputVal);

      inputRedact(inputVal, !!retryErr);
      inputRedact = null;

      if (retryErr)
        return cli.input(msg, def, {
          info: options.info,
          silent: options.silent,
          edit: options.edit,
          clearOnType: options.clearOnType,
          completer: options.completer,
          validate: options.validate,
          validationError: retryErr.toString()
        }, true).then(resolve, reject);

      // bump the input queue
      var next = inputQueue.shift();
      if (next)
        cli.input.apply(null, next.args).then(next.resolve, next.reject);
      else {
        process.stdout.write(logBuffer);
        process.stderr.write(errBuffer);
        logBuffer = '';
        errBuffer = '';
        logging = true;
        logged = false;
      }

      resolve(inputVal);
    });

    inputRedact = function(inputVal, retry) {
      inputVal = inputVal || '';
      var inputText = !retry ? format.q(msg, !options.edit && def || '') + inputVal + '\n' : '';
      // erase the help text, rewrite the question above, and begin the next question
      process.stdout.write('\033[' + infoLines + 'A\033[J' + inputText);
    };

    mutableStdout.muted = false;

    if (options.silent)
      mutableStdout.muted = true;

    if (def && options.edit)
      rl.write(def);
  });
};
<|endoftext|>