<commit_before>"use strict";
var _u = require('underscore'), invoker = require('./inject'),
    util = require('./util'),
    _q = require('./query'),
    domain = require('domain'),
    populate = _q.populate,
    sort = _q.sort,
    paginate = _q.paginate,
    clean = _q.clean,
    filter = _q.filter,
    idObj = _q.idObj,
    streams = require('./streams'),
    Stream = require('stream').Stream,
    w = require('./when'), promise = w.promise, domain = require('domain');
;
module.exports = function MongooseRestRouter(base, router, options) {
    var ResponseStream = options.responseStream,
        conn = options.conn || options.mongoose && options.mongoose.connection || options.mongoose,
        ModelType = conn.Model || conn.base.Model,
        reGet = new RegExp('^\\' + base + '\/(.*)$'), ppush = Function.apply.bind(Array.prototype.push);

    var api = router.route(reGet);
    api.all(function (req, res, next) {
        var p = req.params = normalize(req.params);
        var Model = m(p.shift());
        if (!Model) {
            return next('No model of type ' + req.params.type);
        }
        req.Model = Model;
        delete req.params.type;
        var d = req._domain = domain.create();
        d.on('error', function (e) {
            console.log('error', e);
            return options.error(e, req, res);
        });
        d.add(req);
        d.add(res);
        d.run(next);
    });

    function m(model) {
        return _q.m(conn, model);
    }

    function transform(stream, ctx) {
        return options.transformer.pump(stream, ctx);
    }

    function run(ctx, res) {

        return function run$return(e, o) {
            if (e) {
                return options.error(e, null, res);
            }
            o = o instanceof Stream ? o : streams.ToStream(o);
            transform(o, ctx).pipe(options.responseStream({
                response: {},
                single: true
            })).pipe(streams.ToJson()).pipe(streams.Stringify()).pipe(res);
        }
    }

    var finder = function (Model, find, req, res, next) {
        var single = req.query.single;
        return function (err, count) {
            if (err) return next(err);
            populate(find, req.query);
            paginate(find, req.query);
            sort(find, req.query);
            var stream = transform(createstream(find, req, res, next), req.query, Model);
            if (req.query.filter && Model) {
                filter(find, req.query, Model);
                delete find.options.sort;
                find.count(function (err, fcount) {
                    if (err) return next(err);
                    stream.pipe(new ResponseStream({total: count, filterTotal: fcount}, single)).pipe(res);
                });
            } else {
                stream.pipe(new ResponseStream({total: count}, single)).pipe(res);
            }
        }
    };

    /* router.get(base + '/:type', function (req, res, next) {
     var Model = req.Model;
     Model.count({}, finder(Model, Model.find({}), req, res, next));
     });
     */

    function normalize(params) {
        var ret = [];
        if (!params) return ret;
        if (_u.isArray(params)) {
            for (var i = 0, l = params.length; i < l; i++) {
                var v = params[i];

                ppush(ret, v && v.split ? v.split(/\//g) : v == null ? [] : [v]);
            }
        } else {
            Object.keys(params).sort().forEach(function (k) {
                var v = params[k];

                ppush(ret, v && v.split ? v.split(/\//g) : v == null ? [] : [v]);
            })
        }
        ret = _u.compact(ret);
        return ret;
    }

    function ctx(req) {
        return _u.pick(req, 'query', 'session', 'params', 'body', 'Model', 'meta');
    }

    api.get(function (req, res, next) {
            var params = req.params, find, Model = req.Model, id = params.shift(), single = req.query.single;
            //    var single = 'single' in req.query ? req.query.single : (params.length + (id == "finder" ? -1 : 0) % 2 == 0);
            var _ctx = ctx(req);

            //    transform(createstream(cbs, req, res, next), req.query, Model).pipe(new ResponseStream(null, single)).pipe(res);
            var single = params.length % 2 === 0;

            //swallow finders
            if (id === 'finder' || typeof Model[params[0]] === 'function') {
               // find = Model;
                var m = params.shift();
                find = function(){
                    return invoker.resolve(Model[m], Model, _ctx);
                }
            } else {

                if (id) {
                    single = params.length === 0 ? true : single;

                    find = function () {
                        return Model.findOne(idObj(id));
                    }
                    // find = Model.findOne.bind(Model, idObj(id));
                } else {
                    // find = Model.find.bind(Model, {});
                    single = params.length === 0 ? false : single;
                    find = function () {
                        return Model.find({});
                    }

                }
                if (params.length) {
                    populate(find, {populate: [params.join('.')]});
                }
            }
            invoker.invoke(find, params, _ctx, get$advice, function (e, o) {
                if (e) {
                    return options.error(e, null, res);
                }
                o = o instanceof Stream ? o : streams.ToStream(o);
                transform(o, _ctx).pipe(options.responseStream({
                    single: ('single' in req.query) ? req.query.single : single,
                    response: {total: _ctx.body.count}
                })).pipe(streams.ToJson()).pipe(streams.Stringify()).pipe(res);
            });

        }
    );

    function get$advice(str, ctx, obj, advice, nValue) {
        if (isF(nValue, 'where')) {
            _q.filter(nValue, ctx.query, ctx.Model);
        }

        if (ctx.query.single == true) {
            if (isF(nValue, 'stream')) {
                return advice(nValue.stream());
            }
            return advice(nValue);
        }
        if (str.length === 0 && isF(nValue, 'count')) {
            populate(nValue, ctx.query);
            paginate(nValue, ctx.query);
         //   var _sort = nValue.options.sort;
         //  delete nValue.options.sort;
            var op = nValue.op;
            nValue.count(function (e, c) {
                if (e) return advice(e);
                nValue.op = op;
                sort(nValue, ctx.query);
                ctx.body.count = c;

                advice(nValue.stream());
            });
        } else {
            advice(nValue);
        }

    }

    function isF(obj, f) {
        var args = Array.prototype.slice.call(arguments, 1);
        if (!obj) return false;
        while (args.length) {
            if (!(typeof obj[args.shift()] === 'function'))
                return false;
        }
        return true;
    }

    function put$advice(str, ctx, obj, advice, nValue) {
        //Can't save embedded documents, so we will do this.  There is a possibility that somewhere in the chain we get a model that is not the current var Model.
        // We should be OK.
        var saveTo = nValue instanceof ModelType ? (ctx.model = nValue) : ctx.model;
        if (str.length === 0) {
            nValue.set(ctx.body);
            return saveTo.save(function (e, o) {
                return advice(e || nValue);
            });
        }
        advice(nValue);
    }

    api.put(function (req, res, next) {
        var params = req.params, Model = req.Model, _ctx = ctx(req);
        invoker.invoke(Model.findOne(idObj(params.shift())), params, _ctx, put$advice, run(_ctx, res));
    });

    function post$advice(str, ctx, obj, advice, nValue) {
        //Can't save embedded documents, so we will do this.  There is a possibility that somewhere in the chain we get a model that is not the current var Model.
        // We should be OK.
        var saveTo;
        if (nValue instanceof ModelType) {
            saveTo = ctx.model = nValue;
        } else {
            saveTo = ctx.model;
        }

        if (saveTo && str.length === 0) {
            var idx = str.shift();
            if (Array.isArray(nValue)) {
                if (/^\d+?$/.test(idx)) {
                    nValue.splice(idx, ctx.body, 0);
                } else {
                    nValue.push(ctx.body);
                    idx = nValue.length - 1;
                }
            } else {
                if (idx) {
                    obj.set(idx, ctx.body)
                } else {
                    try {
                        saveTo.set(ctx.body);
                    } catch (e) {
                        console.log('error ', e);
                    }
                }
            }
            return saveTo.save(function (e, o) {
                if (e)
                    return advice(e);
                return advice(idx !== void(0) ? nValue[idx] : o);
            });
        }
        advice(nValue);

    }

    api.delete(function (req, res, next) {
        var path = (req.params[0] || '').split(/\/+?/g);
        var type = req.Model;

        type.findOne(idObj(req), function (err, doc) {
            if (err)           return next(err);
            var obj = doc;
            while (path.length > 1) {
                obj = obj[path.shift()];
            }
            if (obj)
                obj.pull(path.shift());

            doc.save(function (e) {
                if (e)           return next(err);
                res.send({
                    status: 0
                })

            })
        });

    });
    api.post(function (req, res, next) {
        var params = req.params, id = params.shift(), Model = req.Model, _ctx = ctx(req), cb = run(_ctx, res);
        if (id) {
            if (!Model[id]) {
                if (!params.length) {
                    return cb({
                        message: 'Invalid Post'
                    });
                }
                params.unshift(function(){
                 return   Model.findOne(idObj(id));
                });
            } else {
                params.unshift(id);
            }
        }
        invoker.invoke(Model, params, _ctx, post$advice, cb);

    });
    /*
     router.post(base + '/:type', function (req, res) {
     var Model = m(req);
     new Model(clean(req.body)).save(run({total: 1}, res, Model, req.query));
     });
     */
    /*
     api.route('/:id')
     .put(function (req, res, next) {
     var Model = req.Model;
     Model.findOne(idObj(req), function (err, obj) {
     if (err)return next(err);
     _u.extend(obj, clean(req.body));
     obj.save(run({total: 1}, res, Model, req.query));
     });

     }).delete(function (req, res, next) {
     req.Model.findOneAndRemove(idObj(req), function (err) {
     if (err)           return next(err);
     res.send({
     status: 0,
     payload: []
     })
     });
     });
     */
};
<commit_msg>83 of 85 pass<commit_after>"use strict";
var _u = require('underscore'), invoker = require('./inject'),
    util = require('./util'),
    _q = require('./query'),
    domain = require('domain'),
    populate = _q.populate,
    sort = _q.sort,
    paginate = _q.paginate,
    clean = _q.clean,
    filter = _q.filter,
    idObj = _q.idObj,
    streams = require('./streams'),
    Stream = require('stream').Stream,
    w = require('./when'), promise = w.promise, domain = require('domain');
;
module.exports = function MongooseRestRouter(base, router, options) {
    var ResponseStream = options.responseStream,
        conn = options.conn || options.mongoose && options.mongoose.connection || options.mongoose,
        ModelType = conn.Model || conn.base.Model,
        reGet = new RegExp('^\\' + base + '\/(.*)$'), ppush = Function.apply.bind(Array.prototype.push);

    var api = router.route(reGet);
    api.all(function (req, res, next) {
        var p = req.params = normalize(req.params);
        var Model = m(p.shift());
        if (!Model) {
            return next('No model of type ' + req.params.type);
        }
        req.Model = Model;
        delete req.params.type;
        var d = req._domain = domain.create();
        d.on('error', function (e) {
            console.log('error', e);
            return options.error(e, req, res);
        });
        d.add(req);
        d.add(res);
        d.run(next);
    });

    function m(model) {
        return _q.m(conn, model);
    }

    function transform(stream, ctx) {
        return options.transformer.pump(stream, ctx);
    }

    function run(ctx, res) {

        return function run$return(e, o) {
            if (e) {
                return options.error(e, null, res);
            }
            o = o instanceof Stream ? o : streams.ToStream(o);
            transform(o, ctx).pipe(options.responseStream({
                response: {},
                single: true
            })).pipe(streams.ToJson()).pipe(streams.Stringify()).pipe(res);
        }
    }

    var finder = function (Model, find, req, res, next) {
        var single = req.query.single;
        return function (err, count) {
            if (err) return next(err);
            populate(find, req.query);
            paginate(find, req.query);
            sort(find, req.query);
            var stream = transform(createstream(find, req, res, next), req.query, Model);
            if (req.query.filter && Model) {
                filter(find, req.query, Model);
                delete find.options.sort;
                find.count(function (err, fcount) {
                    if (err) return next(err);
                    stream.pipe(new ResponseStream({total: count, filterTotal: fcount}, single)).pipe(res);
                });
            } else {
                stream.pipe(new ResponseStream({total: count}, single)).pipe(res);
            }
        }
    };

    /* router.get(base + '/:type', function (req, res, next) {
     var Model = req.Model;
     Model.count({}, finder(Model, Model.find({}), req, res, next));
     });
     */

    function normalize(params) {
        var ret = [];
        if (!params) return ret;
        if (_u.isArray(params)) {
            for (var i = 0, l = params.length; i < l; i++) {
                var v = params[i];

                ppush(ret, v && v.split ? v.split(/\//g) : v == null ? [] : [v]);
            }
        } else {
            Object.keys(params).sort().forEach(function (k) {
                var v = params[k];

                ppush(ret, v && v.split ? v.split(/\//g) : v == null ? [] : [v]);
            })
        }
        ret = _u.compact(ret);
        return ret;
    }

    function ctx(req) {
        return _u.pick(req, 'query', 'session', 'params', 'body', 'Model', 'meta');
    }

    api.get(function (req, res, next) {
            var params = req.params, find, Model = req.Model, id = params.shift(), single = req.query.single;
            //    var single = 'single' in req.query ? req.query.single : (params.length + (id == "finder" ? -1 : 0) % 2 == 0);
            var _ctx = ctx(req);

            //    transform(createstream(cbs, req, res, next), req.query, Model).pipe(new ResponseStream(null, single)).pipe(res);
            var single = params.length % 2 === 0;

            //swallow finders
            if (id === 'finder' || typeof Model[params[0]] === 'function') {
               // find = Model;
                if (id === 'finder') single = false;
                var m = params.shift();
                find = function(){
                    return invoker.resolve(Model[m], Model, _ctx);
                }
            } else {

                if (id) {
                    single = params.length === 0 ? true : single;

                    find = function () {
                        return Model.findOne(idObj(id));
                    }
                    // find = Model.findOne.bind(Model, idObj(id));
                } else {
                    // find = Model.find.bind(Model, {});
                    single = params.length === 0 ? false : single;
                    find = function () {
                        return Model.find({});
                    }

                }
                if (params.length) {
                    populate(find, {populate: [params.join('.')]});
                }
            }
            invoker.invoke(find, params, _ctx, get$advice, function (e, o) {
                if (e) {
                    return options.error(e, null, res);
                }
                o = o instanceof Stream ? o : streams.ToStream(o);
                transform(o, _ctx).pipe(options.responseStream({
                    single: ('single' in req.query) ? req.query.single : single,
                    response: {total: _ctx.body.count}
                })).pipe(streams.ToJson()).pipe(streams.Stringify()).pipe(res);
            });

        }
    );

    function get$advice(str, ctx, obj, advice, nValue) {
        if (isF(nValue, 'where')) {
            _q.filter(nValue, ctx.query, ctx.Model);
        }

        if (ctx.query.single == true) {
            if (isF(nValue, 'stream')) {
                return advice(nValue.stream());
            }
            return advice(nValue);
        }
        if (str.length === 0 && isF(nValue, 'count')) {
            populate(nValue, ctx.query);
            paginate(nValue, ctx.query);
            filter(nValue, ctx.query, ctx.Model);
         //   var _sort = nValue.options.sort;
         //  delete nValue.options.sort;
            var op = nValue.op;
            nValue.count(function (e, c) {
                if (e) return advice(e);
                nValue.op = op;
                sort(nValue, ctx.query);
                ctx.body.count = c;

                advice(nValue.stream());
            });
        } else {
            advice(nValue);
        }

    }

    function isF(obj, f) {
        var args = Array.prototype.slice.call(arguments, 1);
        if (!obj) return false;
        while (args.length) {
            if (!(typeof obj[args.shift()] === 'function'))
                return false;
        }
        return true;
    }

    function put$advice(str, ctx, obj, advice, nValue) {
        //Can't save embedded documents, so we will do this.  There is a possibility that somewhere in the chain we get a model that is not the current var Model.
        // We should be OK.
        var saveTo = nValue instanceof ModelType ? (ctx.model = nValue) : ctx.model;
        if (str.length === 0) {
            nValue.set(ctx.body);
            return saveTo.save(function (e, o) {
                return advice(e || nValue);
            });
        }
        advice(nValue);
    }

    api.put(function (req, res, next) {
        var params = req.params, Model = req.Model, _ctx = ctx(req);
        invoker.invoke(Model.findOne(idObj(params.shift())), params, _ctx, put$advice, run(_ctx, res));
    });

    function post$advice(str, ctx, obj, advice, nValue) {
        //Can't save embedded documents, so we will do this.  There is a possibility that somewhere in the chain we get a model that is not the current var Model.
        // We should be OK.
        var saveTo;
        if (nValue instanceof ModelType) {
            saveTo = ctx.model = nValue;
        } else {
            saveTo = ctx.model;
        }

        if (saveTo && str.length === 0) {
            var idx = str.shift();
            if (Array.isArray(nValue)) {
                if (/^\d+?$/.test(idx)) {
                    nValue.splice(idx, ctx.body, 0);
                } else {
                    nValue.push(ctx.body);
                    idx = nValue.length - 1;
                }
            } else {
                if (idx) {
                    obj.set(idx, ctx.body)
                } else {
                    try {
                        saveTo.set(ctx.body);
                    } catch (e) {
                        console.log('error ', e);
                    }
                }
            }
            return saveTo.save(function (e, o) {
                if (e)
                    return advice(e);
                return advice(idx !== void(0) ? nValue[idx] : o);
            });
        }
        advice(nValue);

    }

    api.delete(function (req, res, next) {
        var path = (req.params[0] || '').split(/\/+?/g);
        var type = req.Model;

        type.findOne(idObj(req), function (err, doc) {
            if (err)           return next(err);
            var obj = doc;
            while (path.length > 1) {
                obj = obj[path.shift()];
            }
            if (obj)
                obj.pull(path.shift());

            doc.save(function (e) {
                if (e)           return next(err);
                res.send({
                    status: 0
                })

            })
        });

    });
    api.post(function (req, res, next) {
        var params = req.params, id = params.shift(), Model = req.Model, _ctx = ctx(req), cb = run(_ctx, res);
        if (id) {
            if (!Model[id]) {
                if (!params.length) {
                    return cb({
                        message: 'Invalid Post'
                    });
                }
                params.unshift(function(){
                 return   Model.findOne(idObj(id));
                });
            } else {
                params.unshift(id);
            }
        }
        invoker.invoke(Model, params, _ctx, post$advice, cb);

    });
    /*
     router.post(base + '/:type', function (req, res) {
     var Model = m(req);
     new Model(clean(req.body)).save(run({total: 1}, res, Model, req.query));
     });
     */
    /*
     api.route('/:id')
     .put(function (req, res, next) {
     var Model = req.Model;
     Model.findOne(idObj(req), function (err, obj) {
     if (err)return next(err);
     _u.extend(obj, clean(req.body));
     obj.save(run({total: 1}, res, Model, req.query));
     });

     }).delete(function (req, res, next) {
     req.Model.findOneAndRemove(idObj(req), function (err) {
     if (err)           return next(err);
     res.send({
     status: 0,
     payload: []
     })
     });
     });
     */
};
<|endoftext|>


<commit_before>var handlers = require('./handlers'),
    models   = require('./models'),
    helpers  = require('./helpers'),
    errors     = require('./errors'),
    spike      = require('./spike');

module.exports = function (app) {
  // A sandbox object to contain some specific objects that are commonly used by
  // handlers. In future it would be ideal that each handler only receives the
  // objects that it requires.
  var sandbox = {
    store:   app.store,
    models:  models.createModels(app.store),
    mailer:  app.mailer,
    helpers: helpers.createHelpers(app)
  }, binHandler, sessionHandler;

  // Create handlers for accepting incoming requests.
  binHandler = new handlers.BinHandler(sandbox);
  sessionHandler = new handlers.SessionHandler(sandbox);

  // Handler Events

  binHandler.on('updated', spike.ping.bind(spike));
  binHandler.on('render-scripts', spike.appendScripts.bind(spike));

  // Load the bin from the store when encountered in the url. Also handles the
  // "latest" url action.
  app.param('bin', binHandler.loadBin);
  app.param('name', sessionHandler.loadUser);

  // Set up the routes.
  app.get('/', binHandler.getDefault);

  // Login/Create account.
  app.post('/sethome', sessionHandler.routeSetHome);
  app.post('/logout', sessionHandler.logoutUser);
  app.post('/forgot', sessionHandler.forgotPassword);
  app.get('/forgot', sessionHandler.requestToken);
  app.get('/reset', sessionHandler.resetPassword);

  // List
  app.get('/list', binHandler.getUserBins);

  // Latest
  app.get('/:bin/latest((.|\/):format)?', binHandler.redirectToLatest);
  app.get('/:bin/latest/edit', binHandler.redirectToLatest);

  // Quick and easy urls for test - allows me to do /rem/last on my mobile devices
  app.get('/:name/last?/:quiet(quiet)?', binHandler.getLatestForUser, spike.getStream, binHandler.getBinPreview);
  app.get('/:name/last/edit', binHandler.getLatestForUser, binHandler.getBin);
  app.get('/:name/last/watch', binHandler.getLatestForUser, binHandler.getLiveEditBin);

  // Edit
  app.get('/:bin/:rev?/edit', binHandler.getBin);
  app.get('/:bin/:rev?/watch', binHandler.getLiveEditBin);

  // Save
  app.post('/save', binHandler.createBin);
  app.post('/:bin/:rev?/save', binHandler.createRevision);

  // Download
  app.get('/:bin/:rev?/download', binHandler.downloadBin);

  // Source
  app.get('/:bin/:rev?/source', binHandler.getBinSource);
  app.get('/:bin/:rev?.:format(js|json|css|html)', binHandler.getBinSourceFile);
  app.get('/:bin/:rev?/:format(js)', function (req, res) {
    // Redirect legacy /js suffix to the new .js extension.
    res.redirect(301, req.path.replace(/\/js$/, '.js'));
  });

  // Log
  app.get('/:bin/:rev/log', spike.getLog);
  app.post('/:bin/:rev/log', spike.postLog);

  // Preview
  app.get('/:bin/:quiet(quiet)?', binHandler.getBinPreview);
  app.get('/:bin/:rev?/:quiet(quiet)?', spike.getStream, binHandler.getBinPreview);

  // Error handler.
  app.use(function (err, req, res, next) {
    if (err instanceof errors.NotFound) {
      return binHandler.notFound(req, res);
    } else if (err instanceof errors.HTTPError) {
      res.statusCode = err.status;

      if (req.accepts('html')) {
        res.contentType('html');
        res.send(err.toHTMLString());
      } else if (req.accepts('json')) {
        res.json(err);
      } else {
        res.contentType('txt');
        res.send(err.toString());
      }

      return;
    }
    next(err);
  });
};
<commit_msg>Implement the error handlers in the routes.js file<commit_after>var handlers = require('./handlers'),
    models   = require('./models'),
    helpers  = require('./helpers'),
    errors     = require('./errors'),
    spike      = require('./spike');

module.exports = function (app) {
  // A sandbox object to contain some specific objects that are commonly used by
  // handlers. In future it would be ideal that each handler only receives the
  // objects that it requires.
  var sandbox = {
    store:   app.store,
    models:  models.createModels(app.store),
    mailer:  app.mailer,
    helpers: helpers.createHelpers(app)
  }, binHandler, sessionHandler, errorHandler;

  // Create handlers for accepting incoming requests.
  binHandler = new handlers.BinHandler(sandbox);
  sessionHandler = new handlers.SessionHandler(sandbox);
  errorHandler = new handlers.ErrorHandler(sandbox);

  // Handler Events

  binHandler.on('updated', spike.ping.bind(spike));
  binHandler.on('render-scripts', spike.appendScripts.bind(spike));

  // Load the bin from the store when encountered in the url. Also handles the
  // "latest" url action.
  app.param('bin', binHandler.loadBin);
  app.param('name', sessionHandler.loadUser);

  // Set up the routes.
  app.get('/', binHandler.getDefault);

  // Login/Create account.
  app.post('/sethome', sessionHandler.routeSetHome);
  app.post('/logout', sessionHandler.logoutUser);
  app.post('/forgot', sessionHandler.forgotPassword);
  app.get('/forgot', sessionHandler.requestToken);
  app.get('/reset', sessionHandler.resetPassword);

  // List
  app.get('/list', binHandler.getUserBins);

  // Latest
  app.get('/:bin/latest((.|\/):format)?', binHandler.redirectToLatest);
  app.get('/:bin/latest/edit', binHandler.redirectToLatest);

  // Quick and easy urls for test - allows me to do /rem/last on my mobile devices
  app.get('/:name/last?/:quiet(quiet)?', binHandler.getLatestForUser, spike.getStream, binHandler.getBinPreview);
  app.get('/:name/last/edit', binHandler.getLatestForUser, binHandler.getBin);
  app.get('/:name/last/watch', binHandler.getLatestForUser, binHandler.getLiveEditBin);

  // Edit
  app.get('/:bin/:rev?/edit', binHandler.getBin);
  app.get('/:bin/:rev?/watch', binHandler.getLiveEditBin);

  // Save
  app.post('/save', binHandler.createBin);
  app.post('/:bin/:rev?/save', binHandler.createRevision);

  // Download
  app.get('/:bin/:rev?/download', binHandler.downloadBin);

  // Source
  app.get('/:bin/:rev?/source', binHandler.getBinSource);
  app.get('/:bin/:rev?.:format(js|json|css|html)', binHandler.getBinSourceFile);
  app.get('/:bin/:rev?/:format(js)', function (req, res) {
    // Redirect legacy /js suffix to the new .js extension.
    res.redirect(301, req.path.replace(/\/js$/, '.js'));
  });

  // Log
  app.get('/:bin/:rev/log', spike.getLog);
  app.post('/:bin/:rev/log', spike.postLog);

  // Preview
  app.get('/:bin/:quiet(quiet)?', binHandler.getBinPreview);
  app.get('/:bin/:rev?/:quiet(quiet)?', spike.getStream, binHandler.getBinPreview);

  // Error handler.
  app.use(errorHandler.httpError);

  // Final connect error handler when in development.
  app.configure('development', function () {
    app.use(express.errorHandler({showStack: true, dumpExceptions: true}));
  });

  // Final connect error handler when in development.
  app.configure('production', function () {
    app.use(errorHandler.uncaughtError);
  });
};
<|endoftext|>


<commit_before>var viewParams = require('./view-params')
var moment = require('moment')
var async = require('async')
var _ = require('underscore')
var Doclet = require('./models/doclet')
var User = require('./models/user')
var Repo = require('./models/repo')

var Routes = function () {}

Routes.prototype.authGithub = function () {}

Routes.prototype.authGithubCallback = function (req, res) {
  req.session.save(function () {
    User.findOne({passportId: req.session.passport.user}, function (err, user) {
      if (err) {
        res.status(500).send('Sorry, login failed')
      } else {
        res.redirect('/' + user._id)
      }
    })
  })
}

Routes.prototype.logout = function (req, res) {
  req.session.destroy(function () {
    res.redirect('/')
  })
}

Routes.prototype.ensureAuthenticated = function (req, res, next) {
  if (req.isAuthenticated()) {
    return next()
  }
  res.redirect('/')
}

Routes.prototype.badge = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  console.log('badge for', key)
  Doclet.findById(key, function (err, doclet) {
    console.log(err, doclet)
    if (err) {
      res.status(500).send('Err ' + err)
      return
    }
    if (!doclet) {
      res.status(404).send()
      return
    }
    if (doclet.error) {
      res.redirect('https://img.shields.io/badge/API--Doc_' + req.params.version + '-failed-red.svg')
      return
    } else {
      res.redirect('https://img.shields.io/badge/API--Doc_' + req.params.version + '-ready-green.svg')
      return
    }
  })
}

Routes.prototype.addRepo = function (req, res) {
  var fullname = req.body.repo
  console.log('adding repo', fullname)
  var done = function (err) {
    if (err) {
      req.flash('error', err)
      res.redirect('/' + req.params.user)
    } else {
      req.flash('result', 'Added repository ' + fullname + ' to Doclets.io')
      res.redirect('/' + req.params.user)
    }
  }
  Repo.findById(fullname, function (err, repo) {
    if (err) {
      done(err)
    } else {
      repo.enableWebHook(done)
    }
  })
}

Routes.prototype.changeRepo = function (req, res) {
  async.parallel([
    Doclet.find.bind(Doclet, {owner: req.user._id, repo: req.params.repo}),
    Repo.findById.bind(Repo, req.user._id + '/' + req.params.repo)
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    } else {
      var doclets = results[0]
      var repo = results[1]
      var actions = _.map(doclets, function (doclet) {
        return doclet.updateIsPublic.bind(doclet, req.body)
      })
      actions.push(repo.updateWebHook.bind(repo, req.body))
      async.parallel(actions, function (err) {
        if (err) {
          req.flash('error', err)
        } else {
          req.flash('result', true)
        }
        res.redirect('/' + req.user._id + '/' + req.params.repo)
      })
    }
  })
}

Routes.prototype.index = function (req, res) {
  Doclet.find({isPublic: true})
    .limit(10)
    .sort({createdAt: -1})
    .populate('_repo')
    .exec(function (err, doclets) {
      if (err) {
        res.status(500).send('Err ' + err)
        return
      } else {
        doclets = doclets.map(function (doclet) {
          return viewParams.getApiParams(doclet)
        })
        res.render('index.jade', {
          doclets: doclets,
          user: req.user,
          moment: moment,
          path: '/'
        })
      }
    })
}

Routes.prototype.search = function (req, res) {
  var query = req.query.q.toLowerCase()
  Doclet.find({repo: {$regex: new RegExp(query, 'i')}}, function (err, doclets) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    } else {
      res.render('search.jade', {
        doclets: doclets,
        query: query,
        user: req.user
      })
    }
  })
}

Routes.prototype.sync = function (req, res) {
  console.log('syncing user', req.user._id)
  Repo.sync(req.user._id, {type: 'oauth', token: req.user.token}, function (err) {
    if (err) {
      res.status(500).send('Err ' + err)
    } else {
      res.redirect('/' + req.user._id)
    }
  })
}

Routes.prototype.user = function (req, res) {
  async.parallel([
    Doclet.findByOwner.bind(Doclet, req.params.user),
    User.findById.bind(User, req.params.user),
    function (callback) {
      if (req.user) {
        Repo.findOrSyncByUser(req.user._id, {type: 'oauth', token: req.user.token}, callback)
      } else {
        Repo.find({owner: req.params.user}, callback)
      }
    }
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    }
    var doclets = results[0]
    var owner = results[1]
    var repos = results[2]
    if (!owner) {
      res.status(404).send('not found')
      return
    }
    doclets = _.filter(doclets, function (doclet) {
      return doclet.hasUserAccess(req.user)
    })
    res.render('user.jade', {
      doclets: doclets,
      user: req.user,
      repos: repos,
      owner: owner,
      username: req.params.user,
      moment: moment,
      path: '/' + req.params.user,
      _: _
    })
  })
}

Routes.prototype.repo = function (req, res) {
  var fullname = [req.params.user, req.params.repo].join('/')
  async.parallel([
    function (callback) {
      Repo
        .findById(fullname)
        .populate('_owner')
        .exec(callback)
    },
    function (callback) {
      Doclet
        .find({repo: req.params.repo, owner: req.params.user})
        .exec(callback)
    }
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err' + err)
      return
    }
    var repo = results[0]
    var doclets = results[1]
    if (!repo && !req.user) {
      res.status(404).send('not found')
      return
    } else if (!repo && req.user) {
      res.redirect('/' + req.user._id)
      return
    }
    doclets = _.filter(doclets, function (doclet) {
      return doclet.hasUserAccess(req.user)
    })
    var params = {
      _: _,
      user: req.user,
      owner: req.user || repo._owner,
      username: req.params.user,
      repo: repo,
      versions: doclets,
      moment: moment,
      error: req.flash('error')[0],
      result: req.flash('result')[0]
    }
    res.render('repo.jade', params)
  })
}

Routes.prototype.api = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  Doclet
    .findById(key)
    .populate('_repo')
    .populate('_owner')
    .exec(function (err, doclet) {
      if (err || !doclet) {
        res.status(500).send('Err ' + err)
        return
      } else if (!doclet.hasUserAccess(req.user)) {
        res.status(404).send('Not public')
        return
      } else {
        var params = viewParams.getApiParams(doclet)
        params.user = req.user
        params.username = req.params.user
        res.render('api.jade', params)
      }
    })
}

Routes.prototype.article = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  Doclet
    .findById(key)
    .populate('_repo')
    .populate('_owner')
    .exec(function (err, doclet) {
      if (err || !doclet) {
        res.status(500).send('Err ' + err)
        return
      } else if (!doclet.hasUserAccess(req.user)) {
        res.status(404).send('Not public')
        return
      } else {
        var params = viewParams.getArticleParams(doclet, req.params.article)
        params.user = req.user
        params.username = req.params.user
        params._owner = doclet._owner
        res.render('article.jade', params)
      }
    })
}

Routes.prototype.serializeUser = function (ghUser, done) {
  var id = ghUser.profile.id
  User.findOne({passportId: id}, function (err, user) {
    if (err) {
      done(err)
    } else if (!user) {
      User.createFromGitHubPassport(ghUser, function (err) {
        done(err, id)
      })
    } else {
      done(null, id)
    }
  })
}

Routes.prototype.createPassportUser = function (accessToken, refreshToken, profile, done) {
  return done(null, {
    profile: profile,
    token: accessToken,
    refreshToken: refreshToken
  })
}

Routes.prototype.deserializeUser = function (id, done) {
  User.findOne({passportId: id}, done)
}

module.exports.Routes = Routes
<commit_msg>fix redirect after addRepo<commit_after>var viewParams = require('./view-params')
var moment = require('moment')
var async = require('async')
var _ = require('underscore')
var Doclet = require('./models/doclet')
var User = require('./models/user')
var Repo = require('./models/repo')

var Routes = function () {}

Routes.prototype.authGithub = function () {}

Routes.prototype.authGithubCallback = function (req, res) {
  req.session.save(function () {
    User.findOne({passportId: req.session.passport.user}, function (err, user) {
      if (err) {
        res.status(500).send('Sorry, login failed')
      } else {
        res.redirect('/' + user._id)
      }
    })
  })
}

Routes.prototype.logout = function (req, res) {
  req.session.destroy(function () {
    res.redirect('/')
  })
}

Routes.prototype.ensureAuthenticated = function (req, res, next) {
  if (req.isAuthenticated()) {
    return next()
  }
  res.redirect('/')
}

Routes.prototype.badge = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  console.log('badge for', key)
  Doclet.findById(key, function (err, doclet) {
    console.log(err, doclet)
    if (err) {
      res.status(500).send('Err ' + err)
      return
    }
    if (!doclet) {
      res.status(404).send()
      return
    }
    if (doclet.error) {
      res.redirect('https://img.shields.io/badge/API--Doc_' + req.params.version + '-failed-red.svg')
      return
    } else {
      res.redirect('https://img.shields.io/badge/API--Doc_' + req.params.version + '-ready-green.svg')
      return
    }
  })
}

Routes.prototype.addRepo = function (req, res) {
  var fullname = req.body.repo
  console.log('adding repo', fullname)
  var done = function (err) {
    if (err) {
      req.flash('error', err)
      res.redirect('/' + fullname.split('/')[0])
    } else {
      res.redirect('/' + fullname)
    }
  }
  Repo.findById(fullname, function (err, repo) {
    if (err) {
      done(err)
    } else {
      repo.enableWebHook(done)
    }
  })
}

Routes.prototype.changeRepo = function (req, res) {
  async.parallel([
    Doclet.find.bind(Doclet, {owner: req.user._id, repo: req.params.repo}),
    Repo.findById.bind(Repo, req.user._id + '/' + req.params.repo)
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    } else {
      var doclets = results[0]
      var repo = results[1]
      var actions = _.map(doclets, function (doclet) {
        return doclet.updateIsPublic.bind(doclet, req.body)
      })
      actions.push(repo.updateWebHook.bind(repo, req.body))
      async.parallel(actions, function (err) {
        if (err) {
          req.flash('error', err)
        } else {
          req.flash('result', true)
        }
        res.redirect('/' + req.user._id + '/' + req.params.repo)
      })
    }
  })
}

Routes.prototype.index = function (req, res) {
  Doclet.find({isPublic: true})
    .limit(10)
    .sort({createdAt: -1})
    .populate('_repo')
    .exec(function (err, doclets) {
      if (err) {
        res.status(500).send('Err ' + err)
        return
      } else {
        doclets = doclets.map(function (doclet) {
          return viewParams.getApiParams(doclet)
        })
        res.render('index.jade', {
          doclets: doclets,
          user: req.user,
          moment: moment,
          path: '/'
        })
      }
    })
}

Routes.prototype.search = function (req, res) {
  var query = req.query.q.toLowerCase()
  Doclet.find({repo: {$regex: new RegExp(query, 'i')}}, function (err, doclets) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    } else {
      res.render('search.jade', {
        doclets: doclets,
        query: query,
        user: req.user
      })
    }
  })
}

Routes.prototype.sync = function (req, res) {
  console.log('syncing user', req.user._id)
  Repo.sync(req.user._id, {type: 'oauth', token: req.user.token}, function (err) {
    if (err) {
      res.status(500).send('Err ' + err)
    } else {
      res.redirect('/' + req.user._id)
    }
  })
}

Routes.prototype.user = function (req, res) {
  async.parallel([
    Doclet.findByOwner.bind(Doclet, req.params.user),
    User.findById.bind(User, req.params.user),
    function (callback) {
      if (req.user) {
        Repo.findOrSyncByUser(req.user._id, {type: 'oauth', token: req.user.token}, callback)
      } else {
        Repo.find({owner: req.params.user}, callback)
      }
    }
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err ' + err)
      return
    }
    var doclets = results[0]
    var owner = results[1]
    var repos = results[2]
    if (!owner) {
      res.status(404).send('not found')
      return
    }
    doclets = _.filter(doclets, function (doclet) {
      return doclet.hasUserAccess(req.user)
    })
    res.render('user.jade', {
      doclets: doclets,
      user: req.user,
      repos: repos,
      owner: owner,
      username: req.params.user,
      moment: moment,
      path: '/' + req.params.user,
      _: _
    })
  })
}

Routes.prototype.repo = function (req, res) {
  var fullname = [req.params.user, req.params.repo].join('/')
  async.parallel([
    function (callback) {
      Repo
        .findById(fullname)
        .populate('_owner')
        .exec(callback)
    },
    function (callback) {
      Doclet
        .find({repo: req.params.repo, owner: req.params.user})
        .exec(callback)
    }
  ], function (err, results) {
    if (err) {
      res.status(500).send('Err' + err)
      return
    }
    var repo = results[0]
    var doclets = results[1]
    if (!repo && !req.user) {
      res.status(404).send('not found')
      return
    } else if (!repo && req.user) {
      res.redirect('/' + req.user._id)
      return
    }
    doclets = _.filter(doclets, function (doclet) {
      return doclet.hasUserAccess(req.user)
    })
    var params = {
      _: _,
      user: req.user,
      owner: req.user || repo._owner,
      username: req.params.user,
      repo: repo,
      versions: doclets,
      moment: moment,
      error: req.flash('error')[0],
      result: req.flash('result')[0]
    }
    res.render('repo.jade', params)
  })
}

Routes.prototype.api = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  Doclet
    .findById(key)
    .populate('_repo')
    .populate('_owner')
    .exec(function (err, doclet) {
      if (err || !doclet) {
        res.status(500).send('Err ' + err)
        return
      } else if (!doclet.hasUserAccess(req.user)) {
        res.status(404).send('Not public')
        return
      } else {
        var params = viewParams.getApiParams(doclet)
        params.user = req.user
        params.username = req.params.user
        res.render('api.jade', params)
      }
    })
}

Routes.prototype.article = function (req, res) {
  var key = [req.params.user, req.params.repo, req.params.version].join('/')
  Doclet
    .findById(key)
    .populate('_repo')
    .populate('_owner')
    .exec(function (err, doclet) {
      if (err || !doclet) {
        res.status(500).send('Err ' + err)
        return
      } else if (!doclet.hasUserAccess(req.user)) {
        res.status(404).send('Not public')
        return
      } else {
        var params = viewParams.getArticleParams(doclet, req.params.article)
        params.user = req.user
        params.username = req.params.user
        params._owner = doclet._owner
        res.render('article.jade', params)
      }
    })
}

Routes.prototype.serializeUser = function (ghUser, done) {
  var id = ghUser.profile.id
  User.findOne({passportId: id}, function (err, user) {
    if (err) {
      done(err)
    } else if (!user) {
      User.createFromGitHubPassport(ghUser, function (err) {
        done(err, id)
      })
    } else {
      done(null, id)
    }
  })
}

Routes.prototype.createPassportUser = function (accessToken, refreshToken, profile, done) {
  return done(null, {
    profile: profile,
    token: accessToken,
    refreshToken: refreshToken
  })
}

Routes.prototype.deserializeUser = function (id, done) {
  User.findOne({passportId: id}, done)
}

module.exports.Routes = Routes
<|endoftext|>


<commit_before>
var compose = require('koa-compose')
var isGenFun = require('is-gen-fn')
var flatten = require('flatten')
var assert = require('assert')
var METHODS = require('methods').map(function (method) {
  return method.toUpperCase()
})

var Router = require('./router')

Router.prototype.routes = function () {
  var router = this
  var app = this.app

  // app.route(paths).get(gen).post(gen)
  app.route = function () {
    return new Route(router, [].slice.call(arguments))
  }

  // app.get(paths, gen, gen ,gen)
  METHODS.forEach(function (method) {
    app[method.toLowerCase()] = function (paths) {
      new Route(router, paths)
        ._append(method, [].slice.call(arguments, 1))

      return app
    }
  })

  app.del = app.delete
}

function Route(router, paths) {
  this.router = router

  if (!Array.isArray(paths)) paths = [paths]
  paths = flatten(paths).filter(Boolean)

  var nodes = this.nodes = []

  if (!paths.length) throw new Error('Route must have a path')

  paths.forEach(function (path) {
    if (typeof path !== 'string')
      throw new TypeError('Paths must be strings: ' + path)
    if (path !== '/' && path[0] !== '/')
      throw new Error('Paths must start with a "/": ' + path)

    router.trie.define(path).forEach(function (node) {
      if (!~nodes.indexOf(node)) nodes.push(node)
    })
  })

  if (!nodes.length) throw new Error('No routes defined. Something went wrong.')
}

Route.prototype._append = function (method, generators) {
  var implementedMethods = this.router.methods
  var _method = method.toLowerCase()

  // For 501 Not Implemented support
  implementedMethods[method] = true
  if (method === 'GET') implementedMethods.HEAD = true

  // Take out all the falsey middleware
  var stack = flatten(generators).filter(Boolean).filter(assertGenFun)
  if (!stack.length) return

  this.nodes.forEach(function (node) {
    // Push the generators to the generator stack
    node[_method] = (node[_method] || []).concat(stack)

    // Builds the list of supported methods for this route
    // for OPTIONS and 405 responses
    node.methods = METHODS.filter(function (method) {
      return node[method.toLowerCase()]
    })
    if (~node.methods.indexOf('GET') && !~node.methods.indexOf('HEAD'))
      node.methods.push('HEAD')
    if (!~node.methods.indexOf('OPTIONS'))
      node.methods.push('OPTIONS')
    node.methods = node.methods.join(',')

    // Builds the actual composed generator
    node[method] = node[_method].length > 1
      ? compose(node[_method])
      : node[_method][0]
  })
}

Router.prototype.end = function () {
  return this.router.app
}

// For `app.get(path, gen, gen, gen)` support
METHODS.forEach(function (method) {
  Route.prototype[method.toLowerCase()] = function () {
    this._append(method, flatten([].slice.call(arguments)))
    return this
  }
})

Route.prototype.del = Route.prototype.delete

function assertGenFun(fn) {
  assert(isGenFun(fn), 'all middleware must be generator functions')
  return fn
}
<commit_msg>use assertion for better code coverage<commit_after>
var compose = require('koa-compose')
var isGenFun = require('is-gen-fn')
var flatten = require('flatten')
var assert = require('assert')
var METHODS = require('methods').map(function (method) {
  return method.toUpperCase()
})

var Router = require('./router')

Router.prototype.routes = function () {
  var router = this
  var app = this.app

  // app.route(paths).get(gen).post(gen)
  app.route = function () {
    return new Route(router, [].slice.call(arguments))
  }

  // app.get(paths, gen, gen ,gen)
  METHODS.forEach(function (method) {
    app[method.toLowerCase()] = function (paths) {
      new Route(router, paths)
        ._append(method, [].slice.call(arguments, 1))

      return app
    }
  })

  app.del = app.delete
}

function Route(router, paths) {
  this.router = router

  if (!Array.isArray(paths)) paths = [paths]
  paths = flatten(paths).filter(Boolean)

  var nodes = this.nodes = []

  assert(paths.length, 'Route must have a path')

  paths.forEach(function (path) {
    assert(typeof path === 'string', 'Paths must be strings: ' + path)
    assert(path[0] === '/', 'Paths must start with a "/": ' + path)
    router.trie.define(path).forEach(function (node) {
      if (!~nodes.indexOf(node)) nodes.push(node)
    })
  })

  assert(nodes.length, 'No routes defined. Something went wrong.')
}

Route.prototype._append = function (method, generators) {
  var implementedMethods = this.router.methods
  var _method = method.toLowerCase()

  // For 501 Not Implemented support
  implementedMethods[method] = true
  if (method === 'GET') implementedMethods.HEAD = true

  // Take out all the falsey middleware
  var stack = flatten(generators).filter(Boolean).filter(assertGenFun)
  if (!stack.length) return

  this.nodes.forEach(function (node) {
    // Push the generators to the generator stack
    node[_method] = (node[_method] || []).concat(stack)

    // Builds the list of supported methods for this route
    // for OPTIONS and 405 responses
    node.methods = METHODS.filter(function (method) {
      return node[method.toLowerCase()]
    })
    if (~node.methods.indexOf('GET') && !~node.methods.indexOf('HEAD'))
      node.methods.push('HEAD')
    if (!~node.methods.indexOf('OPTIONS'))
      node.methods.push('OPTIONS')
    node.methods = node.methods.join(',')

    // Builds the actual composed generator
    node[method] = node[_method].length > 1
      ? compose(node[_method])
      : node[_method][0]
  })
}

Router.prototype.end = function () {
  return this.router.app
}

// For `app.get(path, gen, gen, gen)` support
METHODS.forEach(function (method) {
  Route.prototype[method.toLowerCase()] = function () {
    this._append(method, flatten([].slice.call(arguments)))
    return this
  }
})

Route.prototype.del = Route.prototype.delete

function assertGenFun(fn) {
  assert(isGenFun(fn), 'all middleware must be generator functions')
  return fn
}
<|endoftext|>


<commit_before>// run-time control channel
var agent = require('./agent');
var agentVersion = require('strong-agent/package.json').version;
var async = require('async');
var cluster = require('cluster');
var debug = require('./debug')('runctl');
var EventEmitter = require('events').EventEmitter;
var fs = require('fs');
var master = require('strong-cluster-control');
var npmls = require('strong-npm-ls');
var os = require('os');
var targetctl = require('./targetctl');
var util = require('util');
var wsChannel = require('strong-control-channel/ws-channel').connect;
var processChannel = require('strong-control-channel/process').attach;

exports.start = start;
exports.onRequest = onRequest; // For testing

// Return the number of event listeners.
var listenerCount = EventEmitter.listenerCount || function(emitter, event) {
  return emitter.listeners(event).length;
};

// sent on startup and with every status message
var osVersion = {
  platform: os.platform(),
  arch: os.arch(),
  release: os.release()
};

var ctlChannel = {
  notify: function notify(req) {
    debug('notify: ipcctl? %j %s', !!ipcctl, debug.json(req));
    if (ipcctl) {
      ipcctl.notify(req);
    }
    server.notify(req);
  },
};
// Expose runctl using local domain server
var server = require('strong-control-channel/server').create(onRequest);

// Expose runctl using node IPC only in the master/supervisor (workers
// attach targetctl to the master).
var ipcctl = null;
if (cluster.isMaster && !process.env.SL_RUN_SKIP_IPCCTL) {
  var wsUrl = process.env.STRONGLOOP_CONTROL;
  if (wsUrl) {
    debug('runctl connect: %s', wsUrl);
    ipcctl = wsChannel(onRequest, wsUrl);
    ipcctl.unref();
  } else if (process.send) {
    debug('runctl connect: <process channel>');
    ipcctl = processChannel(onRequest);
  }
}

if (cluster.isMaster) {
  exports.notifyStarted = function() {
    // Must not be done synchronously in the require, we don't know the app name
    // until after run has found the app, and change to its directory.
    ctlChannel.notify({
      wid: 0,
      cmd: 'started',
      pid: process.pid,
      ppid: 0,  // Value is not known for the master process
      pst: master.startTime,
      appName: agent().config.appName,
      agentVersion: agentVersion,
      nodeVersion: process.version,
      osVersion: osVersion,
      setSize: master.size,
    });

    // Store pid for later use in fork messages
    master.pid = process.pid;

    // Status notifications usually come after every fork and exit, but if there
    // are no forks, we still want at least one status notification sent. Note
    // that size may be undefined, which isn't > or < than 1.
    if (!(master.options.size >= 1)) {
      notifyStatus();
    }
  };

  cluster.on('listening', function(worker, address) {
    debug('listening');
    ctlChannel.notify({
      cmd: 'listening',
      wid: worker.id,
      pid: worker.process.pid,
      pst: worker.startTime,
      address: address,
    });
  });

  exports.parentCtl = ctlChannel;

  master.on('fork', function(worker) {
    debug('fork');
    ctlChannel.notify({
      cmd: 'fork',
      wid: worker.id,
      pid: worker.process.pid,
      ppid: master.pid,
      pst: worker.startTime,
    });
    notifyStatus();
  });

  cluster.on('exit', function(worker, code, signal) {
    ctlChannel.notify({
      cmd: 'exit',
      wid: worker.id,
      pid: worker.process.pid,
      pst: worker.startTime,
      suicide: worker.suicide,
      reason: signal || code,
    });
    notifyStatus();
  });

  function notifyStatus() {
    var status = clusterStatus();
    status.cmd = 'status';
    ctlChannel.notify(status);
  }
}

function start(options) {
  var logger = options.logger;

  // XXX(sam) I don't like this 'last one wins' approach, but its impossible to
  // prevent the channel outliving sl-run under all conditions, this is the only
  // robust way I've found.
  try {
    fs.unlinkSync(options.channel);
  } catch (er) {
    /* eslint no-empty:0 */
    // Didn't exist
  }

  server.listen(options.channel);

  server.on('error', function(er) {
    logger.error('control channel failed to listen on `%s`: %s',
                 options.channel, er);
    throw er;
  });

  master.on('stop', function() {
    server.close();
  });
  return server;
}

function onRequest(req, callback) {
  /* eslint no-redeclare:0 */
  debug('request %s', debug.json(req));

  var cmd = req.cmd;
  var rsp = {
  };

  if (cmd === 'status') {
    rsp = clusterStatus();
  } else if (cmd === 'npm-ls') {
    rsp = null;
    npmls.read('.', function(er, data) {
      return callback(er ? {error: er.message} : data);
    });
  } else if (cmd === 'set-size') {
    try {
      if (/cpu/i.test(req.size))
        req.size = os.cpus().length;
      else
        req.size = +req.size;
      master.setSize(req.size);
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'stop') {
    try {
      master.stop();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'restart') {
    try {
      master.restart();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'disconnect') {
    cluster.disconnect();

  } else if (cmd === 'fork') {
    rsp = null;
    var worker = cluster.fork();
    // worker emits its own 'fork' _after_ s-c-c has augmented it
    worker.once('fork', function() {
      var rsp = {
        workerID: worker.id,
        processID: worker.process.pid,
        pst: worker.startTime,
      };
      callback(rsp);
    });

  } else if (cmd === 'env-set') {
    for (var k in req.env) {
      process.env[k] = req.env[k];
    }
    rsp = requestAllTargets(req, callback);

  } else if (cmd === 'env-unset') {
    for (var k in req.env) {
      delete process.env[req.env[k]];
    }
    rsp = requestAllTargets(req, callback);

  } else if (cmd === 'tracing') {
    if (req.enabled) {
      process.env.STRONGLOOP_TRACING = 1;
    } else if (process.env.STRONGLOOP_TRACING !== null) {
      delete process.env.STRONGLOOP_TRACING;
    }

    try {
      master.restart();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'signal') {
    if (req.pid !== process.pid) {
     // Do nothing
    } else if (listenerCount(process, req.signame) > 0) {
      // If there are any listeners for this signal, emit it on the
      // process object.
      process.emit(req.signame);
    } else {
      // If there are no listeners, use the default action.
      process.kill(process.pid, req.signame);
    }

  } else {
    // Pass any others off to the target
    rsp = requestOfTarget(req, rsp, callback);
  }

  if (callback && rsp) {
    process.nextTick(callback.bind(null, rsp));
  }
}

function requestAllTargets(req, callback) {
  console.log('forwarding to all workers:', req);
  var wIds = [];
  for (var w in cluster.workers) {
    wIds.push(cluster.workers[w].id);
  }
  async.map(wIds, forwardToWorker, makeResponse);
  return null;

  function forwardToWorker(wid, next) {
    var wreq = JSON.parse(JSON.stringify(req));
    wreq.target = wid;
    console.log('forwarding %j', wreq);
    requestOfTarget(wreq, {}, function(rsp) {
      next(null, rsp);
    });
  }

  function makeResponse(_, res) {
    callback(res);
  }
}

function requestOfTarget(req, rsp, callback) {
  debug('requestOfTarget %j', req);

  if (targetctl.request(+req.target, req, wrapCallback)) {
    rsp = null;
  } else {
    rsp.error = util.format('target %s not found', req.target);
  }

  return rsp;

  function wrapCallback(rsp) {
    // add pst field in master since workers don't know their own pst
    var target = cluster.workers[+rsp.target] || master;
    rsp.pst = target.startTime;
    if (rsp.notify) {
      ctlChannel.notify(rsp.notify);
      delete rsp.notify;
    }

    callback(rsp);
  }
}

function clusterStatus() {
  var mStatus = master.status();
  // pst is used by PM, startTime is set by strong-cluster-control
  mStatus.master.pst = mStatus.master.pst || mStatus.master.startTime;
  mStatus.appName = agent().config.appName;
  mStatus.agentVersion = agentVersion;
  mStatus.nodeVersion = process.version;
  mStatus.osVersion = osVersion;
  return mStatus;
}
<commit_msg>run: print cleaner error when WS channel dies<commit_after>// run-time control channel
var agent = require('./agent');
var agentVersion = require('strong-agent/package.json').version;
var async = require('async');
var cluster = require('cluster');
var debug = require('./debug')('runctl');
var EventEmitter = require('events').EventEmitter;
var fs = require('fs');
var master = require('strong-cluster-control');
var npmls = require('strong-npm-ls');
var os = require('os');
var targetctl = require('./targetctl');
var util = require('util');
var wsChannel = require('strong-control-channel/ws-channel').connect;
var processChannel = require('strong-control-channel/process').attach;

exports.start = start;
exports.onRequest = onRequest; // For testing

// Return the number of event listeners.
var listenerCount = EventEmitter.listenerCount || function(emitter, event) {
  return emitter.listeners(event).length;
};

// sent on startup and with every status message
var osVersion = {
  platform: os.platform(),
  arch: os.arch(),
  release: os.release()
};

var ctlChannel = {
  notify: function notify(req) {
    debug('notify: ipcctl? %j %s', !!ipcctl, debug.json(req));
    if (ipcctl) {
      ipcctl.notify(req);
    }
    server.notify(req);
  },
};
// Expose runctl using local domain server
var server = require('strong-control-channel/server').create(onRequest);

// Expose runctl using node IPC only in the master/supervisor (workers
// attach targetctl to the master).
var ipcctl = null;
if (cluster.isMaster && !process.env.SL_RUN_SKIP_IPCCTL) {
  var wsUrl = process.env.STRONGLOOP_CONTROL;
  if (wsUrl) {
    debug('runctl connect: %s', wsUrl);
    ipcctl = wsChannel(onRequest, wsUrl);
    ipcctl.on('error', function(err) {
      console.error('strong-supervisor: lost connection to control channel: %s',
                    err.message);
      process.exit(1);
    });
    ipcctl.unref();
  } else if (process.send) {
    debug('runctl connect: <process channel>');
    ipcctl = processChannel(onRequest);
  }
}

if (cluster.isMaster) {
  exports.notifyStarted = function() {
    // Must not be done synchronously in the require, we don't know the app name
    // until after run has found the app, and change to its directory.
    ctlChannel.notify({
      wid: 0,
      cmd: 'started',
      pid: process.pid,
      ppid: 0,  // Value is not known for the master process
      pst: master.startTime,
      appName: agent().config.appName,
      agentVersion: agentVersion,
      nodeVersion: process.version,
      osVersion: osVersion,
      setSize: master.size,
    });

    // Store pid for later use in fork messages
    master.pid = process.pid;

    // Status notifications usually come after every fork and exit, but if there
    // are no forks, we still want at least one status notification sent. Note
    // that size may be undefined, which isn't > or < than 1.
    if (!(master.options.size >= 1)) {
      notifyStatus();
    }
  };

  cluster.on('listening', function(worker, address) {
    debug('listening');
    ctlChannel.notify({
      cmd: 'listening',
      wid: worker.id,
      pid: worker.process.pid,
      pst: worker.startTime,
      address: address,
    });
  });

  exports.parentCtl = ctlChannel;

  master.on('fork', function(worker) {
    debug('fork');
    ctlChannel.notify({
      cmd: 'fork',
      wid: worker.id,
      pid: worker.process.pid,
      ppid: master.pid,
      pst: worker.startTime,
    });
    notifyStatus();
  });

  cluster.on('exit', function(worker, code, signal) {
    ctlChannel.notify({
      cmd: 'exit',
      wid: worker.id,
      pid: worker.process.pid,
      pst: worker.startTime,
      suicide: worker.suicide,
      reason: signal || code,
    });
    notifyStatus();
  });

  function notifyStatus() {
    var status = clusterStatus();
    status.cmd = 'status';
    ctlChannel.notify(status);
  }
}

function start(options) {
  var logger = options.logger;

  // XXX(sam) I don't like this 'last one wins' approach, but its impossible to
  // prevent the channel outliving sl-run under all conditions, this is the only
  // robust way I've found.
  try {
    fs.unlinkSync(options.channel);
  } catch (er) {
    /* eslint no-empty:0 */
    // Didn't exist
  }

  server.listen(options.channel);

  server.on('error', function(er) {
    logger.error('control channel failed to listen on `%s`: %s',
                 options.channel, er);
    throw er;
  });

  master.on('stop', function() {
    server.close();
  });
  return server;
}

function onRequest(req, callback) {
  /* eslint no-redeclare:0 */
  debug('request %s', debug.json(req));

  var cmd = req.cmd;
  var rsp = {
  };

  if (cmd === 'status') {
    rsp = clusterStatus();
  } else if (cmd === 'npm-ls') {
    rsp = null;
    npmls.read('.', function(er, data) {
      return callback(er ? {error: er.message} : data);
    });
  } else if (cmd === 'set-size') {
    try {
      if (/cpu/i.test(req.size))
        req.size = os.cpus().length;
      else
        req.size = +req.size;
      master.setSize(req.size);
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'stop') {
    try {
      master.stop();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'restart') {
    try {
      master.restart();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'disconnect') {
    cluster.disconnect();

  } else if (cmd === 'fork') {
    rsp = null;
    var worker = cluster.fork();
    // worker emits its own 'fork' _after_ s-c-c has augmented it
    worker.once('fork', function() {
      var rsp = {
        workerID: worker.id,
        processID: worker.process.pid,
        pst: worker.startTime,
      };
      callback(rsp);
    });

  } else if (cmd === 'env-set') {
    for (var k in req.env) {
      process.env[k] = req.env[k];
    }
    rsp = requestAllTargets(req, callback);

  } else if (cmd === 'env-unset') {
    for (var k in req.env) {
      delete process.env[req.env[k]];
    }
    rsp = requestAllTargets(req, callback);

  } else if (cmd === 'tracing') {
    if (req.enabled) {
      process.env.STRONGLOOP_TRACING = 1;
    } else if (process.env.STRONGLOOP_TRACING !== null) {
      delete process.env.STRONGLOOP_TRACING;
    }

    try {
      master.restart();
    } catch (er) {
      rsp.error = er.message;
    }

  } else if (cmd === 'signal') {
    if (req.pid !== process.pid) {
     // Do nothing
    } else if (listenerCount(process, req.signame) > 0) {
      // If there are any listeners for this signal, emit it on the
      // process object.
      process.emit(req.signame);
    } else {
      // If there are no listeners, use the default action.
      process.kill(process.pid, req.signame);
    }

  } else {
    // Pass any others off to the target
    rsp = requestOfTarget(req, rsp, callback);
  }

  if (callback && rsp) {
    process.nextTick(callback.bind(null, rsp));
  }
}

function requestAllTargets(req, callback) {
  console.log('forwarding to all workers:', req);
  var wIds = [];
  for (var w in cluster.workers) {
    wIds.push(cluster.workers[w].id);
  }
  async.map(wIds, forwardToWorker, makeResponse);
  return null;

  function forwardToWorker(wid, next) {
    var wreq = JSON.parse(JSON.stringify(req));
    wreq.target = wid;
    console.log('forwarding %j', wreq);
    requestOfTarget(wreq, {}, function(rsp) {
      next(null, rsp);
    });
  }

  function makeResponse(_, res) {
    callback(res);
  }
}

function requestOfTarget(req, rsp, callback) {
  debug('requestOfTarget %j', req);

  if (targetctl.request(+req.target, req, wrapCallback)) {
    rsp = null;
  } else {
    rsp.error = util.format('target %s not found', req.target);
  }

  return rsp;

  function wrapCallback(rsp) {
    // add pst field in master since workers don't know their own pst
    var target = cluster.workers[+rsp.target] || master;
    rsp.pst = target.startTime;
    if (rsp.notify) {
      ctlChannel.notify(rsp.notify);
      delete rsp.notify;
    }

    callback(rsp);
  }
}

function clusterStatus() {
  var mStatus = master.status();
  // pst is used by PM, startTime is set by strong-cluster-control
  mStatus.master.pst = mStatus.master.pst || mStatus.master.startTime;
  mStatus.appName = agent().config.appName;
  mStatus.agentVersion = agentVersion;
  mStatus.nodeVersion = process.version;
  mStatus.osVersion = osVersion;
  return mStatus;
}
<|endoftext|>


<commit_before>// Copyright 2012 Pedro P. Candel <kusorbox@gmail.com> All rights reserved.
var util = require('util'),
    uuid = require('node-uuid'),
    fs = require('fs'),
    path = require('path'),
    async = require('async'),
    WorkflowJobRunner = require('./job-runner');

// - opts - configuration options:
//    - identifier: Unique identifier for this runner.
//    - forks: Max number of child processes to fork at the same time.
//    - run_interval: Check for new jobs every 'run_interval' minutes.
//                    (By default, every 2 minutes).
//    - sandbox: Collection of node modules to pass to the sandboxed tasks
//               execution. Object with the form:
//               {
//                  'module_global_var_name': 'node-module-name'
//               }
//               By default, only the global timeouts are passed to the tasks
//               sandbox.
var WorkflowRunner = module.exports = function (backend, opts) {
  this.backend = backend;
  if (!opts) {
    opts = {};
  }
  this.identifier = opts.identifier || null;
  this.forks = opts.forks || 10;
  this.run_interval = (opts.run_interval || 2) * 60 * 1000;
  this.interval = null;
  this.shutting_down = false;
  this.child_processes = [];
  this.sandbox = opts.sandbox || {};
  this.trace = opts.trace || false;
  this.log = opts.log || console;
};

WorkflowRunner.prototype.init = function (callback) {
  var self = this,
      series = [];
  // Using async.series we can kind of write this sequentially and run some
  // methods only when needed:

  // 1) We need an unique identifier for this runner, also something
  //    human friendly to identify the runner from a bunch of them would be
  //    helpful.
  if (!self.identifier) {
    series.push(function (cb) {
      self.getIdentifier(function (err, uuid) {
        if (err) {
          return cb(err);
        }
        self.identifier = uuid;
        // Not that we really care about the return value here, anyway:
        return cb(null, uuid);
      });
    });
  }
  // The runner will register itself on the backend, with its unique id.
  series.push(function (cb) {
    self.backend.registerRunner(self.identifier, function (err) {
      if (err) {
        return cb(err);
      }
      return cb(null, null);
    });
  });
  // 2) On init, the runner will check for any job flagged as running with
  //    the runner identifier. This means a previous failure so, first thing
  //    will be take care of such failure.
  //    NOTE: It would be desirable to "hide" the next queued Job to the
  //    other runners when we fetch it for the current one.
  async.series(series, function (err, results) {
    // Note we don't care at all about the results.
    if (err) {
      return callback(err);
    }
    return callback();
  });
};

// Wait for children to finish, do not began any other child process.
// Call callback on done.
WorkflowRunner.prototype.quit = function (callback) {
  var self = this;
  self.shutting_down = true;
  clearInterval(self.interval);
  if (Object.keys(self.child_processes).length > 0) {
    process.nextTick(function () {
      self.quit(callback);
    });
  } else {
    callback();
  }
};

// This is the main runner method, where jobs execution takes place.
// Every call to this method will result into a child_process being forked.
WorkflowRunner.prototype.runJob = function (job, callback) {
  var self = this,
  wf_job_runner = new WorkflowJobRunner({
    runner: self,
    backend: self.backend,
    job: job,
    trace: self.trace
  });
  wf_job_runner.run(callback);
};

WorkflowRunner.prototype.getIdentifier = function (callback) {
  var cfg_file = path.resolve(__dirname, '../config/workflow-indentifier');

  path.exists(cfg_file, function (exist) {
    if (exist) {
      fs.readFile(cfg_file, 'utf-8', function (err, data) {
        if (err) {
          return callback(err);
        }
        return callback(null, data);
      });
    } else {
      var id = uuid();
      fs.writeFile(cfg_file, id, 'utf-8', function (err) {
        if (err) {
          return callback(err);
        }
        return callback(null, id);
      });
    }
  });
};

WorkflowRunner.prototype.runNow = function (job) {
  return (new Date().getTime() >= new Date(job.exec_after).getTime());
};

// Just in case we need to kill it without leaving child processes around:
WorkflowRunner.prototype.kill = function (callback) {
  var self = this;
  self.shutting_down = true;
  clearInterval(self.interval);
  if (Object.keys(self.child_processes).length > 0) {
    Object.keys(self.child_processes).forEach(function (p) {
      process.kill(p, 'SIGKILL');
    });
    self.child_processes = [];
  }
  if (callback) {
    callback();
  }
};

WorkflowRunner.prototype.run = function () {
  var self = this,
  // Queue worker. Tries to run a job, including "hiding" it from other
  // runners:
  worker = function (uuid, callback) {
    self.backend.getJob(uuid, function (err, job) {
      if (err) {
        callback(err);
      }

      if (self.runNow(job)) {
        self.backend.runJob(uuid, self.identifier, function (err) {
          if (err) {
            callback(err);
          }
          // Given backend.runJob returns nothing, we need to update the
          // execution status here:
          if (self.trace) {
            console.time('JOB ' + job.uuid);
          }
          job.execution = 'running';
          self.runJob(job, function (err) {
            if (self.trace) {
              console.timeEnd('JOB ' + job.uuid);
            }
            if (err) {
              callback(err);
            }
            callback();
          });
        });
      }
    });
  },
  // We keep a queue with concurrency limit where we'll be pushing new jobs
  queue = async.queue(worker, self.forks);

  self.interval = setInterval(function () {
    self.backend.runnerActive(self.identifier, function (err) {
      if (err) {
        self.log.error('Error reporting runner activity: ' + err);
        return;
      }
      self.backend.isRunnerIdle(self.identifier, function (idle) {
        if (idle === false) {
          var fetch = self.forks - Object.keys(self.child_processes).length;
          self.backend.nextJobs(0, fetch - 1, function (err, jobs) {
            // Error fetching jobs
            if (err) {
              self.log.error('Error fetching jobs: ' + err);
              return;
            }
            // No queued jobs
            if (!jobs) {
              self.log.info('No jobs queued');
              return;
            }
            // Got jobs, let's see if we can run them:
            jobs.forEach(function (job) {
              queue.push(job, function (err) {
                // To be called once queue worker finished processing the job
                if (err) {
                  self.log.error('Error running job: ' + err);
                }
                self.log.info('Job with uuid ' + job + ' ran successfully');
              });
            });
          });
        } else {
          self.log.info('Runner idle.');
        }
      });
    });
  }, self.run_interval);
};

WorkflowRunner.prototype.childUp = function (job_uuid, child_pid) {
  var self = this;
  self.child_processes[child_pid] = job_uuid;
};

WorkflowRunner.prototype.childDown = function (job_uuid, child_pid) {
  var self = this;
  delete self.child_processes[child_pid];
};

// Check for inactive runners.
// "Inactive runner" - a runner with latest status report older than
//  3 times the runners' run_interval.
//  - callback - f(err, runners): `err` always means backend error.
//    `runners` will be an array, even empty.
WorkflowRunner.prototype.inactiveRunners = function (callback) {
  var self = this,
      inactiveRunners = [];
  self.backend.getRunners(function (err, runners) {
    if (err) {
      return callback(err);
    }
    Object.keys(runners).forEach(function (id) {
      var outdated = new Date().getTime() - (self.run_interval * 3000);
      if (
        id !== self.identifier &&
        new Date(runners[id]).getTime() < outdated) {
        inactiveRunners.push(id);
      }
    });
    return callback(null, inactiveRunners);
  });
};

// TODO:
// - Runners should be able to pick any job on a "weird" status when
//   we detect a runner isn't upgrading its status after Y seconds
//   (also configurable), obviously Y needs to be greater than X.

if (require.main === module) {
  var Logger = require('bunyan');
  var log = new Logger({
    name: 'workflow-runner',
    streams: [ {
      level: 'info',
      stream: process.stdout
    }, {
      level: 'trace',
      path: path.resolve(__dirname, '../logs/runner.log')
    }],
    serializers: {
      err: Logger.stdSerializers.err
    }
  });
  var config_file = path.normalize(__dirname + '/../config/config.json');
  fs.readFile(config_file, 'utf8', function (err, data) {
    if (err) {
      throw err;
    }
    var config, backend, Backend, runner;
    config = JSON.parse(data);
    config.logger = log;
    Backend = require(config.backend.module);
    backend = new Backend(config.backend.opts);
    backend.init(function () {
      runner = new WorkflowRunner(backend, config.runner);
      runner.run();
    });
  });
}
<commit_msg>GH-35: Found nice bug while writing the examples.<commit_after>// Copyright 2012 Pedro P. Candel <kusorbox@gmail.com> All rights reserved.
var util = require('util'),
    uuid = require('node-uuid'),
    fs = require('fs'),
    path = require('path'),
    async = require('async'),
    WorkflowJobRunner = require('./job-runner');

// - opts - configuration options:
//    - identifier: Unique identifier for this runner.
//    - forks: Max number of child processes to fork at the same time.
//    - run_interval: Check for new jobs every 'run_interval' minutes.
//                    (By default, every 2 minutes).
//    - sandbox: Collection of node modules to pass to the sandboxed tasks
//               execution. Object with the form:
//               {
//                  'module_global_var_name': 'node-module-name'
//               }
//               By default, only the global timeouts are passed to the tasks
//               sandbox.
var WorkflowRunner = module.exports = function (backend, opts) {
  this.backend = backend;
  if (!opts) {
    opts = {};
  }
  this.identifier = opts.identifier || null;
  this.forks = opts.forks || 10;
  this.run_interval = (opts.run_interval || 2) * 60 * 1000;
  this.interval = null;
  this.shutting_down = false;
  this.child_processes = [];
  this.sandbox = opts.sandbox || {};
  this.trace = opts.trace || false;
  this.log = opts.log || console;
};

WorkflowRunner.prototype.init = function (callback) {
  var self = this,
      series = [];
  // Using async.series we can kind of write this sequentially and run some
  // methods only when needed:

  // 1) We need an unique identifier for this runner, also something
  //    human friendly to identify the runner from a bunch of them would be
  //    helpful.
  if (!self.identifier) {
    series.push(function (cb) {
      self.getIdentifier(function (err, uuid) {
        if (err) {
          return cb(err);
        }
        self.identifier = uuid;
        // Not that we really care about the return value here, anyway:
        return cb(null, uuid);
      });
    });
  }
  // The runner will register itself on the backend, with its unique id.
  series.push(function (cb) {
    self.backend.registerRunner(self.identifier, function (err) {
      if (err) {
        return cb(err);
      }
      return cb(null, null);
    });
  });
  // 2) On init, the runner will check for any job flagged as running with
  //    the runner identifier. This means a previous failure so, first thing
  //    will be take care of such failure.
  //    NOTE: It would be desirable to "hide" the next queued Job to the
  //    other runners when we fetch it for the current one.
  async.series(series, function (err, results) {
    // Note we don't care at all about the results.
    if (err) {
      return callback(err);
    }
    return callback();
  });
};

// Wait for children to finish, do not began any other child process.
// Call callback on done.
WorkflowRunner.prototype.quit = function (callback) {
  var self = this;
  self.shutting_down = true;
  clearInterval(self.interval);
  if (Object.keys(self.child_processes).length > 0) {
    process.nextTick(function () {
      self.quit(callback);
    });
  } else {
    callback();
  }
};

// This is the main runner method, where jobs execution takes place.
// Every call to this method will result into a child_process being forked.
WorkflowRunner.prototype.runJob = function (job, callback) {
  var self = this,
  wf_job_runner = new WorkflowJobRunner({
    runner: self,
    backend: self.backend,
    job: job,
    trace: self.trace,
    sandbox: self.sandbox
  });
  wf_job_runner.run(callback);
};

WorkflowRunner.prototype.getIdentifier = function (callback) {
  var cfg_file = path.resolve(__dirname, '../config/workflow-indentifier');

  path.exists(cfg_file, function (exist) {
    if (exist) {
      fs.readFile(cfg_file, 'utf-8', function (err, data) {
        if (err) {
          return callback(err);
        }
        return callback(null, data);
      });
    } else {
      var id = uuid();
      fs.writeFile(cfg_file, id, 'utf-8', function (err) {
        if (err) {
          return callback(err);
        }
        return callback(null, id);
      });
    }
  });
};

WorkflowRunner.prototype.runNow = function (job) {
  return (new Date().getTime() >= new Date(job.exec_after).getTime());
};

// Just in case we need to kill it without leaving child processes around:
WorkflowRunner.prototype.kill = function (callback) {
  var self = this;
  self.shutting_down = true;
  clearInterval(self.interval);
  if (Object.keys(self.child_processes).length > 0) {
    Object.keys(self.child_processes).forEach(function (p) {
      process.kill(p, 'SIGKILL');
    });
    self.child_processes = [];
  }
  if (callback) {
    callback();
  }
};

WorkflowRunner.prototype.run = function () {
  var self = this,
  // Queue worker. Tries to run a job, including "hiding" it from other
  // runners:
  worker = function (uuid, callback) {
    self.backend.getJob(uuid, function (err, job) {
      if (err) {
        callback(err);
      }

      if (self.runNow(job)) {
        self.backend.runJob(uuid, self.identifier, function (err) {
          if (err) {
            callback(err);
          }
          // Given backend.runJob returns nothing, we need to update the
          // execution status here:
          if (self.trace) {
            console.time('JOB ' + job.uuid);
          }
          job.execution = 'running';
          self.runJob(job, function (err) {
            if (self.trace) {
              console.timeEnd('JOB ' + job.uuid);
            }
            if (err) {
              callback(err);
            }
            callback();
          });
        });
      }
    });
  },
  // We keep a queue with concurrency limit where we'll be pushing new jobs
  queue = async.queue(worker, self.forks);

  self.interval = setInterval(function () {
    self.backend.runnerActive(self.identifier, function (err) {
      if (err) {
        self.log.error('Error reporting runner activity: ' + err);
        return;
      }
      self.backend.isRunnerIdle(self.identifier, function (idle) {
        if (idle === false) {
          var fetch = self.forks - Object.keys(self.child_processes).length;
          self.backend.nextJobs(0, fetch - 1, function (err, jobs) {
            // Error fetching jobs
            if (err) {
              self.log.error('Error fetching jobs: ' + err);
              return;
            }
            // No queued jobs
            if (!jobs) {
              self.log.info('No jobs queued');
              return;
            }
            // Got jobs, let's see if we can run them:
            jobs.forEach(function (job) {
              queue.push(job, function (err) {
                // To be called once queue worker finished processing the job
                if (err) {
                  self.log.error('Error running job: ' + err);
                }
                self.log.info('Job with uuid ' + job + ' ran successfully');
              });
            });
          });
        } else {
          self.log.info('Runner idle.');
        }
      });
    });
  }, self.run_interval);
};

WorkflowRunner.prototype.childUp = function (job_uuid, child_pid) {
  var self = this;
  self.child_processes[child_pid] = job_uuid;
};

WorkflowRunner.prototype.childDown = function (job_uuid, child_pid) {
  var self = this;
  delete self.child_processes[child_pid];
};

// Check for inactive runners.
// "Inactive runner" - a runner with latest status report older than
//  3 times the runners' run_interval.
//  - callback - f(err, runners): `err` always means backend error.
//    `runners` will be an array, even empty.
WorkflowRunner.prototype.inactiveRunners = function (callback) {
  var self = this,
      inactiveRunners = [];
  self.backend.getRunners(function (err, runners) {
    if (err) {
      return callback(err);
    }
    Object.keys(runners).forEach(function (id) {
      var outdated = new Date().getTime() - (self.run_interval * 3000);
      if (
        id !== self.identifier &&
        new Date(runners[id]).getTime() < outdated) {
        inactiveRunners.push(id);
      }
    });
    return callback(null, inactiveRunners);
  });
};

// TODO:
// - Runners should be able to pick any job on a "weird" status when
//   we detect a runner isn't upgrading its status after Y seconds
//   (also configurable), obviously Y needs to be greater than X.

if (require.main === module) {
  var Logger = require('bunyan');
  var log = new Logger({
    name: 'workflow-runner',
    streams: [ {
      level: 'info',
      stream: process.stdout
    }, {
      level: 'trace',
      path: path.resolve(__dirname, '../logs/runner.log')
    }],
    serializers: {
      err: Logger.stdSerializers.err
    }
  });
  var config_file = path.normalize(__dirname + '/../config/config.json');
  fs.readFile(config_file, 'utf8', function (err, data) {
    if (err) {
      throw err;
    }
    var config, backend, Backend, runner;
    config = JSON.parse(data);
    config.logger = log;
    Backend = require(config.backend.module);
    backend = new Backend(config.backend.opts);
    backend.init(function () {
      runner = new WorkflowRunner(backend, config.runner);
      runner.run();
    });
  });
}
<|endoftext|>