<commit_before>'use strict';

var _ = require('underscore');
var JuttleMoment = require('../../runtime/types/juttle-moment');
var windowMaker = require('./windowmaker');
var periodic_fanin = require('./periodic-fanin');
var errors = require('../../errors');
var Promise = require('bluebird');
var utils = require('../juttle-utils');
var Groups = require('../groups');
var values = require('../values');

var EMIT_BATCH_SIZE = 20000;

var INFO = {
    type: 'proc',
    options: {   // documented, non-deprecated options only
        acc: {},
        every: {},
        forget: {},
        from: {},
        on: {},
        over: {},
        to: {},
        reset: {}
    }
};

class reduce_base extends periodic_fanin {
    // basic reducer runner functions to feed points to reducers and
    // evaluate at epochs.
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var self = this;
        // call this from initialize() if you mix in these functions.
        if (options.over && !options.over.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'over',
                value: values.inspect(options.over)
            });
        }
        if (options.from && !options.from.moment) {
            throw this.compile_error('MOMENT-ERROR', {
                option: 'from',
                value: values.inspect(options.from)
            });
        }
        if (options.to && !options.to.moment) {
            throw this.compile_error('MOMENT-ERROR', {
                option: 'to',
                value: values.inspect(options.to)
            });
        }
        if ((options.from || options.to) && !options.over) {
            throw this.compile_error('FROM-TO-OVER-ERROR', {
                proc: 'reduce'
            });
        }
        this.expr = params.expr || {};
        this.reducer_lhs = _.map(params.reducer_index, function get(i) { return params.lhs[i]; });
        this.accumulate = options.acc || (options.reset !== undefined && !options.reset);
        this.groups = new Groups(this, options, params.funcMaker);
        if (options.forget && this.accumulate) {
            // if we never reset, forget is meaningless
            throw this.compile_error('FORGET-RESET-ERROR', {
                proc: 'reduce'
            });
        }
        if (options.forget && this.groups.by.length === 0) {
            // suppress empties for non-grouped reduce. behavior not yet implemented
            throw this.compile_error('FORGET-BY-ERROR', {
                proc: 'reduce'
            });
        }
        this.forget = !this.accumulate && (options.forget === undefined) ? (this.groups.by.length > 0) : !!options.forget;
        this.interval = options.every;
        this.batch_size = 0;
        this.over = options.over;
        this.from = options.from;
        this.min_epoch = this.from ? this.from.add(this.over) : null;
        this.to = options.to;
        if (this.over) {
            this.groups.funcMakerMaker = function() {
                // each group row will need its own window of points.
                return windowMaker(self.groups.funcMaker, self.over, self.location);
            };
        }
        this.last_epoch = JuttleMoment.epsMoment(-Infinity);
    }
    witness_timestamp(time) {
        // is time in the half-open interval [from...to), or timeless?
        // return true so it can be processed. Else possibly advance
        // an out-of-window reducer to its final time and return false.
        if (!time || (!this.from || this.from.lte(time)) && (!this.to || this.to.gt(time))) {
            return true;
        }
        if (this.to && this.to.lte(time)) {
            var out = this.advance_epoch(this.to);
            this.emit(out);
            this.emit_mark({ time: this.to });
        }
        return false;
    }
    process_points(points) {
        var k;
        for (k = 0; k < points.length; ++k) {
            if (this.witness_timestamp(points[k].time)) {
                this.process_pt(points[k]);
                this.batch_size += 1;
            }
        }
    }
    process_pt(pt) {
        // advance/update each reducer on the point
        var row = this.groups.lookup(pt);
        for (var k = 0; k < row.fns.length; ++k) {
            row.fns[k].update(pt);
        }
    }
    good_epoch(time) {
        // is time in the closed interval [min...to]?
        var min = this.min_epoch || this.from;
        return ((!min || min.lte(time)) &&
                (!this.to || this.to.gte(time)));
    }
    advance_epoch(epoch) {
        // we are crossing an epoch timestamp, so execute the reduce at the epoch.
        // output some points, and reset for the next one.
        var that = this, out = [];
        if (!epoch || (this.good_epoch(epoch) && this.last_epoch.lt(epoch))) {
            // new points get the batch end timestamp, eps behind the epoch
            var eob = epoch ? JuttleMoment.epsMoment(epoch) : null;
            this.last_epoch = epoch;
            if (this.over && eob) {
                // advance all group windows and age out old points
                this.groups.apply(function(keyID) {
                    var row = that.groups.get_row(keyID);
                    row.fns.forEach(function(fn) {
                        fn.advance(eob);
                    });
                    if (that.forget && row.fns.is_empty()) {
                        // in windowed group mode, delete any rows
                        // that have no data in their window.  (in
                        // non-windowed mode, regular reset takes care
                        // of this.  in windowed non-group mode, the
                        // non-group lives on to run on an empty
                        // batch)
                        that.groups.delete_group(keyID);
                    }
                });
            }
            this.groups.apply(function(keyID) {
                var row = that.groups.get_row(keyID);
                var fns = row.fns;
                var pt = { };
                // new points get the batch end timestamp
                if (eob) { pt.time = eob; }
                that.groups.mixin_keys(pt, keyID);
                try {
                    that.locate_juttle_errors(function() {
                        that.expr(pt, fns);
                    });
                    out.push(pt);
                } catch(e) {
                    if (e instanceof errors.RuntimeError) {
                        that.trigger('warning', e);
                    } else {
                        throw e;
                    }
                }
            });
        }

        if (!this.accumulate) {
            this.reset();
        }
        this.batch_size = 0;
        if (this.watch_oops && this.groups.by.length > 0) {
            // this.groups.apply() does not iterate groups in time order, so sort them here.
            out.sort(utils.pointSortFunc('time'));
        }
        return out;
    }
    reset() {
        if (this.over || !this.forget) {
            // windowed or remember-groups mode: reset individual rows by calling their funcMakers
            this.groups.reset_fns();
        } else {
            // non-windowed forgetful mode: tear down the whole group table and start over
            this.groups.reset_groups();
        }
    }
    tick(time) {
        // give the tick a chance to advance the reducer if it should.
        if (this.periodic && time.gt(this.program.now)) {
            this.advance(time);
        } else if (!this.periodic && !this.has_first_mark && time.gt(this.program.now)) {
            // For one-shot windowed reducers, a tick may advance if this.to has passed.
            this.witness_timestamp(time);
        }
        this.emit_tick(time);
    }
}

// reduce driven by upstream batches or EOF.
// Mix in grouping and reduce runner behaviors
class reduce_batch extends reduce_base {
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var allowed_options = ['acc', 'every', 'forget', 'from', 'groupby', 'on', 'over', 'reset', 'to', 'no_groupby_warning'];
        this.validate_options(allowed_options);

        // Disable the base class periodic behavior
        this.periodic = false;

        if (options.on) {
            // if we are here, it is because -every is null (likely a
            // default value for an optional -every parameter), or was
            // not specified, but -on was specified with a non-null
            // value.
            throw this.compile_error('ON-NO-EVERY-ERROR', {
                proc: this.procName()
            });
        }
        this.has_first_mark = false;
    }
    procName() {
        return 'reduce';
    }
    process(points) {
        this.process_points(points);
    }
    mark(mark, from) {
        if (this.has_first_mark) {
            var out = this.advance_epoch(mark.time);
            this.emit(out, from);
        } else {
            this.has_first_mark = true;
        }
        // output this mark after any generated points for this batch.
        // those points will get timestamps epsilon less than this
        // batch end mark if they came from a sort/reduce/join
        // processor.
        this.emit_mark({ time: mark.time }, from);
    }
    eof(from) {
        var self = this;
        if (!this.has_first_mark) {
            // unbatched: run once. If -to was specified the one-shot
            // result gets it as the final timestamp (and if it was
            // previously triggered by data or ticks beyond to, this will
            // do nothing)
            var out = this.advance_epoch(this.to);
            return Promise.each(_.range(out.length / EMIT_BATCH_SIZE), function() {
                return Promise.try(function() {
                    self.emit(out.splice(0, EMIT_BATCH_SIZE));
                }).delay(1);
            })
            .then(function() {
                self.emit_eof();
            });
        }
        this.emit_eof();
    }
}

// data-driven reduce. This works like proc._reduce, but with epochs
// specified by -every and -on, and reduce operations triggered when
// timestamps cross epoch boundaries. It ignores any received batch
// marks (except to advance time) and does not emit marks with its results.
// Mix in periodic, grouping, and reduce runner behaviors
class reduce_every extends reduce_base {
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var allowed_options = ['acc', 'every', 'forget', 'from', 'groupby', 'on', 'over', 'reset', 'to', 'no_groupby_warning'];
        this.validate_options(allowed_options);

        if (!options.every.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'every',
                value: values.inspect(options.every)
            });
        }
        if (options.every.seconds() <= 0) {
            throw this.compile_error('REDUCE-EVERY-ERROR');
        }
        if (options.over && !options.over.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'over',
                value: values.inspect(options.over)
            });
        }
        if (options.on) {
            if (options.on.duration) {
                if (options.on.gt(options.every)) {
                    throw this.compile_error('ON-EVERY-ERROR', {
                        proc: this.procName()
                    });
                }
                this.on = (new JuttleMoment(0)).add(options.on);
            } else if (options.on.moment) {
                this.on = options.on;
            } else {
                throw this.compile_error('DURATION-OR-MOMENT-ERROR', {
                    option: 'on',
                    value: values.inspect(options.on)
                });
            }
        } else {
            // if -from or -to are specified, align on them by default.
            this.on = options.from || options.to ;
        }
        this.after_first_epoch = false;
        this.init_warnings();
    }
    procName() {
        return 'reduce';
    }
    first_epoch(epoch) {
        // emit initial mark (before any points arrive)
        this.emit_mark({ time: epoch });
        this.after_first_epoch = true;
    }
    //
    // mark() and process() are from periodic. tick() from reduce_runner_funcs
    //

    other_epoch(epoch) {
        super.other_epoch(epoch);
        this.emit_mark({ time: epoch });
    }

    eof(from) {
        // Edge case - 0 points followed immediately by eof. To be consistent
        // with `batch | reduce`, run aggregation once and emit the result.
        if (!this.batch_size && !this.after_first_epoch) {
            var out = this.advance_epoch(this.next_epoch);
            this.emit(out);
        }
        super.eof(from);
    }

    //
    // mark() and process() are from periodic. tick() from reduce_runner_funcs
    //
    periodic_eof(from) {
        if (this.batch_size) {
            // run once more for a trailing partial batch of points
            var out = this.advance_epoch(this.next_epoch);
            this.emit(out);
            this.emit_mark({ time: this.next_epoch });
        }
        this.emit_eof();
    }
}

// juttle reduce. Whether you get a batch-driven reducer or a
// data-driven reducer with its own epochs is decided by the presence
// of an -every option to reduce.
function reduce(options, params, location, program) {
    return options.every? new reduce_every(options, params, location, program) : new reduce_batch(options, params, location, program);
}

reduce.info = INFO;

module.exports = reduce;
<commit_msg>Reduce -every no longer emits marks<commit_after>'use strict';

var _ = require('underscore');
var JuttleMoment = require('../../runtime/types/juttle-moment');
var windowMaker = require('./windowmaker');
var periodic_fanin = require('./periodic-fanin');
var errors = require('../../errors');
var Promise = require('bluebird');
var utils = require('../juttle-utils');
var Groups = require('../groups');
var values = require('../values');

var EMIT_BATCH_SIZE = 20000;

var INFO = {
    type: 'proc',
    options: {   // documented, non-deprecated options only
        acc: {},
        every: {},
        forget: {},
        from: {},
        on: {},
        over: {},
        to: {},
        reset: {}
    }
};

class reduce_base extends periodic_fanin {
    // basic reducer runner functions to feed points to reducers and
    // evaluate at epochs.
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var self = this;
        // call this from initialize() if you mix in these functions.
        if (options.over && !options.over.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'over',
                value: values.inspect(options.over)
            });
        }
        if (options.from && !options.from.moment) {
            throw this.compile_error('MOMENT-ERROR', {
                option: 'from',
                value: values.inspect(options.from)
            });
        }
        if (options.to && !options.to.moment) {
            throw this.compile_error('MOMENT-ERROR', {
                option: 'to',
                value: values.inspect(options.to)
            });
        }
        if ((options.from || options.to) && !options.over) {
            throw this.compile_error('FROM-TO-OVER-ERROR', {
                proc: 'reduce'
            });
        }
        this.expr = params.expr || {};
        this.reducer_lhs = _.map(params.reducer_index, function get(i) { return params.lhs[i]; });
        this.accumulate = options.acc || (options.reset !== undefined && !options.reset);
        this.groups = new Groups(this, options, params.funcMaker);
        if (options.forget && this.accumulate) {
            // if we never reset, forget is meaningless
            throw this.compile_error('FORGET-RESET-ERROR', {
                proc: 'reduce'
            });
        }
        if (options.forget && this.groups.by.length === 0) {
            // suppress empties for non-grouped reduce. behavior not yet implemented
            throw this.compile_error('FORGET-BY-ERROR', {
                proc: 'reduce'
            });
        }
        this.forget = !this.accumulate && (options.forget === undefined) ? (this.groups.by.length > 0) : !!options.forget;
        this.interval = options.every;
        this.batch_size = 0;
        this.over = options.over;
        this.from = options.from;
        this.min_epoch = this.from ? this.from.add(this.over) : null;
        this.to = options.to;
        if (this.over) {
            this.groups.funcMakerMaker = function() {
                // each group row will need its own window of points.
                return windowMaker(self.groups.funcMaker, self.over, self.location);
            };
        }
        this.last_epoch = JuttleMoment.epsMoment(-Infinity);
    }
    witness_timestamp(time) {
        // is time in the half-open interval [from...to), or timeless?
        // return true so it can be processed. Else possibly advance
        // an out-of-window reducer to its final time and return false.
        if (!time || (!this.from || this.from.lte(time)) && (!this.to || this.to.gt(time))) {
            return true;
        }
        if (this.to && this.to.lte(time)) {
            var out = this.advance_epoch(this.to);
            this.emit(out);
        }
        return false;
    }
    process_points(points) {
        var k;
        for (k = 0; k < points.length; ++k) {
            if (this.witness_timestamp(points[k].time)) {
                this.process_pt(points[k]);
                this.batch_size += 1;
            }
        }
    }
    process_pt(pt) {
        // advance/update each reducer on the point
        var row = this.groups.lookup(pt);
        for (var k = 0; k < row.fns.length; ++k) {
            row.fns[k].update(pt);
        }
    }
    good_epoch(time) {
        // is time in the closed interval [min...to]?
        var min = this.min_epoch || this.from;
        return ((!min || min.lte(time)) &&
                (!this.to || this.to.gte(time)));
    }
    advance_epoch(epoch) {
        // we are crossing an epoch timestamp, so execute the reduce at the epoch.
        // output some points, and reset for the next one.
        var that = this, out = [];
        if (!epoch || (this.good_epoch(epoch) && this.last_epoch.lt(epoch))) {
            // new points get the batch end timestamp, eps behind the epoch
            var eob = epoch ? JuttleMoment.epsMoment(epoch) : null;
            this.last_epoch = epoch;
            if (this.over && eob) {
                // advance all group windows and age out old points
                this.groups.apply(function(keyID) {
                    var row = that.groups.get_row(keyID);
                    row.fns.forEach(function(fn) {
                        fn.advance(eob);
                    });
                    if (that.forget && row.fns.is_empty()) {
                        // in windowed group mode, delete any rows
                        // that have no data in their window.  (in
                        // non-windowed mode, regular reset takes care
                        // of this.  in windowed non-group mode, the
                        // non-group lives on to run on an empty
                        // batch)
                        that.groups.delete_group(keyID);
                    }
                });
            }
            this.groups.apply(function(keyID) {
                var row = that.groups.get_row(keyID);
                var fns = row.fns;
                var pt = { };
                // new points get the batch end timestamp
                if (eob) { pt.time = eob; }
                that.groups.mixin_keys(pt, keyID);
                try {
                    that.locate_juttle_errors(function() {
                        that.expr(pt, fns);
                    });
                    out.push(pt);
                } catch(e) {
                    if (e instanceof errors.RuntimeError) {
                        that.trigger('warning', e);
                    } else {
                        throw e;
                    }
                }
            });
        }

        if (!this.accumulate) {
            this.reset();
        }
        this.batch_size = 0;
        if (this.watch_oops && this.groups.by.length > 0) {
            // this.groups.apply() does not iterate groups in time order, so sort them here.
            out.sort(utils.pointSortFunc('time'));
        }
        return out;
    }
    reset() {
        if (this.over || !this.forget) {
            // windowed or remember-groups mode: reset individual rows by calling their funcMakers
            this.groups.reset_fns();
        } else {
            // non-windowed forgetful mode: tear down the whole group table and start over
            this.groups.reset_groups();
        }
    }
    tick(time) {
        // give the tick a chance to advance the reducer if it should.
        if (this.periodic && time.gt(this.program.now)) {
            this.advance(time);
        } else if (!this.periodic && !this.has_first_mark && time.gt(this.program.now)) {
            // For one-shot windowed reducers, a tick may advance if this.to has passed.
            this.witness_timestamp(time);
        }
        this.emit_tick(time);
    }
}

// reduce driven by upstream batches or EOF.
// Mix in grouping and reduce runner behaviors
class reduce_batch extends reduce_base {
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var allowed_options = ['acc', 'every', 'forget', 'from', 'groupby', 'on', 'over', 'reset', 'to', 'no_groupby_warning'];
        this.validate_options(allowed_options);

        // Disable the base class periodic behavior
        this.periodic = false;

        if (options.on) {
            // if we are here, it is because -every is null (likely a
            // default value for an optional -every parameter), or was
            // not specified, but -on was specified with a non-null
            // value.
            throw this.compile_error('ON-NO-EVERY-ERROR', {
                proc: this.procName()
            });
        }
        this.has_first_mark = false;
    }
    procName() {
        return 'reduce';
    }
    process(points) {
        this.process_points(points);
    }
    mark(mark, from) {
        if (this.has_first_mark) {
            var out = this.advance_epoch(mark.time);
            this.emit(out, from);
        } else {
            this.has_first_mark = true;
        }
        // output this mark after any generated points for this batch.
        // those points will get timestamps epsilon less than this
        // batch end mark if they came from a sort/reduce/join
        // processor.
        this.emit_mark({ time: mark.time }, from);
    }
    eof(from) {
        var self = this;
        if (!this.has_first_mark) {
            // unbatched: run once. If -to was specified the one-shot
            // result gets it as the final timestamp (and if it was
            // previously triggered by data or ticks beyond to, this will
            // do nothing)
            var out = this.advance_epoch(this.to);
            return Promise.each(_.range(out.length / EMIT_BATCH_SIZE), function() {
                return Promise.try(function() {
                    self.emit(out.splice(0, EMIT_BATCH_SIZE));
                }).delay(1);
            })
            .then(function() {
                self.emit_eof();
            });
        }
        this.emit_eof();
    }
}

// data-driven reduce. This works like proc._reduce, but with epochs
// specified by -every and -on, and reduce operations triggered when
// timestamps cross epoch boundaries. It ignores any received batch
// marks (except to advance time) and does not emit marks with its results.
// Mix in periodic, grouping, and reduce runner behaviors
class reduce_every extends reduce_base {
    constructor(options, params, location, program) {
        super(options, params, location, program);
        var allowed_options = ['acc', 'every', 'forget', 'from', 'groupby', 'on', 'over', 'reset', 'to', 'no_groupby_warning'];
        this.validate_options(allowed_options);

        if (!options.every.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'every',
                value: values.inspect(options.every)
            });
        }
        if (options.every.seconds() <= 0) {
            throw this.compile_error('REDUCE-EVERY-ERROR');
        }
        if (options.over && !options.over.duration) {
            throw this.compile_error('DURATION-ERROR', {
                option: 'over',
                value: values.inspect(options.over)
            });
        }
        if (options.on) {
            if (options.on.duration) {
                if (options.on.gt(options.every)) {
                    throw this.compile_error('ON-EVERY-ERROR', {
                        proc: this.procName()
                    });
                }
                this.on = (new JuttleMoment(0)).add(options.on);
            } else if (options.on.moment) {
                this.on = options.on;
            } else {
                throw this.compile_error('DURATION-OR-MOMENT-ERROR', {
                    option: 'on',
                    value: values.inspect(options.on)
                });
            }
        } else {
            // if -from or -to are specified, align on them by default.
            this.on = options.from || options.to ;
        }
        this.after_first_epoch = false;
        this.init_warnings();
    }
    procName() {
        return 'reduce';
    }
    first_epoch(epoch) {
        this.after_first_epoch = true;
    }
    //
    // mark() and process() are from periodic. tick() from reduce_runner_funcs
    //

    eof(from) {
        // Edge case - 0 points followed immediately by eof. To be consistent
        // with `batch | reduce`, run aggregation once and emit the result.
        if (!this.batch_size && !this.after_first_epoch) {
            var out = this.advance_epoch(this.next_epoch);
            this.emit(out);
        }
        super.eof(from);
    }

    //
    // mark() and process() are from periodic. tick() from reduce_runner_funcs
    //
    periodic_eof(from) {
        if (this.batch_size) {
            // run once more for a trailing partial batch of points
            var out = this.advance_epoch(this.next_epoch);
            this.emit(out);
        }
        this.emit_eof();
    }
}

// juttle reduce. Whether you get a batch-driven reducer or a
// data-driven reducer with its own epochs is decided by the presence
// of an -every option to reduce.
function reduce(options, params, location, program) {
    return options.every? new reduce_every(options, params, location, program) : new reduce_batch(options, params, location, program);
}

reduce.info = INFO;

module.exports = reduce;
<|endoftext|>


<commit_before>'use strict';

const moment = require('moment');
const inherits = require('util').inherits;
const Router = require('./index');
const middleware = require('storj-service-middleware');
const rawbody = middleware.rawbody;
const authenticate = middleware.authenticate;
const constants = require('../../constants');
const stripe = require('../vendor/stripe');

/**
 * Handles endpoints for all user related operations
 */
function DebitsRouter(options) {
  if (!(this instanceof DebitsRouter)) {
    return new DebitsRouter(options);
  }

  Router.apply(this, arguments);

  this.models = options.storage.models;
  this._verify = authenticate(this.storage);
}

inherits(DebitsRouter, Router);

DebitsRouter.prototype.createDebit = function(req, res) {
  const user = req.body.user;
  const type = req.body.type;
  const amount = req.body.amount;
  const created = req.body.created;

  if (!user || !type || !amount) {
    res.sendStatus(400);
  }

  this.models.PaymentProcessor.findOne({user: user, default: true})
      .then((paymentProcessor) => {
        const debit = new this.models.Debit({
          amount: amount,
          type: type,
          user: user,
          created: created
        });
        return Promise.all([
          debit.save(),
          Promise.resolve(paymentProcessor)
        ]);
      }, (err) => {
        console.error(`User lookup failed for user: ${user}`);
        return Promise.reject(err);
      })
      .then((results) => {
        const debit = results[0];
        const paymentProcessor = results[1];

        if (!paymentProcessor) {
          console.log(`Creating debit for ${user} with no default payment processor`);
          return res.status(201).json({debit: debit, warning: 'no default payment processors!'}).end();
        } else {
          // TODO: refactor this to be payment processor agnostic!
          const customerId = paymentProcessor.data.customer.id;

          stripe.invoiceItems.create({
            customer: customerId,
            amount: debit.amount,
            currency: 'usd',
            description: [
              'Storj.io Usage Charge - ',
              debit.type
            ].join('')
          }, (err, invoiceItem) => {
            if (err) {
              console.error(err);
              // TODO Decide whether to send actual error or customer error message
              return res.status(202).json({debit: debit, error: err})
            }

            return res.status(201).json({debit: debit, invoice: invoiceItem}).end();
          });
        }
      })
      .catch((err) => {
        console.error(err);
        // TODO Decide whether to send actual error or customer error message
        return res.status(400).json({error: err}).end();
      })
};

/**
 * Export definitions
 * @private
 */
DebitsRouter.prototype._definitions = function() {
  return [
    ['POST', '/debits', this._verify, this.createDebit]
  ];
};

module.exports = DebitsRouter;
<commit_msg>changes for buffering stripe invoice items to nearest, min cent<commit_after>'use strict';

const moment = require('moment');
const inherits = require('util').inherits;
const Router = require('./index');
const middleware = require('storj-service-middleware');
const authenticate = middleware.authenticate;
const constants = require('../../constants');
const stripe = require('../vendor/stripe');
const graphqlService = require('../graphql');

/**
 * Handles endpoints for all user related operations
 */
function DebitsRouter(options) {
  if (!(this instanceof DebitsRouter)) {
    return new DebitsRouter(options);
  }

  Router.apply(this, arguments);

  this.models = options.storage.models;
  this._verify = authenticate(this.storage);
}

inherits(DebitsRouter, Router);

DebitsRouter.prototype.createDebit = function(req, res) {
  const user = req.body.user;
  const type = req.body.type;
  const amount = req.body.amount;
  const created = req.body.created;

  if (!user || !type || !amount) {
    res.sendStatus(400);
  }

  graphqlService.defaultPaymentProcessor
    .then((paymentProcessor) => {
      if (!paymentProcessor) {
        console.log(`Creating debit for ${user} with no default payment processor`);
        return res.status(201).json({debit: debit, warning: 'no default payment processors!'}).end();
      } else {

        // NB: moment#month() is 0 indexed, but #year() & #date() are not
        //     and the array constructor (e.g. moment.utc([<year>, <month>, <date>])
        //     expects this!
        try {
          // TODO: refactor this to be payment processor agnostic!
          const customerId = paymentProcessor.data.customer.id;
          const billingDate = paymentProcessor.data.billingDate;
          const today = moment.utc();
          const endOfBillingMonth = moment
            .utc([today.year(), today.month()])
            .add(1, 'month')
            .subtract(1, 'day')
            .date();

          const adjustedBillingDate = (billingDate > endOfBillingMonth) ?
            endOfBillingMonth : billingDate;

          const startDate = moment.utc([today.year(), today.month(), adjustedBillingDate]);
          const endDate = moment.utc(startDate).add(1, 'month');

          return this.models.Debit
            .find({
              user: user,
              created: {
                $gte: startDate.toDate(),
                $lt: endDate.toDate()
              }
            })
            .then(debits => {
              return this.models.Debit.create({
                amount: amount,
                type: type,
                user: user,
                created: created
              })
                .then(debit => debits.concat([debit]));
            })
            .then((debits) => {
              paymentProcessor.balanceDebits(debits)
                .then((invoiceItem) => res.status(201).json({debit: debit, invoice: invoiceItem}).end())
                .catch((err) => res.status(202).json({debut: debit, error: err}).end());
            });
        } catch (err) {
          return Promise.reject(err);
        }
      }
    })
    .catch((err) => {
      console.error(err);
      // TODO Decide whether to send actual error or customer error message
      return res.status(500).json({error: err}).end();
    })
};

/**
 * Export definitions
 * @private
 */
DebitsRouter.prototype._definitions = function() {
  return [
    ['POST', '/debits', this._verify, this.createDebit]
  ];
};

module.exports = DebitsRouter;
<|endoftext|>


<commit_before>// const REPOS_CONFIG_FILENAME = './repos.json';
var debug = require('debug')('revo:repoService'),
    Promise = require('bluebird'),
    fs = Promise.promisifyAll(require('fs')),
    config = require('config'),
    request = require('request'),
    uuid = require('uuid'),
    S = require('string'),
    util = require('../util'),
    fs = require('fs'),
    mkdirp = require('mkdirp'),
    ncp = require('ncp').ncp,
    log = require('../logging'),
    Zip = require('adm-zip'),
    npm = require("npm"),
    _ = require('underscore');

// const REVO_HOME = config.home || __dirname.substring(0, __dirname.lastIndexOf('/')).substring(0, __dirname.lastIndexOf('/'));
var dir = __dirname;
dir = dir.substring(0, dir.lastIndexOf('/'));
dir = dir.substring(0, dir.lastIndexOf('/'));
const REVO_HOME = dir;
const LOCAL_REPO = REVO_HOME+'/repo/components';
const THEMES_DIR = REVO_HOME+'/repo/themes';
const COMPONENTS_DIR = REVO_HOME+'/repo/components';
const DOWNLOAD_DIR = REVO_HOME+'/repo/download';
const DOWNLOAD_DIR_SHORT = "repo/download";
const DEFAULT_COMPONENTS_REPO = 'http://localhost:10102';
const LOCAL_RECIPES_REPO = REVO_HOME+'/repo/recipes';
const LOCAL_APPS_REPO = REVO_HOME+'/repo/apps';

const ERROR = require('chalk').red.bold;
const WARN = require('chalk').yellow.bold;
const INFO = require('chalk').green;

var repos = [];

module.exports = {

    app: {
        list: function(){
            return util.getDirs(LOCAL_APPS_REPO);
        }
    },

    recipe: {
        // list local & optionally central recipes
        list: function(){
            return _.map(util.getFiles(LOCAL_RECIPES_REPO), function(`){
                return filename.replace(/\.yaml/, '');
            };
        },
        // check if a recipe exists in local repo or in the hub
        check: function(name) {
            var ret = 'unavailable';
            // check local recipes
            var recipes = util.getFiles(LOCAL_RECIPES_REPO);
            var fullRecipeName = S(name).ensureRight('.yaml').s;
            if(recipes.indexOf(fullRecipeName) >=0 ) return 'local';
            // todo: check for recipes in the hub
            return ret;
        }
    },

    web: {
        //todo: extract into theme model
        fetchTheme: function(theme){
            return new Promise(function (resolve) {
                var themeName = theme.name || 'noname';
                var themeTargetDir = [THEMES_DIR, themeName].join('/');
                // var ret = (theme.zip_path) ? [theme.name, theme.zip_path].join('/') : theme.name;
                var ret = false;
                fs.statAsync(themeTargetDir).then(function(){
                    debug('theme cached locally');
                    return resolve(ret);
                }).catch(function(){
                    ret = true;
                    debug('theme doesn\'t exist, fetching theme from', theme.url);
                    var targetFile = [DOWNLOAD_DIR, uuid.v4()].join('/');
                    var r = request(theme.url).pipe(fs.createWriteStream(targetFile));
                    r.on('finish', function () {
                        debug('download complete, unzipping...');
                        zip = new Zip(targetFile);
                        zip.extractAllTo(themeTargetDir, true);//overwrite existing
                        return resolve(ret);
                    });
                });
            });
        }
    },

    fetchComponent: function(component){
        return new Promise(function (resolve, reject) {
            var componentName = Object.keys(component)[0];
            var opts = component[componentName];
            var componentTargetDir = [COMPONENTS_DIR, componentName].join('/');
            var ret = componentName;
            fs.statAsync(componentTargetDir).then(function(){
                debug(componentName + ' component cached locally');
                resolve(ret);
            }).catch(function(){
                var componentRepo = opts.repo || DEFAULT_COMPONENTS_REPO;
                var componentVersion = opts.version || 'latest';
                var componentUrl = [componentRepo, componentName, componentVersion+'.zip'].join('/');
                if(opts.repo && (opts.repo == 'github')) {
                    componentUrl = ["https://github.com",componentName,"archive","master.zip"].join('/');
                }
                if(opts.uri) {
                    componentUrl = opts.uri;
                }
                debug(componentName+' doesn\'t exist in local repo, fetching component from '+componentUrl);
                var targetFile = [DOWNLOAD_DIR, uuid.v4()].join('/');
                var isError = false;
                var err;
                request({uri: componentUrl})
                // .get(componentUrl)
                .pipe(fs.createWriteStream(targetFile))
                .on('response', function (response) {
                    isError = response.statusCode != '200';
                    if(isError) {
                        err = {
                            component: componentName,
                            message:  "cannot download from "+componentUrl
                        }
                        if(response.headers.status) err.message += ' ['+response.headers.status+']';
                        return reject(err);
                    }
                })
                .on('error', function () {
                    isError = true;
                    err = "cannot download from "+componentUrl;
                    return reject(err);
                })
                .on('close', function () {
                    debug('download complete, adding component to local repo');
                    try {
                        zip = new Zip(targetFile);
                        zip.extractAllTo(componentTargetDir, true);//overwrite existing
                        // remove the intermediary directory.
                        var subdirs = util.getSubdirs(componentTargetDir);
                        var files = util.getFiles(componentTargetDir);
                        if((files == 0) && (subdirs.length == 1)) {
                            var deepTargetDir = [componentTargetDir, subdirs[0]].join('/');
                            debug(WARN('moving '+deepTargetDir+' to '+componentTargetDir))
                            ncp(deepTargetDir, componentTargetDir, function (err) {
                                if (err) return reject(err);
                                else {
                                    util.rimraf(deepTargetDir)
                                    .then(function(){
                                        return resolve(name);
                                    })
                                    .catch(function(){
                                        return reject(err);
                                    })
                                }
                            });
                        }

                        return resolve(ret);
                    } catch(err) {
                        return reject(err);
                    }
                })
            });
        });
    },

    getNpmCache: function(module){
        return new Promise(function (resolve, reject) {
            npm.load({ prefix: DOWNLOAD_DIR_SHORT }, function (er, npm) {
                npm.commands.list([module], true, function(err, info){
                    if(info.dependencies[module]) {
                        debug(module, 'cached at', info.dependencies[module].realPath);
                        if(err) return reject(err);
                        return resolve(info.dependencies[module].realPath);
                    } else {
                        debug(module, 'not installed. installing now');
                        npm.commands.install([module], function(err){
                            npm.commands.list([module], true, function(err, info){
                                if(info.dependencies[module]) {
                                    debug(module+' is now available at '+info.dependencies[module].realPath);
                                    if(err) return reject(err);
                                    else return resolve(info.dependencies[module].realPath);
                                }
                            });
                        });
                    }
                });
            });
        });
    },

//     fetchComponent: function(name) {
//         var componentFound = false;
//         var componentFullPath;
//         for(var i=0; i<repos.length; i++) {
//             var repo = repos[i];
//             if(repo.type && (repo.type == 'local')) {
//                 try {
//                     componentFullPath = S(repo.path).ensureRight('/').s + 'components/' + name;
//                     require.resolve(componentFullPath);
//                     componentFound = true;
//                     if(componentFound) break;
//                 } catch(err) {
//                     // component not found, try next repo
//                 }
// // console.log('TODO: checking component at', repos.length, 'repositories')
//             }
//         }
//         if(componentFound) {
// console.log('component found at', componentFullPath);
//         } else {
// console.log('no component', name, 'found');
//         }
//     },

    // installComponent: function(path, name) {
    //     var source = REVO_HOME + '/components/' + name;
    //     var destination = path + '/components/' + name;
    //     if(fs.existsSync(source)) {
    //         ncp(source, destination, function (err) {
    //             if (err) throw err
    //             log.info(name, 6);
    //         });
    //     } else {
    //         this.fetchComponent(name);
    //         // log.err(name + ' : unknown component', 6);
    //     }
    // },

    installComponent: function(path, name, opts) {
        // debug('installing component', name, 'with options', opts, 'to', path);
        debug('installing component ' + name);
        return new Promise(function(resolve, reject){
            var componentType = opts.type || 'web';
            var source = [COMPONENTS_DIR, name].join('/');
            if(!util.isDir(source)) {
                // return reject(source+' doesn\'t exist');
            } else {
                var subdir = 'components';// untyped components are handled as common components
                if(opts.type == 'web') subdir = ['public', 'components'].join('/');
                var componentName = name.replace('\/', '_');
                var destination = [path, subdir, componentName].join('/');
                debug('installing component from ['+source+'] to ['+destination+']');
                mkdirp(destination, function(err){
                    if(err) reject(err);
                    else {
                        ncp(source, destination, function (err) {
                            if (err) return reject(err);
                            else {
                                if(opts.type == 'web') {
                                    // interpolate index.html in web components
                                    // var templateFilename = destination + '/index.html';
                                    var originalTemplateFilename = destination + '/index.html';
                                    var templateFilename = destination + '/index.revo';
                                    fs.rename(originalTemplateFilename, templateFilename, function(err) {
                                        if (err) throw err;
                                        else {
                                            fs.readFile(templateFilename, {encoding: 'utf8'}, function (err, content) {
                                                if (err) throw err;
                                                var outputContent = util.compileTemplate(content, opts.locals);
                                                fs.writeFile(templateFilename, outputContent, function(err){
                                                    if (err) throw err;
                                                    return resolve(name);
                                                });
                                            })
                                        }
                                    });
                                }
                                else return resolve(name);
                            }
                        });
                    }
                });
            }
        });
    },

    installTheme: function(path, theme) {
        return new Promise(function(resolve, reject){
            var themeName = theme.name || 'noname';
            var templateRelativePath = (theme.zip_path) ? [theme.name, theme.zip_path].join('/') : theme.name;
            var source = [THEMES_DIR, templateRelativePath].join('/');
            var destination = [path, 'public'].join('/');
            debug('installing theme from', source);
            ncp(source, destination, function (err) {
                if (err) reject(err);
                else {
                    return resolve(destination);
                }
            });
        });
    },

    installDependency: function(path, module, moduleCachedPath) {
        return new Promise(function(resolve, reject){
            var source = moduleCachedPath;//[DOWNLOAD_DIR, "node_modules", module].join('/');
            var destination = [path, 'node_modules', module].join('/');
            debug('installing dependency from', source);
            ncp(source, destination, function (err) {
                if (err) reject(err);
                else {
                    return resolve(destination);
                }
            });
        });
    },

    publishComponentToLocalRepo: function(componentPath, repoPath) {
        var source = componentPath;
        var name = this.shortFilename(componentPath);
        var destination = repoPath + '/components/' + name;
        if(fs.existsSync(source)) {
            ncp(source, destination, function (err) {
                if (err) throw err
                log.info(name+' component installed in the local repo at '+repoPath, 6);
            });
        }
    },

    getComponentJson: function(name) {
        var ret = {};
        var componentJsonFilename = [COMPONENTS_DIR, name,'component'].join('/')+'.json';
        if (fs.existsSync(componentJsonFilename)) {
            var content = fs.readFileSync(componentJsonFilename);
            ret = JSON.parse(content);
        }
        return ret;
    }

}

function installFile(name, opts) {
    var inputFilename = 'templates/component/' + name;
    var outputFilename = __dirname + '/components/' + opts.name + '/' + name;
    fs.readFile(inputFilename, {encoding: 'utf8'}, function (err, content) {
        if (err) throw err;
        var outputContent = util.compileTemplate(content, opts);
        fs.writeFile(outputFilename, outputContent);
        if (opts.mode) fs.chmod(outputFilename, opts.mode);
    })
}
<commit_msg>cli: recipe list<commit_after>// const REPOS_CONFIG_FILENAME = './repos.json';
var debug = require('debug')('revo:repoService'),
    Promise = require('bluebird'),
    fs = Promise.promisifyAll(require('fs')),
    config = require('config'),
    request = require('request'),
    uuid = require('uuid'),
    S = require('string'),
    util = require('../util'),
    fs = require('fs'),
    mkdirp = require('mkdirp'),
    ncp = require('ncp').ncp,
    log = require('../logging'),
    Zip = require('adm-zip'),
    npm = require("npm"),
    _ = require('underscore');

// const REVO_HOME = config.home || __dirname.substring(0, __dirname.lastIndexOf('/')).substring(0, __dirname.lastIndexOf('/'));
var dir = __dirname;
dir = dir.substring(0, dir.lastIndexOf('/'));
dir = dir.substring(0, dir.lastIndexOf('/'));
const REVO_HOME = dir;
const LOCAL_REPO = REVO_HOME+'/repo/components';
const THEMES_DIR = REVO_HOME+'/repo/themes';
const COMPONENTS_DIR = REVO_HOME+'/repo/components';
const DOWNLOAD_DIR = REVO_HOME+'/repo/download';
const DOWNLOAD_DIR_SHORT = "repo/download";
const DEFAULT_COMPONENTS_REPO = 'http://localhost:10102';
const LOCAL_RECIPES_REPO = REVO_HOME+'/repo/recipes';
const LOCAL_APPS_REPO = REVO_HOME+'/repo/apps';

const ERROR = require('chalk').red.bold;
const WARN = require('chalk').yellow.bold;
const INFO = require('chalk').green;

var repos = [];

module.exports = {

    app: {
        list: function(){
            return util.getDirs(LOCAL_APPS_REPO);
        }
    },

    recipe: {
        // list local & optionally central recipes
        list: function(){
            return _.map(util.getFiles(LOCAL_RECIPES_REPO), function(filename){
                return filename.replace(/\.yaml/, '');
            });
        },
        // check if a recipe exists in local repo or in the hub
        check: function(name) {
            var ret = 'unavailable';
            // check local recipes
            var recipes = util.getFiles(LOCAL_RECIPES_REPO);
            var fullRecipeName = S(name).ensureRight('.yaml').s;
            if(recipes.indexOf(fullRecipeName) >=0 ) return 'local';
            // todo: check for recipes in the hub
            return ret;
        }
    },

    web: {
        //todo: extract into theme model
        fetchTheme: function(theme){
            return new Promise(function (resolve) {
                var themeName = theme.name || 'noname';
                var themeTargetDir = [THEMES_DIR, themeName].join('/');
                // var ret = (theme.zip_path) ? [theme.name, theme.zip_path].join('/') : theme.name;
                var ret = false;
                fs.statAsync(themeTargetDir).then(function(){
                    debug('theme cached locally');
                    return resolve(ret);
                }).catch(function(){
                    ret = true;
                    debug('theme doesn\'t exist, fetching theme from', theme.url);
                    var targetFile = [DOWNLOAD_DIR, uuid.v4()].join('/');
                    var r = request(theme.url).pipe(fs.createWriteStream(targetFile));
                    r.on('finish', function () {
                        debug('download complete, unzipping...');
                        zip = new Zip(targetFile);
                        zip.extractAllTo(themeTargetDir, true);//overwrite existing
                        return resolve(ret);
                    });
                });
            });
        }
    },

    fetchComponent: function(component){
        return new Promise(function (resolve, reject) {
            var componentName = Object.keys(component)[0];
            var opts = component[componentName];
            var componentTargetDir = [COMPONENTS_DIR, componentName].join('/');
            var ret = componentName;
            fs.statAsync(componentTargetDir).then(function(){
                debug(componentName + ' component cached locally');
                resolve(ret);
            }).catch(function(){
                var componentRepo = opts.repo || DEFAULT_COMPONENTS_REPO;
                var componentVersion = opts.version || 'latest';
                var componentUrl = [componentRepo, componentName, componentVersion+'.zip'].join('/');
                if(opts.repo && (opts.repo == 'github')) {
                    componentUrl = ["https://github.com",componentName,"archive","master.zip"].join('/');
                }
                if(opts.uri) {
                    componentUrl = opts.uri;
                }
                debug(componentName+' doesn\'t exist in local repo, fetching component from '+componentUrl);
                var targetFile = [DOWNLOAD_DIR, uuid.v4()].join('/');
                var isError = false;
                var err;
                request({uri: componentUrl})
                // .get(componentUrl)
                .pipe(fs.createWriteStream(targetFile))
                .on('response', function (response) {
                    isError = response.statusCode != '200';
                    if(isError) {
                        err = {
                            component: componentName,
                            message:  "cannot download from "+componentUrl
                        }
                        if(response.headers.status) err.message += ' ['+response.headers.status+']';
                        return reject(err);
                    }
                })
                .on('error', function () {
                    isError = true;
                    err = "cannot download from "+componentUrl;
                    return reject(err);
                })
                .on('close', function () {
                    debug('download complete, adding component to local repo');
                    try {
                        zip = new Zip(targetFile);
                        zip.extractAllTo(componentTargetDir, true);//overwrite existing
                        // remove the intermediary directory.
                        var subdirs = util.getSubdirs(componentTargetDir);
                        var files = util.getFiles(componentTargetDir);
                        if((files == 0) && (subdirs.length == 1)) {
                            var deepTargetDir = [componentTargetDir, subdirs[0]].join('/');
                            debug(WARN('moving '+deepTargetDir+' to '+componentTargetDir))
                            ncp(deepTargetDir, componentTargetDir, function (err) {
                                if (err) return reject(err);
                                else {
                                    util.rimraf(deepTargetDir)
                                    .then(function(){
                                        return resolve(name);
                                    })
                                    .catch(function(){
                                        return reject(err);
                                    })
                                }
                            });
                        }

                        return resolve(ret);
                    } catch(err) {
                        return reject(err);
                    }
                })
            });
        });
    },

    getNpmCache: function(module){
        return new Promise(function (resolve, reject) {
            npm.load({ prefix: DOWNLOAD_DIR_SHORT }, function (er, npm) {
                npm.commands.list([module], true, function(err, info){
                    if(info.dependencies[module]) {
                        debug(module, 'cached at', info.dependencies[module].realPath);
                        if(err) return reject(err);
                        return resolve(info.dependencies[module].realPath);
                    } else {
                        debug(module, 'not installed. installing now');
                        npm.commands.install([module], function(err){
                            npm.commands.list([module], true, function(err, info){
                                if(info.dependencies[module]) {
                                    debug(module+' is now available at '+info.dependencies[module].realPath);
                                    if(err) return reject(err);
                                    else return resolve(info.dependencies[module].realPath);
                                }
                            });
                        });
                    }
                });
            });
        });
    },

//     fetchComponent: function(name) {
//         var componentFound = false;
//         var componentFullPath;
//         for(var i=0; i<repos.length; i++) {
//             var repo = repos[i];
//             if(repo.type && (repo.type == 'local')) {
//                 try {
//                     componentFullPath = S(repo.path).ensureRight('/').s + 'components/' + name;
//                     require.resolve(componentFullPath);
//                     componentFound = true;
//                     if(componentFound) break;
//                 } catch(err) {
//                     // component not found, try next repo
//                 }
// // console.log('TODO: checking component at', repos.length, 'repositories')
//             }
//         }
//         if(componentFound) {
// console.log('component found at', componentFullPath);
//         } else {
// console.log('no component', name, 'found');
//         }
//     },

    // installComponent: function(path, name) {
    //     var source = REVO_HOME + '/components/' + name;
    //     var destination = path + '/components/' + name;
    //     if(fs.existsSync(source)) {
    //         ncp(source, destination, function (err) {
    //             if (err) throw err
    //             log.info(name, 6);
    //         });
    //     } else {
    //         this.fetchComponent(name);
    //         // log.err(name + ' : unknown component', 6);
    //     }
    // },

    installComponent: function(path, name, opts) {
        // debug('installing component', name, 'with options', opts, 'to', path);
        debug('installing component ' + name);
        return new Promise(function(resolve, reject){
            var componentType = opts.type || 'web';
            var source = [COMPONENTS_DIR, name].join('/');
            if(!util.isDir(source)) {
                // return reject(source+' doesn\'t exist');
            } else {
                var subdir = 'components';// untyped components are handled as common components
                if(opts.type == 'web') subdir = ['public', 'components'].join('/');
                var componentName = name.replace('\/', '_');
                var destination = [path, subdir, componentName].join('/');
                debug('installing component from ['+source+'] to ['+destination+']');
                mkdirp(destination, function(err){
                    if(err) reject(err);
                    else {
                        ncp(source, destination, function (err) {
                            if (err) return reject(err);
                            else {
                                if(opts.type == 'web') {
                                    // interpolate index.html in web components
                                    // var templateFilename = destination + '/index.html';
                                    var originalTemplateFilename = destination + '/index.html';
                                    var templateFilename = destination + '/index.revo';
                                    fs.rename(originalTemplateFilename, templateFilename, function(err) {
                                        if (err) throw err;
                                        else {
                                            fs.readFile(templateFilename, {encoding: 'utf8'}, function (err, content) {
                                                if (err) throw err;
                                                var outputContent = util.compileTemplate(content, opts.locals);
                                                fs.writeFile(templateFilename, outputContent, function(err){
                                                    if (err) throw err;
                                                    return resolve(name);
                                                });
                                            })
                                        }
                                    });
                                }
                                else return resolve(name);
                            }
                        });
                    }
                });
            }
        });
    },

    installTheme: function(path, theme) {
        return new Promise(function(resolve, reject){
            var themeName = theme.name || 'noname';
            var templateRelativePath = (theme.zip_path) ? [theme.name, theme.zip_path].join('/') : theme.name;
            var source = [THEMES_DIR, templateRelativePath].join('/');
            var destination = [path, 'public'].join('/');
            debug('installing theme from', source);
            ncp(source, destination, function (err) {
                if (err) reject(err);
                else {
                    return resolve(destination);
                }
            });
        });
    },

    installDependency: function(path, module, moduleCachedPath) {
        return new Promise(function(resolve, reject){
            var source = moduleCachedPath;//[DOWNLOAD_DIR, "node_modules", module].join('/');
            var destination = [path, 'node_modules', module].join('/');
            debug('installing dependency from', source);
            ncp(source, destination, function (err) {
                if (err) reject(err);
                else {
                    return resolve(destination);
                }
            });
        });
    },

    publishComponentToLocalRepo: function(componentPath, repoPath) {
        var source = componentPath;
        var name = this.shortFilename(componentPath);
        var destination = repoPath + '/components/' + name;
        if(fs.existsSync(source)) {
            ncp(source, destination, function (err) {
                if (err) throw err
                log.info(name+' component installed in the local repo at '+repoPath, 6);
            });
        }
    },

    getComponentJson: function(name) {
        var ret = {};
        var componentJsonFilename = [COMPONENTS_DIR, name,'component'].join('/')+'.json';
        if (fs.existsSync(componentJsonFilename)) {
            var content = fs.readFileSync(componentJsonFilename);
            ret = JSON.parse(content);
        }
        return ret;
    }

}

function installFile(name, opts) {
    var inputFilename = 'templates/component/' + name;
    var outputFilename = __dirname + '/components/' + opts.name + '/' + name;
    fs.readFile(inputFilename, {encoding: 'utf8'}, function (err, content) {
        if (err) throw err;
        var outputContent = util.compileTemplate(content, opts);
        fs.writeFile(outputFilename, outputContent);
        if (opts.mode) fs.chmod(outputFilename, opts.mode);
    })
}
<|endoftext|>


<commit_before>var async = require('async');
var _ = require('underscore');
var DatasetClient = require('./DatasetClient');
var util = require('util');
var syncUtil = require('./util');
var SYNC_UPDATE_TYPES = require('./pending-processor').SYNC_UPDATE_TYPES;
var convertToObject = syncUtil.convertToObject;
var syncStorage, pendingQueue;
var debug = syncUtil.debug;

/**
 * List the records for the given datasetClient
 * @param {DatasetClient} datasetClient
 * @param {Function} cb
 */
function listLocalDatasetClientData(datasetClient, cb) {
  syncStorage.readDatasetClientWithRecords(datasetClient.getId(), function(err, datasetClientsWithRecords) {
    if (err) {
      return cb(err);
    }
    //no one sync loop has completed yet, return null
    if (!datasetClientsWithRecords.syncCompleted) {
      return cb(null, null);
    } else {
      return cb(null, datasetClientsWithRecords);
    }
  });
}

/**
 * List the applied changes since the last sync loop for the given client
 * @param {String} datasetId the id of the dataset
 * @param {Number} lastSyncEndTime the timestamp of when the last sync loop completed
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listAppliedChangeSinceLastSync(datasetId, lastSyncEndTime, clientInfo, cb) {
  syncStorage.listUpdates(datasetId, {
    type: SYNC_UPDATE_TYPES.APPLIED,
    cuid: clientInfo.cuid,
    timestamp: {$gt: lastSyncEndTime}
  }, cb);
}

/**
 * List the pending changes for the given client that are still waiting to be processed
 * @param {String} datasetId
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listPendingChangesForClient(datasetId, clientInfo, cb) {
  pendingQueue.search({datasetId: datasetId, cuid: clientInfo.cuid}, cb);
}

/**
 * List all the possible changes that are not in the current local dataset copy yet.
 * It includes:
 * - the updates that are applied since last sync loop run from the given client
 * - the pending changes that are still waiting to be processed for the given client
 * @param {String} datasetId
 * @param {Number} lastSyncTime the timestamp of when the last sync loop completed
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listChangesNotInLocalDataset(datasetId, lastSyncTime, clientInfo, cb) {
  async.parallel({
    appliedUpdatesSinceLastSync: async.apply(listAppliedChangeSinceLastSync, datasetId, lastSyncTime, clientInfo),
    pendingChangesForClient: async.apply(listPendingChangesForClient, datasetId, clientInfo)
  }, function(err, results) {
    if (err) {
      return cb(err);
    }
    return cb(null, results.appliedUpdatesSinceLastSync, results.pendingChangesForClient);
  });
}

/**
 * We have some updates that are applied to the backed since last sync run. However, those updates are not in the
 * localDatasetClient yet (they will in after next sync run). But those updates are applied on the client already.
 * This means when we compute the delta between the client records and the localDatasetClient, those changes will be reverted from the client.
 * To avoid that, we loop through those updates and remove them both from the clientRecords and localDatasetClient.
 * This way there will be no delta for those records and the client will keep there records.
 *
 * @param {any} clientRecords
 * @param {any} localDatasetClient
 * @param {any} appliedUpdates
 */
function removeAppliedUpdates(clientRecords, localDatasetClient, appliedUpdates) {
  _.each(appliedUpdates, function(update, recordUid) {
    if (clientRecords[recordUid]) {
      delete clientRecords[recordUid];
    }
    if (localDatasetClient[recordUid]) {
      delete localDatasetClient[recordUid];
    }
    //cover the creation cases. The uid could change. Client may still have the old uid.
    if (update.oldUid) {
      var oldUid = update.oldUid;
      if (clientRecords[oldUid]) {
        delete clientRecords[oldUid];
      }
    }
  });
}

/**
 * We have some pending changes sent in by the client and waiting in the queue. They are not processed yet so the backend or our localDataset don't have those changes.
 * We don't know if those changes will be applied or not. But they are already applied on the client which means if we compute the delta as it is, those changes will be reverted.
 * To avoid that, we will remove those changes from both the client and the localDataset so that the client will keep them for now.
 * @param {any} clientRecordds
 * @param {any} localDatasetClient
 * @param {any} pendingChanges
 */
function removePendingChanges(clientRecords, localDatasetClient, pendingChanges) {
  _.each(pendingChanges, function(pendingChange, uid) {
    if (clientRecords[uid]) {
      delete clientRecords[uid];
    }
    if (localDatasetClient[uid]) {
      delete localDatasetClient[uid];
    }
  });
}

/**
 * Find out the differences between the clientRecords and the serverRecords
 * @param {String} datasetId the id of the dataset
 * @param {Object} clientRecords client records. The key should be the uid of each record, and the value should be the hash of each record
 * @param {Object} serverRecords server records. The key should be the uid of each record, and the value should contain the `data` and the `hash` of each record
 * @returns the records that should be created/updated/deleted from the client
 */
function computeDelta(datasetId, clientRecords, serverRecords) {
  var creates = {};
  var updates = {};
  var deletes = {};
  _.each(serverRecords, function(serverRecord, serverRecordUid) {
    var serverRecHash = serverRecord.hash;
    //record is in both client and server
    if (clientRecords[serverRecordUid]) {
      //hash value doesn't match, needs update
      if (clientRecords[serverRecordUid] !== serverRecHash) {
        debug('[%s] Updating client record %s  client hash=%s', datasetId, serverRecordUid, clientRecords[serverRecordUid]);
        updates[serverRecordUid] = serverRecord;
      }
    } else {
      //record is not in the client, needs create
      debug('[%s] Creating client record %s', datasetId, serverRecordUid);
      creates[serverRecordUid] = serverRecord;
    }
  });

  _.each(clientRecords, function(clientRecordHash, clientRecordUid) {
    if (!serverRecords[clientRecordUid]) {
      //the record is in the client but not in the server, need delete
      debug('[%s] Deleting client record %s', datasetId, clientRecordUid);
      deletes[clientRecordUid] = {};
    }
  });

  return {
    create: creates,
    update: updates,
    delete: deletes
  };
}

/**
 * Sync the data records for the given client
 * @param {String} datasetId
 * @param {Object} params the request body, it normally contain those fields:
 * @param {Object} params.query_params the query parameter for the dataset from the client
 * @param {Object} params.meta_data the meta data for the dataset from the client
 * @param {Object} params._fh an object added by the client sdk. it should have the `cuid` of the client
 * @param {Object} params.clientRecs the client records
 * @param {Function} cb
 */
function syncRecords(datasetId, params, cb) {
  debug('[%s] process syncRecords request', datasetId);
  var queryParams = params.query_params || {};
  var metaData = params.meta_data || {}; //NOTE: the client doesn't send in this value for syncRecords ATM
  var cuid = syncUtil.getCuid(params);
  var datasetClient = new DatasetClient(datasetId, {queryParams: queryParams, metaData: metaData});
  var clientRecords = params.clientRecs;
  async.waterfall([
    function checkDatasetclientStopped(callback) {
      syncStorage.readDatasetClient(datasetClient.getId(), function(err, datasetClientJson) {
        if (err) {
          return callback(err);
        }
        if (!datasetClientJson) {
          var errMsg = "unknown dataset client datasetId = " + datasetId + " :: queryParams = " + util.inspect(queryParams);
          debug("[%s] %s", datasetId, errMsg);
          return callback(errMsg);
        }
        if (datasetClientJson.stopped === true) {
          return callback(new Error('sync stopped for dataset ' + datasetId));
        } else {
          return callback();
        }
      });
    },
    async.apply(listLocalDatasetClientData, datasetClient),
    function listOtherChanges(localDatasetData, callback) {
      if (localDatasetData) {
        listChangesNotInLocalDataset(datasetId, localDatasetData.syncLoopEnd, {cuid: cuid}, function(err, appliedUpdates, clientPendingChanges) {
          return callback(err, localDatasetData, appliedUpdates, clientPendingChanges);
        });
      } else {
        return callback();
      }
    }
  ], function(err, localDatasetData, appliedUpdatesSinceLastSync, pendingUpdates) {
    if (err) {
      return cb(err);
    }
    if (!localDatasetData) {
      return cb(null, {});
    }
    var localDatasetDataObj = convertToObject(localDatasetData.records);
    var appliedUpdatesSinceLastSyncObj = convertToObject(appliedUpdatesSinceLastSync);
    var pendingUpdatesObj = convertToObject(pendingUpdates);

    removeAppliedUpdates(clientRecords, localDatasetDataObj, appliedUpdatesSinceLastSyncObj);
    removePendingChanges(clientRecords, localDatasetDataObj, pendingUpdatesObj);

    var delta = computeDelta(datasetId, clientRecords, localDatasetDataObj);
    var res = delta;
    res.hash = localDatasetData.globalHash;
    debug('[%s] syncRecords API response %j', datasetId, res);
    return cb(null, res);
  });
}

module.exports = function(syncStorageImpl, pendingQueueImpl) {
  syncStorage = syncStorageImpl;
  pendingQueue = pendingQueueImpl;
  return syncRecords;
};
<commit_msg>FH-3460 Handle client records not being in syncRecords request<commit_after>var async = require('async');
var _ = require('underscore');
var DatasetClient = require('./DatasetClient');
var util = require('util');
var syncUtil = require('./util');
var SYNC_UPDATE_TYPES = require('./pending-processor').SYNC_UPDATE_TYPES;
var convertToObject = syncUtil.convertToObject;
var syncStorage, pendingQueue;
var debug = syncUtil.debug;

/**
 * List the records for the given datasetClient
 * @param {DatasetClient} datasetClient
 * @param {Function} cb
 */
function listLocalDatasetClientData(datasetClient, cb) {
  syncStorage.readDatasetClientWithRecords(datasetClient.getId(), function(err, datasetClientsWithRecords) {
    if (err) {
      return cb(err);
    }
    //no one sync loop has completed yet, return null
    if (!datasetClientsWithRecords.syncCompleted) {
      return cb(null, null);
    } else {
      return cb(null, datasetClientsWithRecords);
    }
  });
}

/**
 * List the applied changes since the last sync loop for the given client
 * @param {String} datasetId the id of the dataset
 * @param {Number} lastSyncEndTime the timestamp of when the last sync loop completed
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listAppliedChangeSinceLastSync(datasetId, lastSyncEndTime, clientInfo, cb) {
  syncStorage.listUpdates(datasetId, {
    type: SYNC_UPDATE_TYPES.APPLIED,
    cuid: clientInfo.cuid,
    timestamp: {$gt: lastSyncEndTime}
  }, cb);
}

/**
 * List the pending changes for the given client that are still waiting to be processed
 * @param {String} datasetId
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listPendingChangesForClient(datasetId, clientInfo, cb) {
  pendingQueue.search({datasetId: datasetId, cuid: clientInfo.cuid}, cb);
}

/**
 * List all the possible changes that are not in the current local dataset copy yet.
 * It includes:
 * - the updates that are applied since last sync loop run from the given client
 * - the pending changes that are still waiting to be processed for the given client
 * @param {String} datasetId
 * @param {Number} lastSyncTime the timestamp of when the last sync loop completed
 * @param {Object} clientInfo the client info. Should contain the cuid
 * @param {Function} cb
 */
function listChangesNotInLocalDataset(datasetId, lastSyncTime, clientInfo, cb) {
  async.parallel({
    appliedUpdatesSinceLastSync: async.apply(listAppliedChangeSinceLastSync, datasetId, lastSyncTime, clientInfo),
    pendingChangesForClient: async.apply(listPendingChangesForClient, datasetId, clientInfo)
  }, function(err, results) {
    if (err) {
      return cb(err);
    }
    return cb(null, results.appliedUpdatesSinceLastSync, results.pendingChangesForClient);
  });
}

/**
 * We have some updates that are applied to the backed since last sync run. However, those updates are not in the
 * localDatasetClient yet (they will in after next sync run). But those updates are applied on the client already.
 * This means when we compute the delta between the client records and the localDatasetClient, those changes will be reverted from the client.
 * To avoid that, we loop through those updates and remove them both from the clientRecords and localDatasetClient.
 * This way there will be no delta for those records and the client will keep there records.
 *
 * @param {any} clientRecords
 * @param {any} localDatasetClient
 * @param {any} appliedUpdates
 */
function removeAppliedUpdates(clientRecords, localDatasetClient, appliedUpdates) {
  _.each(appliedUpdates, function(update, recordUid) {
    if (clientRecords[recordUid]) {
      delete clientRecords[recordUid];
    }
    if (localDatasetClient[recordUid]) {
      delete localDatasetClient[recordUid];
    }
    //cover the creation cases. The uid could change. Client may still have the old uid.
    if (update.oldUid) {
      var oldUid = update.oldUid;
      if (clientRecords[oldUid]) {
        delete clientRecords[oldUid];
      }
    }
  });
}

/**
 * We have some pending changes sent in by the client and waiting in the queue. They are not processed yet so the backend or our localDataset don't have those changes.
 * We don't know if those changes will be applied or not. But they are already applied on the client which means if we compute the delta as it is, those changes will be reverted.
 * To avoid that, we will remove those changes from both the client and the localDataset so that the client will keep them for now.
 * @param {any} clientRecordds
 * @param {any} localDatasetClient
 * @param {any} pendingChanges
 */
function removePendingChanges(clientRecords, localDatasetClient, pendingChanges) {
  _.each(pendingChanges, function(pendingChange, uid) {
    if (clientRecords[uid]) {
      delete clientRecords[uid];
    }
    if (localDatasetClient[uid]) {
      delete localDatasetClient[uid];
    }
  });
}

/**
 * Find out the differences between the clientRecords and the serverRecords
 * @param {String} datasetId the id of the dataset
 * @param {Object} clientRecords client records. The key should be the uid of each record, and the value should be the hash of each record
 * @param {Object} serverRecords server records. The key should be the uid of each record, and the value should contain the `data` and the `hash` of each record
 * @returns the records that should be created/updated/deleted from the client
 */
function computeDelta(datasetId, clientRecords, serverRecords) {
  var creates = {};
  var updates = {};
  var deletes = {};
  _.each(serverRecords, function(serverRecord, serverRecordUid) {
    var serverRecHash = serverRecord.hash;
    //record is in both client and server
    if (clientRecords[serverRecordUid]) {
      //hash value doesn't match, needs update
      if (clientRecords[serverRecordUid] !== serverRecHash) {
        debug('[%s] Updating client record %s  client hash=%s', datasetId, serverRecordUid, clientRecords[serverRecordUid]);
        updates[serverRecordUid] = serverRecord;
      }
    } else {
      //record is not in the client, needs create
      debug('[%s] Creating client record %s', datasetId, serverRecordUid);
      creates[serverRecordUid] = serverRecord;
    }
  });

  _.each(clientRecords, function(clientRecordHash, clientRecordUid) {
    if (!serverRecords[clientRecordUid]) {
      //the record is in the client but not in the server, need delete
      debug('[%s] Deleting client record %s', datasetId, clientRecordUid);
      deletes[clientRecordUid] = {};
    }
  });

  return {
    create: creates,
    update: updates,
    delete: deletes
  };
}

/**
 * Sync the data records for the given client
 * @param {String} datasetId
 * @param {Object} params the request body, it normally contain those fields:
 * @param {Object} params.query_params the query parameter for the dataset from the client
 * @param {Object} params.meta_data the meta data for the dataset from the client
 * @param {Object} params._fh an object added by the client sdk. it should have the `cuid` of the client
 * @param {Object} params.clientRecs the client records
 * @param {Function} cb
 */
function syncRecords(datasetId, params, cb) {
  debug('[%s] process syncRecords request', datasetId);
  var queryParams = params.query_params || {};
  var metaData = params.meta_data || {}; //NOTE: the client doesn't send in this value for syncRecords ATM
  var cuid = syncUtil.getCuid(params);
  var datasetClient = new DatasetClient(datasetId, {queryParams: queryParams, metaData: metaData});
  var clientRecords = params.clientRecs || {};
  async.waterfall([
    function checkDatasetclientStopped(callback) {
      syncStorage.readDatasetClient(datasetClient.getId(), function(err, datasetClientJson) {
        if (err) {
          return callback(err);
        }
        if (!datasetClientJson) {
          var errMsg = "unknown dataset client datasetId = " + datasetId + " :: queryParams = " + util.inspect(queryParams);
          debug("[%s] %s", datasetId, errMsg);
          return callback(errMsg);
        }
        if (datasetClientJson.stopped === true) {
          return callback(new Error('sync stopped for dataset ' + datasetId));
        } else {
          return callback();
        }
      });
    },
    async.apply(listLocalDatasetClientData, datasetClient),
    function listOtherChanges(localDatasetData, callback) {
      if (localDatasetData) {
        listChangesNotInLocalDataset(datasetId, localDatasetData.syncLoopEnd, {cuid: cuid}, function(err, appliedUpdates, clientPendingChanges) {
          return callback(err, localDatasetData, appliedUpdates, clientPendingChanges);
        });
      } else {
        return callback();
      }
    }
  ], function(err, localDatasetData, appliedUpdatesSinceLastSync, pendingUpdates) {
    if (err) {
      return cb(err);
    }
    if (!localDatasetData) {
      return cb(null, {});
    }
    var localDatasetDataObj = convertToObject(localDatasetData.records);
    var appliedUpdatesSinceLastSyncObj = convertToObject(appliedUpdatesSinceLastSync);
    var pendingUpdatesObj = convertToObject(pendingUpdates);

    removeAppliedUpdates(clientRecords, localDatasetDataObj, appliedUpdatesSinceLastSyncObj);
    removePendingChanges(clientRecords, localDatasetDataObj, pendingUpdatesObj);

    var delta = computeDelta(datasetId, clientRecords, localDatasetDataObj);
    var res = delta;
    res.hash = localDatasetData.globalHash;
    debug('[%s] syncRecords API response %j', datasetId, res);
    return cb(null, res);
  });
}

module.exports = function(syncStorageImpl, pendingQueueImpl) {
  syncStorage = syncStorageImpl;
  pendingQueue = pendingQueueImpl;
  return syncRecords;
};
<|endoftext|>


<commit_before>'use strict';
import AWS from 'aws-sdk';
import config from 'config';
import BBPromise from 'bluebird';
import logger from '@hoist/logger';
import Moment from 'moment';
import url from 'url';
import request from 'request-promise';
import HealthCheck from '../models/health_check';
import {
  flatten,
  includes,
  find,
  filter
}
from 'lodash';
AWS.config.update({
  accessKeyId: config.get('Hoist.aws.account'),
  secretAccessKey: config.get('Hoist.aws.secret'),
  region: config.get('Hoist.aws.region')
});

class HealthChecker {
  constructor() {
    this._logger = logger.child({
      cls: this.constructor.name
    });
    this.ec2 = BBPromise.promisifyAll(new AWS.EC2());
  }
  loadRabbitConnections() {
    var rabbitUrl = url.parse(config.get('Hoist.rabbit.managementurl'));
    rabbitUrl.pathname = '/api/connections';
    return request.get(url.format(rabbitUrl), {
        json: true
      })
      .then((response) => {
        return response.map((connection) => {
          return connection.peer_host;
        });
      });
  }
  loadRabbitQueues() {
    var rabbitUrl = url.parse(config.get('Hoist.rabbit.managementurl'));
    rabbitUrl.pathname = '/api/queues';
    return request.get(url.format(rabbitUrl), {
        json: true
      })
      .then((response) => {
        return response.filter((queue) => {
          return queue.name.startsWith('executor:commands:');
        }).map((queue) => {
          return queue.name;
        });
      });
  }
  loadEC2Details() {
    this._logger.info('finding instances in EC2');
    return this.ec2.describeInstancesAsync({
      Filters: [{
        Name: 'instance-state-name',
        Values: ['running']
      }, {
        Name: 'tag:aws:autoscaling:groupName',
        Values: ['main-cluster_executor_container_host_autoscale']
      }]
    }).then((result) => {
      var instances = result.Reservations.map((reservation) => {
        return reservation.Instances;
      });
      instances = flatten(instances).map((instance) => {
        return {
          id: instance.InstanceId,
          launchTime: new Moment(instance.LaunchTime),
          privateIpAddress: instance.PrivateIpAddress
        };
      });
      this._logger.info({
        instanceCount: instances.length
      }, 'got response');
      return instances;
    });
  }
  markHealth(instance, connections, queues) {
    if (instance.launchTime.isAfter(new Moment().subtract(15, 'minutes'))) {
      this._logger.info({
        instance: instance.id
      }, 'instance is less than 15 minutes old');
      instance.healthy = true;
    } else if (includes(connections, instance.privateIpAddress)) {
      this._logger.info({
        instance: instance.id
      }, 'instance connected');
      let connectedQueues = filter(queues, (queue) => {
        let matches = new RegExp(`executor-[abcdefghij]-${instance.id}`).test(queue);
        return matches;
      });
      this._logger.info({
        instance: instance.id,
        connectedQueues: connectedQueues.length
      }, 'instance connected');
      if (connectedQueues.length > 9) {
        this._logger.info({
          instance: instance.id
        }, 'executor VM has 10 executors connected');
        instance.healthy = true;
      } else {
        this._logger.info({
          instance: instance.id
        }, 'executor VM has less than 10 executors connected');
        instance.healthy = false;
      }

    } else {
      this._logger.info({
        instance: instance.id
      }, 'instance not connected');
      instance.healthy = false;
    }
  }
  updateHealthCheck(healthCheck, instances) {
    return Promise.resolve()
      .then(() => {
        let previousResults = healthCheck.instances;
        healthCheck.instances = [];
        this._logger.info('updating health check results');
        instances.forEach((instance) => {
          let instanceHealth = find(previousResults, (previousResult) => {
            return previousResult.instanceId === instance.id;
          }) || {
            instanceId: instance.id,
            passedChecks: 0,
            failedSince: null,
            currentState: 'healthy',
            failedChecks: 0

          };
          if (instance.healthy) {
            instanceHealth.failedSince = null;
            instanceHealth.passedChecks = instanceHealth.passedChecks + 1;
            instanceHealth.currentState = 'healthy';
          } else {
            instanceHealth.failedSince = instanceHealth.failedSince || new Moment().utc().toDate();
            instanceHealth.failedChecks = instanceHealth.failedChecks + 1;
            instanceHealth.currentState = 'unhealthy';
          }
          healthCheck.instances.push(instanceHealth);
        });
      });
  }
  terminateFailingInstances(healthCheck) {
    this._logger.info('finding failed instances');
    return Promise.resolve()
      .then(() => {
        let failedInstances = healthCheck.instances.filter((instance) => {
          return instance.currentState === 'unhealthy' &&
            new Moment(instance.failedSince).isBefore(new Moment().subtract(10, 'minutes'));
        });
        this._logger.info({
          failedInstances: failedInstances.length
        }, 'got failed instances');
        if (failedInstances.length > 0) {
          let instanceIds = failedInstances.map((instance) => {
            return instance.instanceId;
          });
          this._logger.info({
            instanceIds
          }, 'terminating instances');
          return this.ec2.terminateInstancesAsync({
            InstanceIds: instanceIds
          });
        }
      });
  }
  check() {
    return this.loadEC2Details()
      .then((instances) => {
        return Promise.all([
            this.loadRabbitConnections(),
            this.loadRabbitQueues()
          ])
          .then((results) => {
            let connections = results[0];
            let queues = results[1];
            this._logger.info({
              connections: connections.length,
              queues: queues.length
            }, 'connections');
            instances.forEach((instance) => {
              this.markHealth(instance, connections, queues);
            });
            this._logger.info('finding health check record');
            return HealthCheck.findOneAsync({}).then((healthCheck) => {
              this._logger.info('record found');
              return healthCheck || new HealthCheck();
            }).then((healthCheck) => {
              this._logger.info('updating health checks');
              return this.updateHealthCheck(healthCheck, instances).then(() => {
                this._logger.info('terminating instances');
                return this.terminateFailingInstances(healthCheck).then(() => {
                  this._logger.info({

                  }, 'saving health check');
                  return healthCheck.saveAsync().then(() => {
                    this._logger.info('saved health checks');
                  }).catch((err) => {
                    this._logger.error(err);
                  });
                });
              });
            });
          });

      }).catch((err) => {
        this._logger.error(err);
        throw err;
      });

  }
}

var mongoConnection = require('../mongoose_connection');

mongoConnection.connect();

let healthChecker = new HealthChecker();
healthChecker.check().then(() => {
  process.exit(0);
});
<commit_msg>only kill executors if less than 6 connected<commit_after>'use strict';
import AWS from 'aws-sdk';
import config from 'config';
import BBPromise from 'bluebird';
import logger from '@hoist/logger';
import Moment from 'moment';
import url from 'url';
import request from 'request-promise';
import HealthCheck from '../models/health_check';
import {
  flatten,
  includes,
  find,
  filter
}
from 'lodash';
AWS.config.update({
  accessKeyId: config.get('Hoist.aws.account'),
  secretAccessKey: config.get('Hoist.aws.secret'),
  region: config.get('Hoist.aws.region')
});

class HealthChecker {
  constructor() {
    this._logger = logger.child({
      cls: this.constructor.name
    });
    this.ec2 = BBPromise.promisifyAll(new AWS.EC2());
  }
  loadRabbitConnections() {
    var rabbitUrl = url.parse(config.get('Hoist.rabbit.managementurl'));
    rabbitUrl.pathname = '/api/connections';
    return request.get(url.format(rabbitUrl), {
        json: true
      })
      .then((response) => {
        return response.map((connection) => {
          return connection.peer_host;
        });
      });
  }
  loadRabbitQueues() {
    var rabbitUrl = url.parse(config.get('Hoist.rabbit.managementurl'));
    rabbitUrl.pathname = '/api/queues';
    return request.get(url.format(rabbitUrl), {
        json: true
      })
      .then((response) => {
        return response.filter((queue) => {
          return queue.name.startsWith('executor:commands:');
        }).map((queue) => {
          return queue.name;
        });
      });
  }
  loadEC2Details() {
    this._logger.info('finding instances in EC2');
    return this.ec2.describeInstancesAsync({
      Filters: [{
        Name: 'instance-state-name',
        Values: ['running']
      }, {
        Name: 'tag:aws:autoscaling:groupName',
        Values: ['main-cluster_executor_container_host_autoscale']
      }]
    }).then((result) => {
      var instances = result.Reservations.map((reservation) => {
        return reservation.Instances;
      });
      instances = flatten(instances).map((instance) => {
        return {
          id: instance.InstanceId,
          launchTime: new Moment(instance.LaunchTime),
          privateIpAddress: instance.PrivateIpAddress
        };
      });
      this._logger.info({
        instanceCount: instances.length
      }, 'got response');
      return instances;
    });
  }
  markHealth(instance, connections, queues) {
    if (instance.launchTime.isAfter(new Moment().subtract(15, 'minutes'))) {
      this._logger.info({
        instance: instance.id
      }, 'instance is less than 15 minutes old');
      instance.healthy = true;
    } else if (includes(connections, instance.privateIpAddress)) {
      this._logger.info({
        instance: instance.id
      }, 'instance connected');
      let connectedQueues = filter(queues, (queue) => {
        let matches = new RegExp(`executor-[abcdefghij]-${instance.id}`).test(queue);
        return matches;
      });
      this._logger.info({
        instance: instance.id,
        connectedQueues: connectedQueues.length
      }, 'instance connected');
      if (connectedQueues.length > 6) {
        this._logger.info({
          instance: instance.id
        }, 'executor VM has most of their executors connected');
        instance.healthy = true;
      } else {
        this._logger.info({
          instance: instance.id
        }, 'executor VM has less than half executors connected');
        instance.healthy = false;
      }

    } else {
      this._logger.info({
        instance: instance.id
      }, 'instance not connected');
      instance.healthy = false;
    }
  }
  updateHealthCheck(healthCheck, instances) {
    return Promise.resolve()
      .then(() => {
        let previousResults = healthCheck.instances;
        healthCheck.instances = [];
        this._logger.info('updating health check results');
        instances.forEach((instance) => {
          let instanceHealth = find(previousResults, (previousResult) => {
            return previousResult.instanceId === instance.id;
          }) || {
            instanceId: instance.id,
            passedChecks: 0,
            failedSince: null,
            currentState: 'healthy',
            failedChecks: 0

          };
          if (instance.healthy) {
            instanceHealth.failedSince = null;
            instanceHealth.passedChecks = instanceHealth.passedChecks + 1;
            instanceHealth.currentState = 'healthy';
          } else {
            instanceHealth.failedSince = instanceHealth.failedSince || new Moment().utc().toDate();
            instanceHealth.failedChecks = instanceHealth.failedChecks + 1;
            instanceHealth.currentState = 'unhealthy';
          }
          healthCheck.instances.push(instanceHealth);
        });
      });
  }
  terminateFailingInstances(healthCheck) {
    this._logger.info('finding failed instances');
    return Promise.resolve()
      .then(() => {
        let failedInstances = healthCheck.instances.filter((instance) => {
          return instance.currentState === 'unhealthy' &&
            new Moment(instance.failedSince).isBefore(new Moment().subtract(10, 'minutes'));
        });
        this._logger.info({
          failedInstances: failedInstances.length
        }, 'got failed instances');
        if (failedInstances.length > 0) {
          let instanceIds = failedInstances.map((instance) => {
            return instance.instanceId;
          });
          this._logger.info({
            instanceIds
          }, 'terminating instances');
          return this.ec2.terminateInstancesAsync({
            InstanceIds: instanceIds
          });
        }
      });
  }
  check() {
    return this.loadEC2Details()
      .then((instances) => {
        return Promise.all([
            this.loadRabbitConnections(),
            this.loadRabbitQueues()
          ])
          .then((results) => {
            let connections = results[0];
            let queues = results[1];
            this._logger.info({
              connections: connections.length,
              queues: queues.length
            }, 'connections');
            instances.forEach((instance) => {
              this.markHealth(instance, connections, queues);
            });
            this._logger.info('finding health check record');
            return HealthCheck.findOneAsync({}).then((healthCheck) => {
              this._logger.info('record found');
              return healthCheck || new HealthCheck();
            }).then((healthCheck) => {
              this._logger.info('updating health checks');
              return this.updateHealthCheck(healthCheck, instances).then(() => {
                this._logger.info('terminating instances');
                return this.terminateFailingInstances(healthCheck).then(() => {
                  this._logger.info({

                  }, 'saving health check');
                  return healthCheck.saveAsync().then(() => {
                    this._logger.info('saved health checks');
                  }).catch((err) => {
                    this._logger.error(err);
                  });
                });
              });
            });
          });

      }).catch((err) => {
        this._logger.error(err);
        throw err;
      });

  }
}

var mongoConnection = require('../mongoose_connection');

mongoConnection.connect();

let healthChecker = new HealthChecker();
healthChecker.check().then(() => {
  process.exit(0);
});
<|endoftext|>