<commit_before>var urllib = require('urllib');
var list = require('./data.json').dataList || [];
var fs = require('fs');

var url = 'https://api.github.com/zen';
var cursor = 0 ;

var option = {
  timeout: 10000
}

function hasEnough(data) {
  if (cursor > 5) {
    return true;
  }
  if (list.indexOf(data) != -1) {
    cursor ++;
  } else {
    cursor = 0;
  }
  return false;
}

function doFetch(cb) {
  urllib.request(url, option, function (err, data) {
    if (err) {
      //console.log(err);
      return cb();
      //TODO
    }
    var result = data.toString();
    if (list.indexOf(result) == -1) {
      list.push(result);
    }
    console.log(result);
    cb && cb(result);
  });
}

function saveData(data) {

  if (hasEnough(data)) {
    fs.writeFileSync('./data.json', JSON.stringify(list, 2, 2));
  } else {
    doFetch(saveData);
  }
}

doFetch(saveData)<commit_msg>add COMMON NUMBER<commit_after>var urllib = require('urllib');
var list = require('./data.json').dataList || [];
var fs = require('fs');

var url = 'https://api.github.com/zen';
var cursor = 0 ;
var MAX_COUNT = 5;

var option = {
  timeout: 10000
}

function hasEnough(data) {
  if (cursor > MAX_COUNT) {
    return true;
  }
  if (list.indexOf(data) != -1) {
    cursor ++;
  } else {
    cursor = 0;
  }
  return false;
}

function doFetch(cb) {
  urllib.request(url, option, function (err, data) {
    if (err) {
      //console.log(err);
      return cb();
      //TODO
    }
    var result = data.toString();
    if (list.indexOf(result) == -1) {
      list.push(result);
    }
    console.log(result);
    cb && cb(result);
  });
}

function saveData(data) {

  if (hasEnough(data)) {
    fs.writeFileSync('./data.json', JSON.stringify(list, 2, 2));
  } else {
    doFetch(saveData);
  }
}

doFetch(saveData)<|endoftext|>


<commit_before>const mongoose = require('mongoose');
mongoose.Promise = Promise;
const Schema = mongoose.Schema;
const ObjectId = mongoose.Types.ObjectId;
const timestamp = require('mongoose-timestamp');
const debug = require('debug')('mongostate');
const jsondiffpatch = require('jsondiffpatch').create();

const states = {
  INIT: 'init',
  PENDING: 'pending',
  COMMITTED: 'committed',
  ROLLBACK: 'rollback',
  CANCELLED: 'cancelled'
};

const operations = {
  CREATE: 'create',
  UPDATE: 'update',
  REMOVE: 'remove'
};

const transactionSchema = new Schema({
  state: {
    type: String,
    enums: [states.INIT, states.PENDING, states.COMMITTED, states.ROLLBACK, states.CANCELLED],
    default: states.INIT,
    required: true
  },
  usedModelNames: [String],
  actions: [{
    operation: {
      type: String, required: true, enums: [
        operations.CREATE,
        operations.UPDATE,
        operations.REMOVE
      ]
    },
    model: { type: String, required: true },
    entity: { type: String, required: true },
    enableHistory: { type: Boolean, default: false }
  }],
  error: {
    message: String,
    stack: String
  },
  biz: {}
});

const lockSchema = new Schema({
  transaction: { type: Schema.ObjectId, required: true, index: true },
  model: { type: String, required: true },
  entity: { type: String, required: true }
});

lockSchema.index({ model: 1, entity: 1 }, { unique: true });

const historySchema = new Schema({
  transaction: { type: Schema.ObjectId, required: true, index: true },
  entity: { type: String, required: true, index: true },
  biz: {},
  prev: {},
  diff: {},
  reverted: { type: Boolean, default: false }
});

historySchema.virtual('current').get(function () {
  return jsondiffpatch.patch(this.prev, this.diff);
});

historySchema.set('toJSON', { virtuals: true });

if (!mongoose.plugins.some(plugin => plugin[0] === timestamp)) {
  transactionSchema.plugin(timestamp);
  lockSchema.plugin(timestamp);
  historySchema.plugin(timestamp);
}

class Transaction {
  constructor ({
    connection,
    transactionModel,
    lockModel,
    id,
    historyConnection
  }) {
    this.id = id;
    this.transactionModel = transactionModel;
    this.lockModel = lockModel;
    this.connection = connection;
    this.usedModel = {};
    this.historyConnection = historyConnection;
  }

  static getTransactionModel (connection, transactionCollectionName = 'transaction') {
    if (!connection) throw new Error('connection is required!');
    return connection.model(transactionCollectionName, transactionSchema);
  }

  static getLockModel (connection, lockCollectionName = 'lock') {
    if (!connection) throw new Error('connection is required!');
    return connection.model(lockCollectionName, lockSchema);
  }

  static * init ({
    connection,
    id,
    transactionCollectionName,
    lockCollectionName,
    historyConnection,
    biz = {}
  } = {}) {
    const transactionModel = this.getTransactionModel(connection, transactionCollectionName);
    const lockModel = this.getLockModel(connection, lockCollectionName);
    let t;
    if (id) t = yield transactionModel.findById(id);
    if (!t) {
      id = id || new ObjectId;
      yield transactionModel.create({ _id: id, biz: JSON.parse(JSON.stringify(biz)) });
    } else {
      if ([states.CANCELLED, states.COMMITTED].includes(t.state)) throw new Error(`The transaction [${t.id}] has [${t.state}].`);
    }
    return new this({
      connection,
      transactionModel,
      lockModel,
      id,
      historyConnection
    });
  }

  * 'try' (wrapper = function * () {
  }) {
    if (wrapper.constructor.name !== 'GeneratorFunction') throw new Error('wrapper should be a generator function.');
    const transaction = yield this.findTransaction();
    if (transaction.state === states.INIT) {
      yield this.transactionModel.findByIdAndUpdate(this.id, {
        $set: { state: states.PENDING }
      });
    }
    let result;
    try {
      const transaction = yield this.findTransaction();
      if (transaction.state === states.PENDING) {
        result = yield wrapper.bind(this)();
        yield this.commit();
      } else {
        yield this.cancel(new Error('Transaction is not pending!'));
      }
    } catch (err) {
      yield this.cancel(err);
      throw err;
    }
    return result;
  }

  use (Model, enableHistory = true) {
    const modelName = Model.modelName;
    const schema = Model.schema;
    schema.add({
      __t: { type: Schema.ObjectId, required: true }
    });
    const SSModel = this.connection.model(`sub_state_${modelName}`, schema);
    if (!this.usedModel[modelName]) {
      this.usedModel[modelName] = { Model, SSModel };
      this.transactionModel.findOneAndUpdate({
        _id: this.id,
        usedModelNames: { $ne: modelName }
      }, {
        $push: {
          usedModelNames: modelName
        }
      }).exec();
    }
    let History;
    if (this.historyConnection) {
      History = this.historyConnection.model(`${modelName}_history`, historySchema);
    }
    return {
      create: function * (...params) {
        return yield this.create.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOneAndUpdate: function * (...params) {
        return yield this.findOneAndUpdate.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findByIdAndUpdate: function * (...params) {
        return yield this.findByIdAndUpdate.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findByIdAndRemove: function * (...params) {
        return yield this.findByIdAndRemove.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOneAndRemove: function * (...params) {
        return yield this.findOneAndRemove.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOne: function * (...params) {
        return yield this.findOne.bind(this)(Model, SSModel, params);
      }.bind(this),
      findById: function * (...params) {
        return yield this.findById.bind(this)(Model, SSModel, params);
      }.bind(this),
      findHistories: function * (...params) {
        if (!History) return [];
        return yield History.find(...params);
      },
      findLatestHistory: function * (...params) {
        if (!History) return null;
        return yield History.findOne(...params).sort({ _id: -1 });
      },
      revertTo: function * (historyId) {
        if(!Transaction.noWarning) {
          console.warn('The `revertTo` method is dangerous, only use it when you know why!');
        }
        if (!History) return null;
        const transaction = yield this.findTransaction();
        const history = yield History.findById(historyId);
        if (!history) throw new Error(`The history [${historyId}] is not exists!`);
        const doc = history.toJSON().prev;
        const prevEntity = yield Model.findById(history.entity);
        let prev;
        if (prevEntity) {
          prev = prevEntity.toJSON();
          delete prev.__v;
        }
        const diff = jsondiffpatch.diff(prev, doc);
        yield History.create({
          transaction: this.id,
          entity: history.entity,
          biz: transaction.biz,
          prev,
          diff,
          reverted: true
        });
        if (doc) {
          return yield Model.findByIdAndUpdate(history.entity, doc, { upsert: true, new: true });
        } else {
          yield Model.findByIdAndRemove(history.entity);
          return null;
        }
      }.bind(this)
    };
  }

  * create (Model, SSModel, [doc], enableHistory) {
    if (!doc._id) {
      doc._id = new ObjectId;
    } else {
      const entity = yield this.findById(Model, SSModel, [doc._id]);
      if (entity) throw new Error('entity has already exists!');
    }
    yield this.pushAction({
      operation: operations.CREATE,
      model: Model.modelName,
      entity: doc._id,
      enableHistory
    });
    yield this.lock({ id: doc._id }, Model);
    if(doc.constructor.name === 'model') {
      doc = doc.toJSON();
    }
    return yield this.initSubStateData(SSModel, doc);
  }

  * findOneAndUpdate (Model, SSModel, [query, doc, options], enableHistory) {
    const entity = yield this.findOne(Model, SSModel, [query]);
    if (!entity) throw new Error('Entity is not exists');
    yield this.pushAction({
      operation: operations.UPDATE,
      model: Model.modelName,
      entity: entity.id,
      enableHistory
    });
    yield this.lock(entity, Model);
    if(!doc.$set) {
      doc.__t = this.id;
    }
    return yield SSModel.findOneAndUpdate(query, doc, options);
  }

  * findByIdAndUpdate (Model, SSModel, [id, doc, options], enableHistory) {
    return yield this.findOneAndUpdate(Model, SSModel, [{ _id: id }, doc, options], enableHistory);
  }

  * findOneAndRemove (Model, SSModel, [query], enableHistory) {
    const entity = yield this.findOne(Model, SSModel, [query]);
    yield this.pushAction({
      operation: operations.REMOVE,
      model: Model.modelName,
      entity: entity.id,
      enableHistory
    });
    yield this.lock(entity, Model);
  }

  * findByIdAndRemove (Model, SSModel, [id], enableHistory) {
    yield this.findOneAndRemove(Model, SSModel, [{ _id: id }], enableHistory);
  }

  * findOne (Model, SSModel, [criteria]) {
    const entity = yield SSModel.findOne(criteria);
    const lock = yield this.checkLock(entity, Model);
    if (lock) {
      return entity;
    }
    if (!entity) {
      const srcEntity = yield Model.findOne(criteria);
      if (srcEntity) {
        const doc = srcEntity.toJSON();
        delete doc.__v;
        yield this.initSubStateData(SSModel, doc);
      }
      return srcEntity;
    }
    return entity;
  }

  * findById (Model, SSModel, [id]) {
    return yield this.findOne(Model, SSModel, [{ _id: id }]);
  }

  * checkLock (entity, Model) {
    if (!entity) return null;
    const lock = yield this.lockModel.findOne({
      model: Model.modelName,
      entity: entity.id
    });
    if (lock && lock.transaction.toString() !== this.id.toString()) {
      throw new Error(`entity [${entity.id}] is locked by transaction [${lock.transaction}]`);
    }
    return lock;
  }

  * lock (entity, Model) {
    if (!entity) throw new Error('Entity is not exists!');
    const lock = yield this.checkLock(entity, Model);
    if (!lock) {
      yield this.lockModel.create({
        transaction: this.id,
        model: Model.modelName,
        entity: entity.id
      });
    }
  }

  * initSubStateData (SSModel, doc) {
    doc.__t = this.id;
    return yield SSModel.create(doc);
  }

  * pushAction (action) {
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $push: {
        actions: action
      }
    });
  }

  * findTransaction () {
    return yield this.transactionModel.findById(this.id);
  }

  * commit () {
    const transaction = yield this.findTransaction();
    if (transaction.state !== states.PENDING) throw new Error(`Expected the transaction [${this.id}] to be pending, but got ${transaction.state}`);

    const entitiesActivated = [];

    for (let action of transaction.actions.reverse()) {
      const { model, entity, enableHistory, operation } = action;
      if (entitiesActivated.includes(`${model}:${entity}`)) continue;
      entitiesActivated.push(`${model}:${entity}`);
      const { Model, SSModel } = this.usedModel[model] || {};
      if (!Model) throw new Error(`Model ${model} has not used, please use the model first`);
      const subStateEntity = yield SSModel.findById(entity);
      let doc;
      if (subStateEntity) {
        doc = subStateEntity.toJSON();
        delete doc.__v;
        delete doc.__t;
      }
      // Record histories
      if (this.historyConnection && enableHistory) {
        const History = this.historyConnection.model(`${model}_history`, historySchema);
        const prevEntity = yield Model.findById(entity);
        let prev;
        if (prevEntity) {
          prev = prevEntity.toJSON();
          delete prev.__v;
        }
        const diff = jsondiffpatch.diff(prev, doc);
        yield History.create({
          transaction: this.id,
          entity,
          biz: transaction.biz,
          prev,
          diff,
        });
      }
      if (doc) {
        const doc = subStateEntity.toJSON();
        delete doc.__v;
        delete doc.__t;
        delete doc.createdAt;
        delete doc.updatedAt;
        yield Model.findByIdAndUpdate(entity, doc, { upsert: true });
      } else {
        yield Model.findByIdAndRemove(entity);
      }
    }
    yield this.clearSubStateData();
    yield this.unlock();
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.COMMITTED
      }
    });
    debug(`Transaction [${this.id}] committed!`);
  }

  * rollback () {
    const transaction = yield this.findTransaction();
    if (![states.PENDING, states.ROLLBACK].includes(transaction.state)) throw new Error(`Expected the transaction [${this.id}] to be pending/rollback, but got ${transaction.state}`);
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.ROLLBACK
      }
    });
    yield this.clearSubStateData();
    debug(`Transaction [${this.id}] rollback success!`);
  }

  * cancel (error) {
    const transaction = yield this.findTransaction();
    if (![states.PENDING, states.ROLLBACK].includes(transaction.state)) throw new Error(`Expected the transaction [${this.id}] to be pending/rollback, but got ${transaction.state}`);
    yield this.rollback();
    yield this.unlock();
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.CANCELLED
      },
      error: {
        message: error.message,
        stack: error.stack
      }
    });
    debug(`Transaction [${this.id}] cancelled!`);
  }

  * clearSubStateData () {
    const usedModelNames = Object.keys(this.usedModel);
    const t = yield this.findTransaction();
    if (t.usedModelNames.some(modelName => !usedModelNames.includes(modelName))) {
      throw new Error(`${t.usedModelNames} should be used first!`);
    }
    for (let modelName of usedModelNames) {
      const { SSModel } = this.usedModel[modelName];
      yield SSModel.remove({ __t: this.id });
    }
  }

  * unlock () {
    yield this.lockModel.remove({ transaction: this.id });
  }

}

module.exports = Transaction;<commit_msg>fixed:   修复了使用$unset时 无法重写的bug<commit_after>const mongoose = require('mongoose');
mongoose.Promise = Promise;
const Schema = mongoose.Schema;
const ObjectId = mongoose.Types.ObjectId;
const timestamp = require('mongoose-timestamp');
const debug = require('debug')('mongostate');
const jsondiffpatch = require('jsondiffpatch').create();

const states = {
  INIT: 'init',
  PENDING: 'pending',
  COMMITTED: 'committed',
  ROLLBACK: 'rollback',
  CANCELLED: 'cancelled'
};

const operations = {
  CREATE: 'create',
  UPDATE: 'update',
  REMOVE: 'remove'
};

const transactionSchema = new Schema({
  state: {
    type: String,
    enums: [states.INIT, states.PENDING, states.COMMITTED, states.ROLLBACK, states.CANCELLED],
    default: states.INIT,
    required: true
  },
  usedModelNames: [String],
  actions: [{
    operation: {
      type: String, required: true, enums: [
        operations.CREATE,
        operations.UPDATE,
        operations.REMOVE
      ]
    },
    model: { type: String, required: true },
    entity: { type: String, required: true },
    enableHistory: { type: Boolean, default: false }
  }],
  error: {
    message: String,
    stack: String
  },
  biz: {}
});

const lockSchema = new Schema({
  transaction: { type: Schema.ObjectId, required: true, index: true },
  model: { type: String, required: true },
  entity: { type: String, required: true }
});

lockSchema.index({ model: 1, entity: 1 }, { unique: true });

const historySchema = new Schema({
  transaction: { type: Schema.ObjectId, required: true, index: true },
  entity: { type: String, required: true, index: true },
  biz: {},
  prev: {},
  diff: {},
  reverted: { type: Boolean, default: false }
});

historySchema.virtual('current').get(function () {
  return jsondiffpatch.patch(this.prev, this.diff);
});

historySchema.set('toJSON', { virtuals: true });

if (!mongoose.plugins.some(plugin => plugin[0] === timestamp)) {
  transactionSchema.plugin(timestamp);
  lockSchema.plugin(timestamp);
  historySchema.plugin(timestamp);
}

class Transaction {
  constructor ({
    connection,
    transactionModel,
    lockModel,
    id,
    historyConnection
  }) {
    this.id = id;
    this.transactionModel = transactionModel;
    this.lockModel = lockModel;
    this.connection = connection;
    this.usedModel = {};
    this.historyConnection = historyConnection;
  }

  static getTransactionModel (connection, transactionCollectionName = 'transaction') {
    if (!connection) throw new Error('connection is required!');
    return connection.model(transactionCollectionName, transactionSchema);
  }

  static getLockModel (connection, lockCollectionName = 'lock') {
    if (!connection) throw new Error('connection is required!');
    return connection.model(lockCollectionName, lockSchema);
  }

  static * init ({
    connection,
    id,
    transactionCollectionName,
    lockCollectionName,
    historyConnection,
    biz = {}
  } = {}) {
    const transactionModel = this.getTransactionModel(connection, transactionCollectionName);
    const lockModel = this.getLockModel(connection, lockCollectionName);
    let t;
    if (id) t = yield transactionModel.findById(id);
    if (!t) {
      id = id || new ObjectId;
      yield transactionModel.create({ _id: id, biz: JSON.parse(JSON.stringify(biz)) });
    } else {
      if ([states.CANCELLED, states.COMMITTED].includes(t.state)) throw new Error(`The transaction [${t.id}] has [${t.state}].`);
    }
    return new this({
      connection,
      transactionModel,
      lockModel,
      id,
      historyConnection
    });
  }

  * 'try' (wrapper = function * () {
  }) {
    if (wrapper.constructor.name !== 'GeneratorFunction') throw new Error('wrapper should be a generator function.');
    const transaction = yield this.findTransaction();
    if (transaction.state === states.INIT) {
      yield this.transactionModel.findByIdAndUpdate(this.id, {
        $set: { state: states.PENDING }
      });
    }
    let result;
    try {
      const transaction = yield this.findTransaction();
      if (transaction.state === states.PENDING) {
        result = yield wrapper.bind(this)();
        yield this.commit();
      } else {
        yield this.cancel(new Error('Transaction is not pending!'));
      }
    } catch (err) {
      yield this.cancel(err);
      throw err;
    }
    return result;
  }

  use (Model, enableHistory = true) {
    const modelName = Model.modelName;
    const schema = Model.schema;
    schema.add({
      __t: { type: Schema.ObjectId, required: true }
    });
    const SSModel = this.connection.model(`sub_state_${modelName}`, schema);
    if (!this.usedModel[modelName]) {
      this.usedModel[modelName] = { Model, SSModel };
      this.transactionModel.findOneAndUpdate({
        _id: this.id,
        usedModelNames: { $ne: modelName }
      }, {
        $push: {
          usedModelNames: modelName
        }
      }).exec();
    }
    let History;
    if (this.historyConnection) {
      History = this.historyConnection.model(`${modelName}_history`, historySchema);
    }
    return {
      create: function * (...params) {
        return yield this.create.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOneAndUpdate: function * (...params) {
        return yield this.findOneAndUpdate.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findByIdAndUpdate: function * (...params) {
        return yield this.findByIdAndUpdate.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findByIdAndRemove: function * (...params) {
        return yield this.findByIdAndRemove.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOneAndRemove: function * (...params) {
        return yield this.findOneAndRemove.bind(this)(Model, SSModel, params, enableHistory);
      }.bind(this),
      findOne: function * (...params) {
        return yield this.findOne.bind(this)(Model, SSModel, params);
      }.bind(this),
      findById: function * (...params) {
        return yield this.findById.bind(this)(Model, SSModel, params);
      }.bind(this),
      findHistories: function * (...params) {
        if (!History) return [];
        return yield History.find(...params);
      },
      findLatestHistory: function * (...params) {
        if (!History) return null;
        return yield History.findOne(...params).sort({ _id: -1 });
      },
      revertTo: function * (historyId) {
        if(!Transaction.noWarning) {
          console.warn('The `revertTo` method is dangerous, only use it when you know why!');
        }
        if (!History) return null;
        const transaction = yield this.findTransaction();
        const history = yield History.findById(historyId);
        if (!history) throw new Error(`The history [${historyId}] is not exists!`);
        const doc = history.toJSON().prev;
        const prevEntity = yield Model.findById(history.entity);
        let prev;
        if (prevEntity) {
          prev = prevEntity.toJSON();
          delete prev.__v;
        }
        const diff = jsondiffpatch.diff(prev, doc);
        yield History.create({
          transaction: this.id,
          entity: history.entity,
          biz: transaction.biz,
          prev,
          diff,
          reverted: true
        });
        if (doc) {
          return yield Model.findByIdAndUpdate(history.entity, doc, { upsert: true, new: true });
        } else {
          yield Model.findByIdAndRemove(history.entity);
          return null;
        }
      }.bind(this)
    };
  }

  * create (Model, SSModel, [doc], enableHistory) {
    if (!doc._id) {
      doc._id = new ObjectId;
    } else {
      const entity = yield this.findById(Model, SSModel, [doc._id]);
      if (entity) throw new Error('entity has already exists!');
    }
    yield this.pushAction({
      operation: operations.CREATE,
      model: Model.modelName,
      entity: doc._id,
      enableHistory
    });
    yield this.lock({ id: doc._id }, Model);
    if(doc.constructor.name === 'model') {
      doc = doc.toJSON();
    }
    return yield this.initSubStateData(SSModel, doc);
  }

  * findOneAndUpdate (Model, SSModel, [query, doc, options], enableHistory) {
    const entity = yield this.findOne(Model, SSModel, [query]);
    if (!entity) throw new Error('Entity is not exists');
    yield this.pushAction({
      operation: operations.UPDATE,
      model: Model.modelName,
      entity: entity.id,
      enableHistory
    });
    yield this.lock(entity, Model);
    if(!doc.$set) {
      doc.__t = this.id;
    }
    return yield SSModel.findOneAndUpdate(query, doc, options);
  }

  * findByIdAndUpdate (Model, SSModel, [id, doc, options], enableHistory) {
    return yield this.findOneAndUpdate(Model, SSModel, [{ _id: id }, doc, options], enableHistory);
  }

  * findOneAndRemove (Model, SSModel, [query], enableHistory) {
    const entity = yield this.findOne(Model, SSModel, [query]);
    yield this.pushAction({
      operation: operations.REMOVE,
      model: Model.modelName,
      entity: entity.id,
      enableHistory
    });
    yield this.lock(entity, Model);
  }

  * findByIdAndRemove (Model, SSModel, [id], enableHistory) {
    yield this.findOneAndRemove(Model, SSModel, [{ _id: id }], enableHistory);
  }

  * findOne (Model, SSModel, [criteria]) {
    const entity = yield SSModel.findOne(criteria);
    const lock = yield this.checkLock(entity, Model);
    if (lock) {
      return entity;
    }
    if (!entity) {
      const srcEntity = yield Model.findOne(criteria);
      if (srcEntity) {
        const doc = srcEntity.toJSON();
        delete doc.__v;
        yield this.initSubStateData(SSModel, doc);
      }
      return srcEntity;
    }
    return entity;
  }

  * findById (Model, SSModel, [id]) {
    return yield this.findOne(Model, SSModel, [{ _id: id }]);
  }

  * checkLock (entity, Model) {
    if (!entity) return null;
    const lock = yield this.lockModel.findOne({
      model: Model.modelName,
      entity: entity.id
    });
    if (lock && lock.transaction.toString() !== this.id.toString()) {
      throw new Error(`entity [${entity.id}] is locked by transaction [${lock.transaction}]`);
    }
    return lock;
  }

  * lock (entity, Model) {
    if (!entity) throw new Error('Entity is not exists!');
    const lock = yield this.checkLock(entity, Model);
    if (!lock) {
      yield this.lockModel.create({
        transaction: this.id,
        model: Model.modelName,
        entity: entity.id
      });
    }
  }

  * initSubStateData (SSModel, doc) {
    doc.__t = this.id;
    return yield SSModel.create(doc);
  }

  * pushAction (action) {
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $push: {
        actions: action
      }
    });
  }

  * findTransaction () {
    return yield this.transactionModel.findById(this.id);
  }

  * commit () {
    const transaction = yield this.findTransaction();
    if (transaction.state !== states.PENDING) throw new Error(`Expected the transaction [${this.id}] to be pending, but got ${transaction.state}`);

    const entitiesActivated = [];

    for (let action of transaction.actions.reverse()) {
      const { model, entity, enableHistory, operation } = action;
      if (entitiesActivated.includes(`${model}:${entity}`)) continue;
      entitiesActivated.push(`${model}:${entity}`);
      const { Model, SSModel } = this.usedModel[model] || {};
      if (!Model) throw new Error(`Model ${model} has not used, please use the model first`);
      const subStateEntity = yield SSModel.findById(entity);
      let doc;
      if (subStateEntity) {
        doc = subStateEntity.toJSON();
        delete doc.__v;
        delete doc.__t;
      }
      // Record histories
      if (this.historyConnection && enableHistory) {
        const History = this.historyConnection.model(`${model}_history`, historySchema);
        const prevEntity = yield Model.findById(entity);
        let prev;
        if (prevEntity) {
          prev = prevEntity.toJSON();
          delete prev.__v;
        }
        const diff = jsondiffpatch.diff(prev, doc);
        yield History.create({
          transaction: this.id,
          entity,
          biz: transaction.biz,
          prev,
          diff,
        });
      }
      if (doc) {
        const doc = subStateEntity.toJSON();
        delete doc.__v;
        delete doc.__t;
        delete doc.createdAt;
        delete doc.updatedAt;
        yield Model.findByIdAndUpdate(entity, doc, { upsert: true , overwrite: true});
      } else {
        yield Model.findByIdAndRemove(entity);
      }
    }
    yield this.clearSubStateData();
    yield this.unlock();
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.COMMITTED
      }
    });
    debug(`Transaction [${this.id}] committed!`);
  }

  * rollback () {
    const transaction = yield this.findTransaction();
    if (![states.PENDING, states.ROLLBACK].includes(transaction.state)) throw new Error(`Expected the transaction [${this.id}] to be pending/rollback, but got ${transaction.state}`);
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.ROLLBACK
      }
    });
    yield this.clearSubStateData();
    debug(`Transaction [${this.id}] rollback success!`);
  }

  * cancel (error) {
    const transaction = yield this.findTransaction();
    if (![states.PENDING, states.ROLLBACK].includes(transaction.state)) throw new Error(`Expected the transaction [${this.id}] to be pending/rollback, but got ${transaction.state}`);
    yield this.rollback();
    yield this.unlock();
    yield this.transactionModel.findByIdAndUpdate(this.id, {
      $set: {
        state: states.CANCELLED
      },
      error: {
        message: error.message,
        stack: error.stack
      }
    });
    debug(`Transaction [${this.id}] cancelled!`);
  }

  * clearSubStateData () {
    const usedModelNames = Object.keys(this.usedModel);
    const t = yield this.findTransaction();
    if (t.usedModelNames.some(modelName => !usedModelNames.includes(modelName))) {
      throw new Error(`${t.usedModelNames} should be used first!`);
    }
    for (let modelName of usedModelNames) {
      const { SSModel } = this.usedModel[modelName];
      yield SSModel.remove({ __t: this.id });
    }
  }

  * unlock () {
    yield this.lockModel.remove({ transaction: this.id });
  }

}

module.exports = Transaction;<|endoftext|>


<commit_before>/*****
This was originally inspired by examples I found googling for (roughly) "NodeJS code user password hash"

Initial Ver:        https://github.com/justsml/escape-from-callback-mountain/commit/f828e863ee04df5b3f7f4f4307681bb3fde5285a?diff=unified
All commits:        https://github.com/justsml/escape-from-callback-mountain/commits/master
PRs Show Refactor:  https://github.com/justsml/escape-from-callback-mountain/pulls?utf8=%E2%9C%93&q=is%3Apr

******/

const {hashString}     = require('./lib/crypto')
const {auditLog}       = require('./lib/log')
const {connection}     = require('./lib/db')

const authValidate = function _validated({username, password, callback}) {
  if (!username || username.length < 4) { return new Error('Invalid username. Required, 4 char minimum.') }
  if (!password || password.length < 4) { return new Error('Invalid password. Required, 4 char minimum.') }
  if (!callback) { return new Error('Callback arg required!') }
  return true
}

function auth({username, password}, callback) {
  let users = null;
  let isValid = authValidate({username, password})

  if (isValid !== true) { return callback(isValid) }

  function _findHandler(err, results) {
    if (err) return callback(err)
    if (!results) {
      return callback(new Error('No users matched. Login failed'))
    }
    // Before returning results, make a non-blocking call to logging function `auditLog`
    auditLog({event: 'login', username}, function _noOp() {/* do nothing */})
    callback(null, results)
  }

  function _hashHandler(err, passHash) {
    if (err) return callback(err)
    users.findOne({username, passHash}, _findHandler)
  }

  function _onConnected(err, {models}) {
    if (err) return callback(err)
    // Get reference to `users` query interface
    users = models.users
    // Hash the password before querying for user
    hashString(password, _hashHandler)
  }

  // Get db connection, hopefully cached? Pooled? - returns models
  connection.open(_onConnected)
}
<commit_msg>DRYer code, as Promised (#3)<commit_after>/*****
This was originally inspired by examples I found googling for (roughly) "NodeJS code user password hash"

Initial Ver:        https://github.com/justsml/escape-from-callback-mountain/commit/f828e863ee04df5b3f7f4f4307681bb3fde5285a?diff=unified
All commits:        https://github.com/justsml/escape-from-callback-mountain/commits/master
PRs Show Refactor:  https://github.com/justsml/escape-from-callback-mountain/pulls?utf8=%E2%9C%93&q=is%3Apr

******/

const Promise           = require('bluebird')
const {hashStringAsync} = Promise.promisifyAll(require('./lib/crypto'))
const {auditLogAsync}   = Promise.promisifyAll(require('./lib/log'))
const {openAsync}       = Promise.promisifyAll(require('./lib/db'))

var _openHandle = openAsync(); // FYI: Promises include memoization (caching) built into same API

function authValidated({username, password}) {
  if (!username || username.length < 4) { return Promise.reject(new Error('Invalid username. Required, 4 char minimum.')) }
  if (!password || password.length < 4) { return Promise.reject(new Error('Invalid password. Required, 4 char minimum.')) }
  return {username, password}
}

function userFound(results) {
  return results ? results : Promise.reject(new Error('No users matched. Login failed'))
}

function usersModel() {
  return _openHandle
    .then(({models}) => models.users)
}

function errorHandler(err) {
  console.error('Failed auth!', err)
}

function auth({username, password}) {
  return Promise
    .resolve({username, password})
    .catch(errorHandler)
    .then(authValidate)
    .tap(() => auditLogAsync({event: 'login', username}))
    .then(usersModel)
    .then(users => {
      // Inner Promise's value will bubble all the way up
      // Note: This can also be further flattened... Keep reading.
      return hashStringAsync(password)
        .then(hashPass => users.findOneAsync({username, passHash}))
        .then(userFound)
    })

}
<|endoftext|>


<commit_before>var cheerio = require('cheerio');
var request = require('request');
var request = require('request').defaults({
     timeout: 30000
});

var api = {};

api.url = 'http://www.supremenewyork.com';

String.prototype.capitalizeEachWord = function() {
    return this.replace(/\w\S*/g, function(txt) {
        return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
    });
}

// getItem('all')
// other options: new, jackets, shirts, tops_sweaters, sweatshirts,
// pants, hats, bags, accessories, shoes, skate

/**
 * Checks for items under desired category
 *
 * @param  {String} category
 * @return {Array}
 */

api.getItems = function(category, callback) {

    var getURL = api.url + '/shop/all/' + category;
    if (category == 'all') {
        getURL = api.url + '/shop/all';
    } else if (category == 'new') {
        getURL = api.url + '/shop/new';
    }

    request(getURL, function(err, resp, html, rrr, body) {
        if (!err && resp.statusCode == 200) {
                if (err) {
                    return callback('No response from website', null);
                } else {
                    var $ = cheerio.load(html);
                }
            var parsedResults = [];
            var count = $('img').length;
            // console.log(len);
            $('img').each(function(i, element) {

                var nextElement = $(this).next();
                var prevElement = $(this).prev();
                var image = "https://" + $(this).attr('src').substring(2);
                var title = $(this).attr('alt');
                var availability = nextElement.text().capitalizeEachWord();
                var link = api.url + this.parent.attribs.href;
                var sizesAvailable;

                if (availability == "") availability = "Available";

                request(link, function(err, resp, html, rrr, body) {

                    if (err) {
                         return callback('No response from website', null);
                    } else {
                        var $ = cheerio.load(html);
                    }

                    var addCartURL = api.url + $('form[id="cart-addf"]').attr('action');

                    if (availability == "Sold Out") {
                        addCartURL = null;
                    }

                    var sizeOptionsAvailable = [];
                    if ($('option')) {
                        $('option').each(function(i, elem) {
                            var size = {
                              id: parseInt($(this).attr('value')),
                              size: $(this).text(),
                            }
                            sizeOptionsAvailable.push(size);
                        });

                        if (sizeOptionsAvailable.length > 0) {
                          sizesAvailable = sizeOptionsAvailable
                        } else {
                          sizesAvailable = null
                        }
                    } else {
                        sizesAvailable = null;
                    }

                    var metadata = {
                        title: $('h1').attr('itemprop', 'name').eq(1).html(),
                        style: $('.style').attr('itemprop', 'model').text(),
                        link: link,
                        description: $('.description').text(),
                        addCartURL: addCartURL,
                        price: parseInt(($('.price')[0].children[0].children[0].data).replace('$', '').replace(',', '')),
                        image: image,
                        sizesAvailable: sizesAvailable,
                        images: [],
                        availability: availability
                    };

                    // Some items don't have extra images (like some of the skateboards)
                    if ($('.styles').length > 0) {
                        var styles = $('.styles')[0].children;
                        for (li in styles) {
                            for (a in styles[li].children) {
                                if (styles[li].children[a].attribs['data-style-name'] == metadata.style) {
                                    metadata.images.push('https:' + JSON.parse(styles[li].children[a].attribs['data-images']).zoomed_url)
                                }
                            }
                        }
                    } else if (title.indexOf('Skateboard') != -1) {
                        // Because fuck skateboards
                        metadata.images.push('https:' + $('#img-main').attr('src'))
                    }

                    parsedResults.push(metadata);

                    if (!--count) {
                        callback(parsedResults, null);
                    }

                })

            });
        } else {
            console.log("Unknown error");
        }
    });
};

api.getItem = function(itemURL, callback) {

    request(itemURL, function(err, resp, html, rrr, body) {

        if (err) {
             return callback('No response from website', null);
        } else {
            var $ = cheerio.load(html);
        }

        var sizeOptionsAvailable = [];
        if ($('option')) {
            $('option').each(function(i, elem) {
              var size = {
                id: parseInt($(this).attr('value')),
                size: $(this).text(),
              }
              sizeOptionsAvailable.push(size);
            });

            if (sizeOptionsAvailable.length > 0) {
              sizesAvailable = sizeOptionsAvailable
            } else {
              sizesAvailable = null
            }
        } else {
            sizesAvailable = null;
        }

        var availability;
        var addCartURL = api.url + $('form[id="cart-addf"]').attr('action');

        var addCartButton = $('input[value="add to cart"]')
        if (addCartButton.attr('type') == '') {
          availability = 'Available'
        } else {
          availability = 'Sold Out'
        }

        if (availability == 'Sold Out') {
          addCartURL = null
        }

        var metadata = {
            title: $('h1').attr('itemprop', 'name').eq(1).html(),
            style: $('.style').attr('itemprop', 'model').text(),
            link: itemURL,
            description: $('.description').text(),
            addCartURL: addCartURL,
            price: parseInt(($('.price')[0].children[0].children[0].data).replace('$', '').replace(',', '')),
            image: 'http:' + $('#img-main').attr('src'),
            sizesAvailable: sizesAvailable,
            images: [],
            availability: availability
        };

        // Some items don't have extra images (like some of the skateboards)
        if ($('.styles').length > 0) {
            var styles = $('.styles')[0].children;
            for (li in styles) {
                for (a in styles[li].children) {
                    if (styles[li].children[a].attribs['data-style-name'] == metadata.style) {
                        metadata.images.push('https:' + JSON.parse(styles[li].children[a].attribs['data-images']).zoomed_url)
                    }
                }
            }
        } else if (title.indexOf('Skateboard') != -1) {
            metadata.images.push('https:' + $('#img-main').attr('src'))
        }

        callback(null, metadata);
    });
};

api.watchOnAllItems = [];
api.watchAllItems = function(interval, category, callback) {
    api.log('Now watching for all items');
    api.watchOnAllItems = setInterval(function() {
        api.getItems(category, function(items) {
            callback(items, null);
        });
    }, 1000 * interval); // Every xx sec
}

api.stopWatchingAllItems = function(callback) {
    clearInterval(api.watchOnAllItems);
    if (api.watchOnAllItems == "") {
        callback(null, 'No watching processes found.');
    } else {
        callback('Watching has stopped.', null);
    }
}

// searches for new item drop TODO
api.onNewItem = function(callback) {
    api.watchAllItems(function(item) {
        // TODO: If new items is find return callback(value);
    });
}

/**
 * Seeks for items on desired category page with specific keywords/styles.
 * @param  {Number} interval
 * @param  {String} category
 * @param  {String} style
 * @param  {String} category
 * @return {Object}
 */
api.seek = function(category, keywords, styleSelection, callback) {
    var productLink = [];
    api.getItems(category, (product, err) => {
        if (err) {
            callback(null, 'Error occured while trying to seek for items.');
        }
        for (i = 0; i < items.length; i++) {
            var title = items[i].title;
            var style = items[i].style;

            if (style === null) {
                // type - style note defined without a match
                if (title.indexOf(keywords) > -1) { // check if the keywords match with the title
                    // found item
                    productLink.push(items[i].link);
                    callback(items[i], null);
                    break;
                } else {
                    continue;
                }
            } else if (style.indexOf(styleSelection) > -1) {
                // type - style defined with match
                if (title.indexOf(keywords) > -1) { // check if the keywords match with the title
                    // found item
                    productLink.push(items[i].link);
                    callback(items[i], null);
                    break;
                } else {
                    continue;
                }
            }
        }

        if (productLink[0] === undefined) {
            callback(null, "Could not find any results matching your keywords.");
        }

    });
}

api.log = function(message) {
    console.log('[supreme api] ' + message);
}

module.exports = api;
<commit_msg>Fix undefined var error<commit_after>var cheerio = require('cheerio');
var request = require('request');
var request = require('request').defaults({
     timeout: 30000
});

var api = {};

api.url = 'http://www.supremenewyork.com';

String.prototype.capitalizeEachWord = function() {
    return this.replace(/\w\S*/g, function(txt) {
        return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
    });
}

// getItem('all')
// other options: new, jackets, shirts, tops_sweaters, sweatshirts,
// pants, hats, bags, accessories, shoes, skate

/**
 * Checks for items under desired category
 *
 * @param  {String} category
 * @return {Array}
 */

api.getItems = function(category, callback) {

    var getURL = api.url + '/shop/all/' + category;
    if (category == 'all') {
        getURL = api.url + '/shop/all';
    } else if (category == 'new') {
        getURL = api.url + '/shop/new';
    }

    request(getURL, function(err, resp, html, rrr, body) {
        if (!err && resp.statusCode == 200) {
                if (err) {
                    return callback('No response from website', null);
                } else {
                    var $ = cheerio.load(html);
                }
            var parsedResults = [];
            var count = $('img').length;
            // console.log(len);
            $('img').each(function(i, element) {

                var nextElement = $(this).next();
                var prevElement = $(this).prev();
                var image = "https://" + $(this).attr('src').substring(2);
                var title = $(this).attr('alt');
                var availability = nextElement.text().capitalizeEachWord();
                var link = api.url + this.parent.attribs.href;
                var sizesAvailable;

                if (availability == "") availability = "Available";

                request(link, function(err, resp, html, rrr, body) {

                    if (err) {
                         return callback('No response from website', null);
                    } else {
                        var $ = cheerio.load(html);
                    }

                    var addCartURL = api.url + $('form[id="cart-addf"]').attr('action');

                    if (availability == "Sold Out") {
                        addCartURL = null;
                    }

                    var sizeOptionsAvailable = [];
                    if ($('option')) {
                        $('option').each(function(i, elem) {
                            var size = {
                              id: parseInt($(this).attr('value')),
                              size: $(this).text(),
                            }
                            sizeOptionsAvailable.push(size);
                        });

                        if (sizeOptionsAvailable.length > 0) {
                          sizesAvailable = sizeOptionsAvailable
                        } else {
                          sizesAvailable = null
                        }
                    } else {
                        sizesAvailable = null;
                    }

                    var metadata = {
                        title: $('h1').attr('itemprop', 'name').eq(1).html(),
                        style: $('.style').attr('itemprop', 'model').text(),
                        link: link,
                        description: $('.description').text(),
                        addCartURL: addCartURL,
                        price: parseInt(($('.price')[0].children[0].children[0].data).replace('$', '').replace(',', '')),
                        image: image,
                        sizesAvailable: sizesAvailable,
                        images: [],
                        availability: availability
                    };

                    // Some items don't have extra images (like some of the skateboards)
                    if ($('.styles').length > 0) {
                        var styles = $('.styles')[0].children;
                        for (li in styles) {
                            for (a in styles[li].children) {
                                if (styles[li].children[a].attribs['data-style-name'] == metadata.style) {
                                    metadata.images.push('https:' + JSON.parse(styles[li].children[a].attribs['data-images']).zoomed_url)
                                }
                            }
                        }
                    } else if (title.indexOf('Skateboard') != -1) {
                        // Because fuck skateboards
                        metadata.images.push('https:' + $('#img-main').attr('src'))
                    }

                    parsedResults.push(metadata);

                    if (!--count) {
                        callback(parsedResults, null);
                    }

                })

            });
        } else {
            console.log("Unknown error");
        }
    });
};

api.getItem = function(itemURL, callback) {

    request(itemURL, function(err, resp, html, rrr, body) {

        if (err) {
             return callback('No response from website', null);
        } else {
            var $ = cheerio.load(html);
        }

        var sizeOptionsAvailable = [];
        if ($('option')) {
            $('option').each(function(i, elem) {
              var size = {
                id: parseInt($(this).attr('value')),
                size: $(this).text(),
              }
              sizeOptionsAvailable.push(size);
            });

            if (sizeOptionsAvailable.length > 0) {
              sizesAvailable = sizeOptionsAvailable
            } else {
              sizesAvailable = null
            }
        } else {
            sizesAvailable = null;
        }

        var availability;
        var addCartURL = api.url + $('form[id="cart-addf"]').attr('action');

        var addCartButton = $('input[value="add to cart"]')
        if (addCartButton.attr('type') == '') {
          availability = 'Available'
        } else {
          availability = 'Sold Out'
        }

        if (availability == 'Sold Out') {
          addCartURL = null
        }

        var metadata = {
            title: $('h1').attr('itemprop', 'name').eq(1).html(),
            style: $('.style').attr('itemprop', 'model').text(),
            link: itemURL,
            description: $('.description').text(),
            addCartURL: addCartURL,
            price: parseInt(($('.price')[0].children[0].children[0].data).replace('$', '').replace(',', '')),
            image: 'http:' + $('#img-main').attr('src'),
            sizesAvailable: sizesAvailable,
            images: [],
            availability: availability
        };

        // Some items don't have extra images (like some of the skateboards)
        if ($('.styles').length > 0) {
            var styles = $('.styles')[0].children;
            for (li in styles) {
                for (a in styles[li].children) {
                    if (styles[li].children[a].attribs['data-style-name'] == metadata.style) {
                        metadata.images.push('https:' + JSON.parse(styles[li].children[a].attribs['data-images']).zoomed_url)
                    }
                }
            }
        } else if (title.indexOf('Skateboard') != -1) {
            metadata.images.push('https:' + $('#img-main').attr('src'))
        }

        callback(null, metadata);
    });
};

api.watchOnAllItems = [];
api.watchAllItems = function(interval, category, callback) {
    api.log('Now watching for all items');
    api.watchOnAllItems = setInterval(function() {
        api.getItems(category, function(items) {
            callback(items, null);
        });
    }, 1000 * interval); // Every xx sec
}

api.stopWatchingAllItems = function(callback) {
    clearInterval(api.watchOnAllItems);
    if (api.watchOnAllItems == "") {
        callback(null, 'No watching processes found.');
    } else {
        callback('Watching has stopped.', null);
    }
}

// searches for new item drop TODO
api.onNewItem = function(callback) {
    api.watchAllItems(function(item) {
        // TODO: If new items is find return callback(value);
    });
}

/**
 * Seeks for items on desired category page with specific keywords/styles.
 * @param  {Number} interval
 * @param  {String} category
 * @param  {String} style
 * @param  {String} category
 * @return {Object}
 */
api.seek = function(category, keywords, styleSelection, callback) {
    var productLink = [];
    api.getItems(category, (product, err) => {
        if (err) {
            callback(null, 'Error occured while trying to seek for items.');
        }
        for (i = 0; i < product.length; i++) {
            var title = product[i].title;
            var style = product[i].style;

            if (style === null) {
                // type - style note defined without a match
                if (title.indexOf(keywords) > -1) { // check if the keywords match with the title
                    // found item
                    productLink.push(product[i].link);
                    callback(product[i], null);
                    break;
                } else {
                    continue;
                }
            } else if (style.indexOf(styleSelection) > -1) {
                // type - style defined with match
                if (title.indexOf(keywords) > -1) { // check if the keywords match with the title
                    // found item
                    productLink.push(product[i].link);
                    callback(product[i], null);
                    break;
                } else {
                    continue;
                }
            }
        }

        if (productLink[0] === undefined) {
            callback(null, "Could not find any results matching your keywords.");
        }

    });
}

api.log = function(message) {
    console.log('[supreme api] ' + message);
}

module.exports = api;
<|endoftext|>


<commit_before>/**
 * Created by maya on 2015/7/18.
 */
import Moment from 'moment';

//This file comes from: http://stackoverflow.com/questions/1638337/the-best-way-to-synchronize-client-side-javascript-clock-with-server-date#answer-22969338

let c = null;

// the NTP algorithm
// t0 is the client's timestamp of the request packet transmission,
// t1 is the server's timestamp of the request packet reception,
// t2 is the server's timestamp of the response packet transmission and
// t3 is the client's timestamp of the response packet reception.
function ntp(t0, t1, t2, t3) {
    return {
        roundtripdelay: (t3 - t0) - (t2 - t1),
        offset: ((t1 - t0) + (t2 - t3)) / 2
    };
}

function onMessage(msg) {
    if (msg.timesync === "Response") {
        // NOTE: t2 isn't entirely accurate because we're assuming that the server spends 0ms on processing.
        // (t1 isn't accurate either, as there's bound to have been some processing before that, but we can't avoid that)
        let t1 = msg.serverTime,
            t2 = msg.serverTime,
            t3 = Date.now();

        // we can get a more accurate version of t2 if the server's response
        // contains a Date header, which it generally will.
        // EDIT: as @Ariel rightly notes, the HTTP Date header only has
        // second resolution, thus using it will actually make the calculated
        // result worse. For higher accuracy, one would thus have to
        // return an extra header with a higher-resolution time. This
        // could be done with nginx for example:
        // http://nginx.org/en/docs/http/ngx_http_core_module.html
        // var date = resp.getResponseHeader("Date");
        // if (date) {
        //     t2 = (new Date(date)).valueOf();
        // }
        c = ntp(t0, t1, t2, t3);

        // log the calculated value rtt and time driff so we can manually verify if they make sense
        console.log("NTP delay: "+c.roundtripdelay+", NTP offset: "+c.offset+", corrected: "+Moment(t3 + c.offset).format("YYYY-MM-DD HH:mm:ss.SSS"));
        MqttClient.offMessage(topic, "timesync", onMessage);
        resolve(c.offset);
    }
}

function getNTPOffset(MqttClient, topic){
    return new Promise(function(resolve, reject){
        if(c){
            console.log("have already got NTP delay: "+c.roundtripdelay+", NTP offset: "+c.offset+", so resolve directly");
            resolve(c.offset);
        }

        // calculate the difference in seconds between the client and server clocks, use
        // the NTP algorithm, see: http://en.wikipedia.org/wiki/Network_Time_Protocol#Clock_synchronization_algorithm
        let t0 = Date.now();
        MqttClient.publish("timesync", {timesync: "Request"});
        MqttClient.onMessage(topic, "timesync", onMessage);
    });
}

export default getNTPOffset;<commit_msg>make t0 as a global var<commit_after>/**
 * Created by maya on 2015/7/18.
 */
import Moment from 'moment';

//This file comes from: http://stackoverflow.com/questions/1638337/the-best-way-to-synchronize-client-side-javascript-clock-with-server-date#answer-22969338

let c = null;
let t0 = 0;

// the NTP algorithm
// t0 is the client's timestamp of the request packet transmission,
// t1 is the server's timestamp of the request packet reception,
// t2 is the server's timestamp of the response packet transmission and
// t3 is the client's timestamp of the response packet reception.
function ntp(t0, t1, t2, t3) {
    return {
        roundtripdelay: (t3 - t0) - (t2 - t1),
        offset: ((t1 - t0) + (t2 - t3)) / 2
    };
}

function onMessage(msg) {
    if (msg.timesync === "Response") {
        // NOTE: t2 isn't entirely accurate because we're assuming that the server spends 0ms on processing.
        // (t1 isn't accurate either, as there's bound to have been some processing before that, but we can't avoid that)
        let t1 = msg.serverTime,
            t2 = msg.serverTime,
            t3 = Date.now();

        // we can get a more accurate version of t2 if the server's response
        // contains a Date header, which it generally will.
        // EDIT: as @Ariel rightly notes, the HTTP Date header only has
        // second resolution, thus using it will actually make the calculated
        // result worse. For higher accuracy, one would thus have to
        // return an extra header with a higher-resolution time. This
        // could be done with nginx for example:
        // http://nginx.org/en/docs/http/ngx_http_core_module.html
        // var date = resp.getResponseHeader("Date");
        // if (date) {
        //     t2 = (new Date(date)).valueOf();
        // }
        c = ntp(t0, t1, t2, t3);

        // log the calculated value rtt and time driff so we can manually verify if they make sense
        console.log("NTP delay: "+c.roundtripdelay+", NTP offset: "+c.offset+", corrected: "+Moment(t3 + c.offset).format("YYYY-MM-DD HH:mm:ss.SSS"));
        MqttClient.offMessage(topic, "timesync", onMessage);
        resolve(c.offset);
    }
}

function getNTPOffset(MqttClient, topic){
    return new Promise(function(resolve, reject){
        if(c){
            console.log("have already got NTP delay: "+c.roundtripdelay+", NTP offset: "+c.offset+", so resolve directly");
            resolve(c.offset);
        }

        // calculate the difference in seconds between the client and server clocks, use
        // the NTP algorithm, see: http://en.wikipedia.org/wiki/Network_Time_Protocol#Clock_synchronization_algorithm
        t0 = Date.now();
        MqttClient.publish("timesync", {timesync: "Request"});
        MqttClient.onMessage(topic, "timesync", onMessage);
    });
}

export default getNTPOffset;<|endoftext|>


<commit_before>#!/usr/bin/env node

var fs			= require("fs"),
	bl			= require("bl"),
	ws			= require("ws"),
	url			= require("url"),
	path		= require("path"),
	http		= require("http"),
	repl		= require("repl"),
	util		= require("util"),
	zlib		= require("zlib"),
	after		= require("after"),
	colors		= require("colors"),
	cheerio		= require("cheerio"),
	chokidar	= require("chokidar"),
	optimist	= require("optimist").argv,
	StreamCache	= require("stream-cache"),

	// We only take one domain for now, whatever's last specified.
	proxyDomain	= (optimist._||[]).pop() || "localhost",
	port		= optimist.port || optimist.p || 3000,
	dir			= optimist.dir || process.cwd() || __dirname,

	// State
	startTime	= Date.now(),
	wsClients	= [],
	cache		= {},

	// Ignore file modification notifications in our watch directory
	// that are triggered before we've been running for this number of
	// miliseconds.
	watchThreshold = 500;

// One time synchronous enumeration of available scripts
var injectAssets = fs.readdirSync(dir),
	cssAssets = injectAssets.filter(function(name) {
		return name.match(/\.css$/i);
	}),
	jsAssets = injectAssets.filter(function(name) {
		return name.match(/\.js$/i);
	});

// REPL/Prompt aware log
var logTimer = null,
	lastWasPrompt = false;

function log() {
	if (lastWasPrompt)
		process.stdout.write("\n\n");

	console.log.apply(console,arguments);
	lastWasPrompt = false;

	if (!prompt) return;

	if (logTimer)
		clearTimeout(logTimer);

	logTimer = setTimeout(function() {
		process.stdout.write("\n");
		prompt.displayPrompt();
		lastWasPrompt = true;
	},100);
}

function started() {
	log("Injecto running for %s, on port %d".blue.bold,proxyDomain,port);
	log("Watching %s for changes.",dir.bold);

	chokidar
		.watch(dir)
		.on("add",reload)
		.on("change",reload)
		.on("unlink",reload);
}

function reload(path) {
	if (Date.now() < startTime + watchThreshold) {
		return;
	}

	if (!wsClients.length) return;

	log("\n\nFilesystem changed: %s",path.red);
	log("Triggering reload for %d clients...".red,wsClients.length);

	wsClients.forEach(function(socket) {
		socket.send("reload");
	});
	wsClients = [];
}

function injecto(req,res) {
	var resPath = path.join(dir, req.url.split(/injecto\//).pop());

	fs.stat(resPath,function(err) {
		if (err) return error(req,res,404);
		log("[INJECTO] %s".green,req.url);

		if (resPath.match(/\.css$/i)) {
			console.log("setting header as css");
			res.setHeader("Content-Type","text/css");
		}

		if (resPath.match(/\.js$/i)) {
			console.log("setting header as js");
			res.setHeader("Content-Type","text/javascript");
		}

		fs.createReadStream(resPath).pipe(res);
	});
}

function rewrite(remoteRes,localRes) {
	remoteRes.pipe(bl(runRewrite));

	function runRewrite(err, data) {
		var $ = cheerio.load(data);

		["href","src"].forEach(function(param) {
			$("[" + param + "]").each(function(i,item) {
				var attr = url.parse($(item).attr(param));
				if (attr.hostname === proxyDomain || attr.host === proxyDomain) {
					$(item).attr(param,
						attr.pathname + (attr.search||"") + (attr.hash||""));
				}
			});
		});

		// Kill HTTP import URLs
		// TODO this is a horrible travesty. make this not suck
		var importRegExp =
			new RegExp("\\@import\\s*url\\(([\"\'])http?\\:\\/\\/" + proxyDomain,"ig");

		$("style").each(function(i,item) {
			$(item).html(
					$(item).html().replace(importRegExp,function(m1,m2) {
						return "@import url(" + m2;
					})
				);
		});

		$("head").append("<script type='text/javascript' src='/--injecto-core'/>");

		cssAssets.forEach(function(asset) {
			$("head")
				.append(
					"<link rel='stylesheet' media='screen' href='/--injecto/" + asset + "' />");
		});

		jsAssets.forEach(function(asset) {
			$("body")
				.append(
					"<script type='text/javascript' src='/--injecto/" + asset + "' />");
		});

		var resultCode = $.html();

		delete remoteRes.headers["content-encoding"];
		delete remoteRes.headers["content-length"];
		delete remoteRes.headers["accept-ranges"];
		delete remoteRes.headers["connection"];

		localRes.writeHead(remoteRes.statusCode,remoteRes.headers);
		localRes.end(resultCode);
	}
}

function proxy(req,res) {
	var timeStarted = Date.now(),
		requestOptions = {
			host: proxyDomain,
			port: 80,
			path: req.url,
			headers: req.headers
		};

	requestOptions.headers.host = proxyDomain;
	requestOptions.headers.referer = "http://" + proxyDomain + "/";

	if (cache[req.url]) {
		return done(cache[req.url], true);
	}

	function done(remoteRes, fromCache) {
		var time = Date.now() - timeStarted;

		if (!remoteRes.headers["content-type"]) {
			remoteRes.headers["content-type"] = "text/html";
		}

		log(
			"%s [%s] %s (%dms)",
			(fromCache ? "CACHE" : "REMOTE"),
			String(remoteRes.statusCode)[remoteRes.statusCode>=400?"red":"green"],
			req.url.white,
			time,
			(remoteRes.headers["content-encoding"] || "uncompressed").yellow);

		if (!fromCache && remoteRes.statusCode < 400) {
			cache[req.url] = new StreamCache();

			if (remoteRes.headers["content-encoding"]) {
				delete remoteRes.headers["content-encoding"];
				remoteRes.pipe(zlib.createUnzip()).pipe(cache[req.url]);
			} else {
				remoteRes.pipe(cache[req.url]);
			}

			cache[req.url].headers = remoteRes.headers;
			cache[req.url].statusCode = remoteRes.statusCode;
			remoteRes = cache[req.url];
		}

		if (remoteRes.headers["content-type"] &&
			remoteRes.headers["content-type"].match(/^text\/html/i)) {
			return rewrite(remoteRes,res);
		}

		remoteRes.pipe(res);
	}

	http.get(requestOptions, done)
		.on("error",function(error) {
			log("Request Error: %s".red,error.message);
			res.end();
		});
}

function consolo(req,res) {
	req.pipe(bl(function(err,data) {
		try {
			var data	= JSON.parse(String(data)),
				context	= data.context,
				error	= data.error,
				message	= data.message;

			if (typeof message !== "array" && !(message instanceof Array))
				message = [message];

		} catch(e) {
			log(e);
			return log(String(data));
		}

		process.stdout.write("\n");

		if (context)
			log("Message from ".red + String(context).blue);

		if (error)
			log("%s, line %d: %s", error.file, error.line, String(error.message).red);

		if (message) {
			message = util.format.apply(util,message);

			if (message.indexOf("\n") < 0)
				message = message.blue;

			log(message);
		}

		process.stdout.write("\n");

		res.writeHead(200)
		res.end();
	}));
}

function injectoCore(req,res) {
	log("[INJECTO CORE]".green);
	res.writeHead(200,{"content-type":"text/javascript"});
	fs.createReadStream(path.join(__dirname,"/error.js"))
		.pipe(res);
}

function error(req,res,code) {
	console.error("[%d] Failed to deliver resource (%s)".red,code,req.url);
	res.writeHead(code);
	res.end("Failed to deliver resource. (" + code + ")");
}

function close() {
	log(
		"Thanks! Injecto was running for %d seconds.".green,
		Math.floor((Date.now() - startTime) / 1000)
	);

	process.exit(0);
}

// Server
var server = http.createServer(),
	wsServer = new ws.Server({ port: port + 1 });

// Handler
server.on("request",function(req,res) {
	if (req.url.match(/^\/\-\-injecto-core/))
		return injectoCore(req,res);

	if (req.url.match(/^\/\-\-injecto-console/))
		return consolo(req,res);

	if (req.url.match(/^\/\-\-injecto\//))
		return injecto(req,res);

	proxy(req,res);
});

// Listen
server.listen(port,started);

server.on("error",function(err) {
	log("ERROR: %s".red,error.message);
});

// Web sockets...
wsServer.on("connection", function(ws) {
	var index = wsClients.length;
	wsClients.push(ws);

	log("Web socket client connected.".red);

	prompt.prompt =
		util.format("broadcast REPL: %d clients> ",wsClients.length);

	ws.on("close",function() {
		log("Web socket client disconnected.".red);
		wsClients.splice(index,1);

		prompt.prompt =
			util.format("broadcast REPL: %d clients> ",wsClients.length);
	});
});

process.on("SIGINT",close);

// REPL
var prompt;
prompt = repl.start({
	prompt: "\n",
	eval: function(cmd, context, filename, callback) {
		cmd = (cmd||"").replace(/^\s*\(/, "").replace(/\)\s*$/, "");

		if (!cmd.trim().length) {
			return callback(null,new Error("No command".red));
		}

		if (!wsClients.length) {
			return callback(null,new Error("No clients connected".red));
		}

		if (cmd.replace(/\s+/ig,"") === "reload") {
			return reload("User command");
		}

		console.log("Broadcasting to %d clients.".blue, wsClients.length);
		var next = after(wsClients.length, function() {
			process.stdout.write("\n");
			prompt.displayPrompt()
		});

		wsClients.forEach(function(socket) {
			var complete = false;

			socket.send("command:" + cmd);
			socket.once("message", function(string, data) {
				if (complete) return;

				complete = true;
				string = string.substr(9);

				if (string.match(/^CYCLIC/)) {
					console.log(string.red);

				} else if (string.match(/^\s*undefined/)) {
					console.log("undefined");

				} else {
					console.log(JSON.parse(string))
				}

				next();
			});

			setTimeout(function() {
				if (!complete) {
					complete = true;
					next();
				}
			},250);
		});
	}
});

prompt.on("exit",function() {
	close();
});<commit_msg>Fixed header which wasn't being passed through.<commit_after>#!/usr/bin/env node

var fs			= require("fs"),
	bl			= require("bl"),
	ws			= require("ws"),
	url			= require("url"),
	path		= require("path"),
	http		= require("http"),
	repl		= require("repl"),
	util		= require("util"),
	zlib		= require("zlib"),
	after		= require("after"),
	colors		= require("colors"),
	cheerio		= require("cheerio"),
	chokidar	= require("chokidar"),
	optimist	= require("optimist").argv,
	StreamCache	= require("stream-cache"),

	// We only take one domain for now, whatever's last specified.
	proxyDomain	= (optimist._||[]).pop() || "localhost",
	port		= optimist.port || optimist.p || 3000,
	dir			= optimist.dir || process.cwd() || __dirname,

	// State
	startTime	= Date.now(),
	wsClients	= [],
	cache		= {},

	// Ignore file modification notifications in our watch directory
	// that are triggered before we've been running for this number of
	// miliseconds.
	watchThreshold = 500;

// One time synchronous enumeration of available scripts
var injectAssets = fs.readdirSync(dir),
	cssAssets = injectAssets.filter(function(name) {
		return name.match(/\.css$/i);
	}),
	jsAssets = injectAssets.filter(function(name) {
		return name.match(/\.js$/i);
	});

// REPL/Prompt aware log
var logTimer = null,
	lastWasPrompt = false;

function log() {
	if (lastWasPrompt)
		process.stdout.write("\n\n");

	console.log.apply(console,arguments);
	lastWasPrompt = false;

	if (!prompt) return;

	if (logTimer)
		clearTimeout(logTimer);

	logTimer = setTimeout(function() {
		process.stdout.write("\n");
		prompt.displayPrompt();
		lastWasPrompt = true;
	},100);
}

function started() {
	log("Injecto running for %s, on port %d".blue.bold,proxyDomain,port);
	log("Watching %s for changes.",dir.bold);

	chokidar
		.watch(dir)
		.on("add",reload)
		.on("change",reload)
		.on("unlink",reload);
}

function reload(path) {
	if (Date.now() < startTime + watchThreshold) {
		return;
	}

	if (!wsClients.length) return;

	log("\n\nFilesystem changed: %s",path.red);
	log("Triggering reload for %d clients...".red,wsClients.length);

	wsClients.forEach(function(socket) {
		socket.send("reload");
	});
	wsClients = [];
}

function injecto(req,res) {
	var resPath = path.join(dir, req.url.split(/injecto\//).pop());

	fs.stat(resPath,function(err) {
		if (err) return error(req,res,404);
		log("[INJECTO] %s".green,req.url);

		if (resPath.match(/\.css$/i)) {
			console.log("setting header as css");
			res.setHeader("Content-Type","text/css");
		}

		if (resPath.match(/\.js$/i)) {
			console.log("setting header as js");
			res.setHeader("Content-Type","text/javascript");
		}

		fs.createReadStream(resPath).pipe(res);
	});
}

function rewrite(remoteRes,localRes) {
	remoteRes.pipe(bl(runRewrite));

	function runRewrite(err, data) {
		var $ = cheerio.load(data);

		["href","src"].forEach(function(param) {
			$("[" + param + "]").each(function(i,item) {
				var attr = url.parse($(item).attr(param));
				if (attr.hostname === proxyDomain || attr.host === proxyDomain) {
					$(item).attr(param,
						attr.pathname + (attr.search||"") + (attr.hash||""));
				}
			});
		});

		// Kill HTTP import URLs
		// TODO this is a horrible travesty. make this not suck
		var importRegExp =
			new RegExp("\\@import\\s*url\\(([\"\'])http?\\:\\/\\/" + proxyDomain,"ig");

		$("style").each(function(i,item) {
			$(item).html(
					$(item).html().replace(importRegExp,function(m1,m2) {
						return "@import url(" + m2;
					})
				);
		});

		$("head").append("<script type='text/javascript' src='/--injecto-core'/>");

		cssAssets.forEach(function(asset) {
			$("head")
				.append(
					"<link rel='stylesheet' media='screen' href='/--injecto/" + asset + "' />");
		});

		jsAssets.forEach(function(asset) {
			$("body")
				.append(
					"<script type='text/javascript' src='/--injecto/" + asset + "' />");
		});

		var resultCode = $.html();

		delete remoteRes.headers["content-encoding"];
		delete remoteRes.headers["content-length"];
		delete remoteRes.headers["accept-ranges"];
		delete remoteRes.headers["connection"];

		localRes.writeHead(remoteRes.statusCode,remoteRes.headers);
		localRes.end(resultCode);
	}
}

function proxy(req,res) {
	var timeStarted = Date.now(),
		requestOptions = {
			host: proxyDomain,
			port: 80,
			path: req.url,
			headers: req.headers
		};

	requestOptions.headers.host = proxyDomain;
	requestOptions.headers.referer = "http://" + proxyDomain + "/";

	if (cache[req.url]) {
		return done(cache[req.url], true);
	}

	function done(remoteRes, fromCache) {
		var time = Date.now() - timeStarted;

		if (!remoteRes.headers["content-type"]) {
			remoteRes.headers["content-type"] = "text/html";
		}

		log(
			"%s [%s] %s (%dms) %s %s",
			(fromCache ? "CACHE" : "REMOTE"),
			String(remoteRes.statusCode)[remoteRes.statusCode>=400?"red":"green"],
			req.url.white,
			time,
			(remoteRes.headers["content-encoding"] || "uncompressed").yellow,
			remoteRes.headers["content-type"]);

		if (!fromCache && remoteRes.statusCode < 400) {
			cache[req.url] = new StreamCache();

			if (remoteRes.headers["content-encoding"]) {
				delete remoteRes.headers["content-encoding"];
				remoteRes.pipe(zlib.createUnzip()).pipe(cache[req.url]);
			} else {
				remoteRes.pipe(cache[req.url]);
			}

			cache[req.url].headers = remoteRes.headers;
			cache[req.url].statusCode = remoteRes.statusCode;
			remoteRes = cache[req.url];
		}

		if (remoteRes.headers["content-type"] &&
			remoteRes.headers["content-type"].match(/^text\/html/i)) {
			return rewrite(remoteRes,res);
		}

		res.setHeader("Content-Type",remoteRes.headers["content-type"]);
		remoteRes.pipe(res);
	}

	http.get(requestOptions, done)
		.on("error",function(error) {
			log("Request Error: %s".red,error.message);
			res.end();
		});
}

function consolo(req,res) {
	req.pipe(bl(function(err,data) {
		try {
			var data	= JSON.parse(String(data)),
				context	= data.context,
				error	= data.error,
				message	= data.message;

			if (typeof message !== "array" && !(message instanceof Array))
				message = [message];

		} catch(e) {
			log(e);
			return log(String(data));
		}

		process.stdout.write("\n");

		if (context)
			log("Message from ".red + String(context).blue);

		if (error)
			log("%s, line %d: %s", error.file, error.line, String(error.message).red);

		if (message) {
			message = util.format.apply(util,message);

			if (message.indexOf("\n") < 0)
				message = message.blue;

			log(message);
		}

		process.stdout.write("\n");

		res.writeHead(200)
		res.end();
	}));
}

function injectoCore(req,res) {
	log("[INJECTO CORE]".green);
	res.writeHead(200,{"content-type":"text/javascript"});
	fs.createReadStream(path.join(__dirname,"/error.js"))
		.pipe(res);
}

function error(req,res,code) {
	console.error("[%d] Failed to deliver resource (%s)".red,code,req.url);
	res.writeHead(code);
	res.end("Failed to deliver resource. (" + code + ")");
}

function close() {
	log(
		"Thanks! Injecto was running for %d seconds.".green,
		Math.floor((Date.now() - startTime) / 1000)
	);

	process.exit(0);
}

// Server
var server = http.createServer(),
	wsServer = new ws.Server({ port: port + 1 });

// Handler
server.on("request",function(req,res) {
	if (req.url.match(/^\/\-\-injecto-core/))
		return injectoCore(req,res);

	if (req.url.match(/^\/\-\-injecto-console/))
		return consolo(req,res);

	if (req.url.match(/^\/\-\-injecto\//))
		return injecto(req,res);

	proxy(req,res);
});

// Listen
server.listen(port,started);

server.on("error",function(err) {
	log("ERROR: %s".red,error.message);
});

// Web sockets...
wsServer.on("connection", function(ws) {
	var index = wsClients.length;
	wsClients.push(ws);

	log("Web socket client connected.".red);

	prompt.prompt =
		util.format("broadcast REPL: %d clients> ",wsClients.length);

	ws.on("close",function() {
		log("Web socket client disconnected.".red);
		wsClients.splice(index,1);

		prompt.prompt =
			util.format("broadcast REPL: %d clients> ",wsClients.length);
	});
});

process.on("SIGINT",close);

// REPL
var prompt;
prompt = repl.start({
	prompt: "\n",
	eval: function(cmd, context, filename, callback) {
		cmd = (cmd||"").replace(/^\s*\(/, "").replace(/\)\s*$/, "");

		if (!cmd.trim().length) {
			return callback(null,new Error("No command".red));
		}

		if (!wsClients.length) {
			return callback(null,new Error("No clients connected".red));
		}

		if (cmd.replace(/\s+/ig,"") === "reload") {
			return reload("User command");
		}

		console.log("Broadcasting to %d clients.".blue, wsClients.length);
		var next = after(wsClients.length, function() {
			process.stdout.write("\n");
			prompt.displayPrompt()
		});

		wsClients.forEach(function(socket) {
			var complete = false;

			socket.send("command:" + cmd);
			socket.once("message", function(string, data) {
				if (complete) return;

				complete = true;
				string = string.substr(9);

				if (string.match(/^CYCLIC/)) {
					console.log(string.red);

				} else if (string.match(/^\s*undefined/)) {
					console.log("undefined");

				} else {
					console.log(JSON.parse(string))
				}

				next();
			});

			setTimeout(function() {
				if (!complete) {
					complete = true;
					next();
				}
			},250);
		});
	}
});

prompt.on("exit",function() {
	close();
});<|endoftext|>


<commit_before>// CALCULATE RPS!
var begin = new Date() // for calculating requests per second
var end = new Date()
var __current_rps = 0 // current request per second
var __req_num = 0 // Global application requests number
var __req_num_local = 0 // application requests number, after each RPS calculation
var redis // redis!

// options that can be overwtieen when calling this module
var options = {
  calcRpsEachNseconds: 300,
  TimeFrameInMinutes: 1,
  maxUserRequestLimitPerTimeFrame: {
    POST: 50,
    GET: 300,
    PUT: 50,
    DELETE: 50
  }
}

/**
 * Checks requests intensivity from single User (if any) OR IP address
 * And skips check in we are in development mode
 * @param req
 * @param res
 * @param next
 * @returns next()
 */
function checkReqsIntensivity (req, res, next) {
  var user_id = req.user && req.user.id ? req.user.id : 0
  var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || req.socket.remoteAddress

  var minute = parseInt(Date.now() / 1000 / (options.TimeFrameInMinutes * 60), 10)
  var requestKey = 'RateLimitter:' + (user_id || ip) + ':' + req.method + ':' + minute

  // next minute
  var resetMinute = minute * (options.TimeFrameInMinutes * 60) + (options.TimeFrameInMinutes * 60)

  function errHandler (err) {
    console.log('RateLimitter Error')
    console.log(err)
    return next()
  }

  function processResponse (info) {
    if (info.too_often) {
      console.log('Please slow down, you request too frequently.')
      return res.sendStatus(429)
    } else {
      // save reqnum into redis
      redis.setex(requestKey, options.TimeFrameInMinutes * 60, ++info.reqnum, function (err) {
        if (err) {
          return errHandler(err)
        }
        return next()
      })
    }
  }

  redis.get(requestKey, function (err, reqnum) {
    if (err) {
      return errHandler(err)
    }

    reqnum = parseInt(reqnum, 10) || 0
    var too_often = false
    var remaining = options.maxUserRequestLimitPerTimeFrame[req.method] - reqnum
    if (remaining < 0) {
      too_often = true
      res.setHeader('X-Rate-Limit-Remaining', Math.max(remaining, 0))
    }

    res.setHeader('X-Rate-Limit-Limit', options.maxUserRequestLimitPerTimeFrame[req.method])
    res.setHeader('X-Rate-Limit-Reset', resetMinute)

    console.log('reqnum', reqnum)
    console.log('X-Rate-Limit-Limit', options.maxUserRequestLimitPerTimeFrame[req.method])
    console.log('X-Rate-Limit-Reset', resetMinute)
    console.log('X-Rate-Limit-Remaining', remaining)
    console.log('\n')
    processResponse({too_often: too_often, reqnum: reqnum || 0})
  })
}

/**
 * Express middlewear to be called on every request!
 * @param req
 * @param res
 * @param next
 * @returns next()
 */
function checker (req, res, next) {
  // Rate limitter per USER
  checkReqsIntensivity(req, res, next)

  // calculate current rps
  if (__req_num_local === options.calcRpsEachNseconds) {
    // calculate requests per second
    end = new Date()
    __current_rps = (__req_num_local / ((end - begin) / 1000)).toFixed(0)

    // print statistics
    console.log('\n\nIncoming Connections: ' + __req_num)
    console.log('Requests per second: ' + __current_rps + '   Time: ' + ((end - begin) / 1000).toFixed(3) + ', Requests: ' + __req_num_local + '\n')

    __req_num_local = 0
    begin = new Date()
  }

  __req_num++
  __req_num_local++
}

/**
 * Overwrites obj1's values with obj2's and adds obj2's if non existent in obj1
 * @param obj1
 * @param obj2
 * @returns obj3 a new object based on obj1 and obj2
 */
function mergeOptions (obj1, obj2) {
  var obj3 = {}
  for (var attrname in obj1) { obj3[attrname] = obj1[attrname] }
  for (var attrname2 in obj2) { obj3[attrname2] = obj2[attrname2] }
  return obj3
}

module.exports = function (redisClient, op) {
  if (op) {
    options = mergeOptions(options, op)
  }

  redis = redisClient
  return checker
}
<commit_msg>remove log<commit_after>// CALCULATE RPS!
var begin = new Date() // for calculating requests per second
var end = new Date()
var __current_rps = 0 // current request per second
var __req_num = 0 // Global application requests number
var __req_num_local = 0 // application requests number, after each RPS calculation
var redis // redis!

// options that can be overwtieen when calling this module
var options = {
  calcRpsEachNseconds: 300,
  TimeFrameInMinutes: 1,
  maxUserRequestLimitPerTimeFrame: {
    POST: 50,
    GET: 300,
    PUT: 50,
    DELETE: 50
  }
}

/**
 * Checks requests intensivity from single User (if any) OR IP address
 * And skips check in we are in development mode
 * @param req
 * @param res
 * @param next
 * @returns next()
 */
function checkReqsIntensivity (req, res, next) {
  var user_id = req.user && req.user.id ? req.user.id : 0
  var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || req.socket.remoteAddress

  var minute = parseInt(Date.now() / 1000 / (options.TimeFrameInMinutes * 60), 10)
  var requestKey = 'RateLimitter:' + (user_id || ip) + ':' + req.method + ':' + minute

  // next minute
  var resetMinute = minute * (options.TimeFrameInMinutes * 60) + (options.TimeFrameInMinutes * 60)

  function errHandler (err) {
    console.log('RateLimitter Error')
    console.log(err)
    return next()
  }

  function processResponse (info) {
    if (info.too_often) {
      console.log('Please slow down, you request too frequently.')
      return res.sendStatus(429)
    } else {
      // save reqnum into redis
      redis.setex(requestKey, options.TimeFrameInMinutes * 60, ++info.reqnum, function (err) {
        if (err) {
          return errHandler(err)
        }
        return next()
      })
    }
  }

  redis.get(requestKey, function (err, reqnum) {
    if (err) {
      return errHandler(err)
    }

    reqnum = parseInt(reqnum, 10) || 0
    var too_often = false
    var remaining = options.maxUserRequestLimitPerTimeFrame[req.method] - reqnum
    if (remaining < 0) {
      too_often = true
      res.setHeader('X-Rate-Limit-Remaining', Math.max(remaining, 0))
    }

    res.setHeader('X-Rate-Limit-Limit', options.maxUserRequestLimitPerTimeFrame[req.method])
    res.setHeader('X-Rate-Limit-Reset', resetMinute)

    processResponse({too_often: too_often, reqnum: reqnum || 0})
  })
}

/**
 * Express middlewear to be called on every request!
 * @param req
 * @param res
 * @param next
 * @returns next()
 */
function checker (req, res, next) {
  // Rate limitter per USER
  checkReqsIntensivity(req, res, next)

  // calculate current rps
  if (__req_num_local === options.calcRpsEachNseconds) {
    // calculate requests per second
    end = new Date()
    __current_rps = (__req_num_local / ((end - begin) / 1000)).toFixed(0)

    // print statistics
    console.log('\n\nIncoming Connections: ' + __req_num)
    console.log('Requests per second: ' + __current_rps + '   Time: ' + ((end - begin) / 1000).toFixed(3) + ', Requests: ' + __req_num_local + '\n')

    __req_num_local = 0
    begin = new Date()
  }

  __req_num++
  __req_num_local++
}

/**
 * Overwrites obj1's values with obj2's and adds obj2's if non existent in obj1
 * @param obj1
 * @param obj2
 * @returns obj3 a new object based on obj1 and obj2
 */
function mergeOptions (obj1, obj2) {
  var obj3 = {}
  for (var attrname in obj1) { obj3[attrname] = obj1[attrname] }
  for (var attrname2 in obj2) { obj3[attrname2] = obj2[attrname2] }
  return obj3
}

module.exports = function (redisClient, op) {
  if (op) {
    options = mergeOptions(options, op)
  }

  redis = redisClient
  return checker
}
<|endoftext|>


<commit_before>var Botkit = require('botkit')

var token = process.env.SLACK_TOKEN

var controller = Botkit.slackbot({
  // reconnect to Slack RTM when connection goes bad
  retry: Infinity,
  debug: true
})

console.log('BEEPBOOP_RESOURCER', BEEPBOOP_RESOURCER,
'BEEPBOOP_TOKEN', BEEPBOOP_TOKEN,
'BEEPBOOP_ID', BEEPBOOP_ID)

controller.setupWebserver(3000, function(err, webserver) {
    controller.createWebhookEndpoints(webserver);
});

// Assume single team mode if we have a SLACK_TOKEN
if (token) {
  console.log('Starting in single-team mode')
  controller.spawn({
    token: token
  }).startRTM(function (err, bot, payload) {
    if (err) {
      throw new Error(err)
    }

    console.log('Connected to Slack RTM')
  })
// Otherwise assume multi-team mode - setup beep boop resourcer connection
} else {
  console.log('Starting in Beep Boop multi-team mode')
  require('beepboop-botkit').start(controller, { debug: true })
}

controller.on('bot_channel_join', function (bot, message) {
  bot.reply(message, "I'm here!")
})

controller.hears(['hello', 'hi'], ['direct_mention'], function (bot, message) {
  bot.reply(message, 'Hello.')
})

controller.hears(['hello', 'hi'], ['direct_message'], function (bot, message) {
  bot.reply(message, 'Hello.')
  bot.reply(message, 'It\'s nice to talk to you directly.')
})

controller.hears('.*', ['mention'], function (bot, message) {
  bot.reply(message, 'You really do care about me. :heart:')
})

controller.hears('help', ['direct_message', 'direct_mention'], function (bot, message) {
  var help = 'I will respond to the following messages: \n' +
      '`bot hi` for a simple message.\n' +
      '`bot attachment` to see a Slack attachment message.\n' +
      '`@<your bot\'s name>` to demonstrate detecting a mention.\n' +
      '`bot help` to see this again.'
  bot.reply(message, help)
})

controller.hears(['attachment'], ['direct_message', 'direct_mention'], function (bot, message) {
  var text = 'Beep Beep Boop is a ridiculously simple hosting platform for your Slackbots.'
  var attachments = [{
    fallback: text,
    pretext: 'We bring bots to life. :sunglasses: :thumbsup:',
    title: 'Host, deploy and share your bot in seconds.',
    image_url: 'https://storage.googleapis.com/beepboophq/_assets/bot-1.22f6fb.png',
    title_link: 'https://beepboophq.com/',
    text: text,
    color: '#7CD197'
  }]

  bot.reply(message, {
    attachments: attachments
  }, function (err, resp) {
    console.log(err, resp)
  })
})

controller.hears('.*', ['direct_message', 'direct_mention'], function (bot, message) {
  bot.reply(message, 'Sorry <@' + message.user + '>, I don\'t understand. \n')
})

controller.on('slash_command', function(bot, message) {
    // check message.command
    // and maybe message.text...
    // use EITHER replyPrivate or replyPublic...
    bot.replyPrivate(message, 'This is a private reply to the ' + message.command + ' slash command!');

    // and then continue to use replyPublicDelayed or replyPrivateDelayed
    bot.replyPublicDelayed(message, 'This is a public reply to the ' + message.command + ' slash command!');

    bot.replyPrivateDelayed(message, ':dash:');

});
<commit_msg>Reference node env vars<commit_after>var Botkit = require('botkit')

var token = process.env.SLACK_TOKEN

var controller = Botkit.slackbot({
  // reconnect to Slack RTM when connection goes bad
  retry: Infinity,
  debug: true
})

console.log('BEEPBOOP_RESOURCER', process.env.BEEPBOOP_RESOURCER,
'BEEPBOOP_TOKEN', process.env.BEEPBOOP_TOKEN,
'BEEPBOOP_ID', process.env.BEEPBOOP_ID)

controller.setupWebserver(3000, function(err, webserver) {
    controller.createWebhookEndpoints(webserver);
});

// Assume single team mode if we have a SLACK_TOKEN
if (token) {
  console.log('Starting in single-team mode')
  controller.spawn({
    token: token
  }).startRTM(function (err, bot, payload) {
    if (err) {
      throw new Error(err)
    }

    console.log('Connected to Slack RTM')
  })
// Otherwise assume multi-team mode - setup beep boop resourcer connection
} else {
  console.log('Starting in Beep Boop multi-team mode')
  require('beepboop-botkit').start(controller, { debug: true })
}

controller.on('bot_channel_join', function (bot, message) {
  bot.reply(message, "I'm here!")
})

controller.hears(['hello', 'hi'], ['direct_mention'], function (bot, message) {
  bot.reply(message, 'Hello.')
})

controller.hears(['hello', 'hi'], ['direct_message'], function (bot, message) {
  bot.reply(message, 'Hello.')
  bot.reply(message, 'It\'s nice to talk to you directly.')
})

controller.hears('.*', ['mention'], function (bot, message) {
  bot.reply(message, 'You really do care about me. :heart:')
})

controller.hears('help', ['direct_message', 'direct_mention'], function (bot, message) {
  var help = 'I will respond to the following messages: \n' +
      '`bot hi` for a simple message.\n' +
      '`bot attachment` to see a Slack attachment message.\n' +
      '`@<your bot\'s name>` to demonstrate detecting a mention.\n' +
      '`bot help` to see this again.'
  bot.reply(message, help)
})

controller.hears(['attachment'], ['direct_message', 'direct_mention'], function (bot, message) {
  var text = 'Beep Beep Boop is a ridiculously simple hosting platform for your Slackbots.'
  var attachments = [{
    fallback: text,
    pretext: 'We bring bots to life. :sunglasses: :thumbsup:',
    title: 'Host, deploy and share your bot in seconds.',
    image_url: 'https://storage.googleapis.com/beepboophq/_assets/bot-1.22f6fb.png',
    title_link: 'https://beepboophq.com/',
    text: text,
    color: '#7CD197'
  }]

  bot.reply(message, {
    attachments: attachments
  }, function (err, resp) {
    console.log(err, resp)
  })
})

controller.hears('.*', ['direct_message', 'direct_mention'], function (bot, message) {
  bot.reply(message, 'Sorry <@' + message.user + '>, I don\'t understand. \n')
})

controller.on('slash_command', function(bot, message) {
    // check message.command
    // and maybe message.text...
    // use EITHER replyPrivate or replyPublic...
    bot.replyPrivate(message, 'This is a private reply to the ' + message.command + ' slash command!');

    // and then continue to use replyPublicDelayed or replyPrivateDelayed
    bot.replyPublicDelayed(message, 'This is a public reply to the ' + message.command + ' slash command!');

    bot.replyPrivateDelayed(message, ':dash:');

});
<|endoftext|>