<commit_before>/**
 * Module dependencies
 */

var client = require('hyperagent');
var emitter = require('hyper-emitter');
var each = require('each');

/**
 * Watch the path
 */

module.exports = Request;

function Request(str) {
  if (!(this instanceof Request)) return new Request(str);

  try {
    this.parse(str);
  } catch (err) {
    return fn(err);
  }

  this._listeners = {};
  this._scope = {};
}

Request.prototype.scope = function(scope) {
  this._scope = scope;
  if (this._fn) this.refresh();
  return this;
};

Request.prototype.on = function(fn) {
  this._fn = fn;
  this.refresh();
  return this;
};

Request.prototype.refresh = function() {
  var self = this;
  var scope = self._scope;
  var fn = self._fn;

  // Clear any previous listeners
  this.off();

  if (!self.isRoot) return self.traverse(scope, 0, fn);

  client()
    .on('error', fn)
    .end(function(res) {
      self.traverse(res.body || scope, 1, fn);
    });
};

/**
 * Parse the string with the following syntax
 *
 *     path.to.my.name
 *
 *     .path.to.my.name
 *
 * @param {String} str
 * @api private
 */

Request.prototype.parse = function(str) {
  var path = this.path = str.split('.');
  this.index = path[0];
  this.isRoot = this.index === '';
  this.target = path[path.length - 1];
};

/**
 * Off
 *
 * unsubscribe from any emitters for a request
 */

Request.prototype.off = function() {
  each(this._listeners, function(href, listener) {
    listener();
  });
};

/**
 * Traverse properties in the api
 *
 * @param {Any} parent
 * @param {Integer} i
 * @param {Function} cb
 * @api private
 */

Request.prototype.traverse = function(parent, i, cb) {
  var request = this;

  // We're done searching
  if (i === request.path.length) return cb(null, parent);

  var key = request.path[i];
  var value = parent[key];

  // We couldn't find the property
  if (typeof value === 'undefined') return cb(null);

  // It's local
  if (!value.href || value[request.path[i + 1]]) return request.traverse(value, i + 1, cb);

  // We're just getting the link
  if (request.path[i + 1] === 'href') return cb(null, value);

  // It's a link
  var href = value.href;

  // Unsubscribe and resubscribe if it was previously requested
  if (request._listeners[href]) request._listeners[href]();

  request._listeners[href] = emitter.get(href, function(err, body) {
    if (err) return cb(err);
    if (!body) return cb(null);

    var next = request.path[i + 1];

    // Return the resource without getting the key inside of the body
    if (next === '') return cb(null, body);

    // It's the same name as what the link was
    if (body[key] && !body[next]) return request.traverse(body[key], i + 1, cb);
    // It's a collection
    if (body.data && !body[next]) {
      var data = body.data;
      data.href = body.href;
      return request.traverse(data, i + 1, cb);
    }

    // We're looking for another property
    request.traverse(body, i + 1, cb);
  });
}
<commit_msg>deprecate including a client<commit_after>/**
 * Module dependencies
 */

var each = require('each');

/**
 * Watch the path
 */

module.exports = Request;

function Request(str, client) {
  if (!(this instanceof Request)) return new Request(str, client);

  // init client
  if (client) this.client = client;
  else this.client = defaultClient();

  this.parse(str);

  this._listeners = {};
  this._scope = {};
}

Request.prototype.scope = function(scope) {
  this._scope = scope;
  if (this._fn) this.refresh();
  return this;
};

Request.prototype.on = function(fn) {
  this._fn = fn;
  this.refresh();
  return this;
};

Request.prototype.refresh = function() {
  var self = this;
  var scope = self._scope;
  var fn = self._fn;

  // Clear any previous listeners
  this.off();

  if (!self.isRoot) return self.traverse(scope, 0, fn);

  self.client(function(err, body) {
    if (err) return fn(err);
    self.traverse(body || scope, 1, fn);
  });
};

/**
 * Parse the string with the following syntax
 *
 *     path.to.my.name
 *
 *     .path.to.my.name
 *
 * @param {String} str
 * @api private
 */

Request.prototype.parse = function(str) {
  var path = this.path = str.split('.');
  this.index = path[0];
  this.isRoot = this.index === '';
  this.target = path[path.length - 1];
};

/**
 * Off
 *
 * unsubscribe from any emitters for a request
 */

Request.prototype.off = function() {
  each(this._listeners, function(href, listener) {
    listener();
  });
};

/**
 * Traverse properties in the api
 *
 * @param {Any} parent
 * @param {Integer} i
 * @param {Function} cb
 * @api private
 */

Request.prototype.traverse = function(parent, i, cb) {
  var request = this;

  // We're done searching
  if (i === request.path.length) return cb(null, parent);

  var key = request.path[i];
  var value = parent[key];

  // We couldn't find the property
  if (typeof value === 'undefined') return cb(null);

  // It's local
  if (!value.href || value[request.path[i + 1]]) return request.traverse(value, i + 1, cb);

  // We're just getting the link
  if (request.path[i + 1] === 'href') return cb(null, value);

  // It's a link
  var href = value.href;

  // Unsubscribe and resubscribe if it was previously requested
  if (request._listeners[href]) request._listeners[href]();

  request._listeners[href] = request.client.get(href, function(err, body) {
    if (err) return cb(err);
    if (!body) return cb(null);

    var next = request.path[i + 1];

    // Return the resource without getting the key inside of the body
    if (next === '') return cb(null, body);

    // It's the same name as what the link was
    if (body[key] && !body[next]) return request.traverse(body[key], i + 1, cb);
    // It's a collection
    if (body.data && !body[next]) {
      var data = body.data;
      data.href = body.href;
      return request.traverse(data, i + 1, cb);
    }

    // We're looking for another property
    request.traverse(body, i + 1, cb);
  });
}

/**
 * Deprecated client
 */

var agent = require('hyperagent');
var emitter = require('hyper-emitter');

function defaultClient() {
  console.warn('DEPRECATED', 'future implementations of hyper-path will not provide a client. Please see docs for details.', (new Error).stack);

  function c(fn) {
    return agent()
      .on('error', fn)
      .end(function(res) {
        fn(null, res.body);
      });
  }

  c.get = emitter.get.bind(emitter);

  return c;
}
<|endoftext|>


<commit_before>// require('./app/cec');
// require('./app/web-app');
// -------------------------------------------------------------------------- //
// Example: basic.js
// For more cec-events: http://www.cec-o-matic.com/
// -------------------------------------------------------------------------- //

var nodecec = require( '../' );

var NodeCec = nodecec.NodeCec;
var CEC     = nodecec.CEC;

var cec = new NodeCec( 'node-cec-monitor' );

// -------------------------------------------------------------------------- //
//- KILL CEC-CLIENT PROCESS ON EXIT

process.on( 'SIGINT', function() {
  if ( cec != null ) {
    cec.stop();
  }
  process.exit();
});

// -------------------------------------------------------------------------- //
//- CEC EVENT HANDLING

cec.once( 'ready', function(client) {
  console.log( ' -- READY -- ' );
  client.sendCommand( 0xf0, CEC.Opcode.GIVE_DEVICE_POWER_STATUS );
});

cec.on( 'REPORT_POWER_STATUS', function (packet, status) {
  var keys = Object.keys( CEC.PowerStatus );

  for (var i = keys.length - 1; i >= 0; i--) {
    if (CEC.PowerStatus[keys[i]] == status) {
      console.log('POWER_STATUS:', keys[i]);
      break;
    }
  }

});

cec.on( 'ROUTING_CHANGE', function(packet, fromSource, toSource) {
  console.log( 'Routing changed from ' + fromSource + ' to ' + toSource + '.' );
});

// -------------------------------------------------------------------------- //
//- START CEC CLIENT

// -m  = start in monitor-mode
// -d8 = set log level to 8 (=TRAFFIC) (-d 8)
// -br = logical address set to `recording device`
cec.start( 'cec-client', '-m', '-d', '8', '-b', 'r' );
<commit_msg>proper include node-cec<commit_after>// require('./app/cec');
// require('./app/web-app');
// -------------------------------------------------------------------------- //
// Example: basic.js
// For more cec-events: http://www.cec-o-matic.com/
// -------------------------------------------------------------------------- //

var nodecec = require('node-cec');

var NodeCec = nodecec.NodeCec;
var CEC     = nodecec.CEC;

var cec = new NodeCec( 'node-cec-monitor' );

// -------------------------------------------------------------------------- //
//- KILL CEC-CLIENT PROCESS ON EXIT

process.on( 'SIGINT', function() {
  if ( cec != null ) {
    cec.stop();
  }
  process.exit();
});

// -------------------------------------------------------------------------- //
//- CEC EVENT HANDLING

cec.once( 'ready', function(client) {
  console.log( ' -- READY -- ' );
  client.sendCommand( 0xf0, CEC.Opcode.GIVE_DEVICE_POWER_STATUS );
});

cec.on( 'REPORT_POWER_STATUS', function (packet, status) {
  var keys = Object.keys( CEC.PowerStatus );

  for (var i = keys.length - 1; i >= 0; i--) {
    if (CEC.PowerStatus[keys[i]] == status) {
      console.log('POWER_STATUS:', keys[i]);
      break;
    }
  }

});

cec.on( 'ROUTING_CHANGE', function(packet, fromSource, toSource) {
  console.log( 'Routing changed from ' + fromSource + ' to ' + toSource + '.' );
});

// -------------------------------------------------------------------------- //
//- START CEC CLIENT

// -m  = start in monitor-mode
// -d8 = set log level to 8 (=TRAFFIC) (-d 8)
// -br = logical address set to `recording device`
cec.start( 'cec-client', '-m', '-d', '8', '-b', 'r' );
<|endoftext|>


<commit_before>'use strict';

var fs = require('fs');
var path = require('path');
var gutil = require('gulp-util');
var through = require('through2');

var _ = require('lodash');

var TEMPLATE_PATH = path.join(__dirname, 'tpls', 'constant.tpl.ejs');
var DEFAULT_WRAP_PATH = path.join(__dirname, 'tpls', 'default-wrapper.tpl.ejs');
var AMD_WRAP_PATH = path.join(__dirname, 'tpls', 'amd-wrapper.tpl.ejs');
var COMMONJS_WRAP_PATH = path.join(__dirname, 'tpls', 'commonjs-wrapper.tpl.ejs');
var defaultWrapper, amdWrapper, commonjsWrapper;

var defaults = {
    space: '\t',
    deps: [],
    wrap: false,
    template: undefined,
    templatePath: TEMPLATE_PATH
};

function ngConstantPlugin(opts) {

    var options = _.merge({}, defaults, opts);
    var template = options.template || readFile(options.templatePath);

    return through.obj(objectStream);

    function objectStream(file, enc, cb) {
        /* jshint validthis: true */

        var _this = this;

        if (file.isStream()) {
            _this.emit('error', pluginError('Streaming not supported'));
            return cb();
        }

        try {
            var data = file.isNull() ? {} : JSON.parse(file.contents);

            // Create the module string
            var result = _.template(template, {
                moduleName: options.name || data.name,
                deps:       options.deps || data.deps || [],
                constants:  getConstants(data, options)
            });

            // Handle wrapping
            if (!options.wrap) { options.wrap = data.wrap; }
            result = wrap(result, options);

            file.path = gutil.replaceExtension(file.path, '.js');
            file.contents = new Buffer(result);
            _this.push(file);
        } catch (err) {
            err.fileName = file.path;
            _this.emit('error', pluginError(err));
        }

        cb();
    }
}

function getConstants(data, options) {
    var opts = options || {};
    if (typeof opts.constants === 'string') {
        opts.constants = JSON.parse(opts.constants);
    }

    var dataCnst = data.constants || data;
    var input = _.extend({}, dataCnst, opts.constants);
    var constants =  _.map(input, function (value, name) {
        return {
            name: name,
            value: stringify(value, opts.space)
        };
    });

    return constants;
}

function pluginError(msg) {
    return new gutil.PluginError('gulp-tslint-log', msg);
}

function wrap(input, options) {
    var wrapper = options.wrap || '<%= __ngModule %>';
    if (wrapper === true) {
        if (!defaultWrapper) { defaultWrapper = readFile(DEFAULT_WRAP_PATH); }
        wrapper = defaultWrapper;
    } else if (wrapper === 'amd') {
        if (!amdWrapper) { amdWrapper = readFile(AMD_WRAP_PATH); }
        wrapper = amdWrapper;
    } else if (wrapper === 'commonjs') {
        if (!commonjsWrapper) { commonjsWrapper = readFile(COMMONJS_WRAP_PATH); }
        wrapper = commonjsWrapper;
    }
    return _.template(wrapper, _.merge({ '__ngModule': input }, options));
}

function readFile(filepath) {
    return fs.readFileSync(filepath, 'utf8');
}

function stringify(value, space) {
    return _.isUndefined(value) ? 'undefined' : JSON.stringify(value, null, space);
}

_.extend(ngConstantPlugin, {
    getConstants: getConstants,
    wrap: wrap
});

module.exports = ngConstantPlugin;
<commit_msg>Check deps field in the json file, if none provided then use the one from the options (which is either rhe one provided through options or [] because of the initial merge with defaults)<commit_after>'use strict';

var fs = require('fs');
var path = require('path');
var gutil = require('gulp-util');
var through = require('through2');

var _ = require('lodash');

var TEMPLATE_PATH = path.join(__dirname, 'tpls', 'constant.tpl.ejs');
var DEFAULT_WRAP_PATH = path.join(__dirname, 'tpls', 'default-wrapper.tpl.ejs');
var AMD_WRAP_PATH = path.join(__dirname, 'tpls', 'amd-wrapper.tpl.ejs');
var COMMONJS_WRAP_PATH = path.join(__dirname, 'tpls', 'commonjs-wrapper.tpl.ejs');
var defaultWrapper, amdWrapper, commonjsWrapper;

var defaults = {
    space: '\t',
    deps: [],
    wrap: false,
    template: undefined,
    templatePath: TEMPLATE_PATH
};

function ngConstantPlugin(opts) {

    var options = _.merge({}, defaults, opts);
    var template = options.template || readFile(options.templatePath);

    return through.obj(objectStream);

    function objectStream(file, enc, cb) {
        /* jshint validthis: true */

        var _this = this;

        if (file.isStream()) {
            _this.emit('error', pluginError('Streaming not supported'));
            return cb();
        }

        try {
            var data = file.isNull() ? {} : JSON.parse(file.contents);

            // Create the module string
            var result = _.template(template, {
                moduleName: options.name || data.name,
                deps:       data.deps || options.deps,
                constants:  getConstants(data, options)
            });

            // Handle wrapping
            if (!options.wrap) { options.wrap = data.wrap; }
            result = wrap(result, options);

            file.path = gutil.replaceExtension(file.path, '.js');
            file.contents = new Buffer(result);
            _this.push(file);
        } catch (err) {
            err.fileName = file.path;
            _this.emit('error', pluginError(err));
        }

        cb();
    }
}

function getConstants(data, options) {
    var opts = options || {};
    if (typeof opts.constants === 'string') {
        opts.constants = JSON.parse(opts.constants);
    }

    var dataCnst = data.constants || data;
    var input = _.extend({}, dataCnst, opts.constants);
    var constants =  _.map(input, function (value, name) {
        return {
            name: name,
            value: stringify(value, opts.space)
        };
    });

    return constants;
}

function pluginError(msg) {
    return new gutil.PluginError('gulp-tslint-log', msg);
}

function wrap(input, options) {
    var wrapper = options.wrap || '<%= __ngModule %>';
    if (wrapper === true) {
        if (!defaultWrapper) { defaultWrapper = readFile(DEFAULT_WRAP_PATH); }
        wrapper = defaultWrapper;
    } else if (wrapper === 'amd') {
        if (!amdWrapper) { amdWrapper = readFile(AMD_WRAP_PATH); }
        wrapper = amdWrapper;
    } else if (wrapper === 'commonjs') {
        if (!commonjsWrapper) { commonjsWrapper = readFile(COMMONJS_WRAP_PATH); }
        wrapper = commonjsWrapper;
    }
    return _.template(wrapper, _.merge({ '__ngModule': input }, options));
}

function readFile(filepath) {
    return fs.readFileSync(filepath, 'utf8');
}

function stringify(value, space) {
    return _.isUndefined(value) ? 'undefined' : JSON.stringify(value, null, space);
}

_.extend(ngConstantPlugin, {
    getConstants: getConstants,
    wrap: wrap
});

module.exports = ngConstantPlugin;
<|endoftext|>


<commit_before>/* jshint ignore:start */
'use strict';

const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const fse = require('fs-extra');
const {spawnSync} = require('child_process');
const zipDirectory = require('./zipService').zipDirectory;

BbPromise.promisifyAll(fse);

/**
 * Plugin for Serverless 1.x that bundles python requirements!
 */
class ServerlessPythonRequirements {
  /**
   * add the vendor helper to the current service tree
   * @return {Promise}
   */
  addVendorHelper() {
    if (this.custom().zip) {
      this.serverless.cli.log('Adding Python requirements helper...');

      return fse.copyAsync(
        path.resolve(__dirname, 'unzip_requirements.py'),
        path.join(this.serverless.config.servicePath, 'unzip_requirements.py'));
    }
  };

  /**
   * remove the vendor helper from the current service tree
   * @return {Promise}
   */
  removeVendorHelper() {
    if (this.custom().zip && this.custom().cleanupZipHelper) {
      this.serverless.cli.log('Removing Python requirements helper...');
      return fse.removeAsync('unzip_requirements.py');
    }
  };

  /**
   * parse requirements.txt into .requirements.txt, leaving out #no-deploy lines
   * @return true
   */
  parseRequirements() {
    if (!fse.existsSync(path.join(this.serverless.config.servicePath,
                                  'requirements.txt'))) {
      return BbPromise.resolve();
    }

    this.serverless.cli.log(
      `Parsing Python requirements.txt`);

    const reqs = fse.readFileSync("requirements.txt").toString().split('\n');

    let newReqs = ''
    for (const req of reqs) {
      if (req.indexOf('#no-deploy') === -1) {
          newReqs += `${req}\n`;
        }
    }
    fse.writeFileSync(".serverless/requirements.txt", newReqs, 'utf8');

    return true
  };

  /**
   * pip install the requirements to the .requirements directory
   * @return {Promise}
   */
  installRequirements() {
    if (!fse.existsSync(path.join(this.serverless.config.servicePath,
                                  '.serverless/requirements.txt'))) {
      return BbPromise.resolve();
    }

    const runtime = this.serverless.service.provider.runtime;
    this.serverless.cli.log(
      `Installing required Python packages for runtime ${runtime}...`);

    return new BbPromise((resolve, reject) => {
      let cmd;
      let options;
      const pipCmd = [
        runtime, '-m', 'pip', '--isolated', 'install',
        '-t', '.requirements', '-r', '.serverless/requirements.txt',
      ];
      if (!this.custom().dockerizePip) {
        const pipTestRes = spawnSync(runtime, ['-m', 'pip', 'help', 'install']);
        if (pipTestRes.stdout.toString().indexOf('--system') >= 0)
          pipCmd.push('--system');
      }
      if (this.custom().dockerizePip) {
        cmd = 'docker';

        const image = this.custom().dockerImage || `lambci/lambda:build-${runtime}`;
        this.serverless.cli.log("Docker Image: " + image)

        options = [
          'run', '--rm',
          '-u', process.getuid() + ':' + process.getgid(),
          '-v', `${this.serverless.config.servicePath}:/var/task:z`,
          `${image}`,
        ];
        options.push(...pipCmd);
      } else {
        cmd = pipCmd[0];
        options = pipCmd.slice(1);
      }
      const res = spawnSync(cmd, options);
      if (res.error) {
        return reject(res.error);
      }
      if (res.status != 0) {
        return reject(res.stderr);
      }
      resolve();
    });
  };

  /**
   * zip up .requirements
   * @return {Promise}
   */
  packRequirements() {
    return this.installRequirements().then(() => {
      if (this.custom().zip) {
        this.serverless.cli.log('Zipping required Python packages...');
        return zipDirectory('.requirements', '.requirements.zip');
      }
    });
  }

  /**
   * link all the files in .requirements to the service directory root
   * @return {undefined}
   */
  linkRequirements() {
    if (!this.custom().zip) {
      this.serverless.cli.log('Linking required Python packages...');
      fse.readdirSync('.requirements').map((file) => {
          this.serverless.service.package.include.push(file);
          this.serverless.service.package.include.push(`${file}/**`);
        try {
          fse.symlinkSync(`.requirements/${file}`, `./${file}`);
        } catch (exception) {
          let linkDest = null;
          try {
            linkDest = fse.readlinkSync(`./${file}`);
          } catch (e) {}
          if (linkDest !== `.requirements/${file}`)
            throw new Error(`Unable to link dependency "${file}" because a file
                             by the same name exists in this service`);
        }
      }
        );
    }
  }

  /**
   * unlink all the files in .requirements from the service directory root
   * @return {undefined}
   */
  unlinkRequirements() {
    if (!this.custom().zip) {
      this.serverless.cli.log('Unlinking required Python packages...');
      fse.readdirSync('.requirements').map((file) => fse.unlinkSync(file));
    }
  }

  /**
   * clean up .requirements and .requirements.zip and unzip_requirements.py
   * @return {Promise}
   */
  cleanup() {
    const artifacts = ['.requirements'];
    if (this.custom().zip) {
      artifacts.push('.requirements.zip');
      artifacts.push('unzip_requirements.py');
    }

    return BbPromise.all(_.map(artifacts, (artifact) => fse.removeAsync(
      path.join(this.serverless.config.servicePath, artifact))));
  };

  /**
   * get the custom.pythonRequirements contents, with defaults set
   * @return {Object}
   */
  custom() {
    return Object.assign({
      zip: false,
      cleanupZipHelper: true,
    }, this.serverless.service.custom &&
    this.serverless.service.custom.pythonRequirements || {});
  }

  /**
   * The plugin constructor
   * @param {Object} serverless
   * @param {Object} options
   * makes
   * @return {undefined}
   */
  constructor(serverless, options) {
    this.serverless = serverless;
    this.options = options;

    if (!_.get(this.serverless.service, 'package.exclude'))
      _.set(this.serverless.service, ['package', 'exclude'], []);
    this.serverless.service.package.exclude.push('.requirements/**');
    if (!_.get(this.serverless.service, 'package.include'))
      _.set(this.serverless.service, ['package', 'include'], []);

    this.commands = {
      'requirements': {
        commands: {
          'clean': {
            usage: 'Remove .requirements and requirements.zip',
            lifecycleEvents: [
              'clean',
            ],
          },
          'install': {
            usage: 'install requirements manually',
            lifecycleEvents: [
              'install',
            ],
          },
        },
      },
    };

    let before = () => BbPromise.bind(this)
        .then(this.addVendorHelper)
        .then(this.parseRequirements)
        .then(this.packRequirements)
        .then(this.linkRequirements);

    let after = () => BbPromise.bind(this)
        .then(this.removeVendorHelper)
        .then(this.unlinkRequirements);

    this.hooks = {
      'before:package:createDeploymentArtifacts': before,
      'after:package:createDeploymentArtifacts': after,
      'before:deploy:function:packageFunction': before,
      'after:deploy:function:packageFunction': after,
      'requirements:install:install': () => BbPromise.bind(this)
        .then(this.addVendorHelper)
        .then(this.parseRequirements)
        .then(this.packRequirements),
      'requirements:clean:clean': () => BbPromise.bind(this)
        .then(this.cleanup)
        .then(this.removeVendorHelper),
    };
  }
}

module.exports = ServerlessPythonRequirements;
<commit_msg>fix linting errors<commit_after>/* jshint ignore:start */
'use strict';

const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const fse = require('fs-extra');
const {spawnSync} = require('child_process');
const zipDirectory = require('./zipService').zipDirectory;

BbPromise.promisifyAll(fse);

/**
 * Plugin for Serverless 1.x that bundles python requirements!
 */
class ServerlessPythonRequirements {
  /**
   * add the vendor helper to the current service tree
   * @return {Promise}
   */
  addVendorHelper() {
    if (this.custom().zip) {
      this.serverless.cli.log('Adding Python requirements helper...');

      return fse.copyAsync(
        path.resolve(__dirname, 'unzip_requirements.py'),
        path.join(this.serverless.config.servicePath, 'unzip_requirements.py'));
    }
  };

  /**
   * remove the vendor helper from the current service tree
   * @return {Promise}
   */
  removeVendorHelper() {
    if (this.custom().zip && this.custom().cleanupZipHelper) {
      this.serverless.cli.log('Removing Python requirements helper...');
      return fse.removeAsync('unzip_requirements.py');
    }
  };

  /**
   * parse requirements.txt into .requirements.txt, leaving out #no-deploy lines
   * @return {true}
   */
  parseRequirements() {
    if (!fse.existsSync(path.join(this.serverless.config.servicePath,
                                  'requirements.txt'))) {
      return true;
    }

    this.serverless.cli.log(
      `Parsing Python requirements.txt`);

    const reqs = fse.readFileSync('requirements.txt').toString().split('\n');

    let newReqs = '';
    for (const req of reqs) {
      if (req.indexOf('#no-deploy') === -1) {
          newReqs += `${req}\n`;
        }
    }
    fse.writeFileSync('.serverless/requirements.txt', newReqs, 'utf8');

    return true;
  };

  /**
   * pip install the requirements to the .requirements directory
   * @return {Promise}
   */
  installRequirements() {
    if (!fse.existsSync(path.join(this.serverless.config.servicePath,
                                  '.serverless/requirements.txt'))) {
      return BbPromise.resolve();
    }

    const runtime = this.serverless.service.provider.runtime;
    this.serverless.cli.log(
      `Installing required Python packages for runtime ${runtime}...`);

    return new BbPromise((resolve, reject) => {
      let cmd;
      let options;
      const pipCmd = [
        runtime, '-m', 'pip', '--isolated', 'install',
        '-t', '.requirements', '-r', '.serverless/requirements.txt',
      ];
      if (!this.custom().dockerizePip) {
        const pipTestRes = spawnSync(runtime, ['-m', 'pip', 'help', 'install']);
        if (pipTestRes.stdout.toString().indexOf('--system') >= 0)
          pipCmd.push('--system');
      }
      if (this.custom().dockerizePip) {
        cmd = 'docker';

        const image = this.custom().dockerImage
         || `lambci/lambda:build-${runtime}`;
        this.serverless.cli.log(`Docker Image: ${image}`);

        options = [
          'run', '--rm',
          '-u', process.getuid() + ':' + process.getgid(),
          '-v', `${this.serverless.config.servicePath}:/var/task:z`,
          `${image}`,
        ];
        options.push(...pipCmd);
      } else {
        cmd = pipCmd[0];
        options = pipCmd.slice(1);
      }
      const res = spawnSync(cmd, options);
      if (res.error) {
        return reject(res.error);
      }
      if (res.status != 0) {
        return reject(res.stderr);
      }
      resolve();
    });
  };

  /**
   * zip up .requirements
   * @return {Promise}
   */
  packRequirements() {
    return this.installRequirements().then(() => {
      if (this.custom().zip) {
        this.serverless.cli.log('Zipping required Python packages...');
        return zipDirectory('.requirements', '.requirements.zip');
      }
    });
  }

  /**
   * link all the files in .requirements to the service directory root
   * @return {undefined}
   */
  linkRequirements() {
    if (!this.custom().zip) {
      this.serverless.cli.log('Linking required Python packages...');
      fse.readdirSync('.requirements').map((file) => {
          this.serverless.service.package.include.push(file);
          this.serverless.service.package.include.push(`${file}/**`);
        try {
          fse.symlinkSync(`.requirements/${file}`, `./${file}`);
        } catch (exception) {
          let linkDest = null;
          try {
            linkDest = fse.readlinkSync(`./${file}`);
          } catch (e) {}
          if (linkDest !== `.requirements/${file}`)
            throw new Error(`Unable to link dependency '${file}' because a file
                             by the same name exists in this service`);
        }
      }
        );
    }
  }

  /**
   * unlink all the files in .requirements from the service directory root
   * @return {undefined}
   */
  unlinkRequirements() {
    if (!this.custom().zip) {
      this.serverless.cli.log('Unlinking required Python packages...');
      fse.readdirSync('.requirements').map((file) => fse.unlinkSync(file));
    }
  }

  /**
   * clean up .requirements and .requirements.zip and unzip_requirements.py
   * @return {Promise}
   */
  cleanup() {
    const artifacts = ['.requirements'];
    if (this.custom().zip) {
      artifacts.push('.requirements.zip');
      artifacts.push('unzip_requirements.py');
    }

    return BbPromise.all(_.map(artifacts, (artifact) => fse.removeAsync(
      path.join(this.serverless.config.servicePath, artifact))));
  };

  /**
   * get the custom.pythonRequirements contents, with defaults set
   * @return {Object}
   */
  custom() {
    return Object.assign({
      zip: false,
      cleanupZipHelper: true,
    }, this.serverless.service.custom &&
    this.serverless.service.custom.pythonRequirements || {});
  }

  /**
   * The plugin constructor
   * @param {Object} serverless
   * @param {Object} options
   * makes
   * @return {undefined}
   */
  constructor(serverless, options) {
    this.serverless = serverless;
    this.options = options;

    if (!_.get(this.serverless.service, 'package.exclude'))
      _.set(this.serverless.service, ['package', 'exclude'], []);
    this.serverless.service.package.exclude.push('.requirements/**');
    if (!_.get(this.serverless.service, 'package.include'))
      _.set(this.serverless.service, ['package', 'include'], []);

    this.commands = {
      'requirements': {
        commands: {
          'clean': {
            usage: 'Remove .requirements and requirements.zip',
            lifecycleEvents: [
              'clean',
            ],
          },
          'install': {
            usage: 'install requirements manually',
            lifecycleEvents: [
              'install',
            ],
          },
        },
      },
    };

    let before = () => BbPromise.bind(this)
        .then(this.addVendorHelper)
        .then(this.parseRequirements)
        .then(this.packRequirements)
        .then(this.linkRequirements);

    let after = () => BbPromise.bind(this)
        .then(this.removeVendorHelper)
        .then(this.unlinkRequirements);

    this.hooks = {
      'before:package:createDeploymentArtifacts': before,
      'after:package:createDeploymentArtifacts': after,
      'before:deploy:function:packageFunction': before,
      'after:deploy:function:packageFunction': after,
      'requirements:install:install': () => BbPromise.bind(this)
        .then(this.addVendorHelper)
        .then(this.parseRequirements)
        .then(this.packRequirements),
      'requirements:clean:clean': () => BbPromise.bind(this)
        .then(this.cleanup)
        .then(this.removeVendorHelper),
    };
  }
}

module.exports = ServerlessPythonRequirements;
<|endoftext|>


<commit_before>'use strict';

var PENDING = 'pending';
var SETTLED = 'settled';
var FULFILLED = 'fulfilled';
var REJECTED = 'rejected';
var NOOP = function () {};

var asyncQueue = [];
var asyncTimer;

function asyncFlush() {
	// run promise callbacks
	for (var i = 0; i < asyncQueue.length; i++) {
		asyncQueue[i][0](asyncQueue[i][1]);
	}

	// reset async asyncQueue
	asyncQueue = [];
	asyncTimer = false;
}

function asyncCall(callback, arg) {
	asyncQueue.push([callback, arg]);

	if (!asyncTimer) {
		asyncTimer = true;
		setImmediate(asyncFlush, 0);
	}
}

function invokeResolver(resolver, promise) {
	function resolvePromise(value) {
		resolve(promise, value);
	}

	function rejectPromise(reason) {
		reject(promise, reason);
	}

	try {
		resolver(resolvePromise, rejectPromise);
	} catch (e) {
		rejectPromise(e);
	}
}

function invokeCallback(subscriber) {
	var owner = subscriber.owner;
	var settled = owner._state;
	var value = owner._data;
	var callback = subscriber[settled];
	var promise = subscriber.then;

	if (typeof callback === 'function') {
		settled = FULFILLED;
		try {
			value = callback(value);
		} catch (e) {
			reject(promise, e);
		}
	}

	if (!handleThenable(promise, value)) {
		if (settled === FULFILLED) {
			resolve(promise, value);
		}

		if (settled === REJECTED) {
			reject(promise, value);
		}
	}
}

function handleThenable(promise, value) {
	var resolved;

	try {
		if (promise === value) {
			throw new TypeError('A promises callback cannot return that same promise.');
		}

		if (value && (typeof value === 'function' || typeof value === 'object')) {
			var then = value.then; // then should be retrived only once

			if (typeof then === 'function') {
				then.call(value, function (val) {
					if (!resolved) {
						resolved = true;

						if (value !== val) {
							resolve(promise, val);
						} else {
							fulfill(promise, val);
						}
					}
				}, function (reason) {
					if (!resolved) {
						resolved = true;

						reject(promise, reason);
					}
				});

				return true;
			}
		}
	} catch (e) {
		if (!resolved) {
			reject(promise, e);
		}

		return true;
	}

	return false;
}

function resolve(promise, value) {
	if (promise === value || !handleThenable(promise, value)) {
		fulfill(promise, value);
	}
}

function fulfill(promise, value) {
	if (promise._state === PENDING) {
		promise._state = SETTLED;
		promise._data = value;

		asyncCall(publishFulfillment, promise);
	}
}

function reject(promise, reason) {
	if (promise._state === PENDING) {
		promise._state = SETTLED;
		promise._data = reason;

		asyncCall(publishRejection, promise);
	}
}

function publish(promise) {
	promise._then = promise._then.forEach(invokeCallback);
}

function publishFulfillment(promise) {
	promise._state = FULFILLED;
	publish(promise);
}

function publishRejection(promise) {
	promise._state = REJECTED;
	publish(promise);
}

/**
 * @class
 */
function Promise(resolver) {
	if (typeof resolver !== 'function') {
		throw new TypeError('Promise resolver ' + resolver + ' is not a function');
	}

	if (this instanceof Promise === false) {
		throw new TypeError('Failed to construct \'Promise\': Please use the \'new\' operator, this object constructor cannot be called as a function.');
	}

	this._then = [];

	invokeResolver(resolver, this);
}

Promise.prototype = {
	constructor: Promise,

	_state: PENDING,
	_then: null,
	_data: undefined,

	then: function (onFulfillment, onRejection) {
		var subscriber = {
			owner: this,
			then: new this.constructor(NOOP),
			fulfilled: onFulfillment,
			rejected: onRejection
		};

		if (this._state === FULFILLED || this._state === REJECTED) {
			// already resolved, call callback async
			asyncCall(invokeCallback, subscriber);
		} else {
			// subscribe
			this._then.push(subscriber);
		}

		return subscriber.then;
	},

	'catch': function (onRejection) {
		return this.then(null, onRejection);
	}
};

Promise.all = function (promises) {
	if (!Array.isArray(promises)) {
		throw new TypeError('You must pass an array to Promise.all().');
	}

	return new Promise(function (resolve, reject) {
		var results = [];
		var remaining = 0;

		function resolver(index) {
			remaining++;
			return function (value) {
				results[index] = value;
				if (!--remaining) {
					resolve(results);
				}
			};
		}

		for (var i = 0, promise; i < promises.length; i++) {
			promise = promises[i];

			if (promise && typeof promise.then === 'function') {
				promise.then(resolver(i), reject);
			} else {
				results[i] = promise;
			}
		}

		if (!remaining) {
			resolve(results);
		}
	});
};

Promise.race = function (promises) {
	if (!Array.isArray(promises)) {
		throw new TypeError('You must pass an array to Promise.race().');
	}

	return new Promise(function (resolve, reject) {
		for (var i = 0, promise; i < promises.length; i++) {
			promise = promises[i];

			if (promise && typeof promise.then === 'function') {
				promise.then(resolve, reject);
			} else {
				resolve(promise);
			}
		}
	});
};

Promise.resolve = function (value) {
	if (value && typeof value === 'object' && value.constructor === Promise) {
		return value;
	}

	return new Promise(function (resolve) {
		resolve(value);
	});
};

Promise.reject = function (reason) {
	return new Promise(function (resolve, reject) {
		reject(reason);
	});
};

module.exports = Promise;
<commit_msg>Revert "Remove setTimeout fallback"<commit_after>'use strict';

var PENDING = 'pending';
var SETTLED = 'settled';
var FULFILLED = 'fulfilled';
var REJECTED = 'rejected';
var NOOP = function () {};

var asyncSetTimer = typeof setImmediate !== 'undefined' ? setImmediate : setTimeout;
var asyncQueue = [];
var asyncTimer;

function asyncFlush() {
	// run promise callbacks
	for (var i = 0; i < asyncQueue.length; i++) {
		asyncQueue[i][0](asyncQueue[i][1]);
	}

	// reset async asyncQueue
	asyncQueue = [];
	asyncTimer = false;
}

function asyncCall(callback, arg) {
	asyncQueue.push([callback, arg]);

	if (!asyncTimer) {
		asyncTimer = true;
		asyncSetTimer(asyncFlush, 0);
	}
}

function invokeResolver(resolver, promise) {
	function resolvePromise(value) {
		resolve(promise, value);
	}

	function rejectPromise(reason) {
		reject(promise, reason);
	}

	try {
		resolver(resolvePromise, rejectPromise);
	} catch (e) {
		rejectPromise(e);
	}
}

function invokeCallback(subscriber) {
	var owner = subscriber.owner;
	var settled = owner._state;
	var value = owner._data;
	var callback = subscriber[settled];
	var promise = subscriber.then;

	if (typeof callback === 'function') {
		settled = FULFILLED;
		try {
			value = callback(value);
		} catch (e) {
			reject(promise, e);
		}
	}

	if (!handleThenable(promise, value)) {
		if (settled === FULFILLED) {
			resolve(promise, value);
		}

		if (settled === REJECTED) {
			reject(promise, value);
		}
	}
}

function handleThenable(promise, value) {
	var resolved;

	try {
		if (promise === value) {
			throw new TypeError('A promises callback cannot return that same promise.');
		}

		if (value && (typeof value === 'function' || typeof value === 'object')) {
			var then = value.then; // then should be retrived only once

			if (typeof then === 'function') {
				then.call(value, function (val) {
					if (!resolved) {
						resolved = true;

						if (value !== val) {
							resolve(promise, val);
						} else {
							fulfill(promise, val);
						}
					}
				}, function (reason) {
					if (!resolved) {
						resolved = true;

						reject(promise, reason);
					}
				});

				return true;
			}
		}
	} catch (e) {
		if (!resolved) {
			reject(promise, e);
		}

		return true;
	}

	return false;
}

function resolve(promise, value) {
	if (promise === value || !handleThenable(promise, value)) {
		fulfill(promise, value);
	}
}

function fulfill(promise, value) {
	if (promise._state === PENDING) {
		promise._state = SETTLED;
		promise._data = value;

		asyncCall(publishFulfillment, promise);
	}
}

function reject(promise, reason) {
	if (promise._state === PENDING) {
		promise._state = SETTLED;
		promise._data = reason;

		asyncCall(publishRejection, promise);
	}
}

function publish(promise) {
	promise._then = promise._then.forEach(invokeCallback);
}

function publishFulfillment(promise) {
	promise._state = FULFILLED;
	publish(promise);
}

function publishRejection(promise) {
	promise._state = REJECTED;
	publish(promise);
}

/**
 * @class
 */
function Promise(resolver) {
	if (typeof resolver !== 'function') {
		throw new TypeError('Promise resolver ' + resolver + ' is not a function');
	}

	if (this instanceof Promise === false) {
		throw new TypeError('Failed to construct \'Promise\': Please use the \'new\' operator, this object constructor cannot be called as a function.');
	}

	this._then = [];

	invokeResolver(resolver, this);
}

Promise.prototype = {
	constructor: Promise,

	_state: PENDING,
	_then: null,
	_data: undefined,

	then: function (onFulfillment, onRejection) {
		var subscriber = {
			owner: this,
			then: new this.constructor(NOOP),
			fulfilled: onFulfillment,
			rejected: onRejection
		};

		if (this._state === FULFILLED || this._state === REJECTED) {
			// already resolved, call callback async
			asyncCall(invokeCallback, subscriber);
		} else {
			// subscribe
			this._then.push(subscriber);
		}

		return subscriber.then;
	},

	'catch': function (onRejection) {
		return this.then(null, onRejection);
	}
};

Promise.all = function (promises) {
	if (!Array.isArray(promises)) {
		throw new TypeError('You must pass an array to Promise.all().');
	}

	return new Promise(function (resolve, reject) {
		var results = [];
		var remaining = 0;

		function resolver(index) {
			remaining++;
			return function (value) {
				results[index] = value;
				if (!--remaining) {
					resolve(results);
				}
			};
		}

		for (var i = 0, promise; i < promises.length; i++) {
			promise = promises[i];

			if (promise && typeof promise.then === 'function') {
				promise.then(resolver(i), reject);
			} else {
				results[i] = promise;
			}
		}

		if (!remaining) {
			resolve(results);
		}
	});
};

Promise.race = function (promises) {
	if (!Array.isArray(promises)) {
		throw new TypeError('You must pass an array to Promise.race().');
	}

	return new Promise(function (resolve, reject) {
		for (var i = 0, promise; i < promises.length; i++) {
			promise = promises[i];

			if (promise && typeof promise.then === 'function') {
				promise.then(resolve, reject);
			} else {
				resolve(promise);
			}
		}
	});
};

Promise.resolve = function (value) {
	if (value && typeof value === 'object' && value.constructor === Promise) {
		return value;
	}

	return new Promise(function (resolve) {
		resolve(value);
	});
};

Promise.reject = function (reason) {
	return new Promise(function (resolve, reject) {
		reject(reason);
	});
};

module.exports = Promise;
<|endoftext|>


<commit_before>var detective = require('detective');
var npmi = require('npmi');
var fs = require('fs');
var async = require('async');
var resolve = require('resolve');

module.exports = function (srcPaths, cb) {
  var seen = {};
  var installed = [];

  async.each(srcPaths, function (srcPath, cb) {
    var src = fs.readFileSync(srcPath, 'utf-8');

    async.each(detective(src), function (required, cb) {
      if (required.match(/^[.]/) || seen[required]) return;

      resolve(required, {basedir: process.cwd()}, function(err) {
        if (err) {
          installed.push(required);
          npmi({name: required}, cb);
        } else {
          cb(null);
        }
        seen[required] = true;
      });
    }, cb);
  }, function (err) {
    if (err) return cb(err);

    cb(null, installed);
  });
};<commit_msg>bugfix: not saving installations to package.json<commit_after>var detective = require('detective');
var npmi = require('npmi');
var fs = require('fs');
var async = require('async');
var resolve = require('resolve');

module.exports = function (srcPaths, cb) {
  var seen = {};
  var installed = [];

  async.each(srcPaths, function (srcPath, cb) {
    var src = fs.readFileSync(srcPath, 'utf-8');

    async.each(detective(src), function (required, cb) {
      if (required.match(/^[.]/) || seen[required]) return;

      resolve(required, {basedir: process.cwd()}, function (err) {
        if (err) {
          installed.push(required);
          npmi({
            name: required,
            npmLoad: {
              save: true
            }
          }, cb);
        } else {
          cb(null);
        }
        seen[required] = true;
      });
    }, cb);
  }, function (err) {
    if (err) return cb(err);

    cb(null, installed);
  });
};<|endoftext|>


<commit_before>/*
 * grunt
 * http://gruntjs.com/
 *
 * Copyright (c) 2016 "Cowboy" Ben Alman
 * Licensed under the MIT license.
 * https://github.com/gruntjs/grunt/blob/master/LICENSE-MIT
 */

'use strict';

// Nodejs libs.
var spawn = require('child_process').spawn;
var nodeUtil = require('util');
var path = require('path');

// The module to be exported.
var util = module.exports = {};

util.namespace = require('getobject');

// External libs.
util.hooker = require('hooker');
util.async = require('async');
var _ = util._ = require('lodash');
var which = require('which').sync;
// Instead of process.exit. See https://github.com/cowboy/node-exit
util.exit = require('exit');

// Mixin Underscore.string methods.
_.str = require('underscore.string');
_.mixin(_.str.exports());

// Return a function that normalizes the given function either returning a
// value or accepting a "done" callback that accepts a single value.
util.callbackify = function(fn) {
  return function callbackable() {
    // Invoke original function, getting its result.
    var result = fn.apply(this, arguments);
    // If the same number or less arguments were specified than fn accepts,
    // assume the "done" callback was already handled.
    var length = arguments.length;
    if (length === fn.length) { return; }
    // Otherwise, if the last argument is a function, assume it is a "done"
    // callback and call it.
    var done = arguments[length - 1];
    if (typeof done === 'function') { done(result); }
  };
};

// Create a new Error object, with an origError property that will be dumped
// if grunt was run with the --debug=9 option.
util.error = function(err, origError) {
  if (!nodeUtil.isError(err)) { err = new Error(err); }
  if (origError) { err.origError = origError; }
  return err;
};

// The line feed char for the current system.
util.linefeed = process.platform === 'win32' ? '\r\n' : '\n';

// Normalize linefeeds in a string.
util.normalizelf = function(str) {
  return str.replace(/\r\n|\n/g, util.linefeed);
};

// What "kind" is a value?
// I really need to rework https://github.com/cowboy/javascript-getclass
var kindsOf = {};
'Number String Boolean Function RegExp Array Date Error'.split(' ').forEach(function(k) {
  kindsOf['[object ' + k + ']'] = k.toLowerCase();
});
util.kindOf = function(value) {
  // Null or undefined.
  if (value == null) { return String(value); }
  // Everything else.
  return kindsOf[kindsOf.toString.call(value)] || 'object';
};

// Coerce something to an Array.
util.toArray = _.toArray;

// Return the string `str` repeated `n` times.
util.repeat = function(n, str) {
  return new Array(n + 1).join(str || ' ');
};

// Given str of "a/b", If n is 1, return "a" otherwise "b".
util.pluralize = function(n, str, separator) {
  var parts = str.split(separator || '/');
  return n === 1 ? (parts[0] || '') : (parts[1] || '');
};

// Recurse through objects and arrays, executing fn for each non-object.
util.recurse = function(value, fn, fnContinue) {
  function recurse(value, fn, fnContinue, state) {
    var error;
    if (state.objs.indexOf(value) !== -1) {
      error = new Error('Circular reference detected (' + state.path + ')');
      error.path = state.path;
      throw error;
    }
    var obj, key;
    if (fnContinue && fnContinue(value) === false) {
      // Skip value if necessary.
      return value;
    } else if (util.kindOf(value) === 'array') {
      // If value is an array, recurse.
      return value.map(function(item, index) {
        return recurse(item, fn, fnContinue, {
          objs: state.objs.concat([value]),
          path: state.path + '[' + index + ']',
        });
      });
    } else if (util.kindOf(value) === 'object' && !Buffer.isBuffer(value)) {
      // If value is an object, recurse.
      obj = {};
      for (key in value) {
        obj[key] = recurse(value[key], fn, fnContinue, {
          objs: state.objs.concat([value]),
          path: state.path + (/\W/.test(key) ? '["' + key + '"]' : '.' + key),
        });
      }
      return obj;
    } else {
      // Otherwise pass value into fn and return.
      return fn(value);
    }
  }
  return recurse(value, fn, fnContinue, {objs: [], path: ''});
};

// Spawn a child process, capturing its stdout and stderr.
util.spawn = function(opts, done) {
  // Build a result object and pass it (among other things) into the
  // done function.
  var callDone = function(code, stdout, stderr) {
    // Remove trailing whitespace (newline)
    stdout = _.rtrim(stdout);
    stderr = _.rtrim(stderr);
    // Create the result object.
    var result = {
      stdout: stdout,
      stderr: stderr,
      code: code,
      toString: function() {
        if (code === 0) {
          return stdout;
        } else if ('fallback' in opts) {
          return opts.fallback;
        } else if (opts.grunt) {
          // grunt.log.error uses standard out, to be fixed in 0.5.
          return stderr || stdout;
        }
        return stderr;
      }
    };
    // On error (and no fallback) pass an error object, otherwise pass null.
    done(code === 0 || 'fallback' in opts ? null : new Error(stderr), result, code);
  };

  var cmd, args;
  var pathSeparatorRe = /[\\\/]/g;
  if (opts.grunt) {
    cmd = process.execPath;
    args = process.execArgv.concat(process.argv[1], opts.args);
  } else {
    // On Windows, child_process.spawn will only file .exe files in the PATH,
    // not other executable types (grunt issue #155).
    try {
      if (!pathSeparatorRe.test(opts.cmd)) {
        // Only use which if cmd has no path component.
        cmd = which(opts.cmd);
      } else {
        cmd = opts.cmd.replace(pathSeparatorRe, path.sep);
      }
    } catch (err) {
      callDone(127, '', String(err));
      return;
    }
    args = opts.args || [];
  }

  var child = spawn(cmd, args, opts.opts);
  var stdout = new Buffer('');
  var stderr = new Buffer('');
  if (child.stdout) {
    child.stdout.on('data', function(buf) {
      stdout = Buffer.concat([stdout, new Buffer(buf)]);
    });
  }
  if (child.stderr) {
    child.stderr.on('data', function(buf) {
      stderr = Buffer.concat([stderr, new Buffer(buf)]);
    });
  }
  child.on('close', function(code) {
    callDone(code, stdout.toString(), stderr.toString());
  });
  return child;
};
<commit_msg>Create our own copy of lodash to avoid underscore.string polluting outside instances.<commit_after>/*
 * grunt
 * http://gruntjs.com/
 *
 * Copyright (c) 2016 "Cowboy" Ben Alman
 * Licensed under the MIT license.
 * https://github.com/gruntjs/grunt/blob/master/LICENSE-MIT
 */

'use strict';

// Nodejs libs.
var spawn = require('child_process').spawn;
var nodeUtil = require('util');
var path = require('path');

// The module to be exported.
var util = module.exports = {};

util.namespace = require('getobject');

// External libs.
util.hooker = require('hooker');
util.async = require('async');
var _ = util._ = Object.create(require('lodash'));
var which = require('which').sync;
// Instead of process.exit. See https://github.com/cowboy/node-exit
util.exit = require('exit');

// Mixin Underscore.string methods.
_.str = require('underscore.string');
_.mixin(_.str.exports());

// Return a function that normalizes the given function either returning a
// value or accepting a "done" callback that accepts a single value.
util.callbackify = function(fn) {
  return function callbackable() {
    // Invoke original function, getting its result.
    var result = fn.apply(this, arguments);
    // If the same number or less arguments were specified than fn accepts,
    // assume the "done" callback was already handled.
    var length = arguments.length;
    if (length === fn.length) { return; }
    // Otherwise, if the last argument is a function, assume it is a "done"
    // callback and call it.
    var done = arguments[length - 1];
    if (typeof done === 'function') { done(result); }
  };
};

// Create a new Error object, with an origError property that will be dumped
// if grunt was run with the --debug=9 option.
util.error = function(err, origError) {
  if (!nodeUtil.isError(err)) { err = new Error(err); }
  if (origError) { err.origError = origError; }
  return err;
};

// The line feed char for the current system.
util.linefeed = process.platform === 'win32' ? '\r\n' : '\n';

// Normalize linefeeds in a string.
util.normalizelf = function(str) {
  return str.replace(/\r\n|\n/g, util.linefeed);
};

// What "kind" is a value?
// I really need to rework https://github.com/cowboy/javascript-getclass
var kindsOf = {};
'Number String Boolean Function RegExp Array Date Error'.split(' ').forEach(function(k) {
  kindsOf['[object ' + k + ']'] = k.toLowerCase();
});
util.kindOf = function(value) {
  // Null or undefined.
  if (value == null) { return String(value); }
  // Everything else.
  return kindsOf[kindsOf.toString.call(value)] || 'object';
};

// Coerce something to an Array.
util.toArray = _.toArray;

// Return the string `str` repeated `n` times.
util.repeat = function(n, str) {
  return new Array(n + 1).join(str || ' ');
};

// Given str of "a/b", If n is 1, return "a" otherwise "b".
util.pluralize = function(n, str, separator) {
  var parts = str.split(separator || '/');
  return n === 1 ? (parts[0] || '') : (parts[1] || '');
};

// Recurse through objects and arrays, executing fn for each non-object.
util.recurse = function(value, fn, fnContinue) {
  function recurse(value, fn, fnContinue, state) {
    var error;
    if (state.objs.indexOf(value) !== -1) {
      error = new Error('Circular reference detected (' + state.path + ')');
      error.path = state.path;
      throw error;
    }
    var obj, key;
    if (fnContinue && fnContinue(value) === false) {
      // Skip value if necessary.
      return value;
    } else if (util.kindOf(value) === 'array') {
      // If value is an array, recurse.
      return value.map(function(item, index) {
        return recurse(item, fn, fnContinue, {
          objs: state.objs.concat([value]),
          path: state.path + '[' + index + ']',
        });
      });
    } else if (util.kindOf(value) === 'object' && !Buffer.isBuffer(value)) {
      // If value is an object, recurse.
      obj = {};
      for (key in value) {
        obj[key] = recurse(value[key], fn, fnContinue, {
          objs: state.objs.concat([value]),
          path: state.path + (/\W/.test(key) ? '["' + key + '"]' : '.' + key),
        });
      }
      return obj;
    } else {
      // Otherwise pass value into fn and return.
      return fn(value);
    }
  }
  return recurse(value, fn, fnContinue, {objs: [], path: ''});
};

// Spawn a child process, capturing its stdout and stderr.
util.spawn = function(opts, done) {
  // Build a result object and pass it (among other things) into the
  // done function.
  var callDone = function(code, stdout, stderr) {
    // Remove trailing whitespace (newline)
    stdout = _.rtrim(stdout);
    stderr = _.rtrim(stderr);
    // Create the result object.
    var result = {
      stdout: stdout,
      stderr: stderr,
      code: code,
      toString: function() {
        if (code === 0) {
          return stdout;
        } else if ('fallback' in opts) {
          return opts.fallback;
        } else if (opts.grunt) {
          // grunt.log.error uses standard out, to be fixed in 0.5.
          return stderr || stdout;
        }
        return stderr;
      }
    };
    // On error (and no fallback) pass an error object, otherwise pass null.
    done(code === 0 || 'fallback' in opts ? null : new Error(stderr), result, code);
  };

  var cmd, args;
  var pathSeparatorRe = /[\\\/]/g;
  if (opts.grunt) {
    cmd = process.execPath;
    args = process.execArgv.concat(process.argv[1], opts.args);
  } else {
    // On Windows, child_process.spawn will only file .exe files in the PATH,
    // not other executable types (grunt issue #155).
    try {
      if (!pathSeparatorRe.test(opts.cmd)) {
        // Only use which if cmd has no path component.
        cmd = which(opts.cmd);
      } else {
        cmd = opts.cmd.replace(pathSeparatorRe, path.sep);
      }
    } catch (err) {
      callDone(127, '', String(err));
      return;
    }
    args = opts.args || [];
  }

  var child = spawn(cmd, args, opts.opts);
  var stdout = new Buffer('');
  var stderr = new Buffer('');
  if (child.stdout) {
    child.stdout.on('data', function(buf) {
      stdout = Buffer.concat([stdout, new Buffer(buf)]);
    });
  }
  if (child.stderr) {
    child.stderr.on('data', function(buf) {
      stderr = Buffer.concat([stderr, new Buffer(buf)]);
    });
  }
  child.on('close', function(code) {
    callDone(code, stdout.toString(), stderr.toString());
  });
  return child;
};
<|endoftext|>


<commit_before>/* global d3, R */

(function (R, d3) {
"use strict"

const palette = {
  195: "#feb2dd",
  196: "#feb2dd",
  197: "#e271b1",
  198: "#c6478f",
  199: "#a4286e",
  200: "#721148",
  201: "#3a0222"
}

const {
  assoc,
  curry,
  filter,
  groupBy,
  identity,
  length,
  map,
  pipe,
  prop,
  propOr,
  reduce,
  replace,
  slice,
  sortBy,
  toLower,
  toPairs,
  trim,
  values
} = R

// utils

const getArtist = song => {
  let a = song.artist.toLowerCase()
  const m = a.match(/(.*) feat[^h]/)
  if (m) a = m[1]
  return a.trim()
}

const getAlbum = pipe(propOr("", "album"), toLower, trim)

const formatXY = (o, xFmt = Number, yFmt = identity) => {
  const a = Array.isArray(o) ? o : toPairs(o)
  return a.map(([k, v]) => ({ x: xFmt(k), y: yFmt(v) }))
}

const reduceConcat = pipe(
  toPairs,
  reduce(
    (a, [k, v]) => assoc(v, (a[v] || []).concat(k), a)
  , {})
)

const kebabCase = pipe(
  toLower,
  replace(/\s/g, '-')
)

const byYear = pipe(groupBy(prop("year")), map(length))

const bySortedYear = pipe(toPairs, sortBy(prop(1)))

// raw data

// Promise<Array>
const songsP = fetch("dump.json").then(res => res.json()).then(prop("songs"))

// Promise<Array>
const datedSongsP = songsP.then(pipe(
  map(s => (s.year = Number(s.date), s)),
  filter(s => !isNaN(s.year) && s.artist)
))

// Promise<POJO>
const songsByYearP = datedSongsP.then(byYear)

// Promise<POJO>
const songsBySortedYearP = songsByYearP.then(bySortedYear)

// Promise<POJO>
const songsByDecadeP = datedSongsP.then(pipe(
  groupBy(pipe(
    prop("date"),
    slice(0, 3)
  )),
  map(length)
))

// Promise<POJO>
const newArtistsByYearP = datedSongsP.then(
  reduce((acc, song) => {
    var a = getArtist(song)
    if (!acc[a] || acc[a] > song.year) acc[a] = song.year
    return acc
  }, {})
).then(reduceConcat)

// Promise<POJO>
const albumsByYearP = datedSongsP
.then(reduce((a, s) => (a[getAlbum(s)] = s.year, a), {}))
.then(reduceConcat)

// Promise<Array>
const singlesP = datedSongsP.then(pipe(
  filter(s => getAlbum(s) === 'singles')
))

const singlesByYearP = singlesP.then(byYear)

const singlesBySortedYearP = singlesByYearP.then(bySortedYear)

// XY chart data

const songsByYearXYP = songsByYearP.then(formatXY)
const songsBySortedYearXYP = songsBySortedYearP.then(formatXY)
const songsByDecadeXYP = songsByDecadeP.then(d => formatXY(d, k => String(Number(k)) + "0s"))

const newArtistsByYearXYP = newArtistsByYearP.then(d => formatXY(d, Number, length))

const albumsByYearXYP = albumsByYearP.then(d => formatXY(d, Number, length))

const singlesByYearXYP = singlesByYearP.then(formatXY)
const singlesBySortedYearXYP = singlesBySortedYearP.then(formatXY)

// visualizations

const addMenu = (songs) => {
  const nav = document.createElement("nav")
  const strong = document.createElement("strong")
  strong.textContent = `${songs.length} songs`

  nav.appendChild(strong)
  document.body.appendChild(nav)
}

const addTitle = title => {
  const id = kebabCase(title)
  const h = document.createElement("h2")
  h.textContent = title
  h.id = id
  document.body.appendChild(h)

  const a = document.createElement("a")
  a.textContent = title
  a.href = `#${id}`

  const nav = document.querySelector("nav")
  nav.appendChild(a)
}

const timeChart = curry((title, data) => {
  addTitle(title)

  const margin = {top: 20, right: 20, bottom: 30, left: 40},
    width = 1900 - margin.left - margin.right,
    height = 800 - margin.top - margin.bottom

  const x = d3.scale.ordinal()
    .rangeRoundBands([0, width], .1)
    .domain(data.map(d => d.x))

  const y = d3.scale.linear()
    .range([height, 0])
    .domain([0, d3.max(data, d => d.y)])

  const xAxis = d3.svg.axis()
    .scale(x)

  const yAxis = d3.svg.axis()
    .scale(y)
    .orient("left")
    .ticks(20)

  const svg = d3.select("body").append("svg")
    .attr("width", width + margin.left + margin.right)
    .attr("height", height + margin.top + margin.bottom)
  .append("g")
    .attr("transform", "translate(" + margin.left + "," + margin.top + ")")

  svg.append("g")
    .attr("class", "x axis")
    .attr("transform", "translate(0," + height + ")")
    .call(xAxis)

  svg.append("g")
    .attr("class", "y axis")
    .call(yAxis)

  svg.selectAll(".bar")
    .data(data)
  .enter().append("rect")
    .attr("class", "bar")
    .attr("title", d => x(d.x))
    .attr("x", d => x(d.x))
    .attr("width", x.rangeBand())
    .attr("y", d => y(d.y))
    .attr("height", d => height - y(d.y))
    .attr("fill", d => palette[String(d.x).slice(0, 3)])

  svg.selectAll(".label")
    .data(data)
  .enter().append("svg:text")
    .attr("class", "label")
    .attr("x", d => x(d.x))
    .attr("y", d => y(d.y) - 5)
    .text(d => d.y)
})

const table = curry((title, data) => {
 addTitle(title)

 const h = document.createElement("h3")
 h.textContent = values(data).reduce((acc, v) => acc += v.length, 0)
 const pre = document.createElement("pre")
 pre.textContent = toPairs(data).map(([k, v]) => `${k} ${v.join(", ")}\n`)
 document.body.appendChild(h)
 document.body.appendChild(pre)
})

// layout

songsP.then((songs) => addMenu(songs))

// connect

songsByYearXYP.then(timeChart("Songs per year"))
songsBySortedYearXYP.then(timeChart("Songs per sorted year"))
songsByDecadeXYP.then(timeChart("Songs per decade"))

newArtistsByYearXYP.then(timeChart("New artists per year"))
newArtistsByYearP.then(table("New artists per year"))

albumsByYearXYP.then(timeChart("Albums per year"))
albumsByYearP.then(table("Albums per year"))

singlesByYearXYP.then(timeChart("Singles per year"))
singlesBySortedYearXYP.then(timeChart("Singles per sorted year"))

}(R, d3))
<commit_msg>refactor: use Object.values<commit_after>/* global d3, R */

(function (R, d3) {
"use strict"

const palette = {
  195: "#feb2dd",
  196: "#feb2dd",
  197: "#e271b1",
  198: "#c6478f",
  199: "#a4286e",
  200: "#721148",
  201: "#3a0222"
}

const {
  assoc,
  curry,
  filter,
  groupBy,
  identity,
  length,
  map,
  pipe,
  prop,
  propOr,
  reduce,
  replace,
  slice,
  sortBy,
  toLower,
  toPairs,
  trim,
} = R

// utils

const getArtist = song => {
  let a = song.artist.toLowerCase()
  const m = a.match(/(.*) feat[^h]/)
  if (m) a = m[1]
  return a.trim()
}

const getAlbum = pipe(propOr("", "album"), toLower, trim)

const formatXY = (o, xFmt = Number, yFmt = identity) => {
  const a = Array.isArray(o) ? o : toPairs(o)
  return a.map(([k, v]) => ({ x: xFmt(k), y: yFmt(v) }))
}

const reduceConcat = pipe(
  toPairs,
  reduce(
    (a, [k, v]) => assoc(v, (a[v] || []).concat(k), a)
  , {})
)

const kebabCase = pipe(
  toLower,
  replace(/\s/g, '-')
)

const byYear = pipe(groupBy(prop("year")), map(length))

const bySortedYear = pipe(toPairs, sortBy(prop(1)))

// raw data

// Promise<Array>
const songsP = fetch("dump.json").then(res => res.json()).then(prop("songs"))

// Promise<Array>
const datedSongsP = songsP.then(pipe(
  map(s => (s.year = Number(s.date), s)),
  filter(s => !isNaN(s.year) && s.artist)
))

// Promise<POJO>
const songsByYearP = datedSongsP.then(byYear)

// Promise<POJO>
const songsBySortedYearP = songsByYearP.then(bySortedYear)

// Promise<POJO>
const songsByDecadeP = datedSongsP.then(pipe(
  groupBy(pipe(
    prop("date"),
    slice(0, 3)
  )),
  map(length)
))

// Promise<POJO>
const newArtistsByYearP = datedSongsP.then(
  reduce((acc, song) => {
    var a = getArtist(song)
    if (!acc[a] || acc[a] > song.year) acc[a] = song.year
    return acc
  }, {})
).then(reduceConcat)

// Promise<POJO>
const albumsByYearP = datedSongsP
.then(reduce((a, s) => (a[getAlbum(s)] = s.year, a), {}))
.then(reduceConcat)

// Promise<Array>
const singlesP = datedSongsP.then(pipe(
  filter(s => getAlbum(s) === 'singles')
))

const singlesByYearP = singlesP.then(byYear)

const singlesBySortedYearP = singlesByYearP.then(bySortedYear)

// XY chart data

const songsByYearXYP = songsByYearP.then(formatXY)
const songsBySortedYearXYP = songsBySortedYearP.then(formatXY)
const songsByDecadeXYP = songsByDecadeP.then(d => formatXY(d, k => String(Number(k)) + "0s"))

const newArtistsByYearXYP = newArtistsByYearP.then(d => formatXY(d, Number, length))

const albumsByYearXYP = albumsByYearP.then(d => formatXY(d, Number, length))

const singlesByYearXYP = singlesByYearP.then(formatXY)
const singlesBySortedYearXYP = singlesBySortedYearP.then(formatXY)

// visualizations

const addMenu = (songs) => {
  const nav = document.createElement("nav")
  const strong = document.createElement("strong")
  strong.textContent = `${songs.length} songs`

  nav.appendChild(strong)
  document.body.appendChild(nav)
}

const addTitle = title => {
  const id = kebabCase(title)
  const h = document.createElement("h2")
  h.textContent = title
  h.id = id
  document.body.appendChild(h)

  const a = document.createElement("a")
  a.textContent = title
  a.href = `#${id}`

  const nav = document.querySelector("nav")
  nav.appendChild(a)
}

const timeChart = curry((title, data) => {
  addTitle(title)

  const margin = {top: 20, right: 20, bottom: 30, left: 40},
    width = 1900 - margin.left - margin.right,
    height = 800 - margin.top - margin.bottom

  const x = d3.scale.ordinal()
    .rangeRoundBands([0, width], .1)
    .domain(data.map(d => d.x))

  const y = d3.scale.linear()
    .range([height, 0])
    .domain([0, d3.max(data, d => d.y)])

  const xAxis = d3.svg.axis()
    .scale(x)

  const yAxis = d3.svg.axis()
    .scale(y)
    .orient("left")
    .ticks(20)

  const svg = d3.select("body").append("svg")
    .attr("width", width + margin.left + margin.right)
    .attr("height", height + margin.top + margin.bottom)
  .append("g")
    .attr("transform", "translate(" + margin.left + "," + margin.top + ")")

  svg.append("g")
    .attr("class", "x axis")
    .attr("transform", "translate(0," + height + ")")
    .call(xAxis)

  svg.append("g")
    .attr("class", "y axis")
    .call(yAxis)

  svg.selectAll(".bar")
    .data(data)
  .enter().append("rect")
    .attr("class", "bar")
    .attr("title", d => x(d.x))
    .attr("x", d => x(d.x))
    .attr("width", x.rangeBand())
    .attr("y", d => y(d.y))
    .attr("height", d => height - y(d.y))
    .attr("fill", d => palette[String(d.x).slice(0, 3)])

  svg.selectAll(".label")
    .data(data)
  .enter().append("svg:text")
    .attr("class", "label")
    .attr("x", d => x(d.x))
    .attr("y", d => y(d.y) - 5)
    .text(d => d.y)
})

const table = curry((title, data) => {
 addTitle(title)

 const h = document.createElement("h3")
 h.textContent = Object.values(data).reduce((acc, v) => acc += v.length, 0)
 const pre = document.createElement("pre")
 pre.textContent = toPairs(data).map(([k, v]) => `${k} ${v.join(", ")}\n`)
 document.body.appendChild(h)
 document.body.appendChild(pre)
})

// layout

songsP.then((songs) => addMenu(songs))

// connect

songsByYearXYP.then(timeChart("Songs per year"))
songsBySortedYearXYP.then(timeChart("Songs per sorted year"))
songsByDecadeXYP.then(timeChart("Songs per decade"))

newArtistsByYearXYP.then(timeChart("New artists per year"))
newArtistsByYearP.then(table("New artists per year"))

albumsByYearXYP.then(timeChart("Albums per year"))
albumsByYearP.then(table("Albums per year"))

singlesByYearXYP.then(timeChart("Singles per year"))
singlesBySortedYearXYP.then(timeChart("Singles per sorted year"))

}(R, d3))
<|endoftext|>


<commit_before>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs')
const low = require('lowdb')
const lowDbFileAsync = require('lowdb/lib/file-async')
const request = require('request-promise')
const Q = require('q')
const fs = require('fs-extra')
const path = require('path')
const cheerio = require('cheerio')
const sleep = require('system-sleep')
const moment = require('moment')
const Slack = require('slack-node')
const Regex = require('xregexp')
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || ''
const dbFile = path.join(__dirname, 'data/db.json')
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.')
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true })

const scrapperUrlBase = 'http://empregos.maringa.com/'
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end'
]

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig,
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
]

const deferred = Q.defer()
const deferredProcessing = Q.defer()
const deferredSlack = Q.defer()

db.defaults({ jobs: [], settings: {} }).value()

let keywordsProcessed = 0
let websiteToken = ''
let websiteCookies = []

request({
  uri: scrapperUrlBase,
  transform: (body, response) => {
    const res = { headers: response.headers, statusCode: response.statusCode, body }

    return res
  }
}).then((response) => {
  if (response.statusCode !== 200) {
    deferred.reject(new Error('Erro ao obter token e cookies'))
    return false
  }

  const $ = cheerio.load(response.body)

  websiteCookies = (response.headers['set-cookie'] || []).map((cookie) => {
    const ck = new request.cookie(cookie)

    return ck
  })

  websiteToken = $('input[name="_token"]').val()

  return websiteToken && websiteCookies.length ? deferred.resolve() : deferred.reject(new Error('Token de busca inválido.'))
}).catch((err) => {
  _log('ERROR')
  _log(err)
  _log('-'.repeat(100))
  process.exit(1)
})

Q.when(deferred.promise).then(() => {
  sleep(1000)

  const cookieJar = request.jar()
  websiteCookies.forEach((cookie) => {
    cookieJar.setCookie(cookie, scrapperUrlBase)
  })

  keywords.forEach((keyword, index) => {
    _log(`Looking for jobs with word "${keyword}`)

    let foundJobs = []

    request.post(
      scrapperUrlBase,
      {
        jar: cookieJar,
        form: {
          _token: websiteToken,
          text: keyword,
          estado: 'PR',
          cidade: 'Maringá',
          area: '',
          faixa_salarial: ''
        }
      }
    ).then((response) => {
      const $ = cheerio.load(response)
      const jobRawList = $('.table-vagas').find('.clickable-row ')
      const foundResults = $('h1.text-center.is-primary.h2.my-4').text().trim() || ''
      const totalFoundResults = foundResults ? Number(foundResults.replace(/\D/g, '')) : 0

      _log(`Found ${totalFoundResults} for the keyword "${keyword}" (raw)`)

      if (!jobRawList.length) {
        if (foundResults) {
          if (totalFoundResults === 0) {
            _log(`No job opportunities was found for keyword "${keyword}"`)
            _log('-'.repeat(100))
          } else {
            _log('ERROR')
            _log(`Error getting jobs for keyword "${keyword}" (total of jobs found: ${totalFoundResults})`)
            _log(`Generated output in /tmp/${keyword}.html`)
            _log('-'.repeat(100))

            const tmp = path.join(__dirname, 'tmp')
            if (!fs.existsSync(tmp)) {
              fs.mkdirSync(tmp)
            }
            fs.writeFileSync(path.join(tmp, `${keyword}.html`), response)
          }
        } else {
          _log('ERROR')
          _log(`Error getting jobs for keyword "${keyword}"`)
          _log(`Generated output in /tmp/${keyword}.html`)
          _log('-'.repeat(100))

          const tmp = path.join(__dirname, 'tmp')
          if (!fs.existsSync(tmp)) {
            fs.mkdirSync(tmp)
          }
          fs.writeFileSync(path.join(tmp, `${keyword}.html`), response)
        }

        return
      }

      jobRawList.each((i, job) => {
        const $job = $(job)

        const title = $job.find('td').eq(1).find('p').eq(0).find('strong').text().trim().replace(/[\r\n]/g, '')
        const company = $job.find('td').eq(1).find('p').eq(1).text().trim().replace(/empresa: /i, '')
        const link = $job.data('href')

        // TODO: checar se existe flag de vaga preenchida no site
        const isFilled = $job.find('td').eq(1).find('p').eq(0).find('.badge-success').length > 0
        const dateProcessed = Date.now()
        const id = (link.match(/vaga-emprego\/(\d+)\//) || []).pop()

        if (isNaN(id)) {
          return
        }

        let row = null
        let date = $job.find('td').eq(2).find('p').eq(0).text().trim()

        if (date.toLowerCase() === 'hoje') {
          date = new Date()
        } else {
          date = new Date(date.toString().split('/').reverse().join('-'))
        }
        date.setHours(0, 0, 0, 0)
        date = date.getTime()

        row = db.get('jobs').find({ id }).value()
        if (row) {
          if (!row.is_filled && isFilled) {
            db.get('jobs').find({ id }).assign({ is_filed: isFilled }).value()
          }

          return
        }

        foundJobs.push({
          id,
          title,
          company,
          date,
          date_processed: dateProcessed,
          is_filled: isFilled,
          keyword,
          bot_processed: false,
          url: link
        })
      })

      foundJobs = foundJobs.filter((job) => {
        const test = blacklist.filter((word) => {
          const regex = word.constructor !== RegExp ? new RegExp(`\\b${word}\\b`, 'igm') : word

          return Regex(regex).test(job.title)
        })

        return test.length === 0
      })

      _log(`Found ${foundJobs.length} job opportunities for "${keyword}"...`)
      _log('-'.repeat(100))

      foundJobs.forEach((job) => {
        db.get('jobs').push(job).value()
      })

      keywordsProcessed = index + 1

      if (keywordsProcessed === keywords.length) {
        setTimeout(() => { deferredProcessing.resolve() }, 1000)
      }
    }).catch((err) => {
      _log('ERROR')
      _log(err)
      _log('-'.repeat(100))
    })
  })
}, (err) => {
  _log('ERROR')
  _log(err)
  _log('-'.repeat(100))
})

Q.when(deferredProcessing.promise).then(() => {
  let cuttingTime = 1484186400245

  _log('generating html...')

  try {
    db.set('settings.updated_at', Date.now()).value()

    const outputFile = path.join(__dirname, 'out/jobs.html')
    const html = fs.readFileSync(path.join(__dirname, 'tpl/jobs.ejs'), 'utf8')

    const jobs = db.get('jobs').filter({ is_filled: false }).sortBy('date').reverse().value() || []
    const result = tpl.render(html, { jobs, moment, updated_at: db.get('settings.updated_at').value() || Date.now() })

    if (!fs.existsSync(path.dirname(outputFile)) && !fs.mkdirsSync(path.dirname(outputFile))) {
      throw new Error('Error creating output directory.')
    }

    fs.writeFileSync(outputFile, result)

 // INTEGRATION WITH DEVPARANA SLACK --------------------------------------------------------------------------------------------
    _log('Done generating HTML file: ', outputFile)
    _log('Cheking for new job entries to send to Slack...')

    const botJobs = db.get('jobs').filter({ bot_processed: false })
      .filter({ is_filled: false }).filter(row => row.date_processed >= cuttingTime)
      .sortBy('date').reverse().value()

    sleep(1000)

    if (SLACK_WEBHOOK && botJobs.length) {
      _log(`Found ${botJobs.length} entries to be posted on slack.`)

      let slack = new Slack()
      slack.setWebhook(SLACK_WEBHOOK)

      const mainTitle = (botJobs.length > 1 ? 'Vagas de trabalho encontradas' : 'Vaga de trabalho encontrada') + ' em *Maringá*. Confira!'
      const slackQueue = botJobs.map((item, index) => {
        return () => new Promise((resolve, reject) => {
          _log(`Processing item ${(index + 1)}:`, `${item.title} / ${item.company}`)

          const params = {
            text: (index === 0 ? `${mainTitle} \n\n\n` : '') +
            `*${item.title} / ${item.company}* - ${item.url}`
          }

          slack.webhook(params, (err, response) => {
            if (err) {
              _log('ERROR: ', err)
              _log('ERROR: ', '-'.repeat(100))
              return reject(err)
            }
            if (response.statusCode === 200) {
              _log('Done posting item ' + (index + 1))
              db.get('jobs').find({ id: item.id }).assign({ bot_processed: true }).value()
              sleep(1000)

              resolve(index)
            } else {
              reject(new Error('Error processing item ' + (index + 1) + ': ' + response.statusCode + ': ' + response.statusMessage))
            }
          })
        })
      })

      Array.from(Array(slackQueue.length).keys()).reduce((promise, next) => {
        return promise.then(() => slackQueue[next]()).catch(err => { throw err })
      }, Promise.resolve())

      deferredSlack.resolve('DONE')
    } else if (!SLACK_WEBHOOK) {
      deferredSlack.reject(new Error('ERROR: Enviroment variable "SLACK_WEBHOOK" is not defined. Aborting slack...'))
    } else {
      deferredSlack.resolve('No new job opening found to send to slack.')
    }
 // /INTEGRATION WITH DEVPARANA SLACK -------------------------------------------------------------------------------------------
    Q.when(deferredSlack.promise).then((msg) => {
      _log(msg)
      _log('-'.repeat(100))
    }, (err) => {
      _log('ERROR')
      _log(err)
      _log('-'.repeat(100))
      process.exit(1)
    })
  } catch (err) {
    _log('ERROR: ', err)
  }
  _log('-'.repeat(100))
}, (err) => {
  _log('ERROR: ', err)
  _log('-'.repeat(100))
  process.exit(1)
})

function _log(...args) {
  console.log(...[].concat([`[${moment().format('DD/MM/YYYY HH:mm:ss')}] =>`], Array.from(args) || []))
}
<commit_msg>added new word in the blacklist<commit_after>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs')
const low = require('lowdb')
const lowDbFileAsync = require('lowdb/lib/file-async')
const request = require('request-promise')
const Q = require('q')
const fs = require('fs-extra')
const path = require('path')
const cheerio = require('cheerio')
const sleep = require('system-sleep')
const moment = require('moment')
const Slack = require('slack-node')
const Regex = require('xregexp')
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || ''
const dbFile = path.join(__dirname, 'data/db.json')
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.')
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true })

const scrapperUrlBase = 'http://empregos.maringa.com/'
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end'
]

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  'bordado',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
]

const deferred = Q.defer()
const deferredProcessing = Q.defer()
const deferredSlack = Q.defer()

db.defaults({ jobs: [], settings: {} }).value()

let keywordsProcessed = 0
let websiteToken = ''
let websiteCookies = []

request({
  uri: scrapperUrlBase,
  transform: (body, response) => {
    const res = { headers: response.headers, statusCode: response.statusCode, body }

    return res
  }
}).then((response) => {
  if (response.statusCode !== 200) {
    deferred.reject(new Error('Erro ao obter token e cookies'))
    return false
  }

  const $ = cheerio.load(response.body)

  websiteCookies = (response.headers['set-cookie'] || []).map((cookie) => {
    const ck = new request.cookie(cookie)

    return ck
  })

  websiteToken = $('input[name="_token"]').val()

  return websiteToken && websiteCookies.length ? deferred.resolve() : deferred.reject(new Error('Token de busca inválido.'))
}).catch((err) => {
  _log('ERROR')
  _log(err)
  _log('-'.repeat(100))
  process.exit(1)
})

Q.when(deferred.promise).then(() => {
  sleep(1000)

  const cookieJar = request.jar()
  websiteCookies.forEach((cookie) => {
    cookieJar.setCookie(cookie, scrapperUrlBase)
  })

  keywords.forEach((keyword, index) => {
    _log(`Looking for jobs with word "${keyword}`)

    let foundJobs = []

    request.post(
      scrapperUrlBase,
      {
        jar: cookieJar,
        form: {
          _token: websiteToken,
          text: keyword,
          estado: 'PR',
          cidade: 'Maringá',
          area: '',
          faixa_salarial: ''
        }
      }
    ).then((response) => {
      const $ = cheerio.load(response)
      const jobRawList = $('.table-vagas').find('.clickable-row ')
      const foundResults = $('h1.text-center.is-primary.h2.my-4').text().trim() || ''
      const totalFoundResults = foundResults ? Number(foundResults.replace(/\D/g, '')) : 0

      _log(`Found ${totalFoundResults} for the keyword "${keyword}" (raw)`)

      if (!jobRawList.length) {
        if (foundResults) {
          if (totalFoundResults === 0) {
            _log(`No job opportunities was found for keyword "${keyword}"`)
            _log('-'.repeat(100))
          } else {
            _log('ERROR')
            _log(`Error getting jobs for keyword "${keyword}" (total of jobs found: ${totalFoundResults})`)
            _log(`Generated output in /tmp/${keyword}.html`)
            _log('-'.repeat(100))

            const tmp = path.join(__dirname, 'tmp')
            if (!fs.existsSync(tmp)) {
              fs.mkdirSync(tmp)
            }
            fs.writeFileSync(path.join(tmp, `${keyword}.html`), response)
          }
        } else {
          _log('ERROR')
          _log(`Error getting jobs for keyword "${keyword}"`)
          _log(`Generated output in /tmp/${keyword}.html`)
          _log('-'.repeat(100))

          const tmp = path.join(__dirname, 'tmp')
          if (!fs.existsSync(tmp)) {
            fs.mkdirSync(tmp)
          }
          fs.writeFileSync(path.join(tmp, `${keyword}.html`), response)
        }

        return
      }

      jobRawList.each((i, job) => {
        const $job = $(job)

        const title = $job.find('td').eq(1).find('p').eq(0).find('strong').text().trim().replace(/[\r\n]/g, '')
        const company = $job.find('td').eq(1).find('p').eq(1).text().trim().replace(/empresa: /i, '')
        const link = $job.data('href')

        // TODO: checar se existe flag de vaga preenchida no site
        const isFilled = $job.find('td').eq(1).find('p').eq(0).find('.badge-success').length > 0
        const dateProcessed = Date.now()
        const id = (link.match(/vaga-emprego\/(\d+)\//) || []).pop()

        if (isNaN(id)) {
          return
        }

        let row = null
        let date = $job.find('td').eq(2).find('p').eq(0).text().trim()

        if (date.toLowerCase() === 'hoje') {
          date = new Date()
        } else {
          date = new Date(date.toString().split('/').reverse().join('-'))
        }
        date.setHours(0, 0, 0, 0)
        date = date.getTime()

        row = db.get('jobs').find({ id }).value()
        if (row) {
          if (!row.is_filled && isFilled) {
            db.get('jobs').find({ id }).assign({ is_filed: isFilled }).value()
          }

          return
        }

        foundJobs.push({
          id,
          title,
          company,
          date,
          date_processed: dateProcessed,
          is_filled: isFilled,
          keyword,
          bot_processed: false,
          url: link
        })
      })

      foundJobs = foundJobs.filter((job) => {
        const test = blacklist.filter((word) => {
          const regex = word.constructor !== RegExp ? new RegExp(`\\b${word}\\b`, 'igm') : word

          return Regex(regex).test(job.title)
        })

        return test.length === 0
      })

      _log(`Found ${foundJobs.length} job opportunities for "${keyword}"...`)
      _log('-'.repeat(100))

      foundJobs.forEach((job) => {
        db.get('jobs').push(job).value()
      })

      keywordsProcessed = index + 1

      if (keywordsProcessed === keywords.length) {
        setTimeout(() => { deferredProcessing.resolve() }, 1000)
      }
    }).catch((err) => {
      _log('ERROR')
      _log(err)
      _log('-'.repeat(100))
    })
  })
}, (err) => {
  _log('ERROR')
  _log(err)
  _log('-'.repeat(100))
})

Q.when(deferredProcessing.promise).then(() => {
  let cuttingTime = 1484186400245

  _log('generating html...')

  try {
    db.set('settings.updated_at', Date.now()).value()

    const outputFile = path.join(__dirname, 'out/jobs.html')
    const html = fs.readFileSync(path.join(__dirname, 'tpl/jobs.ejs'), 'utf8')

    const jobs = db.get('jobs').filter({ is_filled: false }).sortBy('date').reverse().value() || []
    const result = tpl.render(html, { jobs, moment, updated_at: db.get('settings.updated_at').value() || Date.now() })

    if (!fs.existsSync(path.dirname(outputFile)) && !fs.mkdirsSync(path.dirname(outputFile))) {
      throw new Error('Error creating output directory.')
    }

    fs.writeFileSync(outputFile, result)

 // INTEGRATION WITH DEVPARANA SLACK --------------------------------------------------------------------------------------------
    _log('Done generating HTML file: ', outputFile)
    _log('Cheking for new job entries to send to Slack...')

    const botJobs = db.get('jobs').filter({ bot_processed: false })
      .filter({ is_filled: false }).filter(row => row.date_processed >= cuttingTime)
      .sortBy('date').reverse().value()

    sleep(1000)

    if (SLACK_WEBHOOK && botJobs.length) {
      _log(`Found ${botJobs.length} entries to be posted on slack.`)

      let slack = new Slack()
      slack.setWebhook(SLACK_WEBHOOK)

      const mainTitle = (botJobs.length > 1 ? 'Vagas de trabalho encontradas' : 'Vaga de trabalho encontrada') + ' em *Maringá*. Confira!'
      const slackQueue = botJobs.map((item, index) => {
        return () => new Promise((resolve, reject) => {
          _log(`Processing item ${(index + 1)}:`, `${item.title} / ${item.company}`)

          const params = {
            text: (index === 0 ? `${mainTitle} \n\n\n` : '') +
            `*${item.title} / ${item.company}* - ${item.url}`
          }

          slack.webhook(params, (err, response) => {
            if (err) {
              _log('ERROR: ', err)
              _log('ERROR: ', '-'.repeat(100))
              return reject(err)
            }
            if (response.statusCode === 200) {
              _log('Done posting item ' + (index + 1))
              db.get('jobs').find({ id: item.id }).assign({ bot_processed: true }).value()
              sleep(1000)

              resolve(index)
            } else {
              reject(new Error('Error processing item ' + (index + 1) + ': ' + response.statusCode + ': ' + response.statusMessage))
            }
          })
        })
      })

      Array.from(Array(slackQueue.length).keys()).reduce((promise, next) => {
        return promise.then(() => slackQueue[next]()).catch(err => { throw err })
      }, Promise.resolve())

      deferredSlack.resolve('DONE')
    } else if (!SLACK_WEBHOOK) {
      deferredSlack.reject(new Error('ERROR: Enviroment variable "SLACK_WEBHOOK" is not defined. Aborting slack...'))
    } else {
      deferredSlack.resolve('No new job opening found to send to slack.')
    }
 // /INTEGRATION WITH DEVPARANA SLACK -------------------------------------------------------------------------------------------
    Q.when(deferredSlack.promise).then((msg) => {
      _log(msg)
      _log('-'.repeat(100))
    }, (err) => {
      _log('ERROR')
      _log(err)
      _log('-'.repeat(100))
      process.exit(1)
    })
  } catch (err) {
    _log('ERROR: ', err)
  }
  _log('-'.repeat(100))
}, (err) => {
  _log('ERROR: ', err)
  _log('-'.repeat(100))
  process.exit(1)
})

function _log(...args) {
  console.log(...[].concat([`[${moment().format('DD/MM/YYYY HH:mm:ss')}] =>`], Array.from(args) || []))
}
<|endoftext|>


<commit_before>var Pozi = require('./lib')
var bot = new Pozi.Bot()

bot.use(new Pozi.WebhookClient())
bot.use(new Pozi.MemoryStorage())

bot.on('message_received', function(message, session, next) {

  var defaults = {
    totalMessages: 0,
    totalWords: 0
  }

  var context = session.getUserContext(defaults)

  var words = message.text.split(" ")
  var totalMessages = context.totalMessages + 1
  var totalWords = context.totalWords + words.length

  session.updateUserContext({
   totalMessages: totalMessages,
   totalWords: totalWords
  })

  next()
})

bot.hears(/\/stats/, function(message, session) {

  var context = session.getUserContext()

  session.send("Total Messages Sent: " + context.totalMessages)
  session.send("Total Words Sent: " + context.totalWords)
})

bot.hears(/.+/, function(message, session) {
  session.send(message.text)
})

bot.listen(process.env.PORT || 3000)

//module.exports = require('./lib')
<commit_msg>Export module.<commit_after>module.exports = require('./lib')
<|endoftext|>


<commit_before>
var queue   = require('level-queue')
var hooks   = require('level-hooks')
var uuid    = require('node-uuid')
var iterate = require('iterate')

function between(key, range) {
  return (
    (!range.start || range.start <= key) &&
    (!range.end || key <= range.end)
  )
}

module.exports = function (db) {

  if(db.trigger) return db

  var ranges = {}

  db.trigger = {
    add: function (range, job) {
      if(!job) job = range.job
      else     range.job = job

      if(!range.name)
        throw new Error('expects trigger to have name')

      //TODO allow range to be string.
      range.start = range.start || ''
      range.end   = range.end   || '~'

      ranges[range.name] = range

      range.map = range.map || function (e) { return e.key }
      db.queue.add('trigger:'+range.name, job)
    }
  }

  queue()(db)
  hooks()(db)

  db.hooks.pre(function (batch) {
    var insert = []
    //for each operation inside each range,
    //atomically add a record, or run a the job.
    iterate.join(batch, ranges, function (item, range, _, name) {
      var key = ''+item.key, mapped
      if(between(key, range) && (mapped = range.map(item))) {
        insert.push(db.queue('trigger:' + range.name, mapped, false))
      }
    })

    return batch.concat(insert)
  })
}
<commit_msg>rewrite based on level-sublevel<commit_after>var shasum = require('shasum')

//if a job starts, and another is queued before the current job ends,
//delay it, so that the job is only triggered once.

module.exports = function (input, jobs, map, work) {
  if(!work) work = map, map = function (data) { return data.key }
  //create a subsection for the jobs,
  //if you don't pass in a separate db,
  //create a section inside the input
  var pending = {}, running = {}

  if('string' === typeof jobs)
    jobs = input.sublevel(jobs)

  var retry = []

  function doJob (data) {
    //don't process deletes!
    if(!data.value) return
    var hash = shasum(data.value)

    if(!running[hash])
      running[hash] = true
    else return

    var done = false

    work(data.value, function (err) {
      if(done) return
      done = true
      if(err) {
        running[hash]
        return setTimeout(function () {
          doJob(data)
        }, 500)
      }

      jobs.del(data.key, function (err) {
        if(err) return retry.push(data)
        delete running[hash]
        if(pending[hash]) {
          delete pending[hash]
          doJob(data)
        }
      })
    })
  }

  input.pre(function (ch, add) {
    var key = map(ch)
    var hash = shasum(key)
    console.log('KEY', key)
    if(!pending[hash])
      add({key: Date.now(), value: key, type: 'put'}, jobs)
    else
      pending[hash] = (0 || pending[hash]) + 1
  })

  jobs.createReadStream().on('data', doJob)
  jobs.post(doJob)

  return jobs
}

<|endoftext|>


<commit_before>'use latest'
const request = require('request')
const conceptsUrl = 'http://webconcepts.info/concepts.json'

module.exports = function (ctx, cb) {

  const send = (text, attachments) => {
    cb(null, {
      response_type: "ephemeral",
      text: text,
      attachments: attachments
    });
  }

  const sendUsage = () => {
    return send(
      `Usage: "{concept-type} {concept-name}", where {concept-type} can be: ${conceptList}`);
  }

  // authorize
  // yes, I know, not a constant time comparison :(
  if(ctx.secrets['slack-token'] != ctx.data.token){
    return cb(null,{})
  }

  // check for empty ...
  var text = ctx.data.text.trim();
  if(!text){
    return sendUsage()
  }

  // ... or invalid command
  var parts = ctx.data.text.trim().split(' ');
  if(parts.length != 2){
    return sendUsage();
  }

  var conceptType = parts[0];
  var conceptName = parts[1];

  request(conceptsUrl, (error, response, body) => {
    if (error || response.statusCode >= 500){
      return send('Unable to access the http://webconcepts.info, sorry');
    }

    var concepts = JSON.parse(body);
    var concept = concepts.filter(c => new RegExp('^' + conceptType + '$', 'i').test(c.concept))[0];

    if(!concept) {
      var conceptList = concepts.reduce((list, c) => {
        list += ', ' + c.concept;
        return list;
      }, '');
      return send(`Unknown concept-type "${parts[0]}", usage: "{concept-type} {concept-name}", where {concept-type} can be: ${conceptList}`);
    }

    var conceptValue = concept.values.filter(c => new RegExp('^' + conceptName + '$', 'i').test(c.value))[0];

    if(!conceptValue) {
      return send(`Unable to find the "${conceptName}" concept, sorry.`);
    }

    var respText = conceptValue.details[0].description;
    var spec = conceptValue.details[0].documentation;
    var site = conceptValue.id;
    send(
        `*${conceptName}*:\n${respText}`,
        [
          {
            title: "See the detailed specification",
            title_link: spec
          },
          {
            title: "See the http://webconcepts.info page",
            title_link: site
          }
        ]
      );
  });
};
<commit_msg>adds alias and improves usage messages<commit_after>'use latest'
const request = require('request')
const conceptsUrl = 'http://webconcepts.info/concepts.json'
function useAlias(conceptType){
  const alias = {
    'method': 'http-method',
    'status': 'http-status-code',
    'relation': 'link-relation'
  }
  return alias[conceptType] || conceptType;
}

module.exports = function (ctx, cb) {

  const send = (text, attachments) => {
    cb(null, {
      response_type: "ephemeral",
      text: text,
      attachments: attachments
    });
  }

  const sendUsage = (concepts) => {
    let conceptList = concepts.map(c => c.concept).join(', ')
    return send(`Usage: "{concept-type} {concept-name}", where {concept-type} can be: ${conceptList}`);
  }

  const sendUsageForUnknownType = (type, concepts) => {
    let conceptList = concepts.map(c => c.concept).join(', ')
    return send(`Unknown concept-type "${type}", usage: "{concept-type} {concept-name}", where {concept-type} can be: ${conceptList}`);
  }

  // authorize
  // yes, I know, not a constant time comparison :(
  if(ctx.secrets['slack-token'] != ctx.data.token){
    return cb(null,{})
  }

  request(conceptsUrl, (error, response, body) => {
    if (error || response.statusCode >= 500){
      return send('Unable to access the http://webconcepts.info, sorry');
    }

    const concepts = JSON.parse(body);

    // check for empty ...
    const text = ctx.data.text.trim();
    if(!text){
      return sendUsage(concepts);
    }

    // ... or invalid command
    const parts = ctx.data.text.trim().split(' ');
    if(parts.length != 2){
      return sendUsage(concepts);
    }

    const conceptType = useAlias(parts[0]);
    const conceptName = parts[1];

    const concept = concepts.filter(c => new RegExp('^' + conceptType + '$', 'i').test(c.concept))[0];

    if(!concept) {
      return sendUsageForUnknownType(conceptType, concepts);
    }

    const conceptValue = concept.values.filter(c => new RegExp('^' + conceptName + '$', 'i').test(c.value))[0];

    if(!conceptValue) {
      return send(`Unable to find the "${conceptName}" concept, sorry.`);
    }

    const respText = conceptValue.details[0].description;
    const spec = conceptValue.details[0].documentation;
    const site = conceptValue.id;
    send(
        `*${conceptName}*:\n${respText}`,
        [
          {
            title: "See the detailed specification",
            title_link: spec
          },
          {
            title: "See the http://webconcepts.info page",
            title_link: site
          }
        ]
      );
  });
};
<|endoftext|>