<commit_before>/**
 * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
 * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
 * and other contributors
 * Released under the Apache 2.0 License
 */
var crawler = require('./crawler'),
  Analyzer = require('./analyze/analyzer'),
  HTMLRenderer = require('./htmlRenderer'),
  Collector = require('./collector'),
  JUnitRenderer = require('./junitRenderer'),
  Graphite = require('./graphite'),
  logger = require('./log'),
  path = require('path'),
  dateFormat = require('dateformat'),
  fs = require('fs-extra'),
  config = require('./conf'),
  async = require('async'),
  urlParser = require('url'),
  os = require('os'),
  EOL = os.EOL,
  binPath = require('phantomjs').path,
  childProcess = require('child_process'),
  log = require('winston');

function Runner() {
  this.analyzer = new Analyzer();
  this.collector = new Collector();
  this.htmlRenderer = new HTMLRenderer();
  this.junitRenderer = new JUnitRenderer(this.collector);
  this.graphite = new Graphite(config.graphiteHost, config.graphitePort, config
    .graphiteNamespace, this.collector);
}

Runner.prototype.run = function(finshedCb) {
  finshedCb = finshedCb || function() {};
  var self = this;

  async.series([
    function(cb){
      createDataDir(cb);
    },
    function(cb){
      writeConfigurationFile(cb);
    },
    function(cb){
      logVersions(cb);
    }
    ],

    function(err, results){
      if (err) {
        throw err;
      }
        if (config.sites) {
            self._analyzeSites(finshedCb);
          }
        else {
          self._analyzeSite(finshedCb);
        }
    });
};

Runner.prototype._analyzeSites = function(cb) {
  var self = this;
  // store all site data here, use it when parsing
  self.sites = {};

  fs.readFile(config.sites, function(err, data) {
    if (err) {
      throw err;
    }
    var urls = data.toString().split(EOL);
    var queue = async.queue(self._setupConfigurationForSite, 1);
    log.log('info', 'Analyze ' + urls.length + ' sites');
    urls.forEach(function(url) {
      if (url !== '') {
        queue.push({
        'url': url,
        'runner': self
      }, function() {
        log.log('info', 'Finished with site ' + url);
      });
    }
    });

    queue.drain = function() {
      async.parallel({
          copySiteAssets: function(cb) {
            self.htmlRenderer.copyAssets(path.join(config.run.absResultDir, '..'), cb);
          },
          renderSites: function(cb) {
            self.htmlRenderer.renderSites(self.sites, cb);
          }
        },
        function(err, results) {
          if (!err) {
            log.log('info', 'Wrote sites result to ' + config.run.absResultDir);
          }
          cb();
        });
    };
  });
};

Runner.prototype._setupConfigurationForSite = function(args, cb) {
  var url = args.url;
  config.url = url;
  config.urlObject = urlParser.parse(config.url);

  var startPath = (config.resultBaseDir.charAt(0) === path.sep) ? config.resultBaseDir : path.join(__dirname, '../',
    config.resultBaseDir);

  config.run.absResultDir = path.join(startPath, 'sites', dateFormat(config.run.date,
    'yyyy-mm-dd-HH-MM-ss'), config.urlObject.hostname);
  // setup the directories needed
  var dataDir = path.join(config.run.absResultDir, config.dataDir);

  fs.mkdirs(dataDir, function(err) {
    if (err) {
      log.log('error', 'Couldnt create the data dir:' + dataDir + ' ' + err);
      throw err;
    } else {
      args.runner._analyzeSite(cb);
    }
  });

};

Runner.prototype._analyzeSite = function(cb) {
  var self = this;

  /**
  This is the main flow of the application and this is what we do:
    1. Fetch the URL:s that will be analyzed, either we crawl a site using
       a start url or we read the URL:s from a file.
    2. Finetune the URL:s = do other thing that's needed, store them to disk etc.
    3. Let the analyser take a go at the URL:s, the analyzer
       got a lot to do, lets check the analyzer.js file
    4. The analyze is finished, lets create output
  **/
  async.waterfall([
    function(cb) {
      self._fetchUrls(crawler, cb);
    },
    function(okUrls, errorUrls, cb) {
      self._fineTuneUrls(okUrls, errorUrls, cb);
    },
    function(urls, downloadErrors, cb) {
      self._analyze(urls, downloadErrors, cb);
    },
    function(downloadErrors, analysisErrors, cb) {
      self._createOutput(downloadErrors, analysisErrors, cb);
    }
  ], function(err, result) {
    if (err) {
        log.log('error', err);
      }
    cb();
  });
};

Runner.prototype._fineTuneUrls = function (okUrls, errorUrls, callback) {
  var downloadErrors = {};
  Object.keys(errorUrls).forEach(function(url) {
    log.log('error', 'Failed to download ' + url);
    downloadErrors[url] = errorUrls[url];
  });

  // limit
  if (config.maxPagesToTest) {
    if (okUrls.length > config.maxPagesToTest) {
      okUrls.length = config.maxPagesToTest;
    }
  }
  if (okUrls.length === 0) {
    log.log('info', 'Didnt get any URLs');
    callback(new Error('No URLs to analyze'), okUrls, downloadErrors);
  }
  else {
  saveUrls(okUrls);
  callback(null, okUrls, downloadErrors);
  }
};

Runner.prototype._fetchUrls = function (crawler, callback) {
  if (config.url) {
    log.log('info', 'Will crawl from start point ' + config.url +
      ' with crawl depth ' + config.deep);
    crawler.crawl(config.url, function(okUrls, errorUrls) {
      callback(null, okUrls, errorUrls);
    });
  } else {
    fs.readFile(config.file, function (err, data) {
      if (err)  {
        throw err;
      }
      var urls = data.toString().split(EOL);
      urls = urls.filter(function(l) {
        return l.length > 0;
      });

      callback(null, urls, {});
    });
  }
};

Runner.prototype._analyze = function (urls, downloadErrors, callback) {
  var analysisErrors = {};
  var self = this;
  log.log('info', 'Will analyze ' + urls.length + ' pages');
  this.analyzer.analyze(urls, this.collector, downloadErrors, analysisErrors, function(err, url, pageData) {

    if (err) {
      log.log('error', 'Could not analyze ' + url + ' (' + JSON.stringify(err) +
        ')');
      analysisErrors[url] = err;
      return;
    }

    if (config.junit) {
      self.junitRenderer.renderForEachPage(url, pageData);
    }
    self.htmlRenderer.renderPage(url, pageData, function() {});
  }, callback);
};

function saveUrls(urls) {
  fs.writeFile(path.join(config.run.absResultDir, 'data', 'urls.txt'), urls.join(
    EOL), function(err) {
    if (err) {
      throw err;
    }
  });
}

function createDataDir(cb) {
  // create the home data dir
  var dataDir = path.join(config.run.absResultDir, config.dataDir);
  fs.mkdirs(dataDir, function(err) {
    if (err) {
      throw err;
    }
    cb(err, null);
  });
}

function writeConfigurationFile(cb) {
  // write the configuration file
  var confFile = path.join(config.run.absResultDir, 'config.json');
  fs.writeFile(confFile, JSON.stringify(
    config), function(err) {
      if (err) {
        log.log('error','Couldnt write configuration file to disk:'+ confFile + ' ' +err);
      }
    cb(err, null);
  });
}

function storeSummary(aggregates,cb) {
  var summary = path.join(config.run.absResultDir, config.dataDir, 'summary.json');
  fs.writeFile(summary, JSON.stringify(aggregates), function(err) {
    if(err) {
        log.log('error','Couldnt write summary json file to disk:'+ summary + ' ' +err);
    }
    cb();
});
}

Runner.prototype._createOutput = function (downloadErrors, analysisErrors, callBack) {
  log.log('info', 'Done analyzing urls');

  // fetch all the data we need, and then generate the output
  var aggregates = this.collector.createAggregates();
  var assets = this.collector.createCollections().assets;
  var domains = this.collector.createCollections().domains;
  var pages = this.collector.createCollections().pages;

  var self = this;

  if (this.sites) {
    this.sites[config.url] = aggregates;
  }

  /* We got a lot of things to do, lets generate all results
  in parallel and then let us know when we are finished
  */
  async.parallel({
      storeSummary: function(cb) {
          storeSummary(aggregates, cb);
      },
      renderSummary: function(cb) {
        self.htmlRenderer.renderSummary(aggregates, cb);
      },
      renderAssets: function(cb) {
        self.htmlRenderer.renderAssets(assets, cb);
      },
      renderDomains: function(cb) {
        if (config.browser) {
          self.htmlRenderer.renderDomains(domains, cb);
        }
        else {
          cb();
        }
      },
      renderPages: function(cb) {
        self.htmlRenderer.renderPages(pages, cb);
      },
      renderRules: function(cb) {
        // TODO the rules needs to be generated after ...
        self.htmlRenderer.renderRules(cb);
      },
      renderErrors: function(cb) {
        self.htmlRenderer.renderErrors(downloadErrors, analysisErrors, cb);
      },
      copyAssets: function(cb) {
        self.htmlRenderer.copyAssets(config.run.absResultDir, cb);
      },
      renderScreenshots: function(cb) {
        if (config.screenshot) {
          self.htmlRenderer.renderScreenshots(pages, cb);
        } else {
          cb();
        }
      },
      sendToGraphite: function(cb) {
        if (config.graphiteHost) {
          self.graphite.sendPageData(aggregates, pages, cb);
        } else {
          cb();
        }
      },
      renderJUnit: function(cb) {
        if (config.junit) {
          self.junitRenderer.renderAfterFullAnalyse(cb);
        }
        else {
          cb();
        }
      }
    },
    function(err, results) {

      // TODO this can be cleaner
      // We clear the number of pages tested and
      // the collected data, so it is ready for next run
      // used when testing multiple sites
      self.htmlRenderer.numberOfAnalyzedPages = 0;
      self.collector.clear();

      if (!err) {
        log.log('info', 'Wrote results to ' + config.run.absResultDir);
      }
      callBack();
    });
};

function logVersions(cb) {

  async.parallel([
      function(callback) {
        childProcess.execFile(binPath, ['--version'], {
          timeout: 120000
        }, function(err, stdout, stderr) {
          callback(null, stdout);
        });
      },
      function(callback) {
        childProcess.exec('java -version', {
          timeout: 120000
        }, function(err, stdout, stderr) {
          callback(null, stderr);
        });
      }
    ],
    function(err, results) {
      log.log('info', 'OS: ' + os.platform() + ' ' + os.release() + ' sitespeed:' + require('../package.json').version +
        ' ' + ' phantomJs:' + results[0].replace(EOL,'') + ' java:' + results[1].split(EOL)[0]);
      cb(null, null);
    });

}

module.exports = Runner;
<commit_msg>fetch domain data also for wpt<commit_after>/**
 * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
 * Copyright (c) 2014, Peter Hedenskog, Tobias Lidskog
 * and other contributors
 * Released under the Apache 2.0 License
 */
var crawler = require('./crawler'),
  Analyzer = require('./analyze/analyzer'),
  HTMLRenderer = require('./htmlRenderer'),
  Collector = require('./collector'),
  JUnitRenderer = require('./junitRenderer'),
  Graphite = require('./graphite'),
  logger = require('./log'),
  path = require('path'),
  dateFormat = require('dateformat'),
  fs = require('fs-extra'),
  config = require('./conf'),
  async = require('async'),
  urlParser = require('url'),
  os = require('os'),
  EOL = os.EOL,
  binPath = require('phantomjs').path,
  childProcess = require('child_process'),
  log = require('winston');

function Runner() {
  this.analyzer = new Analyzer();
  this.collector = new Collector();
  this.htmlRenderer = new HTMLRenderer();
  this.junitRenderer = new JUnitRenderer(this.collector);
  this.graphite = new Graphite(config.graphiteHost, config.graphitePort, config
    .graphiteNamespace, this.collector);
}

Runner.prototype.run = function(finshedCb) {
  finshedCb = finshedCb || function() {};
  var self = this;

  async.series([
    function(cb){
      createDataDir(cb);
    },
    function(cb){
      writeConfigurationFile(cb);
    },
    function(cb){
      logVersions(cb);
    }
    ],

    function(err, results){
      if (err) {
        throw err;
      }
        if (config.sites) {
            self._analyzeSites(finshedCb);
          }
        else {
          self._analyzeSite(finshedCb);
        }
    });
};

Runner.prototype._analyzeSites = function(cb) {
  var self = this;
  // store all site data here, use it when parsing
  self.sites = {};

  fs.readFile(config.sites, function(err, data) {
    if (err) {
      throw err;
    }
    var urls = data.toString().split(EOL);
    var queue = async.queue(self._setupConfigurationForSite, 1);
    log.log('info', 'Analyze ' + urls.length + ' sites');
    urls.forEach(function(url) {
      if (url !== '') {
        queue.push({
        'url': url,
        'runner': self
      }, function() {
        log.log('info', 'Finished with site ' + url);
      });
    }
    });

    queue.drain = function() {
      async.parallel({
          copySiteAssets: function(cb) {
            self.htmlRenderer.copyAssets(path.join(config.run.absResultDir, '..'), cb);
          },
          renderSites: function(cb) {
            self.htmlRenderer.renderSites(self.sites, cb);
          }
        },
        function(err, results) {
          if (!err) {
            log.log('info', 'Wrote sites result to ' + config.run.absResultDir);
          }
          cb();
        });
    };
  });
};

Runner.prototype._setupConfigurationForSite = function(args, cb) {
  var url = args.url;
  config.url = url;
  config.urlObject = urlParser.parse(config.url);

  var startPath = (config.resultBaseDir.charAt(0) === path.sep) ? config.resultBaseDir : path.join(__dirname, '../',
    config.resultBaseDir);

  config.run.absResultDir = path.join(startPath, 'sites', dateFormat(config.run.date,
    'yyyy-mm-dd-HH-MM-ss'), config.urlObject.hostname);
  // setup the directories needed
  var dataDir = path.join(config.run.absResultDir, config.dataDir);

  fs.mkdirs(dataDir, function(err) {
    if (err) {
      log.log('error', 'Couldnt create the data dir:' + dataDir + ' ' + err);
      throw err;
    } else {
      args.runner._analyzeSite(cb);
    }
  });

};

Runner.prototype._analyzeSite = function(cb) {
  var self = this;

  /**
  This is the main flow of the application and this is what we do:
    1. Fetch the URL:s that will be analyzed, either we crawl a site using
       a start url or we read the URL:s from a file.
    2. Finetune the URL:s = do other thing that's needed, store them to disk etc.
    3. Let the analyser take a go at the URL:s, the analyzer
       got a lot to do, lets check the analyzer.js file
    4. The analyze is finished, lets create output
  **/
  async.waterfall([
    function(cb) {
      self._fetchUrls(crawler, cb);
    },
    function(okUrls, errorUrls, cb) {
      self._fineTuneUrls(okUrls, errorUrls, cb);
    },
    function(urls, downloadErrors, cb) {
      self._analyze(urls, downloadErrors, cb);
    },
    function(downloadErrors, analysisErrors, cb) {
      self._createOutput(downloadErrors, analysisErrors, cb);
    }
  ], function(err, result) {
    if (err) {
        log.log('error', err);
      }
    cb();
  });
};

Runner.prototype._fineTuneUrls = function (okUrls, errorUrls, callback) {
  var downloadErrors = {};
  Object.keys(errorUrls).forEach(function(url) {
    log.log('error', 'Failed to download ' + url);
    downloadErrors[url] = errorUrls[url];
  });

  // limit
  if (config.maxPagesToTest) {
    if (okUrls.length > config.maxPagesToTest) {
      okUrls.length = config.maxPagesToTest;
    }
  }
  if (okUrls.length === 0) {
    log.log('info', 'Didnt get any URLs');
    callback(new Error('No URLs to analyze'), okUrls, downloadErrors);
  }
  else {
  saveUrls(okUrls);
  callback(null, okUrls, downloadErrors);
  }
};

Runner.prototype._fetchUrls = function (crawler, callback) {
  if (config.url) {
    log.log('info', 'Will crawl from start point ' + config.url +
      ' with crawl depth ' + config.deep);
    crawler.crawl(config.url, function(okUrls, errorUrls) {
      callback(null, okUrls, errorUrls);
    });
  } else {
    fs.readFile(config.file, function (err, data) {
      if (err)  {
        throw err;
      }
      var urls = data.toString().split(EOL);
      urls = urls.filter(function(l) {
        return l.length > 0;
      });

      callback(null, urls, {});
    });
  }
};

Runner.prototype._analyze = function (urls, downloadErrors, callback) {
  var analysisErrors = {};
  var self = this;
  log.log('info', 'Will analyze ' + urls.length + ' pages');
  this.analyzer.analyze(urls, this.collector, downloadErrors, analysisErrors, function(err, url, pageData) {

    if (err) {
      log.log('error', 'Could not analyze ' + url + ' (' + JSON.stringify(err) +
        ')');
      analysisErrors[url] = err;
      return;
    }

    if (config.junit) {
      self.junitRenderer.renderForEachPage(url, pageData);
    }
    self.htmlRenderer.renderPage(url, pageData, function() {});
  }, callback);
};

function saveUrls(urls) {
  fs.writeFile(path.join(config.run.absResultDir, 'data', 'urls.txt'), urls.join(
    EOL), function(err) {
    if (err) {
      throw err;
    }
  });
}

function createDataDir(cb) {
  // create the home data dir
  var dataDir = path.join(config.run.absResultDir, config.dataDir);
  fs.mkdirs(dataDir, function(err) {
    if (err) {
      throw err;
    }
    cb(err, null);
  });
}

function writeConfigurationFile(cb) {
  // write the configuration file
  var confFile = path.join(config.run.absResultDir, 'config.json');
  fs.writeFile(confFile, JSON.stringify(
    config), function(err) {
      if (err) {
        log.log('error','Couldnt write configuration file to disk:'+ confFile + ' ' +err);
      }
    cb(err, null);
  });
}

function storeSummary(aggregates,cb) {
  var summary = path.join(config.run.absResultDir, config.dataDir, 'summary.json');
  fs.writeFile(summary, JSON.stringify(aggregates), function(err) {
    if(err) {
        log.log('error','Couldnt write summary json file to disk:'+ summary + ' ' +err);
    }
    cb();
});
}

Runner.prototype._createOutput = function (downloadErrors, analysisErrors, callBack) {
  log.log('info', 'Done analyzing urls');

  // fetch all the data we need, and then generate the output
  var aggregates = this.collector.createAggregates();
  var assets = this.collector.createCollections().assets;
  var domains = this.collector.createCollections().domains;
  var pages = this.collector.createCollections().pages;

  var self = this;

  if (this.sites) {
    this.sites[config.url] = aggregates;
  }

  /* We got a lot of things to do, lets generate all results
  in parallel and then let us know when we are finished
  */
  async.parallel({
      storeSummary: function(cb) {
          storeSummary(aggregates, cb);
      },
      renderSummary: function(cb) {
        self.htmlRenderer.renderSummary(aggregates, cb);
      },
      renderAssets: function(cb) {
        self.htmlRenderer.renderAssets(assets, cb);
      },
      renderDomains: function(cb) {
        if (config.browser || config.wptUrl) {
          self.htmlRenderer.renderDomains(domains, cb);
        }
        else {
          cb();
        }
      },
      renderPages: function(cb) {
        self.htmlRenderer.renderPages(pages, cb);
      },
      renderRules: function(cb) {
        // TODO the rules needs to be generated after ...
        self.htmlRenderer.renderRules(cb);
      },
      renderErrors: function(cb) {
        self.htmlRenderer.renderErrors(downloadErrors, analysisErrors, cb);
      },
      copyAssets: function(cb) {
        self.htmlRenderer.copyAssets(config.run.absResultDir, cb);
      },
      renderScreenshots: function(cb) {
        if (config.screenshot) {
          self.htmlRenderer.renderScreenshots(pages, cb);
        } else {
          cb();
        }
      },
      sendToGraphite: function(cb) {
        if (config.graphiteHost) {
          self.graphite.sendPageData(aggregates, pages, cb);
        } else {
          cb();
        }
      },
      renderJUnit: function(cb) {
        if (config.junit) {
          self.junitRenderer.renderAfterFullAnalyse(cb);
        }
        else {
          cb();
        }
      }
    },
    function(err, results) {

      // TODO this can be cleaner
      // We clear the number of pages tested and
      // the collected data, so it is ready for next run
      // used when testing multiple sites
      self.htmlRenderer.numberOfAnalyzedPages = 0;
      self.collector.clear();

      if (!err) {
        log.log('info', 'Wrote results to ' + config.run.absResultDir);
      }
      callBack();
    });
};

function logVersions(cb) {

  async.parallel([
      function(callback) {
        childProcess.execFile(binPath, ['--version'], {
          timeout: 120000
        }, function(err, stdout, stderr) {
          callback(null, stdout);
        });
      },
      function(callback) {
        childProcess.exec('java -version', {
          timeout: 120000
        }, function(err, stdout, stderr) {
          callback(null, stderr);
        });
      }
    ],
    function(err, results) {
      log.log('info', 'OS: ' + os.platform() + ' ' + os.release() + ' sitespeed:' + require('../package.json').version +
        ' ' + ' phantomJs:' + results[0].replace(EOL,'') + ' java:' + results[1].split(EOL)[0]);
      cb(null, null);
    });

}

module.exports = Runner;
<|endoftext|>


<commit_before>const util = require('util');
const mongoose = require('mongoose');
const Collection = require('discord.js').Collection;
const Logger = require('yappybots-addons').discord.logger;
const Schema = mongoose.Schema;

let logger;

/**
* The channel config Schema
* @typedef {Object} ChannelConfigSchema
* @property {String} guildName - Guild Name
* @property {String} guildID Guild Id
* @property {String} channelName Channel Name
* @property {String} channelID Channel Id
* @property {Array} repos Array of repos to event to channel
* @property {Array} repo Repo to use for gitlab commands in channel
* @property {Array} disabledEvents Github events to disable in this channel
* @property {Boolean} embed Use embeds for events or not
*/
const channelConfigSchema = Schema({
  guildName: String,
  guildID: String,
  channelName: String,
  channelID: String,
  repos: Array,
  repo: String,
  embed: Boolean,
  disabledEvents: {
    type: Array,
    default: [
      'merge_request/update',
    ],
  },
  ignoredUsers: Array,
  ignoredBranches: Array,
});

const channelConfig = mongoose.model('ChannelConfig', channelConfigSchema);

/**
* A Channel Config Item
*/
class ChannelConfigItem {
  constructor(client, config) {
    logger = new Logger(client, 'channelconfig');

    /**
    * The bot client
    * @type Client
    * @private
    */
    this._client = client;
    for (let key in config) {
      if (config.hasOwnProperty(key)) {
        this[key] = config[key];
      }
    }

    this.repos = this.repos || [];
  }
  /**
  * Set a specific config property to a value for this config item
  * @param {String} prop Property to modify
  * @param {String} value The new value for the property
  * @see ChannelConfig#set
  * @return {Promise}
  */
  set(prop, value) {
    return this._client.set(this.channelID, prop, value);
  }
  /**
  * Delete repo events from channel
  * @param {String} repo Repo events to delete from channel
  * @return {Promise}
  */
  deleteRepo(repo) {
    let repos = this.repos;
    repos.splice(repos.indexOf(repo), 1);
    return this.set('repos', repos);
  }
}

/**
* The Channel Config manager
*/
class ChannelConfig {
  constructor() {
    /**
    * All the config
    * @type {Collection}
    * @private
    */
    this._data = new Collection();
    this.setup();
    this.validKeys = [
      'repos',
      'repo',
      'embed',
      'disabledEvents',
      'ignoredUsers',
      'ignoredBranches',
    ];
    this.setupEvents = false;

    /**
    * Loaded
    * @type {Boolean}
    */
    this.loaded = false;
  }
  /**
  * Get config from database and add to this._data
  */
  setup() {
    channelConfig.find({}).then(configs => {
      this.loaded = true;
      configs.forEach(row => {
        this._data.set(row.channelID, new ChannelConfigItem(this, row._doc));
      });
    }).catch(Log.error);
  }
  /**
  * Initialize configuration and Discord bot events
  * @param {external:Client} bot Client instance
  */
  init(bot) {
    if (!this.loaded) {
      setTimeout(() => this.init(bot), 5000);
      return;
    }

    for (const ch of bot.channels) {
      const channel = ch[1];
      if (!channel || channel.type !== 'text') continue;
      if (!this.has(channel.id)) {
        Log.info(`ChannelConf | Adding "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        this.add(channel).catch(e => bot.emit('error', e));
      }
    }

    if (!this.setupEvents) {
      this.setupEvents = true;
      bot.on('channelDelete', channel => {
        if (!channel || channel.type !== 'text') return;
        Log.info(`ChannelConf | Deleting "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        this.delete(channel.id).catch(Log.error);
      });
      bot.on('channelCreate', channel => {
        if (!channel || channel.type !== 'text') return;
        if (this.has(channel.id)) return;
        Log.info(`ChannelConf | Adding "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        if (!channel.name || !channel.guild.name || !channel.id) {
          Log.warning('ChannelConf | Some info for channel not found', channel);
          logger.log('Error when adding channel config to database', util.inspect(channel, { depth: 2 }), 'YELLOW');
        }
        this.add(channel).catch(e => bot.emit('error', e));
      });
    }
  }
  /**
  * Find channels with events for repo
  * @param {String} repo Repo for the events
  * @return {ChannelConfigItem}
  */
  findByRepo(repo) {
    let re = repo.toLowerCase();
    return this._data.filter(e => e.repos.filter(r => r === re)[0]);
  }
  /**
  * Get channel config
  * @param {String} channel Channel ID
  * @return {ChannelConfigItem}
  */
  get(channel) {
    return this._data.get(channel);
  }

  /**
  * Has channel in config
  * @param {String} channel Channel ID
  * @return {Boolean}
  */
  has(channel) {
    return this._data.has(channel);
  }

  /**
  * Delete all repo events from channel
  * @param {String} channelID Channel ID with the events to delete
  * @return {Promise<ChannelConfig>}
  */
  delete(channelID) {
    return channelConfig.findOneAndRemove({
      channelID: channelID,
    }).then(() => {
      this._data.delete(channelID);
      return Promise.resolve(this);
    });
  }

  /**
  * Delete all channels from guild
  * @param  {String} guildID Guild ID to delete channel configs for
  * @return {Promise}
  */
  deleteFromGuild(guildID) {
    return Promise.all(
      this._data
      .filter(c => c.guildID === guildID)
      .map(c => {
        Log.info(`ChannelConf | Deleting "${c.guildName}"'s #${c.channelName} (${c.channelID})`);
        return this.delete(c.channelID);
      })
    ).then(() => this);
  }

  /**
  * Add channel to config
  * @param {Channel} channel Channel to add repo events
  * @return {Promise<ChannelConfig>}
  */
  add(channel) {
    if (!channel || !channel.id) return Promise.reject(`No channel passed!`);
    if (channel && channel.id && this.get(channel.id)) return Promise.reject(`Channel already has an entry in database`);
    const conf = {
      guildID: channel.guild && channel.guild.id,
      guildName: channel.guild && channel.guild.name,
      channelID: channel.id,
      channelName: channel.name,
      repos: [],
      prefix: `GL! `,
      disabledEvents: [
        'merge_request/update',
        'pipeline',
      ],
    };
    return channelConfig.create(conf).then(() => {
      const item = new ChannelConfigItem(this, conf);
      this._data.set(conf.channelID, item);
      return item;
    });
  }

  /**
  * Add all channels from guild
  * @param  {Guild} guild Guild obj to add channel configs for
  * @return {Promise}
  */
  addFromGuild(guild) {
    return Promise.all(
      guild.channels
      .filter(c => c.type === 'text')
      .map(c => {
        Log.info(`ChannelConf | Adding "${c.guildName}"'s #${c.channelName} (${c.channelID})`);
        return this.add(c);
      })
    ).then(() => this);
  }

  /**
  * Replace specific channel config prop with value
  * @param {String} channel Channel with the repo events
  * @param {String} prop Property to set
  * @param {String} value Value to set property to
  * @return {Promise<ChannelConfig>} updated config item
  */
  set(channel, prop, value) {
    return new Promise((resolve, reject) => {
      let oldConfig = this._data.find('channelID', channel);
      let newConfig = oldConfig;
      newConfig[prop] = value;
      channelConfig.findOneAndUpdate({
        channelID: channel,
      }, newConfig, {
        new: true,
      }, (err) => {
        if (err) return reject(err);
        this._data.set(channel, new ChannelConfigItem(this, newConfig));
        resolve(this);
      });
    });
  }

  /**
  * Add repo to channel
  * @param {Channel} channel Channel to add repo to
  * @param {String} repo repo to add to channel
  * @see ChannelConfig#set
  * @return {Promise<ChannelConfig>}
  */
  addRepoToChannel(channel, repo) {
    if (!channel || !repo) return Promise.reject(`Invalid arguments.`);
    let conf = this.get(channel);
    let repos = conf.repos;
    repos.push(repo.toLowerCase());
    return this.set(channel, 'repos', repos);
  }

  /**
  * Delete repo events from specific channel
  * @param {String} channel Channel with the repo events
  * @param {String} repo Repo event to remove from Channel
  * @see ChannelConfig#set
  * @return {Promise<ChannelConfig>}
  */
  deleteRepo(channel, repo) {
    return channelConfig.findOneAndRemove({
      repo,
    }).then(() => {
      let oldRepos = this._data.find('channelID', channel);
      let newRepos = oldRepos.slice(0, oldRepos.indexOf(repo));
      return this.set(channel, 'repos', newRepos);
    });
  }
}

module.exports = new ChannelConfig();
<commit_msg>fix(models: channelconfig): don't add channel config if it's missing a value<commit_after>const util = require('util');
const mongoose = require('mongoose');
const Collection = require('discord.js').Collection;
const Logger = require('yappybots-addons').discord.logger;
const Schema = mongoose.Schema;

let logger;

/**
* The channel config Schema
* @typedef {Object} ChannelConfigSchema
* @property {String} guildName - Guild Name
* @property {String} guildID Guild Id
* @property {String} channelName Channel Name
* @property {String} channelID Channel Id
* @property {Array} repos Array of repos to event to channel
* @property {Array} repo Repo to use for gitlab commands in channel
* @property {Array} disabledEvents Github events to disable in this channel
* @property {Boolean} embed Use embeds for events or not
*/
const channelConfigSchema = Schema({
  guildName: String,
  guildID: String,
  channelName: String,
  channelID: String,
  repos: Array,
  repo: String,
  embed: Boolean,
  disabledEvents: {
    type: Array,
    default: [
      'merge_request/update',
    ],
  },
  ignoredUsers: Array,
  ignoredBranches: Array,
});

const channelConfig = mongoose.model('ChannelConfig', channelConfigSchema);

/**
* A Channel Config Item
*/
class ChannelConfigItem {
  constructor(client, config) {
    logger = new Logger(client, 'channelconfig');

    /**
    * The bot client
    * @type Client
    * @private
    */
    this._client = client;
    for (let key in config) {
      if (config.hasOwnProperty(key)) {
        this[key] = config[key];
      }
    }

    this.repos = this.repos || [];
  }
  /**
  * Set a specific config property to a value for this config item
  * @param {String} prop Property to modify
  * @param {String} value The new value for the property
  * @see ChannelConfig#set
  * @return {Promise}
  */
  set(prop, value) {
    return this._client.set(this.channelID, prop, value);
  }
  /**
  * Delete repo events from channel
  * @param {String} repo Repo events to delete from channel
  * @return {Promise}
  */
  deleteRepo(repo) {
    let repos = this.repos;
    repos.splice(repos.indexOf(repo), 1);
    return this.set('repos', repos);
  }
}

/**
* The Channel Config manager
*/
class ChannelConfig {
  constructor() {
    /**
    * All the config
    * @type {Collection}
    * @private
    */
    this._data = new Collection();
    this.setup();
    this.validKeys = [
      'repos',
      'repo',
      'embed',
      'disabledEvents',
      'ignoredUsers',
      'ignoredBranches',
    ];
    this.setupEvents = false;

    /**
    * Loaded
    * @type {Boolean}
    */
    this.loaded = false;
  }
  /**
  * Get config from database and add to this._data
  */
  setup() {
    channelConfig.find({}).then(configs => {
      this.loaded = true;
      configs.forEach(row => {
        this._data.set(row.channelID, new ChannelConfigItem(this, row._doc));
      });
    }).catch(Log.error);
  }
  /**
  * Initialize configuration and Discord bot events
  * @param {external:Client} bot Client instance
  */
  init(bot) {
    if (!this.loaded) {
      setTimeout(() => this.init(bot), 5000);
      return;
    }

    for (const ch of bot.channels) {
      const channel = ch[1];
      if (!channel || channel.type !== 'text') continue;
      if (!this.has(channel.id)) {
        Log.info(`ChannelConf | Adding "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        this.add(channel).catch(e => bot.emit('error', e));
      }
    }

    if (!this.setupEvents) {
      this.setupEvents = true;
      bot.on('channelDelete', channel => {
        if (!channel || channel.type !== 'text') return;
        Log.info(`ChannelConf | Deleting "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        this.delete(channel.id).catch(Log.error);
      });
      bot.on('channelCreate', channel => {
        if (!channel || channel.type !== 'text') return;
        if (this.has(channel.id)) return;
        Log.info(`ChannelConf | Adding "${channel.guild.name}"'s #${channel.name} (${channel.id})`);
        if (!channel.name || !channel.guild.name || !channel.id) {
          Log.warning('ChannelConf | Some info for channel not found', channel);
          logger.log('Error when adding channel config to database', util.inspect(channel, { depth: 2 }), 'YELLOW');
        } else this.add(channel).catch(e => bot.emit('error', e));
      });
    }
  }
  /**
  * Find channels with events for repo
  * @param {String} repo Repo for the events
  * @return {ChannelConfigItem}
  */
  findByRepo(repo) {
    let re = repo.toLowerCase();
    return this._data.filter(e => e.repos.filter(r => r === re)[0]);
  }
  /**
  * Get channel config
  * @param {String} channel Channel ID
  * @return {ChannelConfigItem}
  */
  get(channel) {
    return this._data.get(channel);
  }

  /**
  * Has channel in config
  * @param {String} channel Channel ID
  * @return {Boolean}
  */
  has(channel) {
    return this._data.has(channel);
  }

  /**
  * Delete all repo events from channel
  * @param {String} channelID Channel ID with the events to delete
  * @return {Promise<ChannelConfig>}
  */
  delete(channelID) {
    return channelConfig.findOneAndRemove({
      channelID: channelID,
    }).then(() => {
      this._data.delete(channelID);
      return Promise.resolve(this);
    });
  }

  /**
  * Delete all channels from guild
  * @param  {String} guildID Guild ID to delete channel configs for
  * @return {Promise}
  */
  deleteFromGuild(guildID) {
    return Promise.all(
      this._data
      .filter(c => c.guildID === guildID)
      .map(c => {
        Log.info(`ChannelConf | Deleting "${c.guildName}"'s #${c.channelName} (${c.channelID})`);
        return this.delete(c.channelID);
      })
    ).then(() => this);
  }

  /**
  * Add channel to config
  * @param {Channel} channel Channel to add repo events
  * @return {Promise<ChannelConfig>}
  */
  add(channel) {
    if (!channel || !channel.id) return Promise.reject(`No channel passed!`);
    if (channel && channel.id && this.get(channel.id)) return Promise.reject(`Channel already has an entry in database`);
    const conf = {
      guildID: channel.guild && channel.guild.id,
      guildName: channel.guild && channel.guild.name,
      channelID: channel.id,
      channelName: channel.name,
      repos: [],
      prefix: `GL! `,
      disabledEvents: [
        'merge_request/update',
        'pipeline',
      ],
    };
    return channelConfig.create(conf).then(() => {
      const item = new ChannelConfigItem(this, conf);
      this._data.set(conf.channelID, item);
      return item;
    });
  }

  /**
  * Add all channels from guild
  * @param  {Guild} guild Guild obj to add channel configs for
  * @return {Promise}
  */
  addFromGuild(guild) {
    return Promise.all(
      guild.channels
      .filter(c => c.type === 'text')
      .map(c => {
        Log.info(`ChannelConf | Adding "${c.guildName}"'s #${c.channelName} (${c.channelID})`);
        return this.add(c);
      })
    ).then(() => this);
  }

  /**
  * Replace specific channel config prop with value
  * @param {String} channel Channel with the repo events
  * @param {String} prop Property to set
  * @param {String} value Value to set property to
  * @return {Promise<ChannelConfig>} updated config item
  */
  set(channel, prop, value) {
    return new Promise((resolve, reject) => {
      let oldConfig = this._data.find('channelID', channel);
      let newConfig = oldConfig;
      newConfig[prop] = value;
      channelConfig.findOneAndUpdate({
        channelID: channel,
      }, newConfig, {
        new: true,
      }, (err) => {
        if (err) return reject(err);
        this._data.set(channel, new ChannelConfigItem(this, newConfig));
        resolve(this);
      });
    });
  }

  /**
  * Add repo to channel
  * @param {Channel} channel Channel to add repo to
  * @param {String} repo repo to add to channel
  * @see ChannelConfig#set
  * @return {Promise<ChannelConfig>}
  */
  addRepoToChannel(channel, repo) {
    if (!channel || !repo) return Promise.reject(`Invalid arguments.`);
    let conf = this.get(channel);
    let repos = conf.repos;
    repos.push(repo.toLowerCase());
    return this.set(channel, 'repos', repos);
  }

  /**
  * Delete repo events from specific channel
  * @param {String} channel Channel with the repo events
  * @param {String} repo Repo event to remove from Channel
  * @see ChannelConfig#set
  * @return {Promise<ChannelConfig>}
  */
  deleteRepo(channel, repo) {
    return channelConfig.findOneAndRemove({
      repo,
    }).then(() => {
      let oldRepos = this._data.find('channelID', channel);
      let newRepos = oldRepos.slice(0, oldRepos.indexOf(repo));
      return this.set(channel, 'repos', newRepos);
    });
  }
}

module.exports = new ChannelConfig();
<|endoftext|>


<commit_before>'use strict';

let co = require('co'),
    db = require('./db'),
    config = require('./config'),
    scheduler = require('./scheduler');

/**
 * @class Runner
 */
class Runner {
    /**
     * @param {object} [options]
     * @param {number} [options.scanInterval] - in seconds, 60 by default
     * @param {number} [options.lockInterval] - in seconds, 60 by default
     * @param {function} [options.taskProcessorFactory] - should return task processor by task name, require() by default
     * @param {number} [options.tasksPerScanning] - count of tasks that should be picked per every scanning. By default 1000
     * @param {string} [options.collection] - "tasks" by default
     */
    constructor(options) {
        this._options = Object.assign({
            scanInterval: 60, // seconds
            lockInterval: 60, // seconds
            taskProcessorFactory: require,
            tasksPerScanning: 1000, // tasks that will be executed per every scanning iteration.
            collection: config.collection
        }, options);

        // true if any scanning iteration is in progress
        this.isInProgress = false;

        // graceful shutdown in progress
        this.isStopping = false;

        // runner is stopped
        this.isStopped = false;

        // timer of new scanning iteration
        this.timer = null;
    }

    /**
     * Start scanning iteration based on options, provided via constructor
     * At the end iteration will be rescheduled in this._options.scanInterval seconds
     */
    * start() {
        try {
            yield this.scan();

        } catch (err) {
            // something goes wrong, probably missed connection to mongo - log error
            config.logger.error('Something goes wrong.', err.stack);
        }

        if (this.isStopped) {
            // graceful shutdown
            return;
        }

        // schedule new scanning
        config.logger.debug('Finished scanning iteration, rescan in ' + this._options.scanInterval + ' seconds.');
        this.timer = setTimeout(co.wrap(this.start.bind(this)), this._options.scanInterval * 1000);
    }

    /**
     * Graceful shutdown
     */
    stop() {
        this.isStopping = true;

        if (this.timer) {
            clearTimeout(this.timer);
        }

        if (!this.isInProgress) {
            this.isStopped = true;
        }
    }

    /**
     * Makes one scanning iteration
     */
    * scan() {
        let task,
            iterationIndex = 0;

        this.isInProgress = true;

        // process tasks until it will be finished or until it will process "tasksPerScanning" items
        do {
            iterationIndex++;
            task = yield db.findTaskToProcess(this._options.collection, this._options.lockInterval);

            if ( !task ) {
                // there are no tasks in queue
                break;
            }

            yield this.processTask(task);

        } while (!this.isStopping && iterationIndex < this._options.tasksPerScanning);

        this.isInProgress = false;

        // graceful shutdown
        if (this.isStopping) {
            this.isStopped = true;
        }
    }

    /**
     * Process one task
     * @param {Object} task
     */
    * processTask(task) {
        let previousTask, taskResult;

        config.logger.debug('Start task "' + task.name + '" with id "' + task.taskId + '"');

        if ( task.group ) {
            previousTask = yield db.findPreviousTask(this._options.collection, task);
            // check if previous task still was not processed - it means that current task is blocked by previous one
            if ( previousTask && previousTask.processedAt === null ) {
                // if previousTask' startAt value greater than startAt of current task - reschedule current task on new date
                if ( previousTask.startAt.getTime() > task.startAt.getTime() ) {
                    let logInfo = {
                            taskId: task.taskId,
                            blockedBy: previousTask.taskId
                        };

                    config.logger.debug('Task is blocked: ' + JSON.stringify(logInfo));
                    yield db.rescheduleTask(this._options.collection, task.taskId, new Date(previousTask.startAt.getTime() + 1000));
                }
                return;
            }
        }

        try {
            let previousTaskResult = previousTask ? previousTask.result : null,
                extendedInfo = {
                    failedAt: task.failedAt,  // date of previous error
                    errorMsg: task.errorMsg,  // message of previous error
                    retries: task.retries,    // count of failed executions
                    createdAt: task.createdAt // creation date of task
                };

            let taskProcessor = this._options.taskProcessorFactory(task.name);
            // use taskProcessor as is if it is a function
            // in other case execute taskProcessor.run()
            if ( 'function' === typeof taskProcessor ) {
                taskResult = yield taskProcessor(task.data, previousTaskResult, extendedInfo);

            } else {
                taskResult = yield taskProcessor.run(task.data, previousTaskResult, extendedInfo);
            }

        } catch (err) {
            // something goes wrong with or within task processor - mark task as failed
            // and reschedule it based on its retryStrategy
            config.logger.error('Task was failed: "' + task.taskId + '"', err.stack);
            yield db.markTaskFailed(this._options.collection, task.taskId, err.message);
            yield scheduler.rescheduleFailedTask(this._options.collection, task);
            return;
        }

        if (taskResult === undefined) {
            taskResult = null;
        }

        if ( task.repeatEvery > 0 ) {
            // task is repeatable and should be rescheduled
            yield scheduler.rescheduleRepeatableTask(this._options.collection, task, taskResult);

        } else {
            // task is not repeatable, finish it and save its result
            yield db.markTaskProcessed(this._options.collection, task.taskId, taskResult);
        }
    }
}

module.exports = Runner;
<commit_msg>Log result of task processing<commit_after>'use strict';

let co = require('co'),
    db = require('./db'),
    config = require('./config'),
    scheduler = require('./scheduler');

/**
 * @class Runner
 */
class Runner {
    /**
     * @param {object} [options]
     * @param {number} [options.scanInterval] - in seconds, 60 by default
     * @param {number} [options.lockInterval] - in seconds, 60 by default
     * @param {function} [options.taskProcessorFactory] - should return task processor by task name, require() by default
     * @param {number} [options.tasksPerScanning] - count of tasks that should be picked per every scanning. By default 1000
     * @param {string} [options.collection] - "tasks" by default
     */
    constructor(options) {
        this._options = Object.assign({
            scanInterval: 60, // seconds
            lockInterval: 60, // seconds
            taskProcessorFactory: require,
            tasksPerScanning: 1000, // tasks that will be executed per every scanning iteration.
            collection: config.collection
        }, options);

        // true if any scanning iteration is in progress
        this.isInProgress = false;

        // graceful shutdown in progress
        this.isStopping = false;

        // runner is stopped
        this.isStopped = false;

        // timer of new scanning iteration
        this.timer = null;
    }

    /**
     * Start scanning iteration based on options, provided via constructor
     * At the end iteration will be rescheduled in this._options.scanInterval seconds
     */
    * start() {
        try {
            yield this.scan();

        } catch (err) {
            // something goes wrong, probably missed connection to mongo - log error
            config.logger.error('Something goes wrong.', err.stack);
        }

        if (this.isStopped) {
            // graceful shutdown
            return;
        }

        // schedule new scanning
        config.logger.debug('Finished scanning iteration, rescan in ' + this._options.scanInterval + ' seconds.');
        this.timer = setTimeout(co.wrap(this.start.bind(this)), this._options.scanInterval * 1000);
    }

    /**
     * Graceful shutdown
     */
    stop() {
        this.isStopping = true;

        if (this.timer) {
            clearTimeout(this.timer);
        }

        if (!this.isInProgress) {
            this.isStopped = true;
        }
    }

    /**
     * Makes one scanning iteration
     */
    * scan() {
        let task,
            iterationIndex = 0;

        this.isInProgress = true;

        // process tasks until it will be finished or until it will process "tasksPerScanning" items
        do {
            iterationIndex++;
            task = yield db.findTaskToProcess(this._options.collection, this._options.lockInterval);

            if ( !task ) {
                // there are no tasks in queue
                break;
            }

            yield this.processTask(task);

        } while (!this.isStopping && iterationIndex < this._options.tasksPerScanning);

        this.isInProgress = false;

        // graceful shutdown
        if (this.isStopping) {
            this.isStopped = true;
        }
    }

    /**
     * Process one task
     * @param {Object} task
     */
    * processTask(task) {
        let previousTask, taskResult;

        config.logger.debug('Start processing of task "' + task.name + '" (' + task.taskId + ')');

        if ( task.group ) {
            previousTask = yield db.findPreviousTask(this._options.collection, task);
            // check if previous task still was not processed - it means that current task is blocked by previous one
            if ( previousTask && previousTask.processedAt === null ) {
                // if previousTask' startAt value greater than startAt of current task - reschedule current task on new date
                if ( previousTask.startAt.getTime() > task.startAt.getTime() ) {
                    let logInfo = {
                            taskId: task.taskId,
                            blockedBy: previousTask.taskId
                        };

                    config.logger.debug('Task is blocked: ' + JSON.stringify(logInfo));
                    yield db.rescheduleTask(this._options.collection, task.taskId, new Date(previousTask.startAt.getTime() + 1000));
                }
                return;
            }
        }

        try {
            let previousTaskResult = previousTask ? previousTask.result : null,
                extendedInfo = {
                    failedAt: task.failedAt,  // date of previous error
                    errorMsg: task.errorMsg,  // message of previous error
                    retries: task.retries,    // count of failed executions
                    createdAt: task.createdAt // creation date of task
                };

            let taskProcessor = this._options.taskProcessorFactory(task.name);
            // use taskProcessor as is if it is a function
            // in other case execute taskProcessor.run()
            if ( 'function' === typeof taskProcessor ) {
                taskResult = yield taskProcessor(task.data, previousTaskResult, extendedInfo);

            } else {
                taskResult = yield taskProcessor.run(task.data, previousTaskResult, extendedInfo);
            }

        } catch (err) {
            // something goes wrong with or within task processor - mark task as failed
            // and reschedule it based on its retryStrategy
            config.logger.error('Task was failed: "' + task.taskId + '"', err.stack);
            yield db.markTaskFailed(this._options.collection, task.taskId, err.message);
            yield scheduler.rescheduleFailedTask(this._options.collection, task);
            return;
        }

        if (taskResult === undefined) {
            taskResult = null;
        }

        config.logger.debug('End processing of task "' + task.name + '" (' + task.taskId + '). Result: ', taskResult);

        if ( task.repeatEvery > 0 ) {
            // task is repeatable and should be rescheduled
            yield scheduler.rescheduleRepeatableTask(this._options.collection, task, taskResult);

        } else {
            // task is not repeatable, finish it and save its result
            yield db.markTaskProcessed(this._options.collection, task.taskId, taskResult);
        }
    }
}

module.exports = Runner;
<|endoftext|>


<commit_before>// Generated by CoffeeScript 1.7.1
(function() {
  var BaseModel, ProjectIssues, Utils,
    __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
    __hasProp = {}.hasOwnProperty,
    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };

  BaseModel = require('../BaseModel');

  Utils = require('../Utils');

  ProjectIssues = (function(_super) {
    __extends(ProjectIssues, _super);

    function ProjectIssues() {
      this.list = __bind(this.list, this);
      return ProjectIssues.__super__.constructor.apply(this, arguments);
    }

    ProjectIssues.prototype.list = function(projectId, params, fn) {
      if (params == null) {
        params = {};
      }
      if (fn == null) {
        fn = null;
      }
      this.debug("ProjectIssues::issues()");
      if ('function' === typeof params) {
        fn = params;
        params = {};
      }
      if (params.page == null) {
        params.page = 1;
      }
      if (params.per_page == null) {
        params.per_page = 100;
      }
      return (function() {
        var cb, data;
        data = [];
        cb = (function(_this) {
          return function(err, retData) {
            if (err) {
              if (fn) {
                return fn(data);
              }
            }
            if (retData.length === params.per_page) {
              _this.debug("Recurse ProjectIssues::list()");
              data = data.concat(retData);
              params.page++;
              return _this.get("projects/" + (Utils.parseProjectId(projectId)) + "/issues", params, cb);
            } else {
              data = data.concat(retData);
              if (fn) {
                return fn(data);
              }
            }
          };
        })(this);
        return this.get("projects/" + (Utils.parseProjectId(projectId)) + "/issues", params, cb);
      }).bind(this)();
    };

    return ProjectIssues;

  })(BaseModel);

  module.exports = function(client) {
    return new ProjectIssues(client);
  };

}).call(this);
<commit_msg>Built libraries<commit_after>// Generated by CoffeeScript 1.7.1
(function() {
  var BaseModel, ProjectIssues, Utils,
    __bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
    __hasProp = {}.hasOwnProperty,
    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };

  BaseModel = require('../BaseModel');

  Utils = require('../Utils');

  ProjectIssues = (function(_super) {
    __extends(ProjectIssues, _super);

    function ProjectIssues() {
      this.show = __bind(this.show, this);
      this.list = __bind(this.list, this);
      return ProjectIssues.__super__.constructor.apply(this, arguments);
    }

    ProjectIssues.prototype.list = function(projectId, params, fn) {
      if (params == null) {
        params = {};
      }
      if (fn == null) {
        fn = null;
      }
      this.debug("ProjectIssues::issues()");
      if ('function' === typeof params) {
        fn = params;
        params = {};
      }
      if (params.page == null) {
        params.page = 1;
      }
      if (params.per_page == null) {
        params.per_page = 100;
      }
      return (function() {
        var cb, data;
        data = [];
        cb = (function(_this) {
          return function(err, retData) {
            if (err) {
              if (fn) {
                return fn(data);
              }
            }
            if (retData.length === params.per_page) {
              _this.debug("Recurse ProjectIssues::list()");
              data = data.concat(retData);
              params.page++;
              return _this.get("projects/" + (Utils.parseProjectId(projectId)) + "/issues", params, cb);
            } else {
              data = data.concat(retData);
              if (fn) {
                return fn(data);
              }
            }
          };
        })(this);
        return this.get("projects/" + (Utils.parseProjectId(projectId)) + "/issues", params, cb);
      }).bind(this)();
    };

    ProjectIssues.prototype.show = function(projectId, issueId, params, fn) {
      if (params == null) {
        params = {};
      }
      if (fn == null) {
        fn = null;
      }
      this.debug("ProjectIssues::show()");
      if (projectId.toString().indexOf("/") !== -1) {
        projectId = encodeURIComponent(projectId);
      } else {
        projectId = parseInt(projectId);
      }
      if (issueId.toString().indexOf("/") !== -1) {
        issueId = encodeURIComponent(issueId);
      } else {
        issueId = parseInt(issueId);
      }
      return this.get("projects/" + projectId + "/issues/" + issueId, params, function(data) {
        if (fn) {
          return fn(data);
        }
      });
    };

    return ProjectIssues;

  })(BaseModel);

  module.exports = function(client) {
    return new ProjectIssues(client);
  };

}).call(this);
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//
// Run a pre-validated and pre-prepared test script across a number of
// (local) workers.
//

'use strict';

const os = require('os');
const path = require('path');
const EventEmitter = require('events');
const fork = require('child_process').fork;

const debug = require('debug')('artillery:runner');
const L = require('lodash');

const stats = require('./dispatcher').stats;
const divideWork = require('./dist');

const A = require('async');
const pidusage = require('pidusage');

module.exports = createRunner;

function createRunner(script, payload, opts) {
  const runner = new Runner(script, payload, opts);
  return runner;
}

function Runner(script, payload, opts) {
  this._script = script;
  this._payload = payload;
  this._opts = opts;
  this._workers = {};

  this.events = new EventEmitter();

  this._intermediates = [];
  this._allIntermediates = [];

  this._currentPhase = -1;

  return this;
}

Runner.prototype.run = function run() {
  //
  // Create worker scripts (distribute the work):
  //
  let self = this;
  let numWorkers = process.env.ARTILLERY_WORKERS || 1 || os.cpus().length;
  let workerScripts = divideWork(this._script, numWorkers);
  // Overwrite statsInterval for workers:
  L.each(workerScripts, function(s) {
    s.config.statsInterval = 1;
  });

  debug(JSON.stringify(workerScripts, null, 4));

  //
  // Create workers:
  //
  const DEBUGGER_PORT = 41986;
  L.each(workerScripts, (script, idx) => {
    let forkOptions = {};
    // If running under a debugger, set non-clashing debugger ports for worker
    // processes.
    // Run with: node --debug --expose_debug_as=v8debug --inspect
    const debugging = (typeof v8debug === 'object');
    if (debugging) {
      forkOptions.execArgv = [`--debug=${DEBUGGER_PORT + idx}`];
    }
    let workerProcess = fork(path.join(__dirname, 'worker.js'), forkOptions);
    this._workers[workerProcess.pid] = {
      proc: workerProcess,
      isDone: false
    };
    if(debugging) {
      debug(`forked worker ${workerProcess.pid}; debugger on port ${DEBUGGER_PORT + idx}`);
    } else {
      debug(`forked worker ${workerProcess.pid}`);
    }
    workerProcess.on('message', this._onWorkerMessage.bind(this));
    workerProcess.once('exit', () => {
      if (!this._workers[workerProcess.pid].isDone) {
        debug(`worker ${workerProcess.pid} terminated prematurely`);
        this._workers[workerProcess.pid].isDone = true;
        console.log('Unexpected error, Artillery shutting down.');
        self.shutdown(function() {
          process.exit(1);
        });
      } else {
        debug(`worker ${workerProcess.pid} exited`);
      }
    });
    workerProcess.send({
      command: 'run',
      opts: {
        script: script,
        payload: this._payload, // FIXME: Inefficient with large payloads
        options: this._opts
      }
    });
  });

  // TODO: Use nanotimer
  this._statsInterval = setInterval(
    this._sendStats.bind(this),
    this._script.config.statsInterval * 1000
  );

  // Watch CPU usage of child processes:
  const MELTING_POINT = process.env.ARTILLERY_CPU_THRESHOLD || 80;
  const CPU_CHECK_INTERVAL_MS = 2500;
  setInterval(function() {
    A.map(Object.keys(self._workers), pidusage.stat, function(err, pidStats) {
      if (err) {
        return;
      }
      const busyPids = pidStats.filter(function(o) {
        return o && o.cpu && o.cpu >= MELTING_POINT;
      });
      if (busyPids.length > 0) {
        self.events.emit('highcpu', busyPids);
      }
      debug('busyPids:', busyPids);
    });
  }, CPU_CHECK_INTERVAL_MS).unref();

  return this;
};

Runner.prototype.shutdown = function(done) {
  let self = this;

  A.eachSeries(
    Object.keys(this._workers),
    function(pid, next) {
      if (!self._workers[pid].isDone) {
        debug('Worker %s not done, sending stop command', pid);
        self._workers[pid].proc.send({ command: 'stop' });
        self._workers[pid].isDone = true;
      }
      return next(null);
    }, function(err) {
      if (err) {
        debug(err);
      }

      // Poll until all workers are done:
      setInterval(function() {
        let allDone = L.every(
          Object.keys(self._workers),
          function isDone(pid) {
            return self._workers[pid].isDone;
          });

        if (allDone) {
          debug('All workers done');
          return done();
        }
      }, 100);
    });
};

Runner.prototype._sendStats = function() {
  // Calculate average concurrency:
  // We are averaging and overwriting the value in the report ourselves because
  // combine() presumes that stats objects come from different workers, but
  // we will have multiple intermediate objects from the same worker.

  // Calculate max concurrency (sampled at one second resolution):
  let maxWorkerConcurrencies = L.reduce(
    this._intermediates,
    function(acc, el) {
      const pid = el[0];
      const intermediate = el[1];
      if (typeof acc[pid] !== 'undefined') {
        acc[pid] = L.max([acc[pid], intermediate._concurrency]);
      } else {
        acc[pid] = intermediate._concurrency;
      }
      return acc;
    }, {});

  debug('max worker concurrency: %j', maxWorkerConcurrencies);

  let averageConcurrency = stats.round(
    L.sum(
      L.map(maxWorkerConcurrencies, function(v) { return v; })),
    1);

  let combined = stats.combine(
    L.map(this._intermediates, function(el) { return el[1]; }));

  combined._concurrency = averageConcurrency;

  this.events.emit('stats', combined);
  this._intermediates = [];
};

Runner.prototype._onWorkerMessage = function _onWorkerMessage(message) {
  if (message.event === 'phaseStarted') {
    if (message.phase.index > this._currentPhase) {
      this.events.emit('phaseStarted', message.phase);
      this._currentPhase = message.phase.index;
    }
  }

  if (message.event === 'phaseCompleted') {
  }

  if (message.event === 'stats') {
    this._intermediates.push([message.pid, message.stats]);
    this._allIntermediates.push(message.stats);
  }

  if (message.event === 'done') {
    clearInterval(this._statsInterval);
    let worker = this._workers[message.pid];
    worker.isDone = true;
    worker.proc.send({command: 'stop'});

    if (this._activeWorkerCount() === 0) {
      this._sendStats();
      this.events.emit('done', stats.combine(this._allIntermediates));
    }
  }
};

Runner.prototype._activeWorkerCount = function _activeWorkerCount() {
  var pids = Object.keys(this._workers);
  var count = pids.length;
  pids.forEach((pid) => {
    if (this._workers[pid].isDone) {
      count--;
    }
  });
  return count;
};
<commit_msg>fix: Prevent multiple callback invocations<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//
// Run a pre-validated and pre-prepared test script across a number of
// (local) workers.
//

'use strict';

const os = require('os');
const path = require('path');
const EventEmitter = require('events');
const fork = require('child_process').fork;

const debug = require('debug')('artillery:runner');
const L = require('lodash');

const stats = require('./dispatcher').stats;
const divideWork = require('./dist');

const A = require('async');
const pidusage = require('pidusage');

module.exports = createRunner;

function createRunner(script, payload, opts) {
  const runner = new Runner(script, payload, opts);
  return runner;
}

function Runner(script, payload, opts) {
  this._script = script;
  this._payload = payload;
  this._opts = opts;
  this._workers = {};

  this.events = new EventEmitter();

  this._intermediates = [];
  this._allIntermediates = [];

  this._currentPhase = -1;

  return this;
}

Runner.prototype.run = function run() {
  //
  // Create worker scripts (distribute the work):
  //
  let self = this;
  let numWorkers = process.env.ARTILLERY_WORKERS || 1 || os.cpus().length;
  let workerScripts = divideWork(this._script, numWorkers);
  // Overwrite statsInterval for workers:
  L.each(workerScripts, function(s) {
    s.config.statsInterval = 1;
  });

  debug(JSON.stringify(workerScripts, null, 4));

  //
  // Create workers:
  //
  const DEBUGGER_PORT = 41986;
  L.each(workerScripts, (script, idx) => {
    let forkOptions = {};
    // If running under a debugger, set non-clashing debugger ports for worker
    // processes.
    // Run with: node --debug --expose_debug_as=v8debug --inspect
    const debugging = (typeof v8debug === 'object');
    if (debugging) {
      forkOptions.execArgv = [`--debug=${DEBUGGER_PORT + idx}`];
    }
    let workerProcess = fork(path.join(__dirname, 'worker.js'), forkOptions);
    this._workers[workerProcess.pid] = {
      proc: workerProcess,
      isDone: false
    };
    if(debugging) {
      debug(`forked worker ${workerProcess.pid}; debugger on port ${DEBUGGER_PORT + idx}`);
    } else {
      debug(`forked worker ${workerProcess.pid}`);
    }
    workerProcess.on('message', this._onWorkerMessage.bind(this));
    workerProcess.once('exit', () => {
      if (!this._workers[workerProcess.pid].isDone) {
        debug(`worker ${workerProcess.pid} terminated prematurely`);
        this._workers[workerProcess.pid].isDone = true;
        console.log('Unexpected error, Artillery shutting down.');
        self.shutdown(function() {
          process.exit(1);
        });
      } else {
        debug(`worker ${workerProcess.pid} exited`);
      }
    });
    workerProcess.send({
      command: 'run',
      opts: {
        script: script,
        payload: this._payload, // FIXME: Inefficient with large payloads
        options: this._opts
      }
    });
  });

  // TODO: Use nanotimer
  this._statsInterval = setInterval(
    this._sendStats.bind(this),
    this._script.config.statsInterval * 1000
  );

  // Watch CPU usage of child processes:
  const MELTING_POINT = process.env.ARTILLERY_CPU_THRESHOLD || 80;
  const CPU_CHECK_INTERVAL_MS = 2500;
  setInterval(function() {
    A.map(Object.keys(self._workers), pidusage.stat, function(err, pidStats) {
      if (err) {
        return;
      }
      const busyPids = pidStats.filter(function(o) {
        return o && o.cpu && o.cpu >= MELTING_POINT;
      });
      if (busyPids.length > 0) {
        self.events.emit('highcpu', busyPids);
      }
      debug('busyPids:', busyPids);
    });
  }, CPU_CHECK_INTERVAL_MS).unref();

  return this;
};

Runner.prototype.shutdown = function(done) {
  let self = this;

  A.eachSeries(
    Object.keys(this._workers),
    function(pid, next) {
      if (!self._workers[pid].isDone) {
        debug('Worker %s not done, sending stop command', pid);
        self._workers[pid].proc.send({ command: 'stop' });
        self._workers[pid].isDone = true;
      }
      return next(null);
    }, function(err) {
      if (err) {
        debug(err);
      }

      // Poll until all workers are done:
      let pollInterval = setInterval(function() {
        let allDone = L.every(
          Object.keys(self._workers),
          function isDone(pid) {
            return self._workers[pid].isDone;
          });

        if (allDone) {
          debug('All workers done');
          clearInterval(pollInterval);
          return done();
        }
      }, 100);
    });
};

Runner.prototype._sendStats = function() {
  // Calculate average concurrency:
  // We are averaging and overwriting the value in the report ourselves because
  // combine() presumes that stats objects come from different workers, but
  // we will have multiple intermediate objects from the same worker.

  // Calculate max concurrency (sampled at one second resolution):
  let maxWorkerConcurrencies = L.reduce(
    this._intermediates,
    function(acc, el) {
      const pid = el[0];
      const intermediate = el[1];
      if (typeof acc[pid] !== 'undefined') {
        acc[pid] = L.max([acc[pid], intermediate._concurrency]);
      } else {
        acc[pid] = intermediate._concurrency;
      }
      return acc;
    }, {});

  debug('max worker concurrency: %j', maxWorkerConcurrencies);

  let averageConcurrency = stats.round(
    L.sum(
      L.map(maxWorkerConcurrencies, function(v) { return v; })),
    1);

  let combined = stats.combine(
    L.map(this._intermediates, function(el) { return el[1]; }));

  combined._concurrency = averageConcurrency;

  this.events.emit('stats', combined);
  this._intermediates = [];
};

Runner.prototype._onWorkerMessage = function _onWorkerMessage(message) {
  if (message.event === 'phaseStarted') {
    if (message.phase.index > this._currentPhase) {
      this.events.emit('phaseStarted', message.phase);
      this._currentPhase = message.phase.index;
    }
  }

  if (message.event === 'phaseCompleted') {
  }

  if (message.event === 'stats') {
    this._intermediates.push([message.pid, message.stats]);
    this._allIntermediates.push(message.stats);
  }

  if (message.event === 'done') {
    clearInterval(this._statsInterval);
    let worker = this._workers[message.pid];
    worker.isDone = true;
    worker.proc.send({command: 'stop'});

    if (this._activeWorkerCount() === 0) {
      this._sendStats();
      this.events.emit('done', stats.combine(this._allIntermediates));
    }
  }
};

Runner.prototype._activeWorkerCount = function _activeWorkerCount() {
  var pids = Object.keys(this._workers);
  var count = pids.length;
  pids.forEach((pid) => {
    if (this._workers[pid].isDone) {
      count--;
    }
  });
  return count;
};
<|endoftext|>


<commit_before>"use strict";
/*eslint max-params: [2, 4]*/

var exec = require("child_process").exec;
var _ = require("lodash");
var async = require("async");
var chalk = require("chalk");
var log = require("./log");
var Tracker = require("./utils/tracker");

/**
 * Run a single task.
 *
 * @param {String}    cmd       Shell command
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Function}  callback  Callback `(err)`
 * @returns {Object}            Child process object
 */
var run = function (cmd, shOpts, opts, callback) {
  // Check if buffered output or piped.
  var buffer = opts.buffer;

  log.info("proc:start", cmd);
  var proc = exec(cmd, shOpts, function (err, stdout, stderr) {
    var code = err ? err.code || 1 : 0;
    var level = code === 0 ? "info" : "warn";

    // Write out buffered output.
    if (buffer) {
      process.stdout.write(stdout);
      process.stderr.write(stderr);
    }

    log[level]("proc:end:" + code, cmd);
    callback(err);
  });

  // Concurrent / "whenever" output.
  if (!buffer) {
    proc.stdout.pipe(process.stdout, { end: false });
    proc.stderr.pipe(process.stderr, { end: false });
  }

  return proc;
};

/**
 * Run with retries.
 *
 * @param {String}    cmd       Shell command
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Function}  callback  Callback `(err)`
 * @returns {void}
 */
var retry = function (cmd, shOpts, opts, callback) {
  // Expand options.
  var tracker = opts.tracker;
  var taskEnv = opts.taskEnv;

  // State.
  var tries = opts.tries;
  var success = false;
  var error;

  // Iterate and retry!
  async.whilst(
    function () {
      return !success && tries > 0;
    },
    function (cb) {
      var proc = run(cmd, shOpts, opts, function (err) {
        // Manage, update state.
        tries--;
        error = err;
        success = !error;

        // Check tries.
        if (error && tries > 0) {
          log.warn("proc:retry", chalk.red(tries) + " tries left, Command: " + chalk.gray(cmd) +
            (taskEnv ? ", Environment: " + chalk.magenta(JSON.stringify(taskEnv)) : ""));
        }

        // Execute without error.
        cb();
      });

      if (tracker) {
        tracker.add(proc);
      }
    },
    function (err) {
      error = error || err;
      if (error) {
        var code = error.code || 1;
        log.error("proc:error", "Code: " + code + ", Command: " + cmd);
      }

      callback(error);
    }
  );
};

/**
 * Add and invoke a setup task if present in options.
 *
 * @param {String}    setup     Setup task
 * @param {Object}    shOpts    Shell options
 * @returns {Object}            Process object or `null`.
 */
var addSetup = function (setup, shOpts) {
  if (!setup) { return null; }

  var done = _.once(function (code) {
    var level = code === 0 ? "info" : "error";
    log[level]("setup:end", "Setup command ended with code: " + code);
  });

  log.info("setup:start", "Starting setup task: " + setup);
  var proc = run("builder run " + setup, shOpts, {}, done);
  proc.on("exit", done);

  return proc;
};

/**
 * Wrap callback with setup termination behavior.
 *
 * - Create and invoke setup.
 * - Early termination on setup error.
 * - Ensure callback only called once.
 *
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Object}    tracker   Process tracker
 * @param {Function}  callback  Callback `(err)`
 * @returns {Function}          Wrapped callback
 */
var createFinish = function (shOpts, opts, tracker, callback) {
  // Wrap callback
  var finish = _.once(function (err) {
    tracker.kill(function () {
      callback(err);
    });
  });

  // Add, invoke, and hook to final callback if setup dies early.
  var setup = tracker.add(addSetup(opts.setup, shOpts));
  if (setup) {
    // If setup exit happens before normal termination, kill everything.
    setup.on("exit", function (code) {
      finish(new Error("Setup exited with code: " + code));
    });
  }

  return finish;
};

/**
 * Task runner.
 */
module.exports = {
  /**
   * Run a single task.
   *
   * @param {String}    cmd       Shell command
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {Object}            Child process object
   */
  run: function (cmd, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    return retry(cmd, shOpts, opts, finish);
  },

  /**
   * Run multiple tasks in parallel.
   *
   * @param {Array}     cmds      List of shell commands
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {void}
   */
  concurrent: function (cmds, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    // Get mapper (queue vs. non-queue)
    var queue = opts.queue;
    var map = queue ?
      async.mapLimit.bind(async, cmds, queue) :
      async.map.bind(async, cmds);

    log.info("concurrent", "Starting with queue size: " + chalk.magenta(queue || "unlimited"));
    map(function (cmd, cb) {
      retry(cmd, shOpts, _.extend({ tracker: tracker }, opts), cb);
    }, finish);
  },

  /**
   * Run a single task with multiple environment variables in parallel.
   *
   * @param {String}    cmd       Shell command
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {void}
   */
  envs: function (cmd, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    // Get mapper (queue vs. non-queue)
    var queue = opts.queue;
    var taskEnvs = opts._envs;
    var map = queue ?
      async.mapLimit.bind(async, taskEnvs, queue) :
      async.map.bind(async, taskEnvs);

    log.info("envs", "Starting with queue size: " + chalk.magenta(queue || "unlimited"));
    map(function (taskEnv, cb) {
      // Add each specific set of environment variables.
      // Clone `shOpts` to turn `env` into a plain object: in Node 4+
      // `process.env` is a special object which changes merge behavior.
      var taskShOpts = _.merge(_.cloneDeep(shOpts), { env: taskEnv });
      var taskOpts = _.extend({ tracker: tracker, taskEnv: taskEnv }, opts);

      log.info("envs", "Starting environment " + chalk.magenta(JSON.stringify(taskEnv)) +
        " run for command: " + chalk.gray(cmd));
      retry(cmd, taskShOpts, taskOpts, cb);
    }, finish);
  }
};
<commit_msg>Add higher maxBuffer to 'exec()' options on invocation. Fixes #62<commit_after>"use strict";
/*eslint max-params: [2, 4]*/

var exec = require("child_process").exec;
var _ = require("lodash");
var async = require("async");
var chalk = require("chalk");
var log = require("./log");
var Tracker = require("./utils/tracker");

// One limitation of `exec()` is that it unconditionally buffers stdout/stderr
// input (whether piped, listened, or whatever) leading to a `maxBuffer` bug:
// https://github.com/FormidableLabs/builder/issues/62
//
// We set a comfortable margin here to up the number. In the future, we could
// just go "whole hog" and bump to `Infinity` if needed.
//
// Longer term, we can consider whether we want to do what npm does and use
// `spawn` with manual OS-compatible `sh` vs. `cmd` detection, cobble together
// our own flags and manage everything so that we can use the much more flexible
// `spawn` instead of `exec`.
// https://github.com/FormidableLabs/builder/issues/20
var MAX_BUFFER = 32 * 1024 * 1024;

/**
 * Run a single task.
 *
 * @param {String}    cmd       Shell command
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Function}  callback  Callback `(err)`
 * @returns {Object}            Child process object
 */
var run = function (cmd, shOpts, opts, callback) {
  // Update shell options.
  shOpts = _.extend({
    maxBuffer: MAX_BUFFER
  }, shOpts);

  // Check if buffered output or piped.
  var buffer = opts.buffer;

  log.info("proc:start", cmd);
  var proc = exec(cmd, shOpts, function (err, stdout, stderr) {
    var code = err ? err.code || 1 : 0;
    var level = code === 0 ? "info" : "warn";

    // Write out buffered output.
    if (buffer) {
      process.stdout.write(stdout);
      process.stderr.write(stderr);
    }

    log[level]("proc:end:" + code, cmd);
    callback(err);
  });

  // Concurrent / "whenever" output.
  if (!buffer) {
    proc.stdout.pipe(process.stdout, { end: false });
    proc.stderr.pipe(process.stderr, { end: false });
  }

  return proc;
};

/**
 * Run with retries.
 *
 * @param {String}    cmd       Shell command
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Function}  callback  Callback `(err)`
 * @returns {void}
 */
var retry = function (cmd, shOpts, opts, callback) {
  // Expand options.
  var tracker = opts.tracker;
  var taskEnv = opts.taskEnv;

  // State.
  var tries = opts.tries;
  var success = false;
  var error;

  // Iterate and retry!
  async.whilst(
    function () {
      return !success && tries > 0;
    },
    function (cb) {
      var proc = run(cmd, shOpts, opts, function (err) {
        // Manage, update state.
        tries--;
        error = err;
        success = !error;

        // Check tries.
        if (error && tries > 0) {
          log.warn("proc:retry", chalk.red(tries) + " tries left, Command: " + chalk.gray(cmd) +
            (taskEnv ? ", Environment: " + chalk.magenta(JSON.stringify(taskEnv)) : ""));
        }

        // Execute without error.
        cb();
      });

      if (tracker) {
        tracker.add(proc);
      }
    },
    function (err) {
      error = error || err;
      if (error) {
        var code = error.code || 1;
        log.error("proc:error", "Code: " + code + ", Command: " + cmd);
      }

      callback(error);
    }
  );
};

/**
 * Add and invoke a setup task if present in options.
 *
 * @param {String}    setup     Setup task
 * @param {Object}    shOpts    Shell options
 * @returns {Object}            Process object or `null`.
 */
var addSetup = function (setup, shOpts) {
  if (!setup) { return null; }

  var done = _.once(function (code) {
    var level = code === 0 ? "info" : "error";
    log[level]("setup:end", "Setup command ended with code: " + code);
  });

  log.info("setup:start", "Starting setup task: " + setup);
  var proc = run("builder run " + setup, shOpts, {}, done);
  proc.on("exit", done);

  return proc;
};

/**
 * Wrap callback with setup termination behavior.
 *
 * - Create and invoke setup.
 * - Early termination on setup error.
 * - Ensure callback only called once.
 *
 * @param {Object}    shOpts    Shell options
 * @param {Object}    opts      Runner options
 * @param {Object}    tracker   Process tracker
 * @param {Function}  callback  Callback `(err)`
 * @returns {Function}          Wrapped callback
 */
var createFinish = function (shOpts, opts, tracker, callback) {
  // Wrap callback
  var finish = _.once(function (err) {
    tracker.kill(function () {
      callback(err);
    });
  });

  // Add, invoke, and hook to final callback if setup dies early.
  var setup = tracker.add(addSetup(opts.setup, shOpts));
  if (setup) {
    // If setup exit happens before normal termination, kill everything.
    setup.on("exit", function (code) {
      finish(new Error("Setup exited with code: " + code));
    });
  }

  return finish;
};

/**
 * Task runner.
 */
module.exports = {
  /**
   * Run a single task.
   *
   * @param {String}    cmd       Shell command
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {Object}            Child process object
   */
  run: function (cmd, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    return retry(cmd, shOpts, opts, finish);
  },

  /**
   * Run multiple tasks in parallel.
   *
   * @param {Array}     cmds      List of shell commands
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {void}
   */
  concurrent: function (cmds, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    // Get mapper (queue vs. non-queue)
    var queue = opts.queue;
    var map = queue ?
      async.mapLimit.bind(async, cmds, queue) :
      async.map.bind(async, cmds);

    log.info("concurrent", "Starting with queue size: " + chalk.magenta(queue || "unlimited"));
    map(function (cmd, cb) {
      retry(cmd, shOpts, _.extend({ tracker: tracker }, opts), cb);
    }, finish);
  },

  /**
   * Run a single task with multiple environment variables in parallel.
   *
   * @param {String}    cmd       Shell command
   * @param {Object}    shOpts    Shell options
   * @param {Object}    opts      Runner options
   * @param {Function}  callback  Callback `(err)`
   * @returns {void}
   */
  envs: function (cmd, shOpts, opts, callback) {
    // Add + invoke setup (if any), bind tracker cleanup, and wrap callback.
    var tracker = new Tracker();
    var finish = createFinish(shOpts, opts, tracker, callback);

    // Get mapper (queue vs. non-queue)
    var queue = opts.queue;
    var taskEnvs = opts._envs;
    var map = queue ?
      async.mapLimit.bind(async, taskEnvs, queue) :
      async.map.bind(async, taskEnvs);

    log.info("envs", "Starting with queue size: " + chalk.magenta(queue || "unlimited"));
    map(function (taskEnv, cb) {
      // Add each specific set of environment variables.
      // Clone `shOpts` to turn `env` into a plain object: in Node 4+
      // `process.env` is a special object which changes merge behavior.
      var taskShOpts = _.merge(_.cloneDeep(shOpts), { env: taskEnv });
      var taskOpts = _.extend({ tracker: tracker, taskEnv: taskEnv }, opts);

      log.info("envs", "Starting environment " + chalk.magenta(JSON.stringify(taskEnv)) +
        " run for command: " + chalk.gray(cmd));
      retry(cmd, taskShOpts, taskOpts, cb);
    }, finish);
  }
};
<|endoftext|>


<commit_before>'use strict';

//configgables
var _DEBUG = 255
var port = 4321

//require all modules
var net = require('net')

//debug stub
var DEBUG = function(level){return (_DEBUG > level)}

//setup server
var server = net.createServer(function(sock){
  console.log('Client connected from ' + sock.remoteAddress)
  //utility functions bound on a socket-by-socket basis so we can use (this)
  sock.sendBanner = function(){
    DEBUG(0) && console.log('[O] %rwhois V-1.5:00ffff:00 spudz76.drunkensailor.org (node-rwhois v0.0.1)')
    this.write('%rwhois V-1.5:003fff:00 spudz76.drunkensailor.org (node-rwhois v0.0.1)\r\n')
  }
  sock.sendOK = function(){
    DEBUG(0) && console.log('[O] %ok')
    this.write('%ok\r\n\r\n')
  }

  sock.on('end',function(){console.log('Client disconnected')})
  sock.on('data',function(buff){
    var line = buff.toString()
    DEBUG(1) &&  console.log('[I] ' + line)
    var m = []
    if(m = line.match(/^(\S+)\s+(.*)$/)){
      DEBUG(2) &&  console.log('[EMIT:'+ m[1] + '] args: ' + m[2])
      sock.emit(m[1],m[2])
    }
  })
  sock.on('-rwhois',function(){
    sock.sendBanner()
    DEBUG(0) && console.log('[O] %ok')
    sock.write('%ok\r\n\r\n')
  })
  //all set up, send back the banner as our signal to continue
  sock.sendBanner()
})
//fire it up
server.listen(port,function(){
  console.log('server listening on port ' + port)
})
<commit_msg>All set, works like a charm.<commit_after>'use strict';

//configgables
var _DEBUG = 255
var port = 4321

//require all modules
var net = require('net')

//debug stub
var DEBUG = function(level){return (_DEBUG > level)}

//setup server
var server = net.createServer(function(sock){
  console.log('Client connected from ' + sock.remoteAddress)
  //utility functions bound on a socket-by-socket basis so we can use (this)
  sock.sendBanner = function(){
    DEBUG(0) && console.log('[O] %rwhois V-1.5:00ffff:00 spudz76.drunkensailor.org (node-rwhois v0.0.1)')
    this.write('%rwhois V-1.5:003fff:00 spudz76.drunkensailor.org (node-rwhois v0.0.1)\r\n')
    return true
  }
  sock.sendOK = function(){
    DEBUG(0) && console.log('[O] %ok')
    this.write('%ok\r\n\r\n')
    return true
  }
  sock.sendError = function(code,msg){
    var str = '%error ' + code + ' ' + msg
    DEBUG(0) && console.log('[O] ' + str)
    this.write(str + '\r\n\r\n')
    return true
  }

  sock.on('end',function(){console.log('Client disconnected')})
  sock.on('data',function(buff){
    var line = buff.toString()
    DEBUG(1) &&  console.log('[I] ' + line)
    var m = []
    if(m = line.match(/^(([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.([a-zA-Z]{2,}|[a-zA-Z0-9-]{2,30}\.[a-zA-Z]{2,})$/)){
      line = 'domain ' + m.input
    }
    if(m = line.match(/^(\S+)\s+(.*)$/)){
      DEBUG(2) &&  console.log('[EMIT:'+ m[1] + '] args: ' + m[2])
      sock.emit(m[1],m[2])
    }
  })
  sock.on('-rwhois',function(){
    sock.sendBanner() && sock.sendOK()
  })
  sock.on('domain',function(d){
    DEBUG(1) && console.log('[LOOKUP:' + d + ']')
    sock.sendError(230,'GO FUCK YOURSELF') //'No Objects Found')
  })
  sock.on('error',function(we,dont,care){we = dont = care})
  //all set up, send back the banner as our signal to continue
  sock.sendBanner()
})
//fire it up
server.listen(port,function(){
  console.log('server listening on port ' + port)
})
<|endoftext|>


<commit_before>var Schema = module.exports = function Schema(schema){
  var _schema = this._schema = {};
  this._virtuals = {};
  this.methods = {};

  for (var i in schema){
    _schema[i] = _createSchema(schema[i]);
  }
};

var _createSchema = function(val){
  var type = val.type || (val.constructor === Function ? val : val.constructor),
    obj = {type: type};

  if (val.default){
    obj.default = val.default;
  } else {
    switch (type){
      case String:
        obj.default = '';
        break;

      case Date:
        obj.default = Date.now;
        break;

      case Boolean:
        obj.default = true;
        break;

      case Number:
        obj.default = 0;
        break;

      case Array:
        obj.default = [];
        break;

      case Object:
        obj.default = {};
        break;
    }
  }

  switch (type){
    case Array:
      var length = val.length;
      if (length){
        var nested = [];
        for (var i=0; i<length; i++){
          nested.push(_createSchema(val[i]));
        }
        obj.nested = nested;
      }
      break;

    case Object:
      var keys = Object.keys(val),
        length = keys.length;
      if (length){
        var nested = {};
        for (var i=0; i<length; i++){
          var item = keys[i];
          nested[item] = _createSchema(val[item]);
        }
        obj.nested = nested;
      }
      break;
  }

  return obj;
};

Schema.prototype.virtual = function(name){
  var virtual = this._virtuals[name] = this._virtuals[name] || new Virtual();
  return virtual;
};

var _inputSchema = function(schema, obj){
  switch (schema.type){
    case String:
      if (obj == null || typeof obj === 'undefined'){
        obj = '';
      } else {
        obj = obj.toString();
      }
      break;

    case Number:
      if (obj == null || typeof obj === 'undefined'){
        obj = 0;
      } else {
        obj = +obj;
      }
      break;

    case Date:
      if (_.isDate(obj)){
        obj = obj.toISOString();
      } else {
        var date = new Date(obj);
        if (_.isNan(date.getTime())){
          obj = new Date().toISOString();
        } else {
          obj = date.toISOString();
        }
      }
      break;

    case Boolean:
      obj = !!obj;
      break;

    case Array:
      if (!_.isArray(obj)){
        if (_.isObject(obj)){
          obj = _.toArray(obj);
        } else {
          obj = [obj];
        }
      }
      var nested = schema.nested,
        lastNested;
      if (nested){
        for (var i=0, len=obj.length; i<len; i++){
          var currentNested = nested[i];
          if (currentNested) lastNested = currentNested;
          obj[i] = _inputSchema(lastNested, obj[i]);
        }
      }
      break;

    case Object:
      var nested = schema.nested;
      if (nested){
        for (var i in nested){
          obj[i] = _inputSchema(nested[i], obj[i]);
        }
      }
      break;
  }

  return obj;
};

Schema.prototype.save = function(data){
  var schema = this._schema,
    virtuals = this._virtuals;

  for (var i in schema){
    var item = schema[i],
      defaultVal = item.default;

    if (data.hasOwnProperty(i)){
      data[i] = _inputSchema(item, data[i]);
    } else if (defaultVal != null && typeof defaultVal !== 'undefined'){
      data[i] = _inputSchema(item, _.isFunction(defaultVal) ? defaultVal() : defaultVal);
    }
  }

  for (var j in virtuals){
    var item = virtuals[j];
    if (item.hasOwnProperty('setter')){
      item.setter.call(data);
      delete data[j];
    }
  }

  return data;
};

var _outputSchema = function(schema, obj){
  switch (schema.type){
    case Date:
      obj = new Date(obj);
      break;

    case Array:
      var nested = schema.nested,
        lastNested;
      if (nested){
        for (var i=0, len=obj.length; i<len; i++){
          var currentNested = nested[i];
          if (currentNested) lastNested = currentNested;
          obj[i] = _outputSchema(lastNested, obj[i]);
        }
      }
      break;

    case Object:
      var nested = schema.nested;
      if (nested){
        for (var i in nested){
          obj[i] = _outputSchema(nested[i], obj[i]);
        }
      }
      break;
  }

  return obj;
};

Schema.prototype.restore = function(data){
  var schema = this._schema,
    virtuals = this._virtuals;

  for (var i in schema){
    var item = schema[i];
    if (data.hasOwnProperty(i)){
      data[i] = _outputSchema(item, data[i]);
    }
  }

  for (var j in virtuals){
    var item = virtuals[j];
    if (item.hasOwnProperty('getter')){
      data[i] = item.getter.call(data);
    }
  }

  return data;
};<commit_msg>Fixed schema dependency<commit_after>var _ = require('underscore');

var Schema = module.exports = function Schema(schema){
  var _schema = this._schema = {};
  this._virtuals = {};
  this.methods = {};

  for (var i in schema){
    _schema[i] = _createSchema(schema[i]);
  }
};

var _createSchema = function(val){
  var type = val.type || (val.constructor === Function ? val : val.constructor),
    obj = {type: type};

  if (val.default){
    obj.default = val.default;
  } else {
    switch (type){
      case String:
        obj.default = '';
        break;

      case Date:
        obj.default = Date.now;
        break;

      case Boolean:
        obj.default = true;
        break;

      case Number:
        obj.default = 0;
        break;

      case Array:
        obj.default = [];
        break;

      case Object:
        obj.default = {};
        break;
    }
  }

  switch (type){
    case Array:
      var length = val.length;
      if (length){
        var nested = [];
        for (var i=0; i<length; i++){
          nested.push(_createSchema(val[i]));
        }
        obj.nested = nested;
      }
      break;

    case Object:
      var keys = Object.keys(val),
        length = keys.length;
      if (length){
        var nested = {};
        for (var i=0; i<length; i++){
          var item = keys[i];
          nested[item] = _createSchema(val[item]);
        }
        obj.nested = nested;
      }
      break;
  }

  return obj;
};

Schema.prototype.virtual = function(name){
  var virtual = this._virtuals[name] = this._virtuals[name] || new Virtual();
  return virtual;
};

var _inputSchema = function(schema, obj){
  switch (schema.type){
    case String:
      if (obj == null || typeof obj === 'undefined'){
        obj = '';
      } else {
        obj = obj.toString();
      }
      break;

    case Number:
      if (obj == null || typeof obj === 'undefined'){
        obj = 0;
      } else {
        obj = +obj;
      }
      break;

    case Date:
      if (_.isDate(obj)){
        obj = obj.toISOString();
      } else {
        var date = new Date(obj);
        if (_.isNan(date.getTime())){
          obj = new Date().toISOString();
        } else {
          obj = date.toISOString();
        }
      }
      break;

    case Boolean:
      obj = !!obj;
      break;

    case Array:
      if (!_.isArray(obj)){
        if (_.isObject(obj)){
          obj = _.toArray(obj);
        } else {
          obj = [obj];
        }
      }
      var nested = schema.nested,
        lastNested;
      if (nested){
        for (var i=0, len=obj.length; i<len; i++){
          var currentNested = nested[i];
          if (currentNested) lastNested = currentNested;
          obj[i] = _inputSchema(lastNested, obj[i]);
        }
      }
      break;

    case Object:
      var nested = schema.nested;
      if (nested){
        for (var i in nested){
          obj[i] = _inputSchema(nested[i], obj[i]);
        }
      }
      break;
  }

  return obj;
};

Schema.prototype.save = function(data){
  var schema = this._schema,
    virtuals = this._virtuals;

  for (var i in schema){
    var item = schema[i],
      defaultVal = item.default;

    if (data.hasOwnProperty(i)){
      data[i] = _inputSchema(item, data[i]);
    } else if (defaultVal != null && typeof defaultVal !== 'undefined'){
      data[i] = _inputSchema(item, _.isFunction(defaultVal) ? defaultVal() : defaultVal);
    }
  }

  for (var j in virtuals){
    var item = virtuals[j];
    if (item.hasOwnProperty('setter')){
      item.setter.call(data);
      delete data[j];
    }
  }

  return data;
};

var _outputSchema = function(schema, obj){
  switch (schema.type){
    case Date:
      obj = new Date(obj);
      break;

    case Array:
      var nested = schema.nested,
        lastNested;
      if (nested){
        for (var i=0, len=obj.length; i<len; i++){
          var currentNested = nested[i];
          if (currentNested) lastNested = currentNested;
          obj[i] = _outputSchema(lastNested, obj[i]);
        }
      }
      break;

    case Object:
      var nested = schema.nested;
      if (nested){
        for (var i in nested){
          obj[i] = _outputSchema(nested[i], obj[i]);
        }
      }
      break;
  }

  return obj;
};

Schema.prototype.restore = function(data){
  var schema = this._schema,
    virtuals = this._virtuals;

  for (var i in schema){
    var item = schema[i];
    if (data.hasOwnProperty(i)){
      data[i] = _outputSchema(item, data[i]);
    }
  }

  for (var j in virtuals){
    var item = virtuals[j];
    if (item.hasOwnProperty('getter')){
      data[i] = item.getter.call(data);
    }
  }

  return data;
};<|endoftext|>