<commit_before>var fs = require("fs");
var path = require("path");

var files = require('./files.js');
var utils = require('./utils.js');
var release = require('./release.js');
var mongoExitCodes = require('./mongo-exit-codes.js');
var inFiber = require('./fiber-helpers.js').inFiber;

var _ = require('underscore');
var unipackage = require('./unipackage.js');
var Fiber = require('fibers');
var Future = require('fibers/future');

// Given a Mongo URL, open an interative Mongo shell on this terminal
// on that database.
var runMongoShell = function (url) {
  var mongoPath = path.join(files.getDevBundle(), 'mongodb', 'bin', 'mongo');
  // XXX mongo URLs are not real URLs (notably, the comma-separation for
  // multiple hosts). We've had a little better luck using the mongodb-uri npm
  // package.
  var mongoUrl = require('url').parse(url);
  var auth = mongoUrl.auth && mongoUrl.auth.split(':');
  var ssl = require('querystring').parse(mongoUrl.query).ssl === "true";

  var args = [];
  if (ssl) args.push('--ssl');
  if (auth) args.push('-u', auth[0]);
  if (auth) args.push('-p', auth[1]);
  args.push(mongoUrl.hostname + ':' + mongoUrl.port + mongoUrl.pathname);

  var child_process = require('child_process');
  var proc = child_process.spawn(mongoPath,
                                 args,
                                 { stdio: 'inherit' });
};

// Find all running Mongo processes that were started by this program
// (even by other simultaneous runs of this program). If passed,
// appDir and port act as filters on the list of running mongos.
//
// Yields. Returns an array of objects with keys pid, port, appDir.
var findMongoPids = function (appDir, port) {
  var fut = new Future;

  // 'ps ax' should be standard across all MacOS and Linux.
  var child_process = require('child_process');
  child_process.exec(
    'ps ax',
    function (error, stdout, stderr) {
      if (error) {
        fut['throw'](new Error("Couldn't run ps ax: " + JSON.stringify(error)));
        return;
      }

      var ret = [];
      _.each(stdout.split('\n'), function (line) {
        // Matches mongos we start. Note that this matches
        // 'fake-mongod' (our mongod stub for automated tests) as well
        // as 'mongod'.
        var m = line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db(?: |$)/);
        if (m && m.length === 4) {
          var foundPid =  parseInt(m[1]);
          var foundPort = parseInt(m[2]);
          var foundPath = m[3];

          if ( (! port || port === foundPort) &&
               (! appDir || appDir === foundPath)) {
            ret.push({
              pid: foundPid,
              port: foundPort,
              appDir: foundPath
            });
          }
        }
      });

      fut['return'](ret);
    });

  return fut.wait();
};

// See if mongo is running already. Yields. Returns the port that
// mongo is running on or null if mongo is not running.
var findMongoPort = function (appDir) {
  var pids = findMongoPids(appDir);

  if (pids.length !== 1) {
    return null;
  }

  var pid = pids[0].pid;
  try {
    process.kill(pid, 0); // make sure it is still alive
  } catch (e) {
    return null;
  }

  return pids[0].port;
};

// Kill any mongos running on 'port'. Yields, and returns once they
// are all dead. Throws an exception on failure.
//
// This is a big hammer for dealing with still running mongos, but
// smaller hammers have failed before and it is getting tiresome.
var findMongoAndKillItDead = function (port) {
  var pids = findMongoPids(null, port);

  if (! pids.length)
    return; // nothing to kill

  // Go through the list serially. There really should only ever be
  // one but we're not taking any chances.
  _.each(pids, function (processInfo) {
    var pid = processInfo.pid;

    // Send kill attempts and wait. First a SIGINT, then if it isn't
    // dead within 2 sec, SIGKILL. Check every 100ms to see if it's
    // dead.
    for (var attempts = 1; attempts <= 40; attempts ++) {
      var signal = 0;
      if (attempts === 1)
        signal = 'SIGINT';
      else if (attempts === 20 || attempts === 30)
        signal = 'SIGKILL';

      try {
        process.kill(pid, signal);
      } catch (e) {
        // it's dead. on to the next one
        return;
      }

      utils.sleepMs(100);
    }

    // give up after 4 seconds.
    // XXX should actually catch this higher up and print a nice
    // error. foreseeable conditions should never result in exceptions
    // for the user.
    throw new Error("Can't kill running mongo (pid " + pid + ").");
  });
};

// Starts a single instance of mongod, and configures it properly. Does not
// yield.
var launchMongo = function (options) {
  var onListen = options.onListen || function () {};
  var onExit = options.onExit || function () {};

  var noOplog = false;
  var mongod_path = path.join(
    files.getDevBundle(), 'mongodb', 'bin', 'mongod');

  // Automated testing: If this is set, instead of starting mongod, we
  // start our stub (fake-mongod) which can then be remote-controlled
  // by the test.
  if (process.env.METEOR_TEST_FAKE_MONGOD_CONTROL_PORT) {
    mongod_path = path.join(files.getCurrentToolsDir(),
                            'tools', 'tests', 'fake-mongod', 'fake-mongod');

    // oplog support requires sending admin commands to mongod, so
    // it'd be hard to make fake-mongod support it.
    noOplog = true;
  }

  // store data in appDir
  var dbPath = path.join(options.appDir, '.meteor', 'local', 'db');
  files.mkdir_p(dbPath, 0755);
  // add .gitignore if needed.
  files.addToGitignore(path.join(options.appDir, '.meteor'), 'local');

  var proc = null;
  var cancelStartup = false;
  var callOnExit;

  var handle = {
    stop: function () {
      if (! proc)
        cancelStartup = true;
      else {
        proc.removeListener('exit', callOnExit);
        proc.kill('SIGINT');
      }
    }
  };

  Fiber(function () {
    findMongoAndKillItDead(options.port);

    var portFile = path.join(dbPath, 'METEOR-PORT');
    var portFileExists = false;
    var createReplSet = true;
    try {
      createReplSet = +(fs.readFileSync(portFile)) !== options.port;
      portFileExists = true;
    } catch (e) {
      if (!e || e.code !== 'ENOENT')
        throw e;
    }

    if (noOplog)
      createReplSet = false;

    // If this is the first time we're using this DB, or we changed
    // port since the last time, then we want to destroy any
    // existing replSet configuration and create a new one. First we
    // delete the "local" database if it exists. (It's a pain and slow
    // to change the port in an existing replSet configuration. It's
    // also a little slow to initiate a new replSet, thus the attempt
    // to not do it unless the port changes.)
    if (createReplSet) {
      // Delete the port file, so we don't mistakenly believe that the DB is
      // still configured.
      if (portFileExists)
        fs.unlinkSync(portFile);

      try {
        var dbFiles = fs.readdirSync(dbPath);
      } catch (e) {
        if (!e || e.code !== 'ENOENT')
          throw e;
      }
      _.each(dbFiles, function (dbFile) {
        if (/^local\./.test(dbFile))
          fs.unlinkSync(path.join(dbPath, dbFile));
      });

      // Load mongo-livedata so we'll be able to talk to it.
      var mongoNpmModule = unipackage.load({
        library: release.current.library,
        packages: [ 'mongo-livedata' ],
        release: release.current.name
      })['mongo-livedata'].MongoInternals.NpmModule;
    }

    // Start mongod with a dummy replSet and wait for it to
    // listen.
    var child_process = require('child_process');
    var replSetName = 'meteor';
    if (cancelStartup)
      return;
    proc = child_process.spawn(mongod_path, [
      // nb: cli-test.sh and findMongoPids make strong assumptions about the
      // order of the arguments! Check them before changing any arguments.
      '--bind_ip', '127.0.0.1',
      '--smallfiles',
      '--nohttpinterface',
      '--port', options.port,
      '--dbpath', dbPath,
      // Use an 8MB oplog rather than 256MB. Uses less space on disk and
      // initializes faster. (Not recommended for production!)
      '--oplogSize', '8',
      '--replSet', replSetName
    ]);

    var stderrOutput = '';
    proc.stderr.setEncoding('utf8');
    proc.stderr.on('data', function (data) {
      stderrOutput += data;
    });

    callOnExit = function (code, signal) {
      onExit(code, signal, stderrOutput);
    };
    proc.on('exit', callOnExit);

    proc.stdout.setEncoding('utf8');
    var listening = false;
    var replSetReady = false;
    var replSetReadyToBeInitiated = false;
    var alreadyInitiatedReplSet = false;
    var alreadyCalledOnListen = false;
    var maybeCallOnListen = function () {
      if (listening && (replSetReady || noOplog) && !alreadyCalledOnListen) {
        if (createReplSet)
          fs.writeFileSync(portFile, options.port);
        alreadyCalledOnListen = true;
        onListen();
      }
    };

    var maybeInitiateReplset = function () {
      // We need to want to create a replset, be confident that the server is
      // listening, be confident that the server's replset implementation is
      // ready to be initiated, and have not already done it.
      if (!(createReplSet && listening && replSetReadyToBeInitiated
            && !alreadyInitiatedReplSet)) {
        return;
      }

      alreadyInitiatedReplSet = true;

      // Connect to it and start a replset.
      var db = new mongoNpmModule.Db(
        'meteor', new mongoNpmModule.Server('127.0.0.1', options.port),
        {safe: true});
      db.open(function(err, db) {
        if (err)
          throw err;
        db.admin().command({
          replSetInitiate: {
            _id: replSetName,
            members: [{_id : 0, host: '127.0.0.1:' + options.port}]
          }
        }, function (err, result) {
          if (err)
              throw err;
          // why this isn't in the error is unclear.
          if (result && result.documents && result.documents[0]
              && result.documents[0].errmsg) {
            throw result.document[0].errmsg;
          }
          db.close(true);
        });
      });
    };

    proc.stdout.on('data', function (data) {
      // note: don't use "else ifs" in this, because 'data' can have multiple
      // lines
      if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) {
        replSetReadyToBeInitiated = true;
        maybeInitiateReplset();
      }

      if (/ \[initandlisten\] waiting for connections on port/.test(data)) {
        listening = true;
        maybeInitiateReplset();
        maybeCallOnListen();
      }

      if (/ \[rsMgr\] replSet PRIMARY/.test(data)) {
        replSetReady = true;
        maybeCallOnListen();
      }
    });
  }).run();

  return handle;
};

// This runs a Mongo process and restarts it whenever it fails. If it
// restarts too often, we give up on restarting it, diagnostics are
// logged, and onFailure is called.
//
// options: appDir, port, runLog, onFailure
var MongoRunner = function (options) {
  var self = this;
  self.appDir = options.appDir;
  self.port = options.port;
  self.runLog = options.runLog;
  self.onFailure = options.onFailure;

  self.handle = null;
  self.shuttingDown = false;
  self.startupFuture = null;

  self.errorCount = 0;
  self.errorTimer = null;
};

_.extend(MongoRunner.prototype, {
  // Blocks (yields) until the server has started for the first time
  // and is accepting connections. (It might subsequently die and be
  // restarted; we won't tell you about that.) Returns true if we were
  // able to get it to start at least once.
  //
  // If the server fails to start for the first time (after a few
  // restarts), we'll print a message and give up, returning false.
  start: function () {
    var self = this;

    if (self.handle)
      throw new Error("already running?");

    self.startupFuture = new Future;
    self._startOrRestart();
    return self.startupFuture.wait();
  },

  _startOrRestart: function () {
    var self = this;

    if (self.handle)
      throw new Error("already running?");

    self.handle = launchMongo({
      appDir: self.appDir,
      port: self.port,
      onExit: _.bind(self._exited, self),
      onListen: function () {
        if (self.startupFuture) {
          // It's come up successfully for the first time. Make
          // start() return.
          self.startupFuture['return'](true);
          self.startupFuture = null;
        }
      }
    });
  },

  _exited: function (code, signal, stderr) {
    var self = this;
    self.handle = null;

    // If Mongo exited because (or rather, anytime after) we told it
    // to exit, great, nothing to do. Otherwise, we'll print an error
    // and try to restart.
    if (self.shuttingDown)
      return;

    // Print the last 20 lines of stderr.
    self.runLog.log(
      stderr.split('\n').slice(-20).join('\n') +
      "Unexpected mongo exit code " + code + ". Restarting.");

    // We'll restart it up to 3 times in a row. The counter is reset
    // when 5 seconds goes without a restart. (Note that by using a
    // timer instead of looking at the current date, we avoid getting
    // confused by time changes.)
    self.errorCount ++;
    if (self.errorTimer)
      clearTimeout(self.errorTimer);
    self.errorTimer = setTimeout(function () {
      self.errorCount = 0;
    }, 5000);

    if (self.errorCount < 3) {
      // Wait a second, then restart.
      setTimeout(inFiber(function () {
        self._startOrRestart();
      }), 1000);
      return;
    }

    // Too many restarts, too quicky. It's dead. Print friendly
    // diagnostics and give up.
    var explanation = mongoExitCodes.Codes[code];
    var message = "Can't start Mongo server.";

    if (explanation)
      message += "\n" + explanation.longText;

    if (explanation === mongoExitCodes.EXIT_NET_ERROR) {
      message += "\n\n" +
"Check for other processes listening on port " + self.port + "\n" +
"or other Meteor instances running in the same project.";
    }

    if (! explanation && /GLIBC/i.test(stderr)) {
      message += "\n\n" +
"Looks like you are trying to run Meteor on an old Linux distribution.\n" +
"Meteor on Linux requires glibc version 2.9 or above. Try upgrading your\n" +
"distribution to the latest version.";
    }

    self.runLog.log(message);
    self.onFailure && self.onFailure();

    if (self.startupFuture) {
      // start() is still blocking.. make it return
      self.startupFuture['return'](false);
      self.startupFuture = null;
    }
  },

  // Idempotent
  stop: function () {
    var self = this;

    if (self.shuttingDown)
      return;

    self.shuttingDown = true;

    if (self.handle) {
      self.handle.stop();
      self.handle = null;
    }
  }
});

exports.runMongoShell = runMongoShell;
exports.findMongoPort = findMongoPort;
exports.MongoRunner = MongoRunner;
<commit_msg>clear mongo timers when shut down<commit_after>var fs = require("fs");
var path = require("path");

var files = require('./files.js');
var utils = require('./utils.js');
var release = require('./release.js');
var mongoExitCodes = require('./mongo-exit-codes.js');
var inFiber = require('./fiber-helpers.js').inFiber;

var _ = require('underscore');
var unipackage = require('./unipackage.js');
var Fiber = require('fibers');
var Future = require('fibers/future');

// Given a Mongo URL, open an interative Mongo shell on this terminal
// on that database.
var runMongoShell = function (url) {
  var mongoPath = path.join(files.getDevBundle(), 'mongodb', 'bin', 'mongo');
  // XXX mongo URLs are not real URLs (notably, the comma-separation for
  // multiple hosts). We've had a little better luck using the mongodb-uri npm
  // package.
  var mongoUrl = require('url').parse(url);
  var auth = mongoUrl.auth && mongoUrl.auth.split(':');
  var ssl = require('querystring').parse(mongoUrl.query).ssl === "true";

  var args = [];
  if (ssl) args.push('--ssl');
  if (auth) args.push('-u', auth[0]);
  if (auth) args.push('-p', auth[1]);
  args.push(mongoUrl.hostname + ':' + mongoUrl.port + mongoUrl.pathname);

  var child_process = require('child_process');
  var proc = child_process.spawn(mongoPath,
                                 args,
                                 { stdio: 'inherit' });
};

// Find all running Mongo processes that were started by this program
// (even by other simultaneous runs of this program). If passed,
// appDir and port act as filters on the list of running mongos.
//
// Yields. Returns an array of objects with keys pid, port, appDir.
var findMongoPids = function (appDir, port) {
  var fut = new Future;

  // 'ps ax' should be standard across all MacOS and Linux.
  var child_process = require('child_process');
  child_process.exec(
    'ps ax',
    function (error, stdout, stderr) {
      if (error) {
        fut['throw'](new Error("Couldn't run ps ax: " + JSON.stringify(error)));
        return;
      }

      var ret = [];
      _.each(stdout.split('\n'), function (line) {
        // Matches mongos we start. Note that this matches
        // 'fake-mongod' (our mongod stub for automated tests) as well
        // as 'mongod'.
        var m = line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db(?: |$)/);
        if (m && m.length === 4) {
          var foundPid =  parseInt(m[1]);
          var foundPort = parseInt(m[2]);
          var foundPath = m[3];

          if ( (! port || port === foundPort) &&
               (! appDir || appDir === foundPath)) {
            ret.push({
              pid: foundPid,
              port: foundPort,
              appDir: foundPath
            });
          }
        }
      });

      fut['return'](ret);
    });

  return fut.wait();
};

// See if mongo is running already. Yields. Returns the port that
// mongo is running on or null if mongo is not running.
var findMongoPort = function (appDir) {
  var pids = findMongoPids(appDir);

  if (pids.length !== 1) {
    return null;
  }

  var pid = pids[0].pid;
  try {
    process.kill(pid, 0); // make sure it is still alive
  } catch (e) {
    return null;
  }

  return pids[0].port;
};

// Kill any mongos running on 'port'. Yields, and returns once they
// are all dead. Throws an exception on failure.
//
// This is a big hammer for dealing with still running mongos, but
// smaller hammers have failed before and it is getting tiresome.
var findMongoAndKillItDead = function (port) {
  var pids = findMongoPids(null, port);

  if (! pids.length)
    return; // nothing to kill

  // Go through the list serially. There really should only ever be
  // one but we're not taking any chances.
  _.each(pids, function (processInfo) {
    var pid = processInfo.pid;

    // Send kill attempts and wait. First a SIGINT, then if it isn't
    // dead within 2 sec, SIGKILL. Check every 100ms to see if it's
    // dead.
    for (var attempts = 1; attempts <= 40; attempts ++) {
      var signal = 0;
      if (attempts === 1)
        signal = 'SIGINT';
      else if (attempts === 20 || attempts === 30)
        signal = 'SIGKILL';

      try {
        process.kill(pid, signal);
      } catch (e) {
        // it's dead. on to the next one
        return;
      }

      utils.sleepMs(100);
    }

    // give up after 4 seconds.
    // XXX should actually catch this higher up and print a nice
    // error. foreseeable conditions should never result in exceptions
    // for the user.
    throw new Error("Can't kill running mongo (pid " + pid + ").");
  });
};

// Starts a single instance of mongod, and configures it properly. Does not
// yield.
var launchMongo = function (options) {
  var onListen = options.onListen || function () {};
  var onExit = options.onExit || function () {};

  var noOplog = false;
  var mongod_path = path.join(
    files.getDevBundle(), 'mongodb', 'bin', 'mongod');

  // Automated testing: If this is set, instead of starting mongod, we
  // start our stub (fake-mongod) which can then be remote-controlled
  // by the test.
  if (process.env.METEOR_TEST_FAKE_MONGOD_CONTROL_PORT) {
    mongod_path = path.join(files.getCurrentToolsDir(),
                            'tools', 'tests', 'fake-mongod', 'fake-mongod');

    // oplog support requires sending admin commands to mongod, so
    // it'd be hard to make fake-mongod support it.
    noOplog = true;
  }

  // store data in appDir
  var dbPath = path.join(options.appDir, '.meteor', 'local', 'db');
  files.mkdir_p(dbPath, 0755);
  // add .gitignore if needed.
  files.addToGitignore(path.join(options.appDir, '.meteor'), 'local');

  var proc = null;
  var cancelStartup = false;
  var callOnExit;

  var handle = {
    stop: function () {
      if (! proc)
        cancelStartup = true;
      else {
        proc.removeListener('exit', callOnExit);
        proc.kill('SIGINT');
      }
    }
  };

  Fiber(function () {
    findMongoAndKillItDead(options.port);

    var portFile = path.join(dbPath, 'METEOR-PORT');
    var portFileExists = false;
    var createReplSet = true;
    try {
      createReplSet = +(fs.readFileSync(portFile)) !== options.port;
      portFileExists = true;
    } catch (e) {
      if (!e || e.code !== 'ENOENT')
        throw e;
    }

    if (noOplog)
      createReplSet = false;

    // If this is the first time we're using this DB, or we changed
    // port since the last time, then we want to destroy any
    // existing replSet configuration and create a new one. First we
    // delete the "local" database if it exists. (It's a pain and slow
    // to change the port in an existing replSet configuration. It's
    // also a little slow to initiate a new replSet, thus the attempt
    // to not do it unless the port changes.)
    if (createReplSet) {
      // Delete the port file, so we don't mistakenly believe that the DB is
      // still configured.
      if (portFileExists)
        fs.unlinkSync(portFile);

      try {
        var dbFiles = fs.readdirSync(dbPath);
      } catch (e) {
        if (!e || e.code !== 'ENOENT')
          throw e;
      }
      _.each(dbFiles, function (dbFile) {
        if (/^local\./.test(dbFile))
          fs.unlinkSync(path.join(dbPath, dbFile));
      });

      // Load mongo-livedata so we'll be able to talk to it.
      var mongoNpmModule = unipackage.load({
        library: release.current.library,
        packages: [ 'mongo-livedata' ],
        release: release.current.name
      })['mongo-livedata'].MongoInternals.NpmModule;
    }

    // Start mongod with a dummy replSet and wait for it to
    // listen.
    var child_process = require('child_process');
    var replSetName = 'meteor';
    if (cancelStartup)
      return;
    proc = child_process.spawn(mongod_path, [
      // nb: cli-test.sh and findMongoPids make strong assumptions about the
      // order of the arguments! Check them before changing any arguments.
      '--bind_ip', '127.0.0.1',
      '--smallfiles',
      '--nohttpinterface',
      '--port', options.port,
      '--dbpath', dbPath,
      // Use an 8MB oplog rather than 256MB. Uses less space on disk and
      // initializes faster. (Not recommended for production!)
      '--oplogSize', '8',
      '--replSet', replSetName
    ]);

    var stderrOutput = '';
    proc.stderr.setEncoding('utf8');
    proc.stderr.on('data', function (data) {
      stderrOutput += data;
    });

    callOnExit = function (code, signal) {
      onExit(code, signal, stderrOutput);
    };
    proc.on('exit', callOnExit);

    proc.stdout.setEncoding('utf8');
    var listening = false;
    var replSetReady = false;
    var replSetReadyToBeInitiated = false;
    var alreadyInitiatedReplSet = false;
    var alreadyCalledOnListen = false;
    var maybeCallOnListen = function () {
      if (listening && (replSetReady || noOplog) && !alreadyCalledOnListen) {
        if (createReplSet)
          fs.writeFileSync(portFile, options.port);
        alreadyCalledOnListen = true;
        onListen();
      }
    };

    var maybeInitiateReplset = function () {
      // We need to want to create a replset, be confident that the server is
      // listening, be confident that the server's replset implementation is
      // ready to be initiated, and have not already done it.
      if (!(createReplSet && listening && replSetReadyToBeInitiated
            && !alreadyInitiatedReplSet)) {
        return;
      }

      alreadyInitiatedReplSet = true;

      // Connect to it and start a replset.
      var db = new mongoNpmModule.Db(
        'meteor', new mongoNpmModule.Server('127.0.0.1', options.port),
        {safe: true});
      db.open(function(err, db) {
        if (err)
          throw err;
        db.admin().command({
          replSetInitiate: {
            _id: replSetName,
            members: [{_id : 0, host: '127.0.0.1:' + options.port}]
          }
        }, function (err, result) {
          if (err)
              throw err;
          // why this isn't in the error is unclear.
          if (result && result.documents && result.documents[0]
              && result.documents[0].errmsg) {
            throw result.document[0].errmsg;
          }
          db.close(true);
        });
      });
    };

    proc.stdout.on('data', function (data) {
      // note: don't use "else ifs" in this, because 'data' can have multiple
      // lines
      if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) {
        replSetReadyToBeInitiated = true;
        maybeInitiateReplset();
      }

      if (/ \[initandlisten\] waiting for connections on port/.test(data)) {
        listening = true;
        maybeInitiateReplset();
        maybeCallOnListen();
      }

      if (/ \[rsMgr\] replSet PRIMARY/.test(data)) {
        replSetReady = true;
        maybeCallOnListen();
      }
    });
  }).run();

  return handle;
};

// This runs a Mongo process and restarts it whenever it fails. If it
// restarts too often, we give up on restarting it, diagnostics are
// logged, and onFailure is called.
//
// options: appDir, port, runLog, onFailure
var MongoRunner = function (options) {
  var self = this;
  self.appDir = options.appDir;
  self.port = options.port;
  self.runLog = options.runLog;
  self.onFailure = options.onFailure;

  self.handle = null;
  self.shuttingDown = false;
  self.startupFuture = null;

  self.errorCount = 0;
  self.errorTimer = null;
  self.restartTimer = null;
};

_.extend(MongoRunner.prototype, {
  // Blocks (yields) until the server has started for the first time
  // and is accepting connections. (It might subsequently die and be
  // restarted; we won't tell you about that.) Returns true if we were
  // able to get it to start at least once.
  //
  // If the server fails to start for the first time (after a few
  // restarts), we'll print a message and give up, returning false.
  start: function () {
    var self = this;

    if (self.handle)
      throw new Error("already running?");

    self.startupFuture = new Future;
    self._startOrRestart();
    return self.startupFuture.wait();
  },

  _startOrRestart: function () {
    var self = this;

    if (self.handle)
      throw new Error("already running?");

    self.handle = launchMongo({
      appDir: self.appDir,
      port: self.port,
      onExit: _.bind(self._exited, self),
      onListen: function () {
        if (self.startupFuture) {
          // It's come up successfully for the first time. Make
          // start() return.
          self.startupFuture['return'](true);
          self.startupFuture = null;
        }
      }
    });
  },

  _exited: function (code, signal, stderr) {
    var self = this;
    self.handle = null;
    self.restartTimer = null;

    // If Mongo exited because (or rather, anytime after) we told it
    // to exit, great, nothing to do. Otherwise, we'll print an error
    // and try to restart.
    if (self.shuttingDown)
      return;

    // Print the last 20 lines of stderr.
    self.runLog.log(
      stderr.split('\n').slice(-20).join('\n') +
      "Unexpected mongo exit code " + code + ". Restarting.");

    // We'll restart it up to 3 times in a row. The counter is reset
    // when 5 seconds goes without a restart. (Note that by using a
    // timer instead of looking at the current date, we avoid getting
    // confused by time changes.)
    self.errorCount ++;
    if (self.errorTimer)
      clearTimeout(self.errorTimer);
    self.errorTimer = setTimeout(function () {
      self.errorCount = 0;
    }, 5000);

    if (self.errorCount < 3) {
      // Wait a second, then restart.
      self.restartTimer = setTimeout(inFiber(function () {
        self._startOrRestart();
      }), 1000);
      return;
    }

    // Too many restarts, too quicky. It's dead. Print friendly
    // diagnostics and give up.
    var explanation = mongoExitCodes.Codes[code];
    var message = "Can't start Mongo server.";

    if (explanation)
      message += "\n" + explanation.longText;

    if (explanation === mongoExitCodes.EXIT_NET_ERROR) {
      message += "\n\n" +
"Check for other processes listening on port " + self.port + "\n" +
"or other Meteor instances running in the same project.";
    }

    if (! explanation && /GLIBC/i.test(stderr)) {
      message += "\n\n" +
"Looks like you are trying to run Meteor on an old Linux distribution.\n" +
"Meteor on Linux requires glibc version 2.9 or above. Try upgrading your\n" +
"distribution to the latest version.";
    }

    self.runLog.log(message);
    self.onFailure && self.onFailure();

    if (self.startupFuture) {
      // start() is still blocking.. make it return
      self.startupFuture['return'](false);
      self.startupFuture = null;
    }
  },

  // Idempotent
  stop: function () {
    var self = this;

    if (self.shuttingDown)
      return;

    self.shuttingDown = true;

    self.errorTimer && clearTimeout(self.errorTimer);
    self.restartTimer && clearTimeout(self.restartTimer);

    if (self.handle) {
      self.handle.stop();
      self.handle = null;
    }
  }
});

exports.runMongoShell = runMongoShell;
exports.findMongoPort = findMongoPort;
exports.MongoRunner = MongoRunner;
<|endoftext|>


<commit_before>/**
 * @author Serhii Maksymchuk from Ubrainians for imCode
 * 22.06.18
 */
Imcms.define(
    'imcms-super-admin-page-builder',
    [
        'imcms-bem-builder', 'imcms-components-builder', 'imcms-super-admin-tabs-builder', 'imcms-i18n-texts', 'jquery'
    ],
    function (BEM, components, superAdminTabs, texts, $) {

        texts = texts.superAdmin;

        var panels$;

        function buildSuperAdminPanels() {
            return superAdminTabs.tabBuilders.map(function (tabBuilder, index) {
                return tabBuilder.buildTab(index);
            });
        }

        function buildHead() {
            return new BEM({
                block: 'imcms-head',
                elements: {
                    'logo': $('<a>', {
                        href: 'https://www.imcms.net/'
                    }),
                    'title': $('<div>', {
                        'class': 'imcms-title',
                        text: texts.head
                    })
                }
            }).buildBlockStructure('<div>', {
                'class': 'imcms-info-head'
            });
        }

        function buildSuperAdmin() {
            panels$ = buildSuperAdminPanels();

            return new BEM({
                block: 'imcms-pop-up-modal',
                elements: {
                    'head': buildHead(),
                    'left-side': superAdminTabs.buildWindowTabs(panels$),
                    'right-side': $('<div>', {'class': 'imcms-right-side'}).append(panels$)
                }
            }).buildBlockStructure('<div>', {
                'class': 'imcms-info-page'
            });
        }

        return {
            build: function () {
                $('body').append(buildSuperAdmin());
            }
        };
    }
);
<commit_msg>IMCMS-300 - New design to superadmin-pages: - Showing first tab by default.<commit_after>/**
 * @author Serhii Maksymchuk from Ubrainians for imCode
 * 22.06.18
 */
Imcms.define(
    'imcms-super-admin-page-builder',
    [
        'imcms-bem-builder', 'imcms-components-builder', 'imcms-super-admin-tabs-builder', 'imcms-i18n-texts', 'jquery'
    ],
    function (BEM, components, superAdminTabs, texts, $) {

        texts = texts.superAdmin;

        var panels$;

        function buildSuperAdminPanels() {
            return superAdminTabs.tabBuilders.map(function (tabBuilder, index) {
                var $tab = tabBuilder.buildTab(index);

                (index === 0) ? $tab.slideDown() : $tab.slideUp();

                return $tab;
            });
        }

        function buildHead() {
            return new BEM({
                block: 'imcms-head',
                elements: {
                    'logo': $('<a>', {
                        href: 'https://www.imcms.net/'
                    }),
                    'title': $('<div>', {
                        'class': 'imcms-title',
                        text: texts.head
                    })
                }
            }).buildBlockStructure('<div>', {
                'class': 'imcms-info-head'
            });
        }

        function buildSuperAdmin() {
            panels$ = buildSuperAdminPanels();

            return new BEM({
                block: 'imcms-pop-up-modal',
                elements: {
                    'head': buildHead(),
                    'left-side': superAdminTabs.buildWindowTabs(panels$),
                    'right-side': $('<div>', {'class': 'imcms-right-side'}).append(panels$)
                }
            }).buildBlockStructure('<div>', {
                'class': 'imcms-info-page'
            });
        }

        return {
            build: function () {
                $('body').append(buildSuperAdmin());
            }
        };
    }
);
<|endoftext|>


<commit_before>'use strict';

var clear       = require('es5-ext/object/clear')
  , d           = require('d/d')
  , getIterator = require('es6-iterator/get')
  , forOf       = require('es6-iterator/for-of')
  , Set         = require('../polyfill')
  , Iterator    = require('./_iterator')

  , isArray = Array.isArray, create = Object.create
  , defineProperties = Object.defineProperties
  , hasOwnProperty = Object.prototype.hasOwnProperty
  , PrimitiveSet;

module.exports = PrimitiveSet = function (/*iterable*/) {
	var iterable = arguments[0];
	if (!(this instanceof PrimitiveSet)) return new PrimitiveSet(iterable);
	if (this.__setData__ !== undefined) {
		throw new TypeError(this + " cannot be reinitialized");
	}
	if (iterable != null) {
		if (!isArray(iterable)) iterable = getIterator(iterable);
	}
	defineProperties(this, {
		__setData__: d('', create(null)),
		__size__: d('w', 0)
	});
	if (!iterable) return;
	forOf(iterable, function (value) { this.add(value); }, this);
};

PrimitiveSet.prototype = Object.create(Set.prototype, {
	constructor: d(PrimitiveSet),
	_serialize: d(function (value) {
		if (value && (typeof value.toString !== 'function')) return;
		return String(value);
	}),
	add: d(function (value) {
		var key = this._serialize(value);
		if (key == null) throw new TypeError(value + " cannot be serialized");
		if (hasOwnProperty.call(this.__setData__, key)) return this;
		this.__setData__[key] = value;
		++this.__size__;
		this.emit('_add', key);
		return this;
	}),
	clear: d(function () {
		clear(this.__setData__);
		this.__size__ = 0;
		this.emit('_clear');
	}),
	delete: d(function (value) {
		var key = this._serialize(value);
		if (key == null) return false;
		if (!hasOwnProperty.call(this.__setData__, key)) return false;
		delete this.__setData__[key];
		--this.__size__;
		this.emit('_delete', key);
		return true;
	}),
	entries: d(function () { return new Iterator(this, 'key+value'); }),
	has: d(function (value) {
		var key = this._serialize(value);
		if (key == null) return false;
		return hasOwnProperty.call(this.__setData__, key);
	}),
	size: d.gs(function () { return this.__size__; }),
	get: d(function (key) { return this.__setData__[key]; }),
	values: d(function () { return new Iterator(this); })
});
<commit_msg>`null` instead of `undefined`<commit_after>'use strict';

var clear       = require('es5-ext/object/clear')
  , d           = require('d/d')
  , getIterator = require('es6-iterator/get')
  , forOf       = require('es6-iterator/for-of')
  , Set         = require('../polyfill')
  , Iterator    = require('./_iterator')

  , isArray = Array.isArray, create = Object.create
  , defineProperties = Object.defineProperties
  , hasOwnProperty = Object.prototype.hasOwnProperty
  , PrimitiveSet;

module.exports = PrimitiveSet = function (/*iterable*/) {
	var iterable = arguments[0];
	if (!(this instanceof PrimitiveSet)) return new PrimitiveSet(iterable);
	if (this.__setData__ !== undefined) {
		throw new TypeError(this + " cannot be reinitialized");
	}
	if (iterable != null) {
		if (!isArray(iterable)) iterable = getIterator(iterable);
	}
	defineProperties(this, {
		__setData__: d('', create(null)),
		__size__: d('w', 0)
	});
	if (!iterable) return;
	forOf(iterable, function (value) { this.add(value); }, this);
};

PrimitiveSet.prototype = Object.create(Set.prototype, {
	constructor: d(PrimitiveSet),
	_serialize: d(function (value) {
		if (value && (typeof value.toString !== 'function')) return null;
		return String(value);
	}),
	add: d(function (value) {
		var key = this._serialize(value);
		if (key == null) throw new TypeError(value + " cannot be serialized");
		if (hasOwnProperty.call(this.__setData__, key)) return this;
		this.__setData__[key] = value;
		++this.__size__;
		this.emit('_add', key);
		return this;
	}),
	clear: d(function () {
		clear(this.__setData__);
		this.__size__ = 0;
		this.emit('_clear');
	}),
	delete: d(function (value) {
		var key = this._serialize(value);
		if (key == null) return false;
		if (!hasOwnProperty.call(this.__setData__, key)) return false;
		delete this.__setData__[key];
		--this.__size__;
		this.emit('_delete', key);
		return true;
	}),
	entries: d(function () { return new Iterator(this, 'key+value'); }),
	has: d(function (value) {
		var key = this._serialize(value);
		if (key == null) return false;
		return hasOwnProperty.call(this.__setData__, key);
	}),
	size: d.gs(function () { return this.__size__; }),
	get: d(function (key) { return this.__setData__[key]; }),
	values: d(function () { return new Iterator(this); })
});
<|endoftext|>


<commit_before>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

const apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';  // The url to ping on completion
const webhookKey = process.env.WEBHOOK_KEY || '123';  // Secret key to auth with API
const processTimeout = process.env.PROCESS_TIMEOUT || 10 * 60 * 1000; // Kill the process after a certain time in case it hangs
let pg;
let st;

// This is a top-level safety mechanism, we'll kill this process after a certain
// time in case it's hanging. Intended to avoid https://github.com/openaq/openaq-fetch/issues/154
setTimeout(() => {
  log.error('Uh oh, process timed out.');
  process.exit(1);
}, processTimeout);

// Flatten the sources into a single array, taking into account sources argument
sources = chain(sources).values().flatten().value();
if (argv.source) {
  sources = find(sources, { name: argv.source });

  // Check here to make sure we have at least one valid source
  if (!sources) {
    log.error('I\'m sorry Dave, I searched all known sources and can\'t ' +
      'find anything for', argv.source);
    process.exit(1);
  }

  // Make it a single element array to play nicely downstream
  sources = [sources];
}

/**
 * Find the adapter for a given source
 * @param {string} name An adapter name
 * @return {Adapter} The associated adapter
 */
var findAdapter = function (name) {
  return find(adapters, function (a) {
    return a.name === name;
  });
};

/**
* Ping openaq-api to let it know data fetching is complete
* @param {function} cb A function of form func(error) called on completion
*/
var sendUpdatedWebhook = function (cb) {
  var form = {
    key: webhookKey,
    action: 'DATABASE_UPDATED'
  };
  request.post(apiURL, {form: form}, function (err, res, body) {
    if (err) {
      cb(err);
    }

    cb(null);
  });
};

/**
 * Build an object that can be inserted into our database.
 * @param {object} m measurement object
 * @return {object} an object capable of being saved into the PostgreSQL database
 */
let buildSQLObject = function (m) {
  let obj = {
    location: m.location,
    value: m.value,
    unit: m.unit,
    parameter: m.parameter,
    country: m.country,
    city: m.city,
    source_name: m.sourceName,
    date_utc: m.date.utc,
    source_type: m.sourceType,
    mobile: m.mobile
  };
  // Copy object JSON to the data field
  obj.data = assign({}, m);
  // If we have coordinates, save them with postgis
  if (m.coordinates) {
    obj.coordinates = st.geomFromText(`Point(${m.coordinates.longitude} ${m.coordinates.latitude})`, 4326);
  }

  return obj;
};

/**
 * Create a function to ask the adapter for data, verify the data and attempt
 * to save to a database when appropriate (i.e., not running with `--dryrun`).
 * @param {object} source A source object
 * @return {function} The function to make the magic happen
 */
var getAndSaveData = function (source) {
  // Generates a formatted message based on fetch results
  let generateResultsMessage = function (measurements, source, failures, fetchStarted, fetchEnded, isDryrun = false) {
    return {
      message: `${isDryrun ? '[Dry Run] ' : ''}New measurements inserted for ${source.name}: ${measurements.length}`,
      failures: failures,
      count: measurements.length,
      duration: (fetchEnded - fetchStarted) / 1000,
      sourceName: source.name
    };
  };

  return function (done) {
    // Get the appropriate adapter
    var adapter = findAdapter(source.adapter);
    if (!adapter) {
      const msg = generateResultsMessage([], source, {'Could not find adapter.': 1}, 0, 0, argv.dryrun);
      return done(null, msg);
    }

    let fetchStarted = Date.now();
    adapter.fetchData(source, function (err, data) {
      let fetchEnded = Date.now();
      // If we have an error
      if (err) {
        const errDict = {};
        const key = err.message || 'Unknown adapter error';
        errDict[key] = 1;
        const msg = generateResultsMessage([], source, errDict, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Verify the data format
      let { isValid, failures: reasons } = utils.verifyDataFormat(data);

      // If the data format is invalid
      if (!isValid) {
        const msg = generateResultsMessage([], source, reasons, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Clean up the measurements a bit before validation
      data.measurements = data.measurements.map((m) => {
        // Set defaults on measurement if needed
        m.location = m.location || data.name; // use existing location if it exists
        m.country = m.country || source.country;
        m.city = m.city || source.city; // use city from measurement, otherwise default to source
        m.sourceName = source.name;

        // Set defaults for sourceType (default to government)
        // and mobile (default to false).
        m.sourceType = m.sourceType || 'government';
        m.mobile = (m.mobile === undefined) ? false : m.mobile;

        // Remove extra fields
        var wanted = ['date', 'parameter', 'location', 'value', 'unit', 'city',
                      'attribution', 'averagingPeriod', 'coordinates',
                      'country', 'sourceName', 'sourceType', 'mobile'];
        return pick(m, wanted);
      });

      // Remove any measurements that don't meet our requirements
      let { pruned, failures } = utils.pruneMeasurements(data.measurements);
      data.measurements = pruned;

      // If we have no measurements to insert, we can exit now
      if (data.measurements && data.measurements.length === 0) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // We can cut out some of the db related tasks if this is a dry run
      if (!argv.dryrun) {
        var inserts = [];
      }
      data.measurements.forEach((m) => {
        // Save or print depending on the state
        if (argv.dryrun) {
          log.info(JSON.stringify(m));
        } else {
          inserts.push(buildSQLObject(m));
        }
      });
      if (argv.dryrun) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        done(null, msg);
      } else {
        // We're running each insert task individually so we can catch any
        // duplicate errors. Good idea? Who knows!
        let insertRecord = function (record) {
          return function (cb) {
            let error;
            pg('measurements')
              .returning('location')
              .insert(record)
              .then((loc) => {
                // Don't need anything thing here for now
              })
              .catch((e) => {
                error = e;
              })
              .done(() => {
                if (error && error.code === '23505') {
                  cb(null, {status: 'duplicate'});
                } else if (error) {
                  // Log out an error if it's not an failed duplicate insert
                  log.error(error);
                  cb(error);
                } else {
                  cb(null, {status: 'new'});
                }
              });
          };
        };
        let tasks = inserts.map((i) => {
          return insertRecord(i);
        });
        async.parallel(tasks, function (err, results) {
          if (err) {
            return done(err);
          }

          // Get rid of duplicates in results array to get actual insert number
          results = filter(results, (r) => {
            return r.status !== 'duplicate';
          });
          let msg = generateResultsMessage(results, source, failures, fetchStarted, fetchEnded, argv.dryrun);
          return done(null, msg);
        });
      }
    });
  };
};

/**
 * Generate tasks to run in parallel, only care about the active sources
 */
let tasks = [];
sources.forEach((source) => {
  if (source.active) {
    tasks.push(getAndSaveData(source));
  }
});

/**
 * Saves information about fetches to the database
 */
const saveFetches = function (timeStarted, timeEnded, itemsInserted, err, results) {
  return function (done) {
    pg('fetches')
      .insert({time_started: timeStarted, time_ended: timeEnded, count: itemsInserted, results: JSON.stringify(err || results)})
      .then((id) => {
        // Insert was successful
        log.info('Fetches table successfully updated');
        done(null);
      })
      .catch((e) => {
        // An error on fetches insert
        log.error(e);
        done(null);
      });
  };
};

/**
 * Save sources information, overwritten any previous results
 *
 */
const saveSources = function () {
  return function (done) {
    pg('sources')
      .del()
      .then(() => {
        // Delete was successful
        log.verbose('Sources table successfully deleted.');

        // Grab data from sources
        const inserts = sources.map((s) => {
          return {data: s};
        });
        pg('sources')
          .insert(inserts)
          .then(() => {
            log.info('Sources table successfully updated.');
            done(null);
          })
          .catch((e) => {
            // An error on sources insert
            log.error(e);
            done(null);
          });
      })
      .catch((e) => {
        // An error on sources delete
        log.error(e);
        done(null);
      });
  };
};

/**
 * Run all the data fetch tasks in parallel, simply logs out results
 */
var runTasks = function () {
  log.info('Running all fetch tasks.');
  let timeStarted = new Date();
  let itemsInserted = 0;
  async.parallel(tasks, (err, results) => {
    let timeEnded = new Date();
    if (err) {
      log.error('Error during fetching of data sources.');
    } else {
      if (!argv.dryrun) {
        log.info('All data grabbed and saved.');
      }
      results.forEach(function (r) {
        // Add to inserted count if response has a count, if there was a failure
        // response will not have a count
        if (r.count !== undefined) {
          itemsInserted += r.count;
        }
        log.info('///////');
        log.info(r.message);
        for (let k of Object.keys(r.failures || {})) {
          log.info(`${r.failures[k]} occurrences of ${k}`);
        }
        log.info('///////');
      });
    }

    // Send out the webhook to openaq-api since we're all done
    if (argv.dryrun) {
      log.info('Dryrun completed, have a good day!');
      process.exit(0);
    } else {
      // Run functions post fetches
      const postFetchFunctions = [
        saveFetches(timeStarted, timeEnded, itemsInserted, err, results),
        saveSources()
      ];
      async.parallel(postFetchFunctions, (err, results) => {
        if (err) {
          log.error(err);
        }

        sendUpdatedWebhook((err) => {
          if (err) {
            log.error(err);
          }

          log.info('Webhook posted, have a good day!');
          process.exit(0);
        });
      });
    }
  });
};

// Branch here depending on whether this is a dryrun or not
if (argv.dryrun) {
  log.info('--- Dry run for Testing, nothing is saved to the database. ---');
  runTasks();
} else {
  // Set up DB and add in postgis features
  pg = knex(knexConfig);
  st = require('knex-postgis')(pg);
  log.info('Connected to database.');

  // Run any needed migrations and away we go
  pg.migrate.latest(knexConfig)
  .then(() => {
    log.info('Database migrations are handled, ready to roll!');
    runTasks();
  })
  .catch((e) => {
    log.error(e);
  });
}
<commit_msg>Undoing insert error hotfixes on fetch<commit_after>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

const apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';  // The url to ping on completion
const webhookKey = process.env.WEBHOOK_KEY || '123';  // Secret key to auth with API
const processTimeout = process.env.PROCESS_TIMEOUT || 10 * 60 * 1000; // Kill the process after a certain time in case it hangs
let pg;
let st;

// This is a top-level safety mechanism, we'll kill this process after a certain
// time in case it's hanging. Intended to avoid https://github.com/openaq/openaq-fetch/issues/154
setTimeout(() => {
  log.error('Uh oh, process timed out.');
  process.exit(1);
}, processTimeout);

// Flatten the sources into a single array, taking into account sources argument
sources = chain(sources).values().flatten().value();
if (argv.source) {
  sources = find(sources, { name: argv.source });

  // Check here to make sure we have at least one valid source
  if (!sources) {
    log.error('I\'m sorry Dave, I searched all known sources and can\'t ' +
      'find anything for', argv.source);
    process.exit(1);
  }

  // Make it a single element array to play nicely downstream
  sources = [sources];
}

/**
 * Find the adapter for a given source
 * @param {string} name An adapter name
 * @return {Adapter} The associated adapter
 */
var findAdapter = function (name) {
  return find(adapters, function (a) {
    return a.name === name;
  });
};

/**
* Ping openaq-api to let it know data fetching is complete
* @param {function} cb A function of form func(error) called on completion
*/
var sendUpdatedWebhook = function (cb) {
  var form = {
    key: webhookKey,
    action: 'DATABASE_UPDATED'
  };
  request.post(apiURL, {form: form}, function (err, res, body) {
    if (err) {
      cb(err);
    }

    cb(null);
  });
};

/**
 * Build an object that can be inserted into our database.
 * @param {object} m measurement object
 * @return {object} an object capable of being saved into the PostgreSQL database
 */
let buildSQLObject = function (m) {
  let obj = {
    location: m.location,
    value: m.value,
    unit: m.unit,
    parameter: m.parameter,
    country: m.country,
    city: m.city,
    source_name: m.sourceName,
    date_utc: m.date.utc,
    source_type: m.sourceType,
    mobile: m.mobile
  };
  // Copy object JSON to the data field
  obj.data = assign({}, m);
  // If we have coordinates, save them with postgis
  if (m.coordinates) {
    obj.coordinates = st.geomFromText(`Point(${m.coordinates.longitude} ${m.coordinates.latitude})`, 4326);
  }

  return obj;
};

/**
 * Create a function to ask the adapter for data, verify the data and attempt
 * to save to a database when appropriate (i.e., not running with `--dryrun`).
 * @param {object} source A source object
 * @return {function} The function to make the magic happen
 */
var getAndSaveData = function (source) {
  // Generates a formatted message based on fetch results
  let generateResultsMessage = function (measurements, source, failures, fetchStarted, fetchEnded, isDryrun = false) {
    return {
      message: `${isDryrun ? '[Dry Run] ' : ''}New measurements inserted for ${source.name}: ${measurements.length}`,
      failures: failures,
      count: measurements.length,
      duration: (fetchEnded - fetchStarted) / 1000,
      sourceName: source.name
    };
  };

  return function (done) {
    // Get the appropriate adapter
    var adapter = findAdapter(source.adapter);
    if (!adapter) {
      const msg = generateResultsMessage([], source, {'Could not find adapter.': 1}, 0, 0, argv.dryrun);
      return done(null, msg);
    }

    let fetchStarted = Date.now();
    adapter.fetchData(source, function (err, data) {
      let fetchEnded = Date.now();
      // If we have an error
      if (err) {
        const errDict = {};
        const key = err.message || 'Unknown adapter error';
        errDict[key] = 1;
        const msg = generateResultsMessage([], source, errDict, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Verify the data format
      let { isValid, failures: reasons } = utils.verifyDataFormat(data);

      // If the data format is invalid
      if (!isValid) {
        const msg = generateResultsMessage([], source, reasons, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Clean up the measurements a bit before validation
      data.measurements = data.measurements.map((m) => {
        // Set defaults on measurement if needed
        m.location = m.location || data.name; // use existing location if it exists
        m.country = m.country || source.country;
        m.city = m.city || source.city; // use city from measurement, otherwise default to source
        m.sourceName = source.name;

        // Set defaults for sourceType (default to government)
        // and mobile (default to false).
        m.sourceType = m.sourceType || 'government';
        m.mobile = (m.mobile === undefined) ? false : m.mobile;

        // Remove extra fields
        var wanted = ['date', 'parameter', 'location', 'value', 'unit', 'city',
                      'attribution', 'averagingPeriod', 'coordinates',
                      'country', 'sourceName', 'sourceType', 'mobile'];
        return pick(m, wanted);
      });

      // Remove any measurements that don't meet our requirements
      let { pruned, failures } = utils.pruneMeasurements(data.measurements);
      data.measurements = pruned;

      // If we have no measurements to insert, we can exit now
      if (data.measurements && data.measurements.length === 0) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // We can cut out some of the db related tasks if this is a dry run
      if (!argv.dryrun) {
        var inserts = [];
      }
      data.measurements.forEach((m) => {
        // Save or print depending on the state
        if (argv.dryrun) {
          log.info(JSON.stringify(m));
        } else {
          inserts.push(buildSQLObject(m));
        }
      });
      if (argv.dryrun) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        done(null, msg);
      } else {
        // We're running each insert task individually so we can catch any
        // duplicate errors. Good idea? Who knows!
        let insertRecord = function (record) {
          return function (done) {
            pg('measurements')
              .returning('location')
              .insert(record)
              .then((loc) => {
                done(null, {status: 'new'});
              })
              .catch((e) => {
                // Log out an error if it's not an failed duplicate insert
                if (e.code === '23505') {
                  return done(null, {status: 'duplicate'});
                }

                log.error(e);
                done(e);
              });
          };
        };
        let tasks = inserts.map((i) => {
          return insertRecord(i);
        });
        async.parallel(tasks, function (err, results) {
          if (err) {
            return done(err);
          }

          // Get rid of duplicates in results array to get actual insert number
          results = filter(results, (r) => {
            return r.status !== 'duplicate';
          });
          let msg = generateResultsMessage(results, source, failures, fetchStarted, fetchEnded, argv.dryrun);
          done(null, msg);
        });
      }
    });
  };
};

/**
 * Generate tasks to run in parallel, only care about the active sources
 */
let tasks = [];
sources.forEach((source) => {
  if (source.active) {
    tasks.push(getAndSaveData(source));
  }
});

/**
 * Saves information about fetches to the database
 */
const saveFetches = function (timeStarted, timeEnded, itemsInserted, err, results) {
  return function (done) {
    pg('fetches')
      .insert({time_started: timeStarted, time_ended: timeEnded, count: itemsInserted, results: JSON.stringify(err || results)})
      .then((id) => {
        // Insert was successful
        log.info('Fetches table successfully updated');
        done(null);
      })
      .catch((e) => {
        // An error on fetches insert
        log.error(e);
        done(null);
      });
  };
};

/**
 * Save sources information, overwritten any previous results
 *
 */
const saveSources = function () {
  return function (done) {
    pg('sources')
      .del()
      .then(() => {
        // Delete was successful
        log.verbose('Sources table successfully deleted.');

        // Grab data from sources
        const inserts = sources.map((s) => {
          return {data: s};
        });
        pg('sources')
          .insert(inserts)
          .then(() => {
            log.info('Sources table successfully updated.');
            done(null);
          })
          .catch((e) => {
            // An error on sources insert
            log.error(e);
            done(null);
          });
      })
      .catch((e) => {
        // An error on sources delete
        log.error(e);
        done(null);
      });
  };
};

/**
 * Run all the data fetch tasks in parallel, simply logs out results
 */
var runTasks = function () {
  log.info('Running all fetch tasks.');
  let timeStarted = new Date();
  let itemsInserted = 0;
  async.parallel(tasks, (err, results) => {
    let timeEnded = new Date();
    if (err) {
      log.error('Error during fetching of data sources.');
    } else {
      if (!argv.dryrun) {
        log.info('All data grabbed and saved.');
      }
      results.forEach(function (r) {
        // Add to inserted count if response has a count, if there was a failure
        // response will not have a count
        if (r.count !== undefined) {
          itemsInserted += r.count;
        }
        log.info('///////');
        log.info(r.message);
        for (let k of Object.keys(r.failures || {})) {
          log.info(`${r.failures[k]} occurrences of ${k}`);
        }
        log.info('///////');
      });
    }

    // Send out the webhook to openaq-api since we're all done
    if (argv.dryrun) {
      log.info('Dryrun completed, have a good day!');
      process.exit(0);
    } else {
      // Run functions post fetches
      const postFetchFunctions = [
        saveFetches(timeStarted, timeEnded, itemsInserted, err, results),
        saveSources()
      ];
      async.parallel(postFetchFunctions, (err, results) => {
        if (err) {
          log.error(err);
        }

        sendUpdatedWebhook((err) => {
          if (err) {
            log.error(err);
          }

          log.info('Webhook posted, have a good day!');
          process.exit(0);
        });
      });
    }
  });
};

// Branch here depending on whether this is a dryrun or not
if (argv.dryrun) {
  log.info('--- Dry run for Testing, nothing is saved to the database. ---');
  runTasks();
} else {
  // Set up DB and add in postgis features
  pg = knex(knexConfig);
  st = require('knex-postgis')(pg);
  log.info('Connected to database.');

  // Run any needed migrations and away we go
  pg.migrate.latest(knexConfig)
  .then(() => {
    log.info('Database migrations are handled, ready to roll!');
    runTasks();
  })
  .catch((e) => {
    log.error(e);
  });
}
<|endoftext|>


<commit_before>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

const apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';  // The url to ping on completion
const webhookKey = process.env.WEBHOOK_KEY || '123';  // Secret key to auth with API
const processTimeout = process.env.PROCESS_TIMEOUT || 10 * 60 * 1000; // Kill the process after a certain time in case it hangs
let pg;
let st;

// This is a top-level safety mechanism, we'll kill this process after a certain
// time in case it's hanging. Intended to avoid https://github.com/openaq/openaq-fetch/issues/154
setTimeout(() => {
  log.error('Uh oh, process timed out.');
  process.exit(1);
}, processTimeout);

// Flatten the sources into a single array, taking into account sources argument
sources = chain(sources).values().flatten().value();
if (argv.source) {
  sources = find(sources, { name: argv.source });

  // Check here to make sure we have at least one valid source
  if (!sources) {
    log.error('I\'m sorry Dave, I searched all known sources and can\'t ' +
      'find anything for', argv.source);
    process.exit(1);
  }

  // Make it a single element array to play nicely downstream
  sources = [sources];
}

/**
 * Find the adapter for a given source
 * @param {string} name An adapter name
 * @return {Adapter} The associated adapter
 */
var findAdapter = function (name) {
  return find(adapters, function (a) {
    return a.name === name;
  });
};

/**
* Ping openaq-api to let it know data fetching is complete
* @param {function} cb A function of form func(error) called on completion
*/
var sendUpdatedWebhook = function (cb) {
  var form = {
    key: webhookKey,
    action: 'DATABASE_UPDATED'
  };
  request.post(apiURL, {form: form}, function (err, res, body) {
    if (err) {
      cb(err);
    }

    cb(null);
  });
};

/**
 * Build an object that can be inserted into our database.
 * @param {object} m measurement object
 * @return {object} an object capable of being saved into the PostgreSQL database
 */
let buildSQLObject = function (m) {
  let obj = {
    location: m.location,
    value: m.value,
    unit: m.unit,
    parameter: m.parameter,
    country: m.country,
    city: m.city,
    source_name: m.sourceName,
    date_utc: m.date.utc,
    source_type: m.sourceType,
    mobile: m.mobile
  };
  // Copy object JSON to the data field
  obj.data = assign({}, m);
  // If we have coordinates, save them with postgis
  if (m.coordinates) {
    obj.coordinates = st.geomFromText(`Point(${m.coordinates.longitude} ${m.coordinates.latitude})`, 4326);
  }

  return obj;
};

/**
 * Create a function to ask the adapter for data, verify the data and attempt
 * to save to a database when appropriate (i.e., not running with `--dryrun`).
 * @param {object} source A source object
 * @return {function} The function to make the magic happen
 */
var getAndSaveData = function (source) {
  // Generates a formatted message based on fetch results
  let generateResultsMessage = function (measurements, source, failures, fetchStarted, fetchEnded, isDryrun = false) {
    return {
      message: `${isDryrun ? '[Dry Run] ' : ''}New measurements inserted for ${source.name}: ${measurements.length}`,
      failures: failures,
      count: measurements.length,
      duration: (fetchEnded - fetchStarted) / 1000,
      sourceName: source.name
    };
  };

  return function (done) {
    // Get the appropriate adapter
    var adapter = findAdapter(source.adapter);
    if (!adapter) {
      const msg = generateResultsMessage([], source, {'Could not find adapter.': 1}, 0, 0, argv.dryrun);
      return done(null, msg);
    }

    let fetchStarted = Date.now();
    adapter.fetchData(source, function (err, data) {
      let fetchEnded = Date.now();
      // If we have an error
      if (err) {
        const errDict = {};
        const key = err.message || 'Unknown adapter error';
        errDict[key] = 1;
        const msg = generateResultsMessage([], source, errDict, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Verify the data format
      let { isValid, failures: reasons } = utils.verifyDataFormat(data);

      // If the data format is invalid
      if (!isValid) {
        const msg = generateResultsMessage([], source, reasons, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Clean up the measurements a bit before validation
      data.measurements = data.measurements.map((m) => {
        // Set defaults on measurement if needed
        m.location = m.location || data.name; // use existing location if it exists
        m.country = m.country || source.country;
        m.city = m.city || source.city; // use city from measurement, otherwise default to source
        m.sourceName = source.name;

        // Set defaults for sourceType (default to government)
        // and mobile (default to false).
        m.sourceType = m.sourceType || 'government';
        m.mobile = (m.mobile === undefined) ? false : m.mobile;

        // Remove extra fields
        var wanted = ['date', 'parameter', 'location', 'value', 'unit', 'city',
                      'attribution', 'averagingPeriod', 'coordinates',
                      'country', 'sourceName', 'sourceType', 'mobile'];
        return pick(m, wanted);
      });

      // Remove any measurements that don't meet our requirements
      let { pruned, failures } = utils.pruneMeasurements(data.measurements);
      data.measurements = pruned;

      // If we have no measurements to insert, we can exit now
      if (data.measurements && data.measurements.length === 0) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // We can cut out some of the db related tasks if this is a dry run
      if (!argv.dryrun) {
        var inserts = [];
      }
      data.measurements.forEach((m) => {
        // Save or print depending on the state
        if (argv.dryrun) {
          log.info(JSON.stringify(m));
        } else {
          inserts.push(buildSQLObject(m));
        }
      });
      if (argv.dryrun) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        done(null, msg);
      } else {
        // We're running each insert task individually so we can catch any
        // duplicate errors. Good idea? Who knows!
        let insertRecord = function (record) {
          return function (done) {
            let error;
            pg('measurements')
              .returning('location')
              .insert(record)
              .then((loc) => {
                // Don't need anything thing here for now
              })
              .catch((e) => {
                error = e;
              })
              .done(() => {
                if (error && error.code === '23505') {
                  done(null, {status: 'duplicate'});
                } else if (error) {
                  // Log out an error if it's not an failed duplicate insert
                  log.error(error);
                  done(error);
                } else {
                  done(null, {status: 'new'});
                }
              });
          };
        };
        let tasks = inserts.map((i) => {
          return insertRecord(i);
        });
        async.parallel(tasks, function (err, results) {
          if (err) {
            return done(err);
          }

          // Get rid of duplicates in results array to get actual insert number
          results = filter(results, (r) => {
            return r.status !== 'duplicate';
          });
          let msg = generateResultsMessage(results, source, failures, fetchStarted, fetchEnded, argv.dryrun);
          done(null, msg);
        });
      }
    });
  };
};

/**
 * Generate tasks to run in parallel, only care about the active sources
 */
let tasks = [];
sources.forEach((source) => {
  if (source.active) {
    tasks.push(getAndSaveData(source));
  }
});

/**
 * Saves information about fetches to the database
 */
const saveFetches = function (timeStarted, timeEnded, itemsInserted, err, results) {
  return function (done) {
    pg('fetches')
      .insert({time_started: timeStarted, time_ended: timeEnded, count: itemsInserted, results: JSON.stringify(err || results)})
      .then((id) => {
        // Insert was successful
        log.info('Fetches table successfully updated');
        done(null);
      })
      .catch((e) => {
        // An error on fetches insert
        log.error(e);
        done(null);
      });
  };
};

/**
 * Save sources information, overwritten any previous results
 *
 */
const saveSources = function () {
  return function (done) {
    pg('sources')
      .del()
      .then(() => {
        // Delete was successful
        log.verbose('Sources table successfully deleted.');

        // Grab data from sources
        const inserts = sources.map((s) => {
          return {data: s};
        });
        pg('sources')
          .insert(inserts)
          .then(() => {
            log.info('Sources table successfully updated.');
            done(null);
          })
          .catch((e) => {
            // An error on sources insert
            log.error(e);
            done(null);
          });
      })
      .catch((e) => {
        // An error on sources delete
        log.error(e);
        done(null);
      });
  };
};

/**
 * Run all the data fetch tasks in parallel, simply logs out results
 */
var runTasks = function () {
  log.info('Running all fetch tasks.');
  let timeStarted = new Date();
  let itemsInserted = 0;
  async.parallel(tasks, (err, results) => {
    let timeEnded = new Date();
    if (err) {
      log.error('Error during fetching of data sources.');
    } else {
      if (!argv.dryrun) {
        log.info('All data grabbed and saved.');
      }
      results.forEach(function (r) {
        // Add to inserted count if response has a count, if there was a failure
        // response will not have a count
        if (r.count !== undefined) {
          itemsInserted += r.count;
        }
        log.info('///////');
        log.info(r.message);
        for (let k of Object.keys(r.failures || {})) {
          log.info(`${r.failures[k]} occurrences of ${k}`);
        }
        log.info('///////');
      });
    }

    // Send out the webhook to openaq-api since we're all done
    if (argv.dryrun) {
      log.info('Dryrun completed, have a good day!');
      process.exit(0);
    } else {
      // Run functions post fetches
      const postFetchFunctions = [
        saveFetches(timeStarted, timeEnded, itemsInserted, err, results),
        saveSources()
      ];
      async.parallel(postFetchFunctions, (err, results) => {
        if (err) {
          log.error(err);
        }

        sendUpdatedWebhook((err) => {
          if (err) {
            log.error(err);
          }

          log.info('Webhook posted, have a good day!');
          process.exit(0);
        });
      });
    }
  });
};

// Branch here depending on whether this is a dryrun or not
if (argv.dryrun) {
  log.info('--- Dry run for Testing, nothing is saved to the database. ---');
  runTasks();
} else {
  // Set up DB and add in postgis features
  pg = knex(knexConfig);
  st = require('knex-postgis')(pg);
  log.info('Connected to database.');

  // Run any needed migrations and away we go
  pg.migrate.latest(knexConfig)
  .then(() => {
    log.info('Database migrations are handled, ready to roll!');
    runTasks();
  })
  .catch((e) => {
    log.error(e);
  });
}
<commit_msg>Trying to fix insert errors<commit_after>/**
 * This is the main code to kick off the data fetching processes, handle their
 * results, saving to a database and repeating the process... forever.
 *
 * There are helpful command line shortcuts, all described with
 * `node index.js --help`.
 */
'use strict';

// Set up command line arguments
var argv = require('yargs')
  .usage('Usage: $0 --dryrun --source \'Beijing US Embassy\'')
  .boolean('dryrun')
  .describe('dryrun', 'Run the fetch process but do not attempt to save to the database and instead print to console, useful for testing.')
  .alias('d', 'dryrun')
  .describe('source', 'Run the fetch process with only the defined source using source name.')
  .alias('s', 'source')
  .nargs('source', 1)
  .help('h')
  .alias('h', 'help')
  .argv;

var async = require('async');
import { assign, filter, pick, chain, find } from 'lodash';
var knex = require('knex');
let knexConfig = require('./knexfile');
var utils = require('./lib/utils');
var request = require('request');
var log = require('./lib/logger');

var adapters = require('./adapters');
var sources = require('./sources');

const apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks';  // The url to ping on completion
const webhookKey = process.env.WEBHOOK_KEY || '123';  // Secret key to auth with API
const processTimeout = process.env.PROCESS_TIMEOUT || 10 * 60 * 1000; // Kill the process after a certain time in case it hangs
let pg;
let st;

// This is a top-level safety mechanism, we'll kill this process after a certain
// time in case it's hanging. Intended to avoid https://github.com/openaq/openaq-fetch/issues/154
setTimeout(() => {
  log.error('Uh oh, process timed out.');
  process.exit(1);
}, processTimeout);

// Flatten the sources into a single array, taking into account sources argument
sources = chain(sources).values().flatten().value();
if (argv.source) {
  sources = find(sources, { name: argv.source });

  // Check here to make sure we have at least one valid source
  if (!sources) {
    log.error('I\'m sorry Dave, I searched all known sources and can\'t ' +
      'find anything for', argv.source);
    process.exit(1);
  }

  // Make it a single element array to play nicely downstream
  sources = [sources];
}

/**
 * Find the adapter for a given source
 * @param {string} name An adapter name
 * @return {Adapter} The associated adapter
 */
var findAdapter = function (name) {
  return find(adapters, function (a) {
    return a.name === name;
  });
};

/**
* Ping openaq-api to let it know data fetching is complete
* @param {function} cb A function of form func(error) called on completion
*/
var sendUpdatedWebhook = function (cb) {
  var form = {
    key: webhookKey,
    action: 'DATABASE_UPDATED'
  };
  request.post(apiURL, {form: form}, function (err, res, body) {
    if (err) {
      cb(err);
    }

    cb(null);
  });
};

/**
 * Build an object that can be inserted into our database.
 * @param {object} m measurement object
 * @return {object} an object capable of being saved into the PostgreSQL database
 */
let buildSQLObject = function (m) {
  let obj = {
    location: m.location,
    value: m.value,
    unit: m.unit,
    parameter: m.parameter,
    country: m.country,
    city: m.city,
    source_name: m.sourceName,
    date_utc: m.date.utc,
    source_type: m.sourceType,
    mobile: m.mobile
  };
  // Copy object JSON to the data field
  obj.data = assign({}, m);
  // If we have coordinates, save them with postgis
  if (m.coordinates) {
    obj.coordinates = st.geomFromText(`Point(${m.coordinates.longitude} ${m.coordinates.latitude})`, 4326);
  }

  return obj;
};

/**
 * Create a function to ask the adapter for data, verify the data and attempt
 * to save to a database when appropriate (i.e., not running with `--dryrun`).
 * @param {object} source A source object
 * @return {function} The function to make the magic happen
 */
var getAndSaveData = function (source) {
  // Generates a formatted message based on fetch results
  let generateResultsMessage = function (measurements, source, failures, fetchStarted, fetchEnded, isDryrun = false) {
    return {
      message: `${isDryrun ? '[Dry Run] ' : ''}New measurements inserted for ${source.name}: ${measurements.length}`,
      failures: failures,
      count: measurements.length,
      duration: (fetchEnded - fetchStarted) / 1000,
      sourceName: source.name
    };
  };

  return function (done) {
    // Get the appropriate adapter
    var adapter = findAdapter(source.adapter);
    if (!adapter) {
      const msg = generateResultsMessage([], source, {'Could not find adapter.': 1}, 0, 0, argv.dryrun);
      return done(null, msg);
    }

    let fetchStarted = Date.now();
    adapter.fetchData(source, function (err, data) {
      let fetchEnded = Date.now();
      // If we have an error
      if (err) {
        const errDict = {};
        const key = err.message || 'Unknown adapter error';
        errDict[key] = 1;
        const msg = generateResultsMessage([], source, errDict, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Verify the data format
      let { isValid, failures: reasons } = utils.verifyDataFormat(data);

      // If the data format is invalid
      if (!isValid) {
        const msg = generateResultsMessage([], source, reasons, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // Clean up the measurements a bit before validation
      data.measurements = data.measurements.map((m) => {
        // Set defaults on measurement if needed
        m.location = m.location || data.name; // use existing location if it exists
        m.country = m.country || source.country;
        m.city = m.city || source.city; // use city from measurement, otherwise default to source
        m.sourceName = source.name;

        // Set defaults for sourceType (default to government)
        // and mobile (default to false).
        m.sourceType = m.sourceType || 'government';
        m.mobile = (m.mobile === undefined) ? false : m.mobile;

        // Remove extra fields
        var wanted = ['date', 'parameter', 'location', 'value', 'unit', 'city',
                      'attribution', 'averagingPeriod', 'coordinates',
                      'country', 'sourceName', 'sourceType', 'mobile'];
        return pick(m, wanted);
      });

      // Remove any measurements that don't meet our requirements
      let { pruned, failures } = utils.pruneMeasurements(data.measurements);
      data.measurements = pruned;

      // If we have no measurements to insert, we can exit now
      if (data.measurements && data.measurements.length === 0) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        return done(null, msg);
      }

      // We can cut out some of the db related tasks if this is a dry run
      if (!argv.dryrun) {
        var inserts = [];
      }
      data.measurements.forEach((m) => {
        // Save or print depending on the state
        if (argv.dryrun) {
          log.info(JSON.stringify(m));
        } else {
          inserts.push(buildSQLObject(m));
        }
      });
      if (argv.dryrun) {
        let msg = generateResultsMessage(data.measurements, source, failures, fetchStarted, fetchEnded, argv.dryrun);
        done(null, msg);
      } else {
        // We're running each insert task individually so we can catch any
        // duplicate errors. Good idea? Who knows!
        let insertRecord = function (record) {
          return function (cb) {
            let error;
            pg('measurements')
              .returning('location')
              .insert(record)
              .then((loc) => {
                // Don't need anything thing here for now
              })
              .catch((e) => {
                error = e;
              })
              .done(() => {
                if (error && error.code === '23505') {
                  cb(null, {status: 'duplicate'});
                } else if (error) {
                  // Log out an error if it's not an failed duplicate insert
                  log.error(error);
                  cb(error);
                } else {
                  cb(null, {status: 'new'});
                }
              });
          };
        };
        let tasks = inserts.map((i) => {
          return insertRecord(i);
        });
        async.parallel(tasks, function (err, results) {
          if (err) {
            return done(err);
          }

          // Get rid of duplicates in results array to get actual insert number
          results = filter(results, (r) => {
            return r.status !== 'duplicate';
          });
          let msg = generateResultsMessage(results, source, failures, fetchStarted, fetchEnded, argv.dryrun);
          done(null, msg);
        });
      }
    });
  };
};

/**
 * Generate tasks to run in parallel, only care about the active sources
 */
let tasks = [];
sources.forEach((source) => {
  if (source.active) {
    tasks.push(getAndSaveData(source));
  }
});

/**
 * Saves information about fetches to the database
 */
const saveFetches = function (timeStarted, timeEnded, itemsInserted, err, results) {
  return function (done) {
    pg('fetches')
      .insert({time_started: timeStarted, time_ended: timeEnded, count: itemsInserted, results: JSON.stringify(err || results)})
      .then((id) => {
        // Insert was successful
        log.info('Fetches table successfully updated');
        done(null);
      })
      .catch((e) => {
        // An error on fetches insert
        log.error(e);
        done(null);
      });
  };
};

/**
 * Save sources information, overwritten any previous results
 *
 */
const saveSources = function () {
  return function (done) {
    pg('sources')
      .del()
      .then(() => {
        // Delete was successful
        log.verbose('Sources table successfully deleted.');

        // Grab data from sources
        const inserts = sources.map((s) => {
          return {data: s};
        });
        pg('sources')
          .insert(inserts)
          .then(() => {
            log.info('Sources table successfully updated.');
            done(null);
          })
          .catch((e) => {
            // An error on sources insert
            log.error(e);
            done(null);
          });
      })
      .catch((e) => {
        // An error on sources delete
        log.error(e);
        done(null);
      });
  };
};

/**
 * Run all the data fetch tasks in parallel, simply logs out results
 */
var runTasks = function () {
  log.info('Running all fetch tasks.');
  let timeStarted = new Date();
  let itemsInserted = 0;
  async.parallel(tasks, (err, results) => {
    let timeEnded = new Date();
    if (err) {
      log.error('Error during fetching of data sources.');
    } else {
      if (!argv.dryrun) {
        log.info('All data grabbed and saved.');
      }
      results.forEach(function (r) {
        // Add to inserted count if response has a count, if there was a failure
        // response will not have a count
        if (r.count !== undefined) {
          itemsInserted += r.count;
        }
        log.info('///////');
        log.info(r.message);
        for (let k of Object.keys(r.failures || {})) {
          log.info(`${r.failures[k]} occurrences of ${k}`);
        }
        log.info('///////');
      });
    }

    // Send out the webhook to openaq-api since we're all done
    if (argv.dryrun) {
      log.info('Dryrun completed, have a good day!');
      process.exit(0);
    } else {
      // Run functions post fetches
      const postFetchFunctions = [
        saveFetches(timeStarted, timeEnded, itemsInserted, err, results),
        saveSources()
      ];
      async.parallel(postFetchFunctions, (err, results) => {
        if (err) {
          log.error(err);
        }

        sendUpdatedWebhook((err) => {
          if (err) {
            log.error(err);
          }

          log.info('Webhook posted, have a good day!');
          process.exit(0);
        });
      });
    }
  });
};

// Branch here depending on whether this is a dryrun or not
if (argv.dryrun) {
  log.info('--- Dry run for Testing, nothing is saved to the database. ---');
  runTasks();
} else {
  // Set up DB and add in postgis features
  pg = knex(knexConfig);
  st = require('knex-postgis')(pg);
  log.info('Connected to database.');

  // Run any needed migrations and away we go
  pg.migrate.latest(knexConfig)
  .then(() => {
    log.info('Database migrations are handled, ready to roll!');
    runTasks();
  })
  .catch((e) => {
    log.error(e);
  });
}
<|endoftext|>


<commit_before>var baltar = require('baltar');
var hyperquest = require('hyperquest');
var concat = require('concat-stream');
var url = require('url');
var path = require('path');
var once = require('one-time');
var parallel = require('parallel-transform');
var async = require('async');
var debug = require('diagnostics')('registry-migrate:fetch');
var mkdirp = require('mkdirp');
var tryParse = require('json-try-parse');

module.exports = Fetch;

/**
 * A stream that fetches all of the packages given from the source registry
 */
function Fetch(source, opts) {
  if (!(this instanceof Fetch)) return new Fetch(source, opts)
  this.source = source;
  //
  // We probably want more than 1 limit here
  //
  this.limit = opts.limit;
  this.dir = opts.dir;

  //
  // Return the parallelized stream so we handle backpressure properly
  //
  this.stream = parallel(this.limit, this.transform.bind(this));

  this.stream._fetch = this;

  return this.stream;
}

/**
 * The transform function for a concurrent transform stream
 */
Fetch.prototype.transform = function transform(data, callback) {
  var name = data.name;
  //
  // 1. Get the full package with all versions and tarball URLs
  //
  this.get(name, (err, pkg) => {
    if (err) return callback(err);
    //
    // 2. Iterate through them all and pull and unpack them onto disk
    //
    mkdirp(path.join(this.dir, name), err => {
      if (err) return callback(err);
      this.iterate(name, pkg.versions, (err, built) => {
        //
        // 3. Return a data structure with the mappings to the full path to all
        // the versions
        // {
        //   name: 'package-name',
        //   versions: {
        //     '0.9.0': '/path/on/disk/0.9.0/package-name'
        //   }
        // }
        //
        callback(err, built);
      });
    });
  });
};

/**
 * Get a single package from the given source and store it at the given path
 */
Fetch.prototype.get = function (name, callback) {
  var fn = once(callback);
  debug('Get full package from registry');

  hyperquest(url.resolve(this.source, name))
    .on('error', fn)
    .pipe(concat({ encoding: 'string' }, function (data) {

      var pkg = tryParse(data);

      if (!pkg) return fn(new Error('Invalid data for ' + name));

      callback(null, pkg);
    }));
};

Fetch.prototype.iterate = function (name, versions, callback) {
  var built = {};
  built.name = name;
  built.versions = {};

  async.eachLimit(versions, this.limit, (pkg, next) => {
    if (!pkg.dist.tarball) {
      debug('bad package version  %s@%d', name, pkg.version);
      return next();
    }
    //
    // Use baltar to put the package into place on disk untarred and all that
    //
    var fullPath = path.join(this.dir, name, pkg.version)
    baltar.pull({
      url: pkg.dist.tarball,
      path: fullPath
    }, (err, entries) => {
      if (err) return next(err);
      //
      // We append package to the path because that is the folder that it
      // `untars` into that will be within the version number folder
      //
      built.versions[pkg.version] = path.join(fullPath, 'package');
      next();
    });
  }, err => {
    callback(err, built);
  });
};
<commit_msg>[fix] optimize receiving fully formed package objects and do not fetch them<commit_after>var baltar = require('baltar');
var hyperquest = require('hyperquest');
var concat = require('concat-stream');
var url = require('url');
var path = require('path');
var once = require('one-time');
var parallel = require('parallel-transform');
var async = require('async');
var debug = require('diagnostics')('registry-migrate:fetch');
var mkdirp = require('mkdirp');
var tryParse = require('json-try-parse');

module.exports = Fetch;

/**
 * A stream that fetches all of the packages given from the source registry
 */
function Fetch(source, opts) {
  if (!(this instanceof Fetch)) return new Fetch(source, opts)
  this.source = source;
  //
  // We probably want more than 1 limit here
  //
  this.limit = opts.limit;
  this.dir = opts.dir;

  //
  // Return the parallelized stream so we handle backpressure properly
  //
  this.stream = parallel(this.limit, this.transform.bind(this));

  this.stream._fetch = this;

  return this.stream;
}

/**
 * The transform function for a concurrent transform stream
 */
Fetch.prototype.transform = function transform(data, callback) {
  var name = data.name;

  //
  // 1. Detect if we are given a data structure
  //
  if (Array.isArray(data.versions)
      && data.versions[0].dist
      && data.versions[0].dist.tarball) {
    //
    // Unpack all the package versions onto disk
    //
    return this.unpack(name, data, callback);
  }

  //
  // 2. Get the full package with all versions and tarball URLs
  //
  this.get(name, (err, pkg) => {
    if (err) return callback(err);
    //
    // 3. Unpack all the versions onto disk
    //
    this.unpack(name, pkg, callback);
  });
};

Fetch.prototype.unpack = function unpack(name, pkg, callback) {
  //
  // 1. Iterate through them all and pull and unpack them onto disk
  //
  mkdirp(path.join(this.dir, name), err => {
    if (err) return callback(err);
    this.iterate(name, pkg.versions, (err, built) => {
      //
      // 2. Return a data structure with the mappings to the full path to all
      // the versions
      // {
      //   name: 'package-name',
      //   versions: {
      //     '0.9.0': '/path/on/disk/0.9.0/package-name'
      //   }
      // }
      //
      callback(err, built);
    });
  });
};

/**
 * Get a single package from the given source and store it at the given path
 */
Fetch.prototype.get = function (name, callback) {
  var fn = once(callback);
  debug('Get full package from registry');

  hyperquest(url.resolve(this.source, name))
    .on('error', fn)
    .pipe(concat({ encoding: 'string' }, function (data) {

      var pkg = tryParse(data);

      if (!pkg) return fn(new Error('Invalid data for ' + name));

      callback(null, pkg);
    }));
};

Fetch.prototype.iterate = function (name, versions, callback) {
  var built = {};
  built.name = name;
  built.versions = {};

  async.eachLimit(versions, this.limit, (pkg, next) => {
    if (!pkg.dist.tarball) {
      debug('bad package version  %s@%d', name, pkg.version);
      return next();
    }
    //
    // Use baltar to put the package into place on disk untarred and all that
    //
    var fullPath = path.join(this.dir, name, pkg.version)
    baltar.pull({
      url: pkg.dist.tarball,
      path: fullPath
    }, (err, entries) => {
      if (err) return next(err);
      //
      // We append package to the path because that is the folder that it
      // `untars` into that will be within the version number folder
      //
      built.versions[pkg.version] = path.join(fullPath, 'package');
      next();
    });
  }, err => {
    callback(err, built);
  });
};
<|endoftext|>