<commit_before>(function (global) {
  var Stash = global.Stash || {};
  var isNode = typeof require !== 'undefined';

  if (isNode) {
    Q = require('q');
  }

  Stash.Item = (function () {
    function Item(key, pool) {
      this.key = key;
      this.pool = pool;

      this._unload_();
    }

    Item.SP_NONE = 1;
    Item.SP_OLD = 2;
    Item.SP_PRECOMPUTE = 4;
    Item.SP_VALUE = 8;

    Item.prototype._unload_ = function () {
      this.value = null;
      this.expiration = false;
    };

    Item.prototype._load_ = function (callback) {
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        return d.get(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype._write_ = function (callback) {
      var deferred = Q.defer();
      this.pool.drivers.reverse().forEach(function (d) {
        return d.put(this.key, this.value, this.expiration).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item._calculateExpiration_ = function (expiration) {
      if (typeof(expiration) === 'number') {
        expiration *= 1000;
        expiration += Date.now();
      } else if (expiration instanceof Date) {
        expiration = expiration.getTime();
      }

      return expiration;
    };

    Item.prototype.get = function (cachePolicy, policyData) {
      var deferred = Q.defer();
      this.cachePolicy = cachePolicy || Stash.Item.SP_NONE;
      this.policyData = policyData;

      if ((cachePolicy & Stash.Item.SP_VALUE) && this.locked) {
        deferred.resolve(policyData);
      }

      this._load_().then(function (data) {
        if (data) {
          this.value = data.value;
          this.expiration = data.expiration;
          data = data.value || null;
        }
        deferred.resolve(data);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.set = function (value, expiration, callback) {
      if (typeof expiration === 'function') {
        callback = expiration;
        expiration = null;
      }

      this._unload_();

      this.expiration = Item._calculateExpiration_(expiration);
      this.value = value;

      return this._write_().then(function () {
        return this.unlock();
      }.bind(this));
    };

    Item.prototype.isMiss = function (callback) {
      var deferred = Q.defer();

      this.isLocked().then(function (locked) {
        var miss;

        if (locked && (this.cachePolicy & Stash.Item.SP_OLD)) {
          miss = false;
        } else if (!locked &&
                   (this.cachePolicy & Stash.Item.SP_PRECOMPUTE) &&
                   this.policyData * 1000 >= this.expiration - Date.now()) {
          miss =  true;
        } else {
          miss = this.value === null ||
            typeof(this.expiration) === 'number' &&
            this.expiration < Date.now();
        }

        deferred.resolve(miss);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.clear = function () {
      var deferred = Q.defer();
      this._unload_();

      this.pool.drivers.forEach(function (driver) {
        driver.delete(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.lock = function () {
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        d.lock(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.isLocked = function () {
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        d.isLocked(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.unlock = function () {
      var deferred = Q.defer();
      this.pool.drivers.forEach(function (d) {
        d.unlock(this.key).then(deferred.resolve);
      }.bind(this));
      return deferred.promise;
    }

    return Item;
  })();

  Stash.Pool = (function () {
    function Pool(drivers) {
      if (!drivers) {
        this.drivers = [new Stash.Drivers.Ephemeral()];
      } else if (Object.prototype.toString.call(drivers) !== '[object Array]') {
        this.drivers = [drivers];
      } else {
        this.drivers = drivers;
      }
    }

    Pool.prototype.getItem = function (key) {
      var item = new Stash.Item(key, this);

      return item;
    };

    Pool.prototype.flush = function () {
      this.drivers.forEach(function (driver) {
        driver.flush();
      });
    };

    Pool.prototype.get = function (key) {
      var item = this.getItem(key);
      var deferred = Q.defer();

      item.get().then(function (data) {
        item.isMiss().then(function (missed) {
          if (missed) {
            item.lock();
            deferred.reject(item.set.bind(item));
          } else {
            deferred.resolve(data);
          }
        });
      });

      return deferred.promise;
    };

    return Pool;
  })();

  Stash.Drivers = {};
  Stash.Drivers.Utils = {
    assemble: function (value, expiration) {
      if (typeof value === 'function') {
        throw new TypeError('Only serializables values can be cached');
      }

      return JSON.stringify({ value: value, expiration: expiration });
    },
    key: function (namespace, key) {
      if (key instanceof Array) {
        key.unshift(namespace);
        return key.filter(String).join('/');
      } else {
       key = key.toString()
        .trim()
        .replace(/^\/+/g, '')
        .replace(/\/+$/g, '');
       return namespace + '/' + key;
      }
    }
  };

  Stash.Drivers.Ephemeral = (function () {
    var cache = {};
    function Ephemeral () {}

    Ephemeral.prototype.get = function (key) {
      key = Stash.Drivers.Utils.key('', key);
      var data = cache[key] || null;

      if (data) {
        data = JSON.parse(data);
      }

      return Q(data);
    };

    Ephemeral.prototype.put = function (key, value, expiration) {
      key = Stash.Drivers.Utils.key('', key);
      var data = Stash.Drivers.Utils.assemble(value, expiration);
      cache[key] = data;
      return Q();
    };

    Ephemeral.prototype.delete = function (key) {
      key = Stash.Drivers.Utils.key('', key);
      var length = key.length;

      Object.keys(cache).forEach(function (_key) {
        if (_key.substr(0, length) === key) {
          cache[_key] = null;
        }
      });

      cache[key] = null;
      return Q();
    };

    Ephemeral.prototype.flush = function () {
      cache = {};
      return Q();
    };

    Ephemeral.prototype.lock = function (key) {
      var key = Stash.Drivers.Utils.key('', key) + '_lock';
      cache[key] = 1;
      return Q();
    };

    Ephemeral.prototype.isLocked = function (key) {
      key = Stash.Drivers.Utils.key('', key) + '_lock';
      return Q(Boolean(cache[key]));
    }

    Ephemeral.prototype.unlock = function (key) {
      key = Stash.Drivers.Utils.key('', key) + '_lock';
      cache[key] = null;
      return Q();
    };

    return Ephemeral;
  })();

  Stash.Drivers.LocalStorage = (function () {
    function LocalStorage (namespace) {
      this.namespace = namespace || 'stash';
    }

    LocalStorage.prototype.get = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var data = localStorage.getItem(key);

      if (data) {
        data = JSON.parse(data);
      }

      return Q(data);
    }

    LocalStorage.prototype.put = function (key, value, expiration) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var data = Stash.Drivers.Utils.assemble(value, expiration);

      return this.putRaw(key, data);
    };

    LocalStorage.prototype.putRaw = function (key, value) {
      localStorage.setItem(key, value);
      return Q();
    }

    LocalStorage.prototype.delete = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var length = key.length;

      for (var i = 0, l = localStorage.length; i < l; i++) {
        var _key = localStorage.key(i);
        if (_key && _key.substr(0, length) === key) {
          localStorage.removeItem(_key);
        }
      }

      localStorage.removeItem(key);
      return Q();
    };

    LocalStorage.prototype.flush = function () {
      return this.delete('');
    };

    LocalStorage.prototype.lock = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key) + '_lock';
      return this.putRaw(key, 1);
    };

    LocalStorage.prototype.isLocked = function (key) {
      var deferred = Q.defer();
      this.get(key + '_lock').then(function (data) {
        deferred.resolve(Boolean(data));
      });

      return deferred.promise;
    };

    LocalStorage.prototype.unlock = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key) + '_lock';
      localStorage.removeItem(key);

      return Q();
    }

    return LocalStorage;
  })();

  if (isNode) {
    module.exports = Stash;
  } else {
    global.Stash = Stash;
  }
})(this);
<commit_msg>cache lock state<commit_after>(function (global) {
  var Stash = global.Stash || {};
  var isNode = typeof require !== 'undefined';

  if (isNode) {
    Q = require('q');
  }

  Stash.Item = (function () {
    function Item(key, pool) {
      this.key = key;
      this.pool = pool;

      this._unload_();
    }

    Item.SP_NONE = 1;
    Item.SP_OLD = 2;
    Item.SP_PRECOMPUTE = 4;
    Item.SP_VALUE = 8;

    Item.prototype._unload_ = function () {
      this.value = null;
      this.expiration = false;
      this.locked = undefined;
    };

    Item.prototype._load_ = function (callback) {
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        return d.get(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype._write_ = function (callback) {
      var deferred = Q.defer();
      this.pool.drivers.reverse().forEach(function (d) {
        return d.put(this.key, this.value, this.expiration).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item._calculateExpiration_ = function (expiration) {
      if (typeof(expiration) === 'number') {
        expiration *= 1000;
        expiration += Date.now();
      } else if (expiration instanceof Date) {
        expiration = expiration.getTime();
      }

      return expiration;
    };

    Item.prototype.get = function (cachePolicy, policyData) {
      var deferred = Q.defer();
      this.cachePolicy = cachePolicy || Stash.Item.SP_NONE;
      this.policyData = policyData;

      if ((cachePolicy & Stash.Item.SP_VALUE) && this.locked) {
        deferred.resolve(policyData);
      }

      this._load_().then(function (data) {
        if (data) {
          this.value = data.value;
          this.expiration = data.expiration;
          data = data.value || null;
        }
        deferred.resolve(data);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.set = function (value, expiration, callback) {
      if (typeof expiration === 'function') {
        callback = expiration;
        expiration = null;
      }

      this._unload_();

      this.expiration = Item._calculateExpiration_(expiration);
      this.value = value;

      this.unlock();
      return this._write_();
    };

    Item.prototype.isMiss = function (callback) {
      var deferred = Q.defer();

      var  resolve = function (locked) {
        var miss;

        if (locked && (this.cachePolicy & Stash.Item.SP_OLD)) {
          miss = false;
        } else if (!locked &&
                   (this.cachePolicy & Stash.Item.SP_PRECOMPUTE) &&
                   this.policyData * 1000 >= this.expiration - Date.now()) {
          miss =  true;
        } else {
          miss = this.value === null ||
            typeof(this.expiration) === 'number' &&
            this.expiration < Date.now();
        }

        deferred.resolve(miss);
      }.bind(this);

      if (this.locked !== undefined) {
        resolve(this.locked);
      } else {
        this.isLocked().then(resolve);
      }

      return deferred.promise;
    };

    Item.prototype.clear = function () {
      var deferred = Q.defer();
      this._unload_();

      this.pool.drivers.forEach(function (driver) {
        driver.delete(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.lock = function () {
      if (this.locked === true) {
        return Q(this.locked);
      }

      this.locked = true;
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        d.lock(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    };

    Item.prototype.isLocked = function () {
      var deferred = Q.defer();

      if (this.locked !== undefined) {
        deferred.resolve(this.locked);
      } else {
        this.pool.drivers.forEach(function (d) {
          d.isLocked(this.key).then(function (locked) {
            this.locked = locked;
            deferred.resolve(locked);
          }.bind(this));
        }.bind(this));
      }

      return deferred.promise;
    };

    Item.prototype.unlock = function () {
      if (this.locked === false) {
        return Q();
      }

      this.locked = false;
      var deferred = Q.defer();

      this.pool.drivers.forEach(function (d) {
        d.unlock(this.key).then(deferred.resolve);
      }.bind(this));

      return deferred.promise;
    }

    return Item;
  })();

  Stash.Pool = (function () {
    function Pool(drivers) {
      if (!drivers) {
        this.drivers = [new Stash.Drivers.Ephemeral()];
      } else if (Object.prototype.toString.call(drivers) !== '[object Array]') {
        this.drivers = [drivers];
      } else {
        this.drivers = drivers;
      }
    }

    Pool.prototype.getItem = function (key) {
      var item = new Stash.Item(key, this);

      return item;
    };

    Pool.prototype.flush = function () {
      this.drivers.forEach(function (driver) {
        driver.flush();
      });
    };

    Pool.prototype.get = function (key) {
      var item = this.getItem(key);
      var deferred = Q.defer();

      item.get().then(function (data) {
        item.isMiss().then(function (missed) {
          if (missed) {
            item.lock();
            deferred.reject(item.set.bind(item));
          } else {
            deferred.resolve(data);
          }
        });
      });

      return deferred.promise;
    };

    return Pool;
  })();

  Stash.Drivers = {};
  Stash.Drivers.Utils = {
    assemble: function (value, expiration) {
      if (typeof value === 'function') {
        throw new TypeError('Only serializables values can be cached');
      }

      return JSON.stringify({ value: value, expiration: expiration });
    },
    key: function (namespace, key) {
      if (key instanceof Array) {
        key.unshift(namespace);
        return key.filter(String).join('/');
      } else {
       key = key.toString()
        .trim()
        .replace(/^\/+/g, '')
        .replace(/\/+$/g, '');
       return namespace + '/' + key;
      }
    }
  };

  Stash.Drivers.Ephemeral = (function () {
    var cache = {};
    function Ephemeral () {}

    Ephemeral.prototype.get = function (key) {
      key = Stash.Drivers.Utils.key('', key);
      var data = cache[key] || null;

      if (data) {
        data = JSON.parse(data);
      }

      return Q(data);
    };

    Ephemeral.prototype.put = function (key, value, expiration) {
      key = Stash.Drivers.Utils.key('', key);
      var data = Stash.Drivers.Utils.assemble(value, expiration);
      cache[key] = data;
      return Q();
    };

    Ephemeral.prototype.delete = function (key) {
      key = Stash.Drivers.Utils.key('', key);
      var length = key.length;

      Object.keys(cache).forEach(function (_key) {
        if (_key.substr(0, length) === key) {
          cache[_key] = null;
        }
      });

      cache[key] = null;
      return Q();
    };

    Ephemeral.prototype.flush = function () {
      cache = {};
      return Q();
    };

    Ephemeral.prototype.lock = function (key) {
      var key = Stash.Drivers.Utils.key('', key) + '_lock';
      cache[key] = 1;
      return Q();
    };

    Ephemeral.prototype.isLocked = function (key) {
      key = Stash.Drivers.Utils.key('', key) + '_lock';
      return Q(Boolean(cache[key]));
    }

    Ephemeral.prototype.unlock = function (key) {
      key = Stash.Drivers.Utils.key('', key) + '_lock';
      cache[key] = null;
      return Q();
    };

    return Ephemeral;
  })();

  Stash.Drivers.LocalStorage = (function () {
    function LocalStorage (namespace) {
      this.namespace = namespace || 'stash';
    }

    LocalStorage.prototype.get = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var data = localStorage.getItem(key);

      if (data) {
        data = JSON.parse(data);
      }

      return Q(data);
    }

    LocalStorage.prototype.put = function (key, value, expiration) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var data = Stash.Drivers.Utils.assemble(value, expiration);

      return this.putRaw(key, data);
    };

    LocalStorage.prototype.putRaw = function (key, value) {
      localStorage.setItem(key, value);
      return Q();
    }

    LocalStorage.prototype.delete = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key);
      var length = key.length;

      for (var i = 0, l = localStorage.length; i < l; i++) {
        var _key = localStorage.key(i);
        if (_key && _key.substr(0, length) === key) {
          localStorage.removeItem(_key);
        }
      }

      localStorage.removeItem(key);
      return Q();
    };

    LocalStorage.prototype.flush = function () {
      return this.delete('');
    };

    LocalStorage.prototype.lock = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key) + '_lock';
      return this.putRaw(key, 1);
    };

    LocalStorage.prototype.isLocked = function (key) {
      var deferred = Q.defer();
      this.get(key + '_lock').then(function (data) {
        deferred.resolve(Boolean(data));
      });

      return deferred.promise;
    };

    LocalStorage.prototype.unlock = function (key) {
      key = Stash.Drivers.Utils.key(this.namespace, key) + '_lock';
      localStorage.removeItem(key);

      return Q();
    }

    return LocalStorage;
  })();

  if (isNode) {
    module.exports = Stash;
  } else {
    global.Stash = Stash;
  }
})(this);
<|endoftext|>


<commit_before>// Whole-script strict mode syntax
"use strict";

var request = require('request');
var blessed = require('blessed');
var contrib = require('blessed-contrib');
var GitHub = require("github");

var github = new GitHub({
    // required
    version: "3.0.0",
    // optional
    debug: true,
    protocol: "https",
    host: "api.github.com", // should be api.github.com for GitHub
    timeout: 5000
});

// set up a grid with two rows and one column

var screen = blessed.screen();
var grid = new contrib.grid({rows: 2, cols: 2, screen: screen});

// create a table for bugzilla bugs

var bugList = grid.set(0, 0, 2, 1, contrib.table, {
    keys: true,
    fg: 'white',
    label: 'wadi bugs',
    columnWidth: [10, 20, 10, 150],
    columnSpacing: 3
});

var bugListData = {
    headers: ['id', 'assignee', 'status', 'title'],
    data: [ ]
};

screen.append(bugList);
bugList.setData(bugListData);

// create table for github activity

var githubActivity = grid.set(0, 1, 2, 1, contrib.table, {
    keys: true,
    fg: 'white',
    label: 'github activity',
    columnWidth: [10, 10, 150],
    columnSpacing: 3
});

var githubActivityData = {
    headers: ['date', 'actor', 'title'],
    data: [ ]
};

screen.append(githubActivity);
githubActivity.setData(githubActivityData);

// make sure its possible to exit

screen.key(['escape', 'q', 'C-c'], function(ch, key) {
    return process.exit(0);
});

// go find all the bugs we are tracking for WADI

// TODO get all history https://bugzilla.mozilla.org/rest/bug/707428/history

request("http://bugzilla.mozilla.org/rest/bug/1201717", function(error, response, body) {
    var tracker = JSON.parse(body);
    var depends_on_list = tracker.bugs[0].depends_on;

    // loop through the list of tracked bug ID's
    for (var bugIndex in depends_on_list) {
        var bugID = depends_on_list[bugIndex];

        // and for each tracked bug ID, go find info for that bug

        request("http://bugzilla.mozilla.org/rest/bug/" + depends_on_list[bugIndex], function(error, response, body) {
            var trackedBug = JSON.parse(body).bugs[0];

            // add info about that tracked bug to the table
            // NOTE: do not update the display now, it will get updated later
            var assignee = '';
            if (trackedBug.assigned_to != 'nobody@mozilla.org') {
                assignee = trackedBug.assigned_to;
            }

            bugListData.data.push([trackedBug.id, assignee, trackedBug.status, trackedBug.summary]);
            bugList.setData(bugListData);
        });
    }

});

// go find all the activity for wadi repo's

// TODO: combine multiple event streams sort by date

github.events.getFromRepo( { 'user': 'mozilla', 'repo': 'oghliner' }, function(err, activities) {
    try {
        if (err) {
            throw new Error(err.message);
        }

        for (var activityIndex in activities) {
            var anActivity = activities[activityIndex];

            var activityDescription = anActivity.type;
            var activityActor = '';

            if (anActivity.type == 'IssueCommentEvent') {
                activityActor = anActivity.payload.comment.user.login;
                activityDescription = anActivity.payload.comment.body;
            } else if (anActivity.type == 'IssuesEvent') {
                activityActor = anActivity.payload.issue.user.login;
                activityDescription = anActivity.payload.issue.body;
            } else if (anActivity.type == 'PullRequestEvent') {
                activityActor = anActivity.payload.pull_request.user.login;
                activityDescription = anActivity.payload.pull_request.title;
            } else if (anActivity.type == 'PullRequestReviewCommentEvent') {
                activityActor = anActivity.payload.comment.user.login;
                activityDescription = anActivity.payload.comment.body;
            } else if (anActivity.type == 'PushEvent') {
                activityActor = anActivity.payload.commits[0].author.email;
                activityDescription = anActivity.payload.commits[0].message;
            } else if (anActivity.type == 'CreateEvent') {
                activityActor = '';
                activityDescription = anActivity.payload.description;
            }

            if (anActivity.type) {
                githubActivityData.data.push([anActivity.created_at.substring(0,10), activityActor, activityDescription]);
                githubActivity.setData(githubActivityData);
            }
        }
    }
    catch(e) {
        console.log('cannot parse events json', e);
        console.log(e.stack);
    }
});

// on a recurring basis, update the display of random values and re-render

setInterval(function() {
    screen.render();
}, 5000);<commit_msg>use personal access token read from environment variable to authenticate<commit_after>// Whole-script strict mode syntax
"use strict";

var request = require('request');
var GitHub = require("github");
var blessed = require('blessed');
var contrib = require('blessed-contrib');

var github = new GitHub({
    // required
    version: "3.0.0",
    // optional
    debug: true,
    protocol: "https",
    host: "api.github.com",
    timeout: 5000
});

console.log();

github.authenticate({
    type: "oauth",
    token: process.env.SEKRIT
});

// set up a grid with two rows and one column

var screen = blessed.screen();
var grid = new contrib.grid({rows: 2, cols: 2, screen: screen});

// create a table for bugzilla bugs

var bugList = grid.set(0, 0, 2, 1, contrib.table, {
    keys: true,
    fg: 'white',
    label: 'wadi bugs',
    columnWidth: [10, 20, 10, 150],
    columnSpacing: 3
});

var bugListData = {
    headers: ['id', 'assignee', 'status', 'title'],
    data: [ ]
};

screen.append(bugList);
bugList.setData(bugListData);

// create table for github activity

var githubActivity = grid.set(0, 1, 2, 1, contrib.table, {
    keys: true,
    fg: 'white',
    label: 'github activity',
    columnWidth: [10, 10, 150],
    columnSpacing: 3
});

var githubActivityData = {
    headers: ['date', 'actor', 'title'],
    data: [ ]
};

screen.append(githubActivity);
githubActivity.setData(githubActivityData);

// make sure its possible to exit

screen.key(['escape', 'q', 'C-c'], function(ch, key) {
    return process.exit(0);
});

// go find all the bugs we are tracking for WADI

// TODO get all history https://bugzilla.mozilla.org/rest/bug/707428/history

request("http://bugzilla.mozilla.org/rest/bug/1201717", function(error, response, body) {
    var tracker = JSON.parse(body);
    var depends_on_list = tracker.bugs[0].depends_on;

    // loop through the list of tracked bug ID's
    for (var bugIndex in depends_on_list) {
        var bugID = depends_on_list[bugIndex];

        // and for each tracked bug ID, go find info for that bug

        request("http://bugzilla.mozilla.org/rest/bug/" + depends_on_list[bugIndex], function(error, response, body) {
            var trackedBug = JSON.parse(body).bugs[0];

            // add info about that tracked bug to the table
            // NOTE: do not update the display now, it will get updated later
            var assignee = '';
            if (trackedBug.assigned_to != 'nobody@mozilla.org') {
                assignee = trackedBug.assigned_to;
            }

            bugListData.data.push([trackedBug.id, assignee, trackedBug.status, trackedBug.summary]);
            bugList.setData(bugListData);
        });
    }

});

// go find all the activity for wadi repo's

// TODO: combine multiple event streams sort by date

github.events.getFromRepo( { 'user': 'mozilla', 'repo': 'oghliner', per_page: 100 }, function(err, activities) {
    try {
        if (err) {
            throw new Error(err.message);
        }

        for (var activityIndex in activities) {
            var anActivity = activities[activityIndex];

            var activityDescription = anActivity.type;
            var activityActor = '';

            if (anActivity.type == 'IssueCommentEvent') {
                activityActor = anActivity.payload.comment.user.login;
                activityDescription = anActivity.payload.comment.body;
            } else if (anActivity.type == 'IssuesEvent') {
                activityActor = anActivity.payload.issue.user.login;
                activityDescription = anActivity.payload.issue.body;
            } else if (anActivity.type == 'PullRequestEvent') {
                activityActor = anActivity.payload.pull_request.user.login;
                activityDescription = anActivity.payload.pull_request.title;
            } else if (anActivity.type == 'PullRequestReviewCommentEvent') {
                activityActor = anActivity.payload.comment.user.login;
                activityDescription = anActivity.payload.comment.body;
            } else if (anActivity.type == 'PushEvent') {
                activityActor = anActivity.payload.commits[0].author.email;
                activityDescription = anActivity.payload.commits[0].message;
            } else if (anActivity.type == 'CreateEvent') {
                activityActor = '';
                activityDescription = anActivity.payload.description;
            }

            if (anActivity.type) {
                githubActivityData.data.push([anActivity.created_at.substring(0,10), activityActor, activityDescription]);
                githubActivity.setData(githubActivityData);
            }
        }
    }
    catch(e) {
        console.log('cannot parse events json', e);
        console.log(e.stack);
    }
});

// on a recurring basis, update the display of random values and re-render

setInterval(function() {
    screen.render();
}, 5000);<|endoftext|>


<commit_before>/**
 * @typedef {Object} WaveSettings
 * @property {number} numSamples
 * @property {number} splashiness
 * @property {number} damping
 * @property {Array.<ChildWaveSettings>} children
 */

/**
 * Defines a sin wave. This is used to create general
 * turbulence when CHURN is called.
 *
 * @typedef {Object} ChildWaveSettings
 * @property {number} amplitude
 * @property {number} period
 */

var DEFAULT_NUM_SAMPLES = 100;
var DEFAULT_SPLASHINESS = 0.01;
var DEFAULT_DAMPING = 0.995;
var DEFAULT_CHILDREN = [
    {period: 0.25, amplitude: 0.2},
    {period: 0.5, amplitude: 0.6},
    {period: 0.3, amplitude: 0.2}
];

/**
 * Defines a single wave with the given settings.
 * @param {WaveSettings=} settings
 */
function wave(settings) {
    if (settings === undefined) {
        settings = {}
    }

    // WAVE SETTINGS
    var _n = settings.numSamples || DEFAULT_NUM_SAMPLES;
    var _children = _normalizeAmplitudes(settings.children) || DEFAULT_CHILDREN;
    var _splashiness = settings.splashiness || DEFAULT_SPLASHINESS;
    var _damping = settings.damping || DEFAULT_DAMPING;

    // VARS FOR THE WAVE PDE
    var u = new Float32Array(_n); // value of wave at each sample idx
    var u_t = new Float32Array(_n); // velocity at each sample idx
    var u_x = new Float32Array(_n - 1);
    var u_tt = new Float32Array(_n - 2);

    /*-----------------*/
    /* PRIVATE METHODS */
    /*-----------------*/

    function _normalizeAmplitudes(children) {
        if (!children) {
            return;
        }

        var totalAmplitude = 0;
        children.forEach(function(childWaveSettings) {
            totalAmplitude += childWaveSettings.amplitude;
        });
        children.forEach(function(childWaveSettings) {
            childWaveSettings.amplitude /= totalAmplitude;
        });
    }

    /**
     * Computes the height of the wave at sample i.
     *
     * @private
     * @param {number} i - index of the sample to compute the wave height for
     * @param {nubmer} crestIdx - index of the crest of the wave
     * @param {WaveSettings} settings
     */
    function _childHeight(i, childWaveSettings, y, phaseShift) {
        // x position of the given sample idx as a float in [0, 1]
        var x = i / _n;

        // sine wave params
        var amplitude = childWaveSettings.amplitude;
        var period = childWaveSettings.period;

        // returns the y value of the child wave at sample i, shifted so that
        // a peak occurs at the current well's index
        return 0.1 * y * amplitude * Math.sin((2 * Math.PI / period) * (x - phaseShift));
    };

    /*-----------------*/
    /*  PUBLIC METHODS */
    /*-----------------*/

    /**
     * Update the height map according to the wave equation.
     * @param {number} dt
     */
    function tick(dt) {
        if (dt === undefined) {
            dt = 0.1;
        }
        var dx = 1. / _n; // change in x per value in u

        // compute dx for each increment of x
        for (var i = 0; i < _n - 1; i++) {
            u_x[i] = (u[i + 1] - u[i]) / dx;
        }

        var c2 = _splashiness * _splashiness; // Constant C^2
        for (var i = 0; i < _n - 2; i++) {
            // compute second derivative wrt x
            var u_xx_i = (u_x[i + 1] - u_x[i]) / dx;

            // then use it to compute second derivative wrt t
            u_tt[i] = u_xx_i * c2;
        }

        // update u_t according to u_tt
        for (var i = 1; i < _n - 1; i++) {
            u_t[i] += u_tt[i - 1] * dt;
            u_t[i] *= _damping;
        }

        // update u
        for (var i = 0; i < _n; i++) {
            u[i] += u_t[i] * dt;
        }

        // cheat to send the first and last sample
        u[_n - 1] = u[_n - 2];
        u[0] = u[1];
    };

    /**
     * Create a splash in the wave.
     *
     * @param {number} x - 0 to 1
     * @param {number} y - -1 to 1
     */
    function splash(x, y, strength) {
        if (strength === undefined) {
            strength = 1;
        }
        var ix = Math.floor(x * _n);
        var totalVolume = 0;
        var peak = _clamp((y - u[ix]) * 0.5, -0.1, 0.1);
        var gaussConstant = Math.max(0.001 / Math.abs(peak), 0.01);
        var halfNumSamples = Math.floor(_n / 2);
        for(var i = -halfNumSamples; i < halfNumSamples; i++) {
            if(ix + i < 0 || ix + i > _n) {
                continue;
            }
            var gauss = strength * (Math.exp(-gaussConstant * i * i) * peak);
            u_t[ix + i] += gauss;
            u[ix + i] += gauss;
            totalVolume += gauss;
        }
        var averageVolume = totalVolume / _n;
        for(var i = 0; i < _n; i++) {
            u_t[i] -= averageVolume;
            u[i] -= averageVolume;
        }
    }

    /**
     * Create general turbulence, centered around the given x, y coords
     *
     * @param {number} x - 0 to 1
     * @param {number} y - -1 to 1
     */
    function churn(x, y, strength) {
        if (strength === undefined) {
            strength = 1;
        }
        // compute shift in phase so that the upwards peak is at the given x
        var phaseShift = x - Math.PI * 0.5;

        // Compute the height of each child wave at each sample
        // and add the offset to the current height map
        var totalVolume = 0;
        for(var i = 0; i < _n; i++) {
            _children.forEach(function(childWaveSettings) {
                var h = strength * _childHeight(i, childWaveSettings, y, phaseShift);
                u_t[i] += h;
                u[i] += h;
                totalVolume += h;
            });
        }

        // Make sure the wave level stays the same. The sum of all
        // heights at each sample should be 0, at the neutral level.
        var averageVolume = totalVolume / _n;
        for (var i = 0; i < _n; i++) {
            u_t[i] -= averageVolume;
            u[i] -= averageVolume;
        }
    }

    /**
     * Grab the height of the wave at the given sample index.
     *
     * @param {number} idx - Index of sample to grab the height for
     * @return {number} y coordinate for the wave at the sample idx. Returns a
     *      Float in [-1, 1], where 0 is the neutral level.
     */
    function height(idx) {
        return u[_clamp(idx, 0, _n - 1)];
    };

    return {
        // actions
        tick: tick,
        splash: splash,
        churn: churn,

        // getters
        height: height
    }
}

/**
 * Clamps x to [min, max]
 */
function _clamp(x, min, max) {
    return Math.max(min, Math.min(max, x))
}
<commit_msg>_normalizeChildren is now fixed. Added a volume normalizer per tick<commit_after>/**
 * @typedef {Object} WaveSettings
 * @property {number} numSamples
 * @property {number} splashiness
 * @property {number} damping
 * @property {Array.<ChildWaveSettings>} children
 */

/**
 * Defines a sin wave. This is used to create general
 * turbulence when CHURN is called.
 *
 * @typedef {Object} ChildWaveSettings
 * @property {number} amplitude
 * @property {number} period
 */

var DEFAULT_NUM_SAMPLES = 100;
var DEFAULT_SPLASHINESS = 0.01;
var DEFAULT_DAMPING = 0.995;
var DEFAULT_CHILDREN = [
    {period: 0.25, amplitude: 0.2},
    {period: 0.5, amplitude: 0.6},
    {period: 0.3, amplitude: 0.2}
];

/**
 * Defines a single wave with the given settings.
 * @param {WaveSettings=} settings
 */
function wave(settings) {
    if (settings === undefined) {
        settings = {}
    }

    // WAVE SETTINGS
    var _n = settings.numSamples || DEFAULT_NUM_SAMPLES;
    var _children = _normalizeAmplitudes(settings.children) || DEFAULT_CHILDREN;
    var _splashiness = settings.splashiness || DEFAULT_SPLASHINESS;
    var _damping = settings.damping || DEFAULT_DAMPING;

    // VARS FOR THE WAVE PDE
    var u = new Float32Array(_n); // value of wave at each sample idx
    var u_t = new Float32Array(_n); // velocity at each sample idx
    var u_x = new Float32Array(_n - 1);
    var u_tt = new Float32Array(_n - 2);

    /*-----------------*/
    /* PRIVATE METHODS */
    /*-----------------*/

    function _normalizeAmplitudes(children) {
        if (!children) {
            return;
        }

        var totalAmplitude = 0;
        children.forEach(function(childWaveSettings) {
            totalAmplitude += childWaveSettings.amplitude;
        });
        children.forEach(function(childWaveSettings) {
            childWaveSettings.amplitude /= totalAmplitude;
        });
        return children;
    }

    function _normalizeVolume() {
        // total height should always add up to 0
        var sum = 0;
        for (var i = 0; i < _n; i++) {
            sum += u[i];
        }
        var avgDiff = sum / _n;
        for (var i = 0; i < _n; i++) {
            u[i] -= avgDiff;
        }
    }

    /**
     * Computes the height of the wave at sample i.
     *
     * @private
     * @param {number} i - index of the sample to compute the wave height for
     * @param {nubmer} crestIdx - index of the crest of the wave
     * @param {WaveSettings} settings
     */
    function _childHeight(i, childWaveSettings, y, phaseShift) {
        // x position of the given sample idx as a float in [0, 1]
        var x = i / _n;

        // sine wave params
        var amplitude = childWaveSettings.amplitude;
        var period = childWaveSettings.period;

        // returns the y value of the child wave at sample i, shifted so that
        // a peak occurs at the current well's index
        return 0.1 * y * amplitude * Math.sin((2 * Math.PI / period) * (x - phaseShift));
    };

    /*-----------------*/
    /*  PUBLIC METHODS */
    /*-----------------*/

    /**
     * Update the height map according to the wave equation.
     * @param {number} dt
     */
    function tick(dt) {
        if (dt === undefined) {
            dt = 0.1;
        }
        _normalizeVolume();
        var dx = 1. / _n; // change in x per value in u

        // compute dx for each increment of x
        for (var i = 0; i < _n - 1; i++) {
            u_x[i] = (u[i + 1] - u[i]) / dx;
        }

        var c2 = _splashiness * _splashiness; // Constant C^2
        for (var i = 0; i < _n - 2; i++) {
            // compute second derivative wrt x
            var u_xx_i = (u_x[i + 1] - u_x[i]) / dx;

            // then use it to compute second derivative wrt t
            u_tt[i] = u_xx_i * c2;
        }

        // update u_t according to u_tt
        for (var i = 1; i < _n - 1; i++) {
            u_t[i] += u_tt[i - 1] * dt;
            u_t[i] *= _damping;
        }

        // update u
        for (var i = 0; i < _n; i++) {
            u[i] += u_t[i] * dt;
        }

        // cheat to send the first and last sample
        u[_n - 1] = u[_n - 2];
        u[0] = u[1];
    };

    /**
     * Create a splash in the wave.
     *
     * @param {number} x - 0 to 1
     * @param {number} y - -1 to 1
     */
    function splash(x, y, strength) {
        if (strength === undefined) {
            strength = 1;
        }
        var ix = Math.floor(x * _n);
        var totalVolume = 0;
        var peak = _clamp((y - u[ix]) * 0.5, -0.1, 0.1);
        var gaussConstant = Math.max(0.001 / Math.abs(peak), 0.01);
        var halfNumSamples = Math.floor(_n / 2);
        for(var i = -halfNumSamples; i < halfNumSamples; i++) {
            if(ix + i < 0 || ix + i > _n) {
                continue;
            }
            var gauss = strength * (Math.exp(-gaussConstant * i * i) * peak);
            u_t[ix + i] += gauss;
            u[ix + i] += gauss;
            totalVolume += gauss;
        }
        var averageVolume = totalVolume / _n;
        for(var i = 0; i < _n; i++) {
            u_t[i] -= averageVolume;
            u[i] -= averageVolume;
        }
    }

    /**
     * Create general turbulence, centered around the given x, y coords
     *
     * @param {number} x - 0 to 1
     * @param {number} y - -1 to 1
     */
    function churn(x, y, strength) {
        if (strength === undefined) {
            strength = 1;
        }
        // compute shift in phase so that the upwards peak is at the given x
        var phaseShift = x - Math.PI * 0.5;

        // Compute the height of each child wave at each sample
        // and add the offset to the current height map
        for(var i = 0; i < _n; i++) {
            _children.forEach(function(childWaveSettings) {
                var h = strength * _childHeight(i, childWaveSettings, y, phaseShift);
                u_t[i] += h;
                u[i] += h;
            });
        }

        _normalizeVolume();
    }

    /**
     * Grab the height of the wave at the given sample index.
     *
     * @param {number} idx - Index of sample to grab the height for
     * @return {number} y coordinate for the wave at the sample idx. Returns a
     *      Float in [-1, 1], where 0 is the neutral level.
     */
    function height(idx) {
        return u[_clamp(idx, 0, _n - 1)];
    };

    return {
        // actions
        tick: tick,
        splash: splash,
        churn: churn,

        // getters
        height: height
    }
}

/**
 * Clamps x to [min, max]
 */
function _clamp(x, min, max) {
    return Math.max(min, Math.min(max, x))
}
<|endoftext|>


<commit_before>// for osm-replication-streams on Node < 8
require('babel-polyfill')
require('core-js/features/array/flat')

const _ = require('highland')
const async = require('async')
const env = require('require-env')
const htmlEntities = require('html-entities').AllHtmlEntities
const OSMParser = require('osm2obj')
const {
  parsers: { AugmentedDiffParser },
  sources: { AugmentedDiffs, Changesets }
} = require('osm-replication-streams')
const { Pool } = require('pg')

const { NOOP } = require('.')
const StatsStream = require('./stats_stream')

const OVERPASS_URL = process.env.OVERPASS_URL || 'http://overpass-api.de';

const pool = new Pool({
  connectionString: env.require('DATABASE_URL')
})

pool.on('error', err => {
  console.warn('Unexpected error on idle client', err)
  throw err
})

const query = (q, params, callback) =>
  pool.connect((err, client, release) => {
    if (typeof params === 'function') {
      callback = params
      params = null
    }

    callback = callback || NOOP

    if (err) {
      console.warn(err)
      release()
      return callback(err)
    }

    return client.query(q, params, (err, results) => {
      release()
      return callback(err, results)
    })
  })

const join = (a, b) => {
  if (typeof a === 'number' || typeof b === 'number') {
    return (a || 0) + (b || 0)
  }

  if (Array.isArray(a) || Array.isArray(b)) {
    return Array.from(new Set(a || [].concat(b)))
  }

  throw new Error(`Don't know how to join: '${a}', '${b}'`)
};

const merge = (a, b) =>
  Object.keys(b).reduce(
    (acc, k) => Object.assign({}, acc, { [k]: join(acc[k], b[k]) }),
    a
  )

const summarizer = batch =>
  batch.reduce(
    (acc, { changeset, stats, uid, user }) =>
      Object.assign({}, acc, {
        [changeset]: Object.assign({}, merge(acc[changeset] || {}, stats), {
          uid,
          user
        })
      }),
    {}
  )

const statUpdater = stats => {
  const lastAugmentedDiff = Math.max.apply(
    null,
    Array.from(
      new Set(
        Object.keys(stats).reduce(
          (acc, k) => acc.concat(stats[k].augmentedDiffs),
          []
        )
      )
    )
  )

  console.warn(`Checkpointing augmented diffs @ ${lastAugmentedDiff}.`)

  const queries = Object.keys(stats)
    .map(changeset =>
      [
        [
          `
INSERT INTO raw_changesets AS c (
  id,
  user_id,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  augmented_diffs,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  roads_added = c.roads_added + $3,
  roads_modified = c.roads_modified + $4,
  waterways_added = c.waterways_added + $5,
  waterways_modified = c.waterways_modified + $6,
  buildings_added = c.buildings_added + $7,
  buildings_modified = c.buildings_modified + $8,
  pois_added = c.pois_added + $9,
  pois_modified = c.pois_modified + $10,
  road_km_added = c.road_km_added + $11,
  road_km_modified = c.road_km_modified + $12,
  waterway_km_added = c.waterway_km_added + $13,
  waterway_km_modified = c.waterway_km_modified + $14,
  augmented_diffs = coalesce(c.augmented_diffs, ARRAY[]::integer[]) || $15,
  updated_at = current_timestamp
WHERE c.id = $1
  AND NOT coalesce(c.augmented_diffs, ARRAY[]::integer[]) && $15
          `,
          [
            Number(changeset),
            Number(stats[changeset].uid),
            stats[changeset].roadsAdded || 0,
            stats[changeset].roadsModified || 0,
            stats[changeset].waterwaysAdded || 0,
            stats[changeset].waterwaysModified || 0,
            stats[changeset].buildingsAdded || 0,
            stats[changeset].buildingsModified || 0,
            stats[changeset].poisAdded || 0,
            stats[changeset].poisModified || 0,
            stats[changeset].roadKmAdded || 0,
            stats[changeset].roadKmModified || 0,
            stats[changeset].waterwayKmAdded || 0,
            stats[changeset].waterwayKmModified || 0,
            stats[changeset].augmentedDiffs
          ]
        ],
        [
          `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
          [Number(stats[changeset].uid), stats[changeset].user]
        ]
      ]
        .concat(
          stats[changeset].countries.map(code => [
            `
INSERT INTO raw_changesets_countries (
  changeset_id,
  country_id
) VALUES (
  $1, (SELECT id FROM raw_countries WHERE code = $2)
)
ON CONFLICT DO NOTHING
          `,
            [Number(changeset), code]
          ])
        )
        .concat([
          [
            `
UPDATE augmented_diff_status
SET id=$1,
  updated_at=current_timestamp
    `,
            [lastAugmentedDiff]
          ]
        ])
    )
    .reduce((acc, a) => acc.concat(a), [])

  return _(push =>
    async.each(
      queries,
      ([q, params], done) => query(q, params, done),
      err => push(err, _.nil)
    )
  )
};

// from https://github.com/openstreetmap/iD/blob/45ffa3b731463bf4eba52519896ad33653def0cd/modules/ui/commit.js
const HASHTAG_REGEX = /(#[^\u2000-\u206F\u2E00-\u2E7F\s\\'!"#$%()*,./:;<=>?@[\]^`{|}~]+)/g

const extractHashtags = tags => {
  tags.comment = htmlEntities.decode(tags.comment)
  return ((tags.comment || "").match(HASHTAG_REGEX) || []).map(x =>
    x.slice(1).toLowerCase());
}

const changesetUpdater = changeset => {
  const {
    id,
    created_at: createdAt,
    closed_at: closedAt,
    uid,
    user
  } = changeset
  let editor = null
  let hashtags = []

  if (changeset.tags != null) {
    editor = changeset.tags.created_by
    hashtags = extractHashtags(changeset.tags)
  }

  const queries = [
    [
      `
INSERT INTO raw_changesets AS c (
  id,
  editor,
  user_id,
  created_at,
  closed_at,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  editor = $2,
  user_id = $3,
  created_at = $4,
  closed_at = $5,
  updated_at = current_timestamp
WHERE c.id = $1
      `,
      [id, editor, uid, createdAt, closedAt]
    ],
    [
      `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
      [uid, user]
    ]
  ]

  return _(push => {
    async.series(
      [
        async.apply(async.each, queries, ([q, params], done) =>
          query(q, params, done)
        ),
        async.apply(async.each, hashtags, (hashtag, done) =>
          async.waterfall(
            [
              async.apply(
                // https://stackoverflow.com/questions/34708509/how-to-use-returning-with-on-conflict-in-postgresql
                query,
                `
WITH input_rows("hashtag") AS (
  VALUES ($1)
),
ins AS (
  INSERT INTO raw_hashtags AS h (hashtag)
  VALUES ($1)
  ON CONFLICT DO NOTHING
  RETURNING id
)
SELECT id
FROM ins
UNION ALL
SELECT id
FROM input_rows
JOIN raw_hashtags USING(hashtag)
                `,
                [hashtag]
              ),
              (result, fin) => {
                const { id: hashtagId } = result.rows[0]

                return query(
                  `
INSERT INTO raw_changesets_hashtags (
  changeset_id,
  hashtag_id
) VALUES ($1, $2)
ON CONFLICT DO NOTHING
                  `,
                  [id, hashtagId],
                  fin
                )
              }
            ],
            done
          )
        )
      ],
      err => push(err, _.nil)
    )
  })
};

const getInitialAugmentedDiffSequenceNumber = callback =>
  query('SELECT id FROM augmented_diff_status', (err, results) => {
    if (err) {
      return callback(err)
    }

    return callback(null, results.rows[0].id)
  })

const checkpointChangesets = sequenceNumber => {
  console.warn(`Checkpointing changesets @ ${sequenceNumber}.`)

  query(
    `
UPDATE changesets_status
SET id=$1,
  updated_at=current_timestamp
  `,
    [Number(sequenceNumber)],
    err => {
      if (err) {
        console.warn(err.stack)
      }
    }
  )
};

const getInitialChangesetSequenceNumber = callback =>
  query('SELECT id FROM changesets_status', (err, results) => {
    if (err) {
      return callback(err)
    }

    return callback(null, results.rows[0].id)
  })

module.exports = (options, callback) => {
  const opts = Object.assign(
    {},
    {
      delay: 15e3,
      infinite: true
    },
    options
  )

  callback = callback || NOOP

  return async.parallel(
    {
      initialAugmentedDiffSequenceNumber: getInitialAugmentedDiffSequenceNumber,
      initialChangesetSequenceNumber: getInitialChangesetSequenceNumber
    },
    (
      err,
      { initialAugmentedDiffSequenceNumber, initialChangesetSequenceNumber }
    ) => {
      if (err) {
        return callback(err)
      }

      return async.parallel(
        [
          done =>
            _(
              AugmentedDiffs({
                baseURL: OVERPASS_URL,
                initialSequence: initialAugmentedDiffSequenceNumber + 1,
                infinite: opts.infinite,
                delay: opts.delay
              })
                .pipe(new AugmentedDiffParser().on('error', console.warn))
                .pipe(new StatsStream())
            )
              // batch by sequence
              .through(s => {
                let batched = []
                let sequence = null

                return s.consume((err, x, push, next) => {
                  if (err) {
                    push(err)
                    return next()
                  }

                  if (x === _.nil) {
                    // end of the stream; flush
                    if (batched.length > 0) {
                      push(null, batched)
                    }

                    return push(null, _.nil)
                  }

                  let activeSequence = sequence

                  if (x.stats != null) {
                    [activeSequence] = x.stats.augmentedDiffs
                  }

                  if (x.type === 'Marker' || sequence !== activeSequence) {
                    // new sequence; flush previous
                    if (batched.length > 0) {
                      push(null, batched)
                    }

                    // reset batch
                    if (x.type !== 'Marker') {
                      batched = [x]
                    } else {
                      batched = []
                    }
                    sequence = activeSequence

                    return next()
                  }

                  // add this item to the batch
                  batched.push(x)

                  return next()
                })
              })
              .map(summarizer)
              .flatMap(statUpdater)
              .done(() => done),
          done =>
            _(
              Changesets({
                initialSequence: initialChangesetSequenceNumber,
                infinite: opts.infinite,
                // TODO checkpoint when a sequence has been fully processed not just seen
                checkpoint: checkpointChangesets
              }).pipe(
                new OSMParser({
                  coerceIds: true
                })
              )
            )
              .flatMap(changesetUpdater)
              .done(() => done)
        ],
        callback
      )
    }
  )
};
<commit_msg>Revert standardJS lint<commit_after>// for osm-replication-streams on Node < 8
require("babel-polyfill");
require("core-js/features/array/flat");

const _ = require("highland");
const async = require("async");
const env = require("require-env");
const htmlEntities = require('html-entities').AllHtmlEntities;
const OSMParser = require("osm2obj");
const {
  parsers: { AugmentedDiffParser },
  sources: { AugmentedDiffs, Changesets }
} = require("osm-replication-streams");
const { Pool } = require("pg");

const { NOOP } = require(".");
const StatsStream = require("./stats_stream");

const OVERPASS_URL = process.env.OVERPASS_URL || "http://overpass-api.de";

const pool = new Pool({
  connectionString: env.require("DATABASE_URL")
});

pool.on("error", err => {
  console.warn("Unexpected error on idle client", err);
  throw err;
});

const query = (q, params, callback) =>
  pool.connect((err, client, release) => {
    if (typeof params === "function") {
      callback = params;
      params = null;
    }

    callback = callback || NOOP;

    if (err) {
      console.warn(err);
      release();
      return callback(err);
    }

    return client.query(q, params, (err, results) => {
      release();
      return callback(err, results);
    });
  });

const join = (a, b) => {
  if (typeof a === "number" || typeof b === "number") {
    return (a || 0) + (b || 0);
  }

  if (Array.isArray(a) || Array.isArray(b)) {
    return Array.from(new Set(a || [].concat(b)));
  }

  throw new Error(`Don't know how to join: '${a}', '${b}'`);
};

const merge = (a, b) =>
  Object.keys(b).reduce(
    (acc, k) => Object.assign({}, acc, { [k]: join(acc[k], b[k]) }),
    a
  );

const summarizer = batch =>
  batch.reduce(
    (acc, { changeset, stats, uid, user }) =>
      Object.assign({}, acc, {
        [changeset]: Object.assign({}, merge(acc[changeset] || {}, stats), {
          uid,
          user
        })
      }),
    {}
  );

const statUpdater = stats => {
  const lastAugmentedDiff = Math.max.apply(
    null,
    Array.from(
      new Set(
        Object.keys(stats).reduce(
          (acc, k) => acc.concat(stats[k].augmentedDiffs),
          []
        )
      )
    )
  );

  console.warn(`Checkpointing augmented diffs @ ${lastAugmentedDiff}.`);

  const queries = Object.keys(stats)
    .map(changeset =>
      [
        [
          `
INSERT INTO raw_changesets AS c (
  id,
  user_id,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  augmented_diffs,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  roads_added = c.roads_added + $3,
  roads_modified = c.roads_modified + $4,
  waterways_added = c.waterways_added + $5,
  waterways_modified = c.waterways_modified + $6,
  buildings_added = c.buildings_added + $7,
  buildings_modified = c.buildings_modified + $8,
  pois_added = c.pois_added + $9,
  pois_modified = c.pois_modified + $10,
  road_km_added = c.road_km_added + $11,
  road_km_modified = c.road_km_modified + $12,
  waterway_km_added = c.waterway_km_added + $13,
  waterway_km_modified = c.waterway_km_modified + $14,
  augmented_diffs = coalesce(c.augmented_diffs, ARRAY[]::integer[]) || $15,
  updated_at = current_timestamp
WHERE c.id = $1
  AND NOT coalesce(c.augmented_diffs, ARRAY[]::integer[]) && $15
          `,
          [
            Number(changeset),
            Number(stats[changeset].uid),
            stats[changeset].roadsAdded || 0,
            stats[changeset].roadsModified || 0,
            stats[changeset].waterwaysAdded || 0,
            stats[changeset].waterwaysModified || 0,
            stats[changeset].buildingsAdded || 0,
            stats[changeset].buildingsModified || 0,
            stats[changeset].poisAdded || 0,
            stats[changeset].poisModified || 0,
            stats[changeset].roadKmAdded || 0,
            stats[changeset].roadKmModified || 0,
            stats[changeset].waterwayKmAdded || 0,
            stats[changeset].waterwayKmModified || 0,
            stats[changeset].augmentedDiffs
          ]
        ],
        [
          `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
          [Number(stats[changeset].uid), stats[changeset].user]
        ]
      ]
        .concat(
          stats[changeset].countries.map(code => [
            `
INSERT INTO raw_changesets_countries (
  changeset_id,
  country_id
) VALUES (
  $1, (SELECT id FROM raw_countries WHERE code = $2)
)
ON CONFLICT DO NOTHING
          `,
            [Number(changeset), code]
          ])
        )
        .concat([
          [
            `
UPDATE augmented_diff_status
SET id=$1,
  updated_at=current_timestamp
    `,
            [lastAugmentedDiff]
          ]
        ])
    )
    .reduce((acc, a) => acc.concat(a), []);

  return _(push =>
    async.each(
      queries,
      ([q, params], done) => query(q, params, done),
      err => push(err, _.nil)
    )
  );
};

// from https://github.com/openstreetmap/iD/blob/45ffa3b731463bf4eba52519896ad33653def0cd/modules/ui/commit.js
const HASHTAG_REGEX = /(#[^\u2000-\u206F\u2E00-\u2E7F\s\\'!"#$%()*,./:;<=>?@[\]^`{|}~]+)/g;

const extractHashtags = tags => {
  tags.comment = htmlEntities.decode(tags.comment);
  return ((tags.comment || "").match(HASHTAG_REGEX) || []).map(x =>
    x.slice(1).toLowerCase());
};

const changesetUpdater = changeset => {
  const {
    id,
    created_at: createdAt,
    closed_at: closedAt,
    uid,
    user
  } = changeset;
  let editor = null;
  let hashtags = [];

  if (changeset.tags != null) {
    editor = changeset.tags.created_by;
    hashtags = extractHashtags(changeset.tags);
  }

  const queries = [
    [
      `
INSERT INTO raw_changesets AS c (
  id,
  editor,
  user_id,
  created_at,
  closed_at,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  editor = $2,
  user_id = $3,
  created_at = $4,
  closed_at = $5,
  updated_at = current_timestamp
WHERE c.id = $1
      `,
      [id, editor, uid, createdAt, closedAt]
    ],
    [
      `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
      [uid, user]
    ]
  ];

  return _(push => {
    async.series(
      [
        async.apply(async.each, queries, ([q, params], done) =>
          query(q, params, done)
        ),
        async.apply(async.each, hashtags, (hashtag, done) =>
          async.waterfall(
            [
              async.apply(
                // https://stackoverflow.com/questions/34708509/how-to-use-returning-with-on-conflict-in-postgresql
                query,
                `
WITH input_rows("hashtag") AS (
  VALUES ($1)
),
ins AS (
  INSERT INTO raw_hashtags AS h (hashtag)
  VALUES ($1)
  ON CONFLICT DO NOTHING
  RETURNING id
)
SELECT id
FROM ins
UNION ALL
SELECT id
FROM input_rows
JOIN raw_hashtags USING(hashtag)
                `,
                [hashtag]
              ),
              (result, fin) => {
                const { id: hashtagId } = result.rows[0];

                return query(
                  `
INSERT INTO raw_changesets_hashtags (
  changeset_id,
  hashtag_id
) VALUES ($1, $2)
ON CONFLICT DO NOTHING
                  `,
                  [id, hashtagId],
                  fin
                );
              }
            ],
            done
          )
        )
      ],
      err => push(err, _.nil)
    );
  });
};

const getInitialAugmentedDiffSequenceNumber = callback =>
  query("SELECT id FROM augmented_diff_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

const checkpointChangesets = sequenceNumber => {
  console.warn(`Checkpointing changesets @ ${sequenceNumber}.`);

  query(
    `
UPDATE changesets_status
SET id=$1,
  updated_at=current_timestamp
  `,
    [Number(sequenceNumber)],
    err => {
      if (err) {
        console.warn(err.stack);
      }
    }
  );
};

const getInitialChangesetSequenceNumber = callback =>
  query("SELECT id FROM changesets_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

module.exports = (options, callback) => {
  const opts = Object.assign(
    {},
    {
      delay: 15e3,
      infinite: true
    },
    options
  );

  callback = callback || NOOP;

  return async.parallel(
    {
      initialAugmentedDiffSequenceNumber: getInitialAugmentedDiffSequenceNumber,
      initialChangesetSequenceNumber: getInitialChangesetSequenceNumber
    },
    (
      err,
      { initialAugmentedDiffSequenceNumber, initialChangesetSequenceNumber }
    ) => {
      if (err) {
        return callback(err);
      }

      return async.parallel(
        [
          done =>
            _(
              AugmentedDiffs({
                baseURL: OVERPASS_URL,
                initialSequence: initialAugmentedDiffSequenceNumber + 1,
                infinite: opts.infinite,
                delay: opts.delay
              })
                .pipe(new AugmentedDiffParser().on("error", console.warn))
                .pipe(new StatsStream())
            )
              // batch by sequence
              .through(s => {
                let batched = [];
                let sequence = null;

                return s.consume((err, x, push, next) => {
                  if (err) {
                    push(err);
                    return next();
                  }

                  if (x === _.nil) {
                    // end of the stream; flush
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    return push(null, _.nil);
                  }

                  let activeSequence = sequence;

                  if (x.stats != null) {
                    [activeSequence] = x.stats.augmentedDiffs;
                  }

                  if (x.type === "Marker" || sequence !== activeSequence) {
                    // new sequence; flush previous
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    // reset batch
                    if (x.type !== "Marker") {
                      batched = [x];
                    } else {
                      batched = [];
                    }
                    sequence = activeSequence;

                    return next();
                  }

                  // add this item to the batch
                  batched.push(x);

                  return next();
                });
              })
              .map(summarizer)
              .flatMap(statUpdater)
              .done(() => done),
          done =>
            _(
              Changesets({
                initialSequence: initialChangesetSequenceNumber,
                infinite: opts.infinite,
                // TODO checkpoint when a sequence has been fully processed not just seen
                checkpoint: checkpointChangesets
              }).pipe(
                new OSMParser({
                  coerceIds: true
                })
              )
            )
              .flatMap(changesetUpdater)
              .done(() => done)
        ],
        callback
      );
    }
  );
};<|endoftext|>


<commit_before>// for osm-replication-streams on Node < 8
require("babel-polyfill");

const _ = require("highland");
const async = require("async");
const env = require("require-env");
const OSMParser = require("osm2obj");
const {
  parsers: { AugmentedDiffParser },
  sources: { AugmentedDiffs, Changesets }
} = require("osm-replication-streams");
const { Pool } = require("pg");

const { NOOP } = require(".");
const StatsStream = require("./stats_stream");

const OVERPASS_URL = process.env.OVERPASS_URL || "http://overpass-api.de";

const pool = new Pool({
  connectionString: env.require("DATABASE_URL")
});

const query = (q, params, callback) =>
  pool.connect((err, client, release) => {
    if (typeof params === "function") {
      callback = params;
      params = null;
    }

    callback = callback || NOOP;

    if (err) {
      console.warn(err);
      release();
      return callback(err);
    }

    return client.query(q, params, (err, results) => {
      release();
      return callback(err, results);
    });
  });

const join = (a, b) => {
  if (typeof a === "number" || typeof b === "number") {
    return (a || 0) + (b || 0);
  }

  if (Array.isArray(a) || Array.isArray(b)) {
    return Array.from(new Set(a || [].concat(b)));
  }

  throw new Error(`Don't know how to join: '${a}', '${b}'`);
};

const merge = (a, b) =>
  Object.keys(b).reduce(
    (acc, k) => Object.assign({}, acc, { [k]: join(acc[k], b[k]) }),
    a
  );

const summarizer = batch =>
  batch.reduce(
    (acc, { changeset, stats, uid, user }) =>
      Object.assign({}, acc, {
        [changeset]: Object.assign({}, merge(acc[changeset] || {}, stats), {
          uid,
          user
        })
      }),
    {}
  );

const statUpdater = stats => {
  const lastAugmentedDiff = Math.max.apply(
    null,
    Array.from(
      new Set(
        Object.keys(stats).reduce(
          (acc, k) => acc.concat(stats[k].augmentedDiffs),
          []
        )
      )
    )
  );

  console.warn(`Checkpointing augmented diffs @ ${lastAugmentedDiff}.`);

  const queries = Object.keys(stats)
    .map(changeset =>
      [
        [
          `
INSERT INTO raw_changesets AS c (
  id,
  user_id,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  augmented_diffs,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  roads_added = c.roads_added + $3,
  roads_modified = c.roads_modified + $4,
  waterways_added = c.waterways_added + $5,
  waterways_modified = c.waterways_modified + $6,
  buildings_added = c.buildings_added + $7,
  buildings_modified = c.buildings_modified + $8,
  pois_added = c.pois_added + $9,
  pois_modified = c.pois_modified + $10,
  road_km_added = c.road_km_added + $11,
  road_km_modified = c.road_km_modified + $12,
  waterway_km_added = c.waterway_km_added + $13,
  waterway_km_modified = c.waterway_km_modified + $14,
  augmented_diffs = coalesce(c.augmented_diffs, ARRAY[]::integer[]) || $15,
  updated_at = current_timestamp
WHERE c.id = $1
  AND NOT coalesce(c.augmented_diffs, ARRAY[]::integer[]) && $15
          `,
          [
            Number(changeset),
            Number(stats[changeset].uid),
            stats[changeset].roadsAdded || 0,
            stats[changeset].roadsModified || 0,
            stats[changeset].waterwaysAdded || 0,
            stats[changeset].waterwaysModified || 0,
            stats[changeset].buildingsAdded || 0,
            stats[changeset].buildingsModified || 0,
            stats[changeset].poisAdded || 0,
            stats[changeset].poisModified || 0,
            stats[changeset].roadKmAdded || 0,
            stats[changeset].roadKmModified || 0,
            stats[changeset].waterwayKmAdded || 0,
            stats[changeset].waterwayKmModified || 0,
            stats[changeset].augmentedDiffs
          ]
        ],
        [
          `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
          [Number(stats[changeset].uid), stats[changeset].user]
        ]
      ]
        .concat(
          stats[changeset].countries.map(code => [
            `
INSERT INTO raw_changesets_countries (
  changeset_id,
  country_id
) VALUES (
  $1, (SELECT id FROM raw_countries WHERE code = $2)
)
ON CONFLICT DO NOTHING
          `,
            [Number(changeset), code]
          ])
        )
        .concat([
          [
            `
UPDATE augmented_diff_status
SET id=$1,
  updated_at=current_timestamp
    `,
            [lastAugmentedDiff]
          ]
        ])
    )
    .reduce((acc, a) => acc.concat(a), []);

  return _(push =>
    async.each(
      queries,
      ([q, params], done) => query(q, params, done),
      err => push(err, _.nil)
    )
  );
};

// from https://github.com/openstreetmap/iD/blob/45ffa3b731463bf4eba52519896ad33653def0cd/modules/ui/commit.js
const HASHTAG_REGEX = /(#[^\u2000-\u206F\u2E00-\u2E7F\s\\'!"#$%()*,./:;<=>?@[\]^`{|}~]+)/g;

const extractHashtags = tags =>
  ((tags.comment || "").match(HASHTAG_REGEX) || []).map(x =>
    x.slice(1).toLowerCase()
  );

const changesetUpdater = changeset => {
  const {
    id,
    created_at: createdAt,
    closed_at: closedAt,
    uid,
    user
  } = changeset;
  let editor = null;
  let hashtags = [];

  if (changeset.tags != null) {
    editor = changeset.tags.created_by;
    hashtags = extractHashtags(changeset.tags);
  }

  const queries = [
    [
      `
INSERT INTO raw_changesets AS c (
  id,
  editor,
  user_id,
  created_at,
  closed_at,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  editor = $2,
  user_id = $3,
  created_at = $4,
  closed_at = $5,
  updated_at = current_timestamp
WHERE c.id = $1
      `,
      [id, editor, uid, createdAt, closedAt]
    ],
    [
      `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
      [uid, user]
    ]
  ];

  return _(push => {
    async.series(
      [
        async.apply(async.each, queries, ([q, params], done) =>
          query(q, params, done)
        ),
        async.apply(async.each, hashtags, (hashtag, done) =>
          async.waterfall(
            [
              async.apply(
                // https://stackoverflow.com/questions/34708509/how-to-use-returning-with-on-conflict-in-postgresql
                query,
                `
WITH input_rows("hashtag") AS (
  VALUES ($1)
),
ins AS (
  INSERT INTO raw_hashtags AS h (hashtag)
  VALUES ($1)
  ON CONFLICT DO NOTHING
  RETURNING id
)
SELECT id
FROM ins
UNION ALL
SELECT id
FROM input_rows
JOIN raw_hashtags USING(hashtag)
                `,
                [hashtag]
              ),
              (result, fin) => {
                const { id: hashtagId } = result.rows[0];

                return query(
                  `
INSERT INTO raw_changesets_hashtags (
  changeset_id,
  hashtag_id
) VALUES ($1, $2)
ON CONFLICT DO NOTHING
                  `,
                  [id, hashtagId],
                  fin
                );
              }
            ],
            done
          )
        )
      ],
      err => push(err, _.nil)
    );
  });
};

const getInitialAugmentedDiffSequenceNumber = callback =>
  query("SELECT id FROM augmented_diff_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

const checkpointChangesets = sequenceNumber => {
  console.warn(`Checkpointing changesets @ ${sequenceNumber}.`);

  query(
    `
UPDATE changesets_status
SET id=$1,
  updated_at=current_timestamp
  `,
    [Number(sequenceNumber)],
    err => {
      if (err) {
        console.warn(err.stack);
      }
    }
  );
};

const getInitialChangesetSequenceNumber = callback =>
  query("SELECT id FROM changesets_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

module.exports = (options, callback) => {
  const opts = Object.assign(
    {},
    {
      delay: 15e3,
      infinite: true
    },
    options
  );

  callback = callback || NOOP;

  return async.parallel(
    {
      initialAugmentedDiffSequenceNumber: getInitialAugmentedDiffSequenceNumber,
      initialChangesetSequenceNumber: getInitialChangesetSequenceNumber
    },
    (
      err,
      { initialAugmentedDiffSequenceNumber, initialChangesetSequenceNumber }
    ) => {
      if (err) {
        return callback(err);
      }

      return async.parallel(
        [
          done =>
            _(
              AugmentedDiffs({
                baseURL: OVERPASS_URL,
                initialSequence: initialAugmentedDiffSequenceNumber + 1,
                infinite: opts.infinite,
                delay: opts.delay
              })
                .pipe(new AugmentedDiffParser().on("error", console.warn))
                .pipe(new StatsStream())
            )
              // batch by sequence
              .through(s => {
                let batched = [];
                let sequence = null;

                return s.consume((err, x, push, next) => {
                  if (err) {
                    push(err);
                    return next();
                  }

                  if (x === _.nil) {
                    // end of the stream; flush
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    return push(null, _.nil);
                  }

                  let activeSequence = sequence;

                  if (x.stats != null) {
                    [activeSequence] = x.stats.augmentedDiffs;
                  }

                  if (x.type === "Marker" || sequence !== activeSequence) {
                    // new sequence; flush previous
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    // reset batch
                    if (x.type !== "Marker") {
                      batched = [x];
                    } else {
                      batched = [];
                    }
                    sequence = activeSequence;

                    return next();
                  }

                  // add this item to the batch
                  batched.push(x);

                  return next();
                });
              })
              .map(summarizer)
              .flatMap(statUpdater)
              .done(() => done),
          done =>
            _(
              Changesets({
                initialSequence: initialChangesetSequenceNumber,
                infinite: opts.infinite,
                // TODO checkpoint when a sequence has been fully processed not just seen
                checkpoint: checkpointChangesets
              }).pipe(
                new OSMParser({
                  coerceIds: true
                })
              )
            )
              .flatMap(changesetUpdater)
              .done(() => done)
        ],
        callback
      );
    }
  );
};
<commit_msg>Throw pool errors<commit_after>// for osm-replication-streams on Node < 8
require("babel-polyfill");

const _ = require("highland");
const async = require("async");
const env = require("require-env");
const OSMParser = require("osm2obj");
const {
  parsers: { AugmentedDiffParser },
  sources: { AugmentedDiffs, Changesets }
} = require("osm-replication-streams");
const { Pool } = require("pg");

const { NOOP } = require(".");
const StatsStream = require("./stats_stream");

const OVERPASS_URL = process.env.OVERPASS_URL || "http://overpass-api.de";

const pool = new Pool({
  connectionString: env.require("DATABASE_URL")
});

pool.on("error", function(err, client) {
  console.warn("Unexpected error on idle client", err);
  throw err;
});

const query = (q, params, callback) =>
  pool.connect((err, client, release) => {
    if (typeof params === "function") {
      callback = params;
      params = null;
    }

    callback = callback || NOOP;

    if (err) {
      console.warn(err);
      release();
      return callback(err);
    }

    return client.query(q, params, (err, results) => {
      release();
      return callback(err, results);
    });
  });

const join = (a, b) => {
  if (typeof a === "number" || typeof b === "number") {
    return (a || 0) + (b || 0);
  }

  if (Array.isArray(a) || Array.isArray(b)) {
    return Array.from(new Set(a || [].concat(b)));
  }

  throw new Error(`Don't know how to join: '${a}', '${b}'`);
};

const merge = (a, b) =>
  Object.keys(b).reduce(
    (acc, k) => Object.assign({}, acc, { [k]: join(acc[k], b[k]) }),
    a
  );

const summarizer = batch =>
  batch.reduce(
    (acc, { changeset, stats, uid, user }) =>
      Object.assign({}, acc, {
        [changeset]: Object.assign({}, merge(acc[changeset] || {}, stats), {
          uid,
          user
        })
      }),
    {}
  );

const statUpdater = stats => {
  const lastAugmentedDiff = Math.max.apply(
    null,
    Array.from(
      new Set(
        Object.keys(stats).reduce(
          (acc, k) => acc.concat(stats[k].augmentedDiffs),
          []
        )
      )
    )
  );

  console.warn(`Checkpointing augmented diffs @ ${lastAugmentedDiff}.`);

  const queries = Object.keys(stats)
    .map(changeset =>
      [
        [
          `
INSERT INTO raw_changesets AS c (
  id,
  user_id,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  augmented_diffs,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  roads_added = c.roads_added + $3,
  roads_modified = c.roads_modified + $4,
  waterways_added = c.waterways_added + $5,
  waterways_modified = c.waterways_modified + $6,
  buildings_added = c.buildings_added + $7,
  buildings_modified = c.buildings_modified + $8,
  pois_added = c.pois_added + $9,
  pois_modified = c.pois_modified + $10,
  road_km_added = c.road_km_added + $11,
  road_km_modified = c.road_km_modified + $12,
  waterway_km_added = c.waterway_km_added + $13,
  waterway_km_modified = c.waterway_km_modified + $14,
  augmented_diffs = coalesce(c.augmented_diffs, ARRAY[]::integer[]) || $15,
  updated_at = current_timestamp
WHERE c.id = $1
  AND NOT coalesce(c.augmented_diffs, ARRAY[]::integer[]) && $15
          `,
          [
            Number(changeset),
            Number(stats[changeset].uid),
            stats[changeset].roadsAdded || 0,
            stats[changeset].roadsModified || 0,
            stats[changeset].waterwaysAdded || 0,
            stats[changeset].waterwaysModified || 0,
            stats[changeset].buildingsAdded || 0,
            stats[changeset].buildingsModified || 0,
            stats[changeset].poisAdded || 0,
            stats[changeset].poisModified || 0,
            stats[changeset].roadKmAdded || 0,
            stats[changeset].roadKmModified || 0,
            stats[changeset].waterwayKmAdded || 0,
            stats[changeset].waterwayKmModified || 0,
            stats[changeset].augmentedDiffs
          ]
        ],
        [
          `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
          [Number(stats[changeset].uid), stats[changeset].user]
        ]
      ]
        .concat(
          stats[changeset].countries.map(code => [
            `
INSERT INTO raw_changesets_countries (
  changeset_id,
  country_id
) VALUES (
  $1, (SELECT id FROM raw_countries WHERE code = $2)
)
ON CONFLICT DO NOTHING
          `,
            [Number(changeset), code]
          ])
        )
        .concat([
          [
            `
UPDATE augmented_diff_status
SET id=$1,
  updated_at=current_timestamp
    `,
            [lastAugmentedDiff]
          ]
        ])
    )
    .reduce((acc, a) => acc.concat(a), []);

  return _(push =>
    async.each(
      queries,
      ([q, params], done) => query(q, params, done),
      err => push(err, _.nil)
    )
  );
};

// from https://github.com/openstreetmap/iD/blob/45ffa3b731463bf4eba52519896ad33653def0cd/modules/ui/commit.js
const HASHTAG_REGEX = /(#[^\u2000-\u206F\u2E00-\u2E7F\s\\'!"#$%()*,./:;<=>?@[\]^`{|}~]+)/g;

const extractHashtags = tags =>
  ((tags.comment || "").match(HASHTAG_REGEX) || []).map(x =>
    x.slice(1).toLowerCase()
  );

const changesetUpdater = changeset => {
  const {
    id,
    created_at: createdAt,
    closed_at: closedAt,
    uid,
    user
  } = changeset;
  let editor = null;
  let hashtags = [];

  if (changeset.tags != null) {
    editor = changeset.tags.created_by;
    hashtags = extractHashtags(changeset.tags);
  }

  const queries = [
    [
      `
INSERT INTO raw_changesets AS c (
  id,
  editor,
  user_id,
  created_at,
  closed_at,
  roads_added,
  roads_modified,
  waterways_added,
  waterways_modified,
  buildings_added,
  buildings_modified,
  pois_added,
  pois_modified,
  road_km_added,
  road_km_modified,
  waterway_km_added,
  waterway_km_modified,
  updated_at
) VALUES (
  $1, $2, $3, $4, $5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, current_timestamp
)
ON CONFLICT (id) DO UPDATE
SET
  editor = $2,
  user_id = $3,
  created_at = $4,
  closed_at = $5,
  updated_at = current_timestamp
WHERE c.id = $1
      `,
      [id, editor, uid, createdAt, closedAt]
    ],
    [
      `
INSERT INTO raw_users AS u (
  id,
  name
) VALUES (
  $1, $2
)
ON CONFLICT (id) DO UPDATE
SET
  name = $2
WHERE u.id = $1
      `,
      [uid, user]
    ]
  ];

  return _(push => {
    async.series(
      [
        async.apply(async.each, queries, ([q, params], done) =>
          query(q, params, done)
        ),
        async.apply(async.each, hashtags, (hashtag, done) =>
          async.waterfall(
            [
              async.apply(
                // https://stackoverflow.com/questions/34708509/how-to-use-returning-with-on-conflict-in-postgresql
                query,
                `
WITH input_rows("hashtag") AS (
  VALUES ($1)
),
ins AS (
  INSERT INTO raw_hashtags AS h (hashtag)
  VALUES ($1)
  ON CONFLICT DO NOTHING
  RETURNING id
)
SELECT id
FROM ins
UNION ALL
SELECT id
FROM input_rows
JOIN raw_hashtags USING(hashtag)
                `,
                [hashtag]
              ),
              (result, fin) => {
                const { id: hashtagId } = result.rows[0];

                return query(
                  `
INSERT INTO raw_changesets_hashtags (
  changeset_id,
  hashtag_id
) VALUES ($1, $2)
ON CONFLICT DO NOTHING
                  `,
                  [id, hashtagId],
                  fin
                );
              }
            ],
            done
          )
        )
      ],
      err => push(err, _.nil)
    );
  });
};

const getInitialAugmentedDiffSequenceNumber = callback =>
  query("SELECT id FROM augmented_diff_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

const checkpointChangesets = sequenceNumber => {
  console.warn(`Checkpointing changesets @ ${sequenceNumber}.`);

  query(
    `
UPDATE changesets_status
SET id=$1,
  updated_at=current_timestamp
  `,
    [Number(sequenceNumber)],
    err => {
      if (err) {
        console.warn(err.stack);
      }
    }
  );
};

const getInitialChangesetSequenceNumber = callback =>
  query("SELECT id FROM changesets_status", (err, results) => {
    if (err) {
      return callback(err);
    }

    return callback(null, results.rows[0].id);
  });

module.exports = (options, callback) => {
  const opts = Object.assign(
    {},
    {
      delay: 15e3,
      infinite: true
    },
    options
  );

  callback = callback || NOOP;

  return async.parallel(
    {
      initialAugmentedDiffSequenceNumber: getInitialAugmentedDiffSequenceNumber,
      initialChangesetSequenceNumber: getInitialChangesetSequenceNumber
    },
    (
      err,
      { initialAugmentedDiffSequenceNumber, initialChangesetSequenceNumber }
    ) => {
      if (err) {
        return callback(err);
      }

      return async.parallel(
        [
          done =>
            _(
              AugmentedDiffs({
                baseURL: OVERPASS_URL,
                initialSequence: initialAugmentedDiffSequenceNumber + 1,
                infinite: opts.infinite,
                delay: opts.delay
              })
                .pipe(new AugmentedDiffParser().on("error", console.warn))
                .pipe(new StatsStream())
            )
              // batch by sequence
              .through(s => {
                let batched = [];
                let sequence = null;

                return s.consume((err, x, push, next) => {
                  if (err) {
                    push(err);
                    return next();
                  }

                  if (x === _.nil) {
                    // end of the stream; flush
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    return push(null, _.nil);
                  }

                  let activeSequence = sequence;

                  if (x.stats != null) {
                    [activeSequence] = x.stats.augmentedDiffs;
                  }

                  if (x.type === "Marker" || sequence !== activeSequence) {
                    // new sequence; flush previous
                    if (batched.length > 0) {
                      push(null, batched);
                    }

                    // reset batch
                    if (x.type !== "Marker") {
                      batched = [x];
                    } else {
                      batched = [];
                    }
                    sequence = activeSequence;

                    return next();
                  }

                  // add this item to the batch
                  batched.push(x);

                  return next();
                });
              })
              .map(summarizer)
              .flatMap(statUpdater)
              .done(() => done),
          done =>
            _(
              Changesets({
                initialSequence: initialChangesetSequenceNumber,
                infinite: opts.infinite,
                // TODO checkpoint when a sequence has been fully processed not just seen
                checkpoint: checkpointChangesets
              }).pipe(
                new OSMParser({
                  coerceIds: true
                })
              )
            )
              .flatMap(changesetUpdater)
              .done(() => done)
        ],
        callback
      );
    }
  );
};
<|endoftext|>