<commit_before>/* Copyright 2016 Streampunk Media Ltd.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

var redioactive = require('../../../util/Redioactive.js');
var util = require('util');
var express = require('express');
var bodyParser = require('body-parser');
var http = require('http');
var https = require('https');
var uuid = require('uuid');
var fs = require('fs');
var Grain = require('../../../model/Grain.js');
var dns = require('dns');
var url = require('url');

// Maximum drift between high water mark and next request in ms
// TODO calculate this from grain rate
var maxDrift = 40 * 8;

function extractVersions(v) {
  var m = v.match(/^([0-9]+):([0-9]+)$/);
  if (m === null) { console.log(v, [Number.MAX_SAFE_INTEGER, 0]); return [Number.MAX_SAFE_INTEGER, 0]; }
  return [+m[1], +m[2]];
}

function compareVersions(l, r) {
  var lm = extractVersions(l);
  var rm = extractVersions(r);
  if (lm[0] < rm[0]) return -1;
  if (lm[0] > rm[0]) return 1;
  if (lm[1] < rm[1]) return -1;
  if (lm[1] > rm[1]) return 1;
  return 0;
}

function versionToMs (v) {
  var e = extractVersions(v);
  if (e[0] === Number.MAX_SAFE_INTEGER) return e[0];
  return (e[0] * 1000) + (e[1] / 1000000|0);
}

function versionDiffMs (smaller, bigger) {
  var smMs = versionToMs(smaller);
  var bgMs = versionToMs(bigger);
  if (smMs === Number.MAX_SAFE_INTEGER || bgMs === Number.MAX_SAFE_INTEGER) return 0;
  return bgMs - smMs;
}

const mimeMatch = /^\s*(\w+)\/(\w+)/;
const paramMatch = /\b(\w+)=(\S+)\b/g;

module.exports = function (RED) {
  function SpmHTTPIn (config) {
    RED.nodes.createNode(this, config);
    redioactive.Funnel.call(this, config);

    if (!this.context().global.get('updated'))
      return this.log('Waiting for global context to be updated.');

    var protocol = (config.protocol === 'HTTP') ? http : https;
    var node = this;
    var total = 0;
    config.pullURL = (config.pullURL.endsWith('/')) ?
      config.pullURL.slice(0, -1) : config.pullURL;
    config.path = (config.path.endsWith('/')) ?
      config.path.slice(0, -1) : config.path;
    var fullPath = config.pullURL + config.path;
    var fullURL = url.parse(fullPath);
    var clientID = 'cid' + Date.now();
    this.baseTime = [ Date.now() / 1000|0, (Date.now() % 1000) * 1000000 ];
    var nodeAPI = this.context().global.get('nodeAPI');
    var ledger = this.context().global.get('ledger');
    var nodeAPI = this.context().global.get('nodeAPI');
    var localName = config.name || `${config.type}-${config.id}`;
    var localDescription = config.description || `${config.type}-${config.id}`;
    var pipelinesID = config.device ?
      RED.nodes.getNode(config.device).nmos_id :
      this.context().global.get('pipelinesID');
    var sourceID = uuid.v4();
    var flowID = uuid.v4();
    var flow = null; var source = null; var recvr = null;
    var tags = {};

    function makeFlowAndSource (headers) {
      var contentType = headers['content-type'];
      var mime = contentType.match(mimeMatch);
      tags = { format : [ mime[1] ], encodingName : [ mime[2] ] };
      var parameters = contentType.match(paramMatch);
      parameters.forEach(function (p) {
         var splitP = p.split('=');
         if (splitP[0] === 'rate') splitP[0] = 'clockRate';
         tags[splitP[0]] = [ splitP[1] ];
      });
      if (headers['arachnid-fourcc']) {
        tags.packing = [ headers['arachnid-fourcc'] ];
      } else if (tags.encodingName[0] === 'raw') {
        tags.clockRate = [ '90000' ];
        tags.packing = [ 'pgroup' ];
      }
      var senderID = headers['arachnid-senderid'];
      senderID = (senderID === undefined) ? null : { sender_id : senderID };
      source = new ledger.Source(sourceID, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], null, null, pipelinesID, null);
      flow = new ledger.Flow(flowID, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], tags, source.id, null);
      recvr = new ledger.Receiver(null, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], null, tags,
        pipelinesID, ledger.transports.dash, senderID);
      nodeAPI.putResource(source)
      .then(function () {
        return nodeAPI.putResource(flow);
      }).then(function () {
        return nodeAPI.putResource(recvr);
      }).catch(function (err) {
        node.error(`Unable to register resource : ${err}`);
      });
    };

    var keepAliveAgent = new protocol.Agent({keepAlive : true });
    var runNext = function (x, push, next) {
      var requestTimer = process.hrtime();
      var req = protocol.request({
          rejectUnauthorized: false,
          hostname: fullURL.hostname,
          port: fullURL.port,
          path: `${fullURL.path}/${nextRequest[x]}`,
          method: 'GET',
          agent: keepAliveAgent,
          headers: {
            'Arachnid-ThreadNumber': x,
            'Arachnid-TotalConcurrent': config.parallel,
            'Arachnid-ClientID': clientID
          }},
          function (res) {
        // console.log('Response received after', process.hrtime(requestTimer));
        var count = 0;
        var position = 0;
        if (res.statusCode === 404) {
          node.warn(`Received not found in thread ${x}, request ${config.path}/${nextRequest[x]} - may be ahead of the game. Retrying.`);
          setTimeout(function () {
            runNext(x, push, next);
          }, 5);
          return;
        }
        if (res.statusCode === 410) {
          node.error(`BANG! Cache miss when reading audio stream ${config.path}/${nextRequest[x]} on thread ${x}.`);
          push(`Request for grain ${config.path}/${nextRequest[x]} that has already gone on thread ${x}. Resetting.`);
          nextRequest =
            [ '-5', '-4', '-3', '-2', '-1', '0' ].slice(-config.parallel);
          activeThreads[x] = false;
          return;
        }
        if (res.statusCode === 200) {
          var grainData = new Buffer(+res.headers['content-length']);
          nextRequest[x] = res.headers['arachnid-ptporigin'];
          if (!flow) makeFlowAndSource(res.headers);
          res.on('data', function (data) {
            data.copy(grainData, position);
            position += data.length;
            count++;
            // console.log(`Data received for ${count} at`, process.hrtime(requestTimer));
          });
          res.on('end', function () {
            console.log('Request time until end', process.hrtime(requestTimer));
            grainData = grainData.slice(0, position);
            nextRequest[x] = res.headers['arachnid-nextbythread'];
            var ptpOrigin = res.headers['arachnid-ptporigin'];
            var ptpSync = res.headers['arachnid-ptpsync'];
            var duration = res.headers['arachnid-grainduration'];
            // TODO fix up regeneration
            var gFlowID = flowID; //(config.regenerate) ? flowID : res.headers['arachnid-flowid'];
            var gSourceID = sourceID; // (config.regenerate) ? sourceID : res.headers['arachnid-sourceid'];
            var tc = res.headers['arachnid-timecode'];
            var g = new Grain([ grainData ], ptpSync, ptpOrigin, tc, gFlowID,
              gSourceID, duration); // regenerate time as emitted
            pushGrains(g, push);
            activeThreads[x] = false;
            next();
          });
        };
        res.on('error', function (e) {
          node.warn(`Received error during streaming of get response on thread ${x}: ${e}.`);
          push(`Received error during streaming of get response on thread ${x}: ${e}.`);
          activeThreads[x] = false;
          next();
        });
      });
      req.on('error', function (e) {
        node.warn(`Received error when requesting frame from server on thread ${x}: ${e}`);
        push(`Received error when requesting frame from server on thread ${x}: ${e}`);
        activeThreads[x] = false;
        next();
      });
      req.end();
      requestTimer = process.hrtime();
    };

    var grainQueue = { };
    var highWaterMark = Number.MAX_SAFE_INTEGER + ':0';
    // Push every grain older than what is in nextRequest, send grains in order
    function pushGrains(g, push) {
      grainQueue[g.formatTimestamp(g.ptpOrigin)] = g;
      console.log('QQQ', nextRequest);
      var nextMin = nextRequest.reduce(function (a, b) {
        return compareVersions(a, b) <= 0 ? a : b;
      });
      Object.keys(grainQueue).filter(function (gts) {
        return compareVersions(gts, nextMin) <= 0;
      })
      .sort(compareVersions)
      .forEach(function (gts) {
        if (!config.regenerate) {
          console.log('>>> PUSHING', gts);
          push(null, grainQueue[gts]);
        } else {
          var g = grainQueue[gts];
          var grainTime = new Buffer(10);
          grainTime.writeUIntBE(this.baseTime[0], 0, 6);
          grainTime.writeUInt32BE(this.baseTime[1], 6);
          var grainDuration = g.getDuration();
          this.baseTime[1] = ( this.baseTime[1] +
            grainDuration[0] * 1000000000 / grainDuration[1]|0 );
          this.baseTime = [ this.baseTime[0] + this.baseTime[1] / 1000000000|0,
            this.baseTime[1] % 1000000000];
          push(null, new Grain(g.buffers, grainTime, g.ptpOrigin, g.timecode,
            flow.id, source.id, g.duration));
        };
        delete grainQueue[gts];
        highWaterMark = gts;
      });
    };

    var activeThreads =
      [ false, false, false, false, false, false].slice(0, config.parallel);
    var nextRequest =
      [ '-5', '-4', '-3', '-2', '-1', '0' ].slice(-config.parallel);

    dns.lookup(fullURL.hostname, function (err, addr, family) {
      if (err) return this.preFlightError(`Unable to resolve DNS for ${fullURL.hostname}: ${err}`);
      node.log(`Resolved URL hostname ${fullURL.hostname} to ${addr}.`);
      fullURL.hostname = addr;
      if (config.mode === 'push') { // TODO push mode
        // var app = express();
        // app.use(bodyParser.raw({ limit : config.payloadLimit || 6000000 }));
        //
        // app.post(config.path, function (req, res) {
        //   console.log(pushCount++, process.hrtime(star
        //     tTime), req.body.length);
        //   res.json({ length : req.body.length }); // Need to add back pressure concept
        // });
        //
        // var server = protocol.createServer((config.protocol === 'HTTP') ? {} : {
        //   key : fs.readFileSync('./certs/dynamorse-key.pem'),
        //   cert : fs.readFileSync('./certs/dynamorse-cert.pem')
        // }, app).listen(config.port);
        // server.on('listening', function () {
        //   this.log(`Dynamorse ${config.protocol} server listening on port ${config.port}.`);
        // });
      } else { // config.mode is set to pull
        this.generator(function (push, next) {
          setTimeout(function() {
            console.log('+++ DEBUG THREADS', activeThreads);
            for ( var i = 0 ; i < activeThreads.length ; i++ ) {
              if (!activeThreads[i]) {
                if (versionDiffMs(highWaterMark, nextRequest[i]) < maxDrift) {
                  runNext.call(this, i, push, next);
                  activeThreads[i] = true;
                } else {
                  node.error(`Not progressing thread ${i} this time due to a drift of ${versionDiffMs(highWaterMark, nextRequest[i])}.`);
                }
              }
            };
          }, (flow === null) ? 1000 : 0);
        });
      }
    }.bind(this));

  }
  util.inherits(SpmHTTPIn, redioactive.Funnel);
  RED.nodes.registerType("spm-http-in", SpmHTTPIn);
}
<commit_msg>Fixed up port number and graceful shutdown at end of stream.<commit_after>/* Copyright 2016 Streampunk Media Ltd.

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

var redioactive = require('../../../util/Redioactive.js');
var util = require('util');
var express = require('express');
var bodyParser = require('body-parser');
var http = require('http');
var https = require('https');
var uuid = require('uuid');
var fs = require('fs');
var Grain = require('../../../model/Grain.js');
var dns = require('dns');
var url = require('url');

// Maximum drift between high water mark and next request in ms
// TODO calculate this from grain rate
var maxDrift = 40 * 8;

function extractVersions(v) {
  var m = v.match(/^([0-9]+):([0-9]+)$/);
  if (m === null) { console.log(v, [Number.MAX_SAFE_INTEGER, 0]); return [Number.MAX_SAFE_INTEGER, 0]; }
  return [+m[1], +m[2]];
}

function compareVersions(l, r) {
  var lm = extractVersions(l);
  var rm = extractVersions(r);
  if (lm[0] < rm[0]) return -1;
  if (lm[0] > rm[0]) return 1;
  if (lm[1] < rm[1]) return -1;
  if (lm[1] > rm[1]) return 1;
  return 0;
}

function versionToMs (v) {
  var e = extractVersions(v);
  if (e[0] === Number.MAX_SAFE_INTEGER) return e[0];
  return (e[0] * 1000) + (e[1] / 1000000|0);
}

function versionDiffMs (smaller, bigger) {
  var smMs = versionToMs(smaller);
  var bgMs = versionToMs(bigger);
  if (smMs === Number.MAX_SAFE_INTEGER || bgMs === Number.MAX_SAFE_INTEGER) return 0;
  return bgMs - smMs;
}

const mimeMatch = /^\s*(\w+)\/(\w+)/;
const paramMatch = /\b(\w+)=(\S+)\b/g;

module.exports = function (RED) {
  function SpmHTTPIn (config) {
    RED.nodes.createNode(this, config);
    redioactive.Funnel.call(this, config);

    if (!this.context().global.get('updated'))
      return this.log('Waiting for global context to be updated.');

    var protocol = (config.protocol === 'HTTP') ? http : https;
    var node = this;
    var total = 0;
    config.pullURL = (config.pullURL.endsWith('/')) ?
      config.pullURL.slice(0, -1) : config.pullURL;
    config.path = (config.path.endsWith('/')) ?
      config.path.slice(0, -1) : config.path;
    var fullPath = `${config.pullURL}:${config.port}${config.path}`;
    var fullURL = url.parse(fullPath);
    var clientID = 'cid' + Date.now();
    this.baseTime = [ Date.now() / 1000|0, (Date.now() % 1000) * 1000000 ];
    var nodeAPI = this.context().global.get('nodeAPI');
    var ledger = this.context().global.get('ledger');
    var nodeAPI = this.context().global.get('nodeAPI');
    var localName = config.name || `${config.type}-${config.id}`;
    var localDescription = config.description || `${config.type}-${config.id}`;
    var pipelinesID = config.device ?
      RED.nodes.getNode(config.device).nmos_id :
      this.context().global.get('pipelinesID');
    var sourceID = uuid.v4();
    var flowID = uuid.v4();
    var flow = null; var source = null; var recvr = null;
    var tags = {};

    function makeFlowAndSource (headers) {
      var contentType = headers['content-type'];
      var mime = contentType.match(mimeMatch);
      tags = { format : [ mime[1] ], encodingName : [ mime[2] ] };
      var parameters = contentType.match(paramMatch);
      parameters.forEach(function (p) {
         var splitP = p.split('=');
         if (splitP[0] === 'rate') splitP[0] = 'clockRate';
         tags[splitP[0]] = [ splitP[1] ];
      });
      if (headers['arachnid-fourcc']) {
        tags.packing = [ headers['arachnid-fourcc'] ];
      } else if (tags.encodingName[0] === 'raw') {
        tags.clockRate = [ '90000' ];
        tags.packing = [ 'pgroup' ];
      }
      var senderID = headers['arachnid-senderid'];
      senderID = (senderID === undefined) ? null : { sender_id : senderID };
      source = new ledger.Source(sourceID, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], null, null, pipelinesID, null);
      flow = new ledger.Flow(flowID, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], tags, source.id, null);
      recvr = new ledger.Receiver(null, null, localName, localDescription,
        "urn:x-nmos:format:" + tags.format[0], null, tags,
        pipelinesID, ledger.transports.dash, senderID);
      nodeAPI.putResource(source)
      .then(function () {
        return nodeAPI.putResource(flow);
      }).then(function () {
        return nodeAPI.putResource(recvr);
      }).catch(function (err) {
        node.error(`Unable to register resource : ${err}`);
      });
    };

    var keepAliveAgent = new protocol.Agent({keepAlive : true });
    var endCount = 0;
    var runNext = function (x, push, next) {
      var requestTimer = process.hrtime();
      var req = protocol.request({
          rejectUnauthorized: false,
          hostname: fullURL.hostname,
          port: fullURL.port,
          path: `${fullURL.path}/${nextRequest[x]}`,
          method: 'GET',
          agent: keepAliveAgent,
          headers: {
            'Arachnid-ThreadNumber': x,
            'Arachnid-TotalConcurrent': config.parallel,
            'Arachnid-ClientID': clientID
          }},
          function (res) {
        // console.log('Response received after', process.hrtime(requestTimer));
        var count = 0;
        var position = 0;
        if (res.statusCode === 404) {
          node.warn(`Received not found in thread ${x}, request ${config.path}/${nextRequest[x]} - may be ahead of the game. Retrying.`);
          setTimeout(function () {
            runNext(x, push, next);
          }, 5);
          return;
        }
        if (res.statusCode === 410) {
          node.error(`BANG! Cache miss when reading audio stream ${config.path}/${nextRequest[x]} on thread ${x}.`);
          push(`Request for grain ${config.path}/${nextRequest[x]} that has already gone on thread ${x}. Resetting.`);
          nextRequest =
            [ '-5', '-4', '-3', '-2', '-1', '0' ].slice(-config.parallel);
          activeThreads[x] = false;
          return;
        }
        if (res.statusCode === 200) {
          var grainData = new Buffer(+res.headers['content-length']);
          nextRequest[x] = res.headers['arachnid-ptporigin'];
          if (!flow) makeFlowAndSource(res.headers);
          res.on('data', function (data) {
            data.copy(grainData, position);
            position += data.length;
            count++;
            // console.log(`Data received for ${count} at`, process.hrtime(requestTimer));
          });
          res.on('end', function () {
            console.log('Request time until end', process.hrtime(requestTimer));
            grainData = grainData.slice(0, position);
            nextRequest[x] = res.headers['arachnid-nextbythread'];
            var ptpOrigin = res.headers['arachnid-ptporigin'];
            var ptpSync = res.headers['arachnid-ptpsync'];
            var duration = res.headers['arachnid-grainduration'];
            // TODO fix up regeneration
            var gFlowID = flowID; //(config.regenerate) ? flowID : res.headers['arachnid-flowid'];
            var gSourceID = sourceID; // (config.regenerate) ? sourceID : res.headers['arachnid-sourceid'];
            var tc = res.headers['arachnid-timecode'];
            var g = new Grain([ grainData ], ptpSync, ptpOrigin, tc, gFlowID,
              gSourceID, duration); // regenerate time as emitted
            pushGrains(g, push);
            activeThreads[x] = false;
            next();
          });
        };
        res.on('error', function (e) {
          node.warn(`Received error during streaming of get response on thread ${x}: ${e}.`);
          push(`Received error during streaming of get response on thread ${x}: ${e}.`);
          activeThreads[x] = false;
          next();
        });
      });
      req.on('error', function (e) {
        // Check for flow !== null is so that shutdown does not happen too early
        if (flow !== null && e.message.indexOf('ECONNREFUSED') >= 0) {
          node.log(`Received connection refused on thread ${x}. Assuming end.`);
          activeThreads[x] = true; // Don't make another request.
          endCount++;
          if (endCount === activeThreads.length) {
            push(null, redioactive.end);
          }
          return;
        }
        node.warn(`Received error when requesting frame from server on thread ${x}: ${e}`);
        push(`Received error when requesting frame from server on thread ${x}: ${e}`);
        activeThreads[x] = false;
        next();
      });
      req.end();
      requestTimer = process.hrtime();
    };

    var grainQueue = { };
    var highWaterMark = Number.MAX_SAFE_INTEGER + ':0';
    // Push every grain older than what is in nextRequest, send grains in order
    function pushGrains(g, push) {
      grainQueue[g.formatTimestamp(g.ptpOrigin)] = g;
      console.log('QQQ', nextRequest);
      var nextMin = nextRequest.reduce(function (a, b) {
        return compareVersions(a, b) <= 0 ? a : b;
      });
      Object.keys(grainQueue).filter(function (gts) {
        return compareVersions(gts, nextMin) <= 0;
      })
      .sort(compareVersions)
      .forEach(function (gts) {
        if (!config.regenerate) {
          console.log('>>> PUSHING', gts);
          push(null, grainQueue[gts]);
        } else {
          var g = grainQueue[gts];
          var grainTime = new Buffer(10);
          grainTime.writeUIntBE(this.baseTime[0], 0, 6);
          grainTime.writeUInt32BE(this.baseTime[1], 6);
          var grainDuration = g.getDuration();
          this.baseTime[1] = ( this.baseTime[1] +
            grainDuration[0] * 1000000000 / grainDuration[1]|0 );
          this.baseTime = [ this.baseTime[0] + this.baseTime[1] / 1000000000|0,
            this.baseTime[1] % 1000000000];
          push(null, new Grain(g.buffers, grainTime, g.ptpOrigin, g.timecode,
            flow.id, source.id, g.duration));
        };
        delete grainQueue[gts];
        highWaterMark = gts;
      });
    };

    var activeThreads =
      [ false, false, false, false, false, false].slice(0, config.parallel);
    var nextRequest =
      [ '-5', '-4', '-3', '-2', '-1', '0' ].slice(-config.parallel);

    dns.lookup(fullURL.hostname, function (err, addr, family) {
      if (err) return this.preFlightError(`Unable to resolve DNS for ${fullURL.hostname}: ${err}`);
      node.log(`Resolved URL hostname ${fullURL.hostname} to ${addr}.`);
      fullURL.hostname = addr;
      if (config.mode === 'push') { // TODO push mode
        // var app = express();
        // app.use(bodyParser.raw({ limit : config.payloadLimit || 6000000 }));
        //
        // app.post(config.path, function (req, res) {
        //   console.log(pushCount++, process.hrtime(star
        //     tTime), req.body.length);
        //   res.json({ length : req.body.length }); // Need to add back pressure concept
        // });
        //
        // var server = protocol.createServer((config.protocol === 'HTTP') ? {} : {
        //   key : fs.readFileSync('./certs/dynamorse-key.pem'),
        //   cert : fs.readFileSync('./certs/dynamorse-cert.pem')
        // }, app).listen(config.port);
        // server.on('listening', function () {
        //   this.log(`Dynamorse ${config.protocol} server listening on port ${config.port}.`);
        // });
      } else { // config.mode is set to pull
        this.generator(function (push, next) {
          setTimeout(function() {
            console.log('+++ DEBUG THREADS', activeThreads);
            for ( var i = 0 ; i < activeThreads.length ; i++ ) {
              if (!activeThreads[i]) {
                if (versionDiffMs(highWaterMark, nextRequest[i]) < maxDrift) {
                  runNext.call(this, i, push, next);
                  activeThreads[i] = true;
                } else {
                  node.error(`Not progressing thread ${i} this time due to a drift of ${versionDiffMs(highWaterMark, nextRequest[i])}.`);
                }
              }
            };
          }, (flow === null) ? 1000 : 0);
        });
      }
    }.bind(this));

  }
  util.inherits(SpmHTTPIn, redioactive.Funnel);
  RED.nodes.registerType("spm-http-in", SpmHTTPIn);
}
<|endoftext|>


<commit_before>(function () {
  "use strict";

  const root = this,
    previous_xo = root.xo,
    xo = {};

  /**
   * @namespace xo
   * @version 0.3.2
  */
  xo.VERSION = '0.4.0';

  function identity(x) {
    return x;
  }

  function is(type) {
    const fastTypes = ['undefined', 'boolean', 'number', 'string', 'symbol', 'function'];
    if (fastTypes.indexOf(type.toLowerCase()) >= 0) {
      type = type.toLowerCase();
      return (test) => typeof test === type;
    }
    type = '[object ' + type + ']';
    return (test) =>  Object.prototype.toString.call(test) === type;
  }

  /**
   * Type check for Boolean
   *
   * @example
   * xo.isBoolean(true); // true
   * xo.isBoolean('true'); // false
   *
   * @function
   * @name xo.isBoolean
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isBoolean = is('Boolean');

  /**
   * Type check for Number
   *
   * @example
   * xo.isNumber(42); // true
   * xo.isNumber('true'); // false
   *
   * @function
   * @name xo.isNumber
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isNumber = is('Number');

  /**
   * Type check for String
   *
   * @example
   * xo.isString('true'); // true
   * xo.isString(true); // false
   *
   * @function
   * @name xo.isString
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isString = is('String');

  /**
   * Type check for Object
   *
   * @example
   * xo.isObject({ a: true }); // true
   * xo.isObject(true); // false
   *
   * @function
   * @name xo.isObject
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isObject = is('Object');

  /**
   * Type check for Array
   *
   * @example
   * xo.isArray([1, 2, 3]); // true
   * xo.isArray(true); // false
   *
   * @function
   * @name xo.isArray
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isArray = is('Array');

  /**
   * Type check for Function
   *
   * @example
   * xo.isFunction(function(){ return true; }); // true
   * xo.isFunction(true); // false
   *
   * @function
   * @name xo.isFunction
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isFunction = is('Function');

  /**
   * Allows users to avoid conflicts over the xo name
   *
   * @example
   * var ox = xo.noConflict();
   *
   * @function
   * @name xo.noConflict
   * @return {Object}
  */
  xo.noConflict = () => {
    root.xo = previous_xo;
    return xo;
  };

  /**
   * Takes a function with zero or more arguments.
   * Returns a function that can be invoked with remaining arguments at a later time
   *
   * @example
   * var greet = function(greeting, name) {
   *   return [greeting, name].join(' ');
   * };
   *
   * var sayHi = xo.partial(greet, 'Hi');
   * sayHi('Bob'); // "Hi Bob"
   *
   * @function
   * @name xo.partial
   * @param {Function} fn - Partially apply this function prefilling some arguments
   * @param {*} [args] - Initial arguments that the partially applied function will be applied to.
   * @return {Function}
  */
  xo.partial = function (fn) {
    const initialArgs = Array.prototype.slice.call(arguments, 1);
    return function () {
      const remainingArgs = Array.prototype.slice.call(arguments);
      return fn.apply(this, initialArgs.concat(remainingArgs));
    };
  };

  /**
   * Takes an n-dimensional nested array.
   * Returns a flattened 1-dimensional array.
   *
   * @example
   * var test = [0, 1, [2, 3], [4, [5, 6]], 7, [8, [9]]];
   * xo.flatten(test); // [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
   *
   * @function
   * @name xo.flatten
   * @param {Array} arr - The array that will be recursively flattened
   * @return {Array}
  */
  xo.flatten = (arr) => {
    let output = [];
    arr.forEach(function (val) {
      output = output.concat(Array.isArray(val) ? xo.flatten(val) : val);
    });
    return output;
  };

  /**
   * Takes an array and a predicate function.
   * Returns an array with only those terms that pass the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var objArr = [
   *   { name: 'a', id: '001' },
   *   { name: 'b', id: '003' },
   *   { name: 'c', id: '003' },
   *   { name: 'd', id: '004' }
   * ];
   * xo.filter(objArr, xo.partial(compare, '003')); // [{ name: 'b', id: '003'},{name: 'c', id: '003'}]
   *
   * @function
   * @name xo.filter
   * @param {Array} arr - The array containing the elements to test
   * @param {Function} predicate - The function against which each element of the array will be tested
   * @return {Array}
  */
  xo.filter = (arr, predicate) => {
    let result = [],
      idx,
      len;
    for (idx = 0, len = arr.length; idx < len; idx += 1) {
      if (predicate(arr[idx])) {
        result.push(arr[idx]);
      }
    }
    return result;
  };

  /**
   * Takes an array.
   * Returns an array with all falsy values removed
   *
   * @example
   * var test = [1, , false, 2, 3, false];
   * xo.compact(test); // [1, 2, 3]
   *
   * @function
   * @name xo.compact
   * @param {Array} arr - The array containing the elements to test
   * @return {Array}
  */
  xo.compact = (arr) => {
    return xo.filter(arr, identity);
  };

  /**
   * Takes an array and a predicate function.
   * Returns the index of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var objArr = [
   *   { name: 'a', id: '001' },
   *   { name: 'b', id: '002' },
   *   { name: 'c', id: '003' },
   *   { name: 'd', id: '004' }
   * ];
   * xo.findIndex(objArr, xo.partial(compare, '003')); // 2
   *
   * @function
   * @name xo.findIndex
   * @param {Array} arr - The array containing the elements to test
   * @param {Function} predicate - The function against which each element of the array will be tested
   * @return {Number}
  */
  xo.findIndex = (arr, predicate) => {
    let idx,
      len;
    for (idx = 0, len = arr.length; idx < len; idx += 1) {
      if (predicate(arr[idx])) {
        return idx;
      }
    }
    return -1;
  };

  /**
   * Takes an object and a predicate function.
   * Returns the key of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var obj = {
   *   hello: { name: 'a', id: '001' },
   *   goodbye: { name: 'b', id: '002' },
   *   yes: { name: 'c', id: '003' },
   *   no: { name: 'd', id: '004' }
   * };
   * xo.findKey(obj, xo.partial(compare, '003')); // yes
   *
   * @function
   * @name xo.findKey
   * @param {Object} obj - The object containing the elements to test
   * @param {Function} predicate - The function against which each property of the object will be tested
   * @return {String}
  */
  xo.findKey = (obj, predicate) => {
    for (let prop in obj) {
      if (obj.hasOwnProperty(prop) && predicate(obj[prop])) {
        return prop;
      }
    }
    return null;
  };

  /**
   * Takes an object or an array  and a predicate function.
   * Returns the value of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var obj = {
   *   hello: { name: 'a', id: '001' },
   *   goodbye: { name: 'b', id: '002' },
   *   yes: { name: 'c', id: '003' },
   *   no: { name: 'd', id: '004' }
   * };
   * xo.find(obj, xo.partial(compare, '003')); // {yes: { name: 'a', id: '003' }}
   *
   * @function
   * @name xo.find
   * @param {Object} {Array} collection - The object or array  containing the elements to test
   * @param {Function} predicate - The function against which each property of the collection will be tested
   * @return {String} {Number}
  */
  xo.find = (collection, predicate) => {
    if (xo.isArray(collection)) {
      return collection[xo.findIndex(collection, predicate)];
    }
    if (xo.isObject(collection)) {
      return collection[xo.findKey(collection, predicate)];
    }
  };

  /**
   * Takes a function and returns a function.
   * Invoking the returned function will return cached results if the same
   * arguments have been provided during previous invocations.
   *
   * @example
   * var upper = function(str) {
   *   return str.toUpperCase();
   * };
   * var memoUpper = xo.memoize(upper);
   * memoUpper('foo'); // "FOO"
   * memoUpper('foo'); // "FOO" (cached version)
   *
   * @function
   * @name xo.memoize
   * @param {Function} fn - The (expensive) function that will have it's return values cached
   * @return {Function}
  */
  xo.memoize = (fn) => {
    let cache = {};
    return function () {
      const key = JSON.stringify(arguments);
      return cache[key] || (cache[key] = fn.apply(this, arguments));
    };
  };

  /**
   * Takes a function and returns a function.
   * The returned function will not be called if supplied with null
   * or undefined arguments
   *
   * @example
   * var sum = function(a, b) {
   *   return a + b;
   * }
   * var maybeSum = xo.maybe(sum);
   * maybeSum(2, 3); // 5
   * maybeSum(null, 3); // doesn't invoke sum
   *
   * @function
   * @name xo.maybe
   * @param {Function} fn - The function to be invoked
   * @return {Function}
  */
  xo.maybe = (fn) => {
    return function () {
      const args = Array.prototype.slice.call(arguments);
      if (!args.length || args.some(function (val) { return val == null; })) {
        return;
      }
      return fn.apply(this, args);
    };
  };

  if (typeof exports !== 'undefined') {
    if (typeof module !== 'undefined' && module.exports) {
      exports = module.exports = xo; // jshint ignore:line
    }
    exports.xo = xo;
  } else {
    root.xo = xo;
  }

}).call(this);
<commit_msg>replaced callbacks with fat arrow functions<commit_after>(function () {
  "use strict";

  const root = this,
    previous_xo = root.xo,
    xo = {};

  /**
   * @namespace xo
   * @version 0.3.2
  */
  xo.VERSION = '0.4.0';

  function identity(x) {
    return x;
  }

  function is(type) {
    const fastTypes = ['undefined', 'boolean', 'number', 'string', 'symbol', 'function'];
    if (fastTypes.indexOf(type.toLowerCase()) >= 0) {
      type = type.toLowerCase();
      return (test) => typeof test === type;
    }
    type = '[object ' + type + ']';
    return (test) =>  Object.prototype.toString.call(test) === type;
  }

  /**
   * Type check for Boolean
   *
   * @example
   * xo.isBoolean(true); // true
   * xo.isBoolean('true'); // false
   *
   * @function
   * @name xo.isBoolean
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isBoolean = is('Boolean');

  /**
   * Type check for Number
   *
   * @example
   * xo.isNumber(42); // true
   * xo.isNumber('true'); // false
   *
   * @function
   * @name xo.isNumber
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isNumber = is('Number');

  /**
   * Type check for String
   *
   * @example
   * xo.isString('true'); // true
   * xo.isString(true); // false
   *
   * @function
   * @name xo.isString
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isString = is('String');

  /**
   * Type check for Object
   *
   * @example
   * xo.isObject({ a: true }); // true
   * xo.isObject(true); // false
   *
   * @function
   * @name xo.isObject
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isObject = is('Object');

  /**
   * Type check for Array
   *
   * @example
   * xo.isArray([1, 2, 3]); // true
   * xo.isArray(true); // false
   *
   * @function
   * @name xo.isArray
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isArray = is('Array');

  /**
   * Type check for Function
   *
   * @example
   * xo.isFunction(function(){ return true; }); // true
   * xo.isFunction(true); // false
   *
   * @function
   * @name xo.isFunction
   * @param {*} test - The argument to be checked
   * @return {Boolean}
  */
  xo.isFunction = is('Function');

  /**
   * Allows users to avoid conflicts over the xo name
   *
   * @example
   * var ox = xo.noConflict();
   *
   * @function
   * @name xo.noConflict
   * @return {Object}
  */
  xo.noConflict = () => {
    root.xo = previous_xo;
    return xo;
  };

  /**
   * Takes a function with zero or more arguments.
   * Returns a function that can be invoked with remaining arguments at a later time
   *
   * @example
   * var greet = function(greeting, name) {
   *   return [greeting, name].join(' ');
   * };
   *
   * var sayHi = xo.partial(greet, 'Hi');
   * sayHi('Bob'); // "Hi Bob"
   *
   * @function
   * @name xo.partial
   * @param {Function} fn - Partially apply this function prefilling some arguments
   * @param {*} [args] - Initial arguments that the partially applied function will be applied to.
   * @return {Function}
  */
  xo.partial = function (fn) {
    const initialArgs = Array.prototype.slice.call(arguments, 1);
    return function () {
      const remainingArgs = Array.prototype.slice.call(arguments);
      return fn.apply(this, initialArgs.concat(remainingArgs));
    };
  };

  /**
   * Takes an n-dimensional nested array.
   * Returns a flattened 1-dimensional array.
   *
   * @example
   * var test = [0, 1, [2, 3], [4, [5, 6]], 7, [8, [9]]];
   * xo.flatten(test); // [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
   *
   * @function
   * @name xo.flatten
   * @param {Array} arr - The array that will be recursively flattened
   * @return {Array}
  */
  xo.flatten = (arr) => {
    let output = [];
    arr.forEach((val) => output = output.concat(Array.isArray(val) ? xo.flatten(val) : val));
    return output;
  };

  /**
   * Takes an array and a predicate function.
   * Returns an array with only those terms that pass the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var objArr = [
   *   { name: 'a', id: '001' },
   *   { name: 'b', id: '003' },
   *   { name: 'c', id: '003' },
   *   { name: 'd', id: '004' }
   * ];
   * xo.filter(objArr, xo.partial(compare, '003')); // [{ name: 'b', id: '003'},{name: 'c', id: '003'}]
   *
   * @function
   * @name xo.filter
   * @param {Array} arr - The array containing the elements to test
   * @param {Function} predicate - The function against which each element of the array will be tested
   * @return {Array}
  */
  xo.filter = (arr, predicate) => {
    let result = [],
      idx,
      len;
    for (idx = 0, len = arr.length; idx < len; idx += 1) {
      if (predicate(arr[idx])) {
        result.push(arr[idx]);
      }
    }
    return result;
  };

  /**
   * Takes an array.
   * Returns an array with all falsy values removed
   *
   * @example
   * var test = [1, , false, 2, 3, false];
   * xo.compact(test); // [1, 2, 3]
   *
   * @function
   * @name xo.compact
   * @param {Array} arr - The array containing the elements to test
   * @return {Array}
  */
  xo.compact = (arr) => {
    return xo.filter(arr, identity);
  };

  /**
   * Takes an array and a predicate function.
   * Returns the index of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var objArr = [
   *   { name: 'a', id: '001' },
   *   { name: 'b', id: '002' },
   *   { name: 'c', id: '003' },
   *   { name: 'd', id: '004' }
   * ];
   * xo.findIndex(objArr, xo.partial(compare, '003')); // 2
   *
   * @function
   * @name xo.findIndex
   * @param {Array} arr - The array containing the elements to test
   * @param {Function} predicate - The function against which each element of the array will be tested
   * @return {Number}
  */
  xo.findIndex = (arr, predicate) => {
    let idx,
      len;
    for (idx = 0, len = arr.length; idx < len; idx += 1) {
      if (predicate(arr[idx])) {
        return idx;
      }
    }
    return -1;
  };

  /**
   * Takes an object and a predicate function.
   * Returns the key of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var obj = {
   *   hello: { name: 'a', id: '001' },
   *   goodbye: { name: 'b', id: '002' },
   *   yes: { name: 'c', id: '003' },
   *   no: { name: 'd', id: '004' }
   * };
   * xo.findKey(obj, xo.partial(compare, '003')); // yes
   *
   * @function
   * @name xo.findKey
   * @param {Object} obj - The object containing the elements to test
   * @param {Function} predicate - The function against which each property of the object will be tested
   * @return {String}
  */
  xo.findKey = (obj, predicate) => {
    for (let prop in obj) {
      if (obj.hasOwnProperty(prop) && predicate(obj[prop])) {
        return prop;
      }
    }
    return null;
  };

  /**
   * Takes an object or an array  and a predicate function.
   * Returns the value of the first term that passes the predicate
   *
   * @example
   * function compare(id, obj) {
   *   return id === obj.id;
   * }
   * var obj = {
   *   hello: { name: 'a', id: '001' },
   *   goodbye: { name: 'b', id: '002' },
   *   yes: { name: 'c', id: '003' },
   *   no: { name: 'd', id: '004' }
   * };
   * xo.find(obj, xo.partial(compare, '003')); // {yes: { name: 'a', id: '003' }}
   *
   * @function
   * @name xo.find
   * @param {Object} {Array} collection - The object or array  containing the elements to test
   * @param {Function} predicate - The function against which each property of the collection will be tested
   * @return {String} {Number}
  */
  xo.find = (collection, predicate) => {
    if (xo.isArray(collection)) {
      return collection[xo.findIndex(collection, predicate)];
    }
    if (xo.isObject(collection)) {
      return collection[xo.findKey(collection, predicate)];
    }
  };

  /**
   * Takes a function and returns a function.
   * Invoking the returned function will return cached results if the same
   * arguments have been provided during previous invocations.
   *
   * @example
   * var upper = function(str) {
   *   return str.toUpperCase();
   * };
   * var memoUpper = xo.memoize(upper);
   * memoUpper('foo'); // "FOO"
   * memoUpper('foo'); // "FOO" (cached version)
   *
   * @function
   * @name xo.memoize
   * @param {Function} fn - The (expensive) function that will have it's return values cached
   * @return {Function}
  */
  xo.memoize = (fn) => {
    let cache = {};
    return function () {
      const key = JSON.stringify(arguments);
      return cache[key] || (cache[key] = fn.apply(this, arguments));
    };
  };

  /**
   * Takes a function and returns a function.
   * The returned function will not be called if supplied with null
   * or undefined arguments
   *
   * @example
   * var sum = function(a, b) {
   *   return a + b;
   * }
   * var maybeSum = xo.maybe(sum);
   * maybeSum(2, 3); // 5
   * maybeSum(null, 3); // doesn't invoke sum
   *
   * @function
   * @name xo.maybe
   * @param {Function} fn - The function to be invoked
   * @return {Function}
  */
  xo.maybe = (fn) => {
    return function () {
      const args = Array.prototype.slice.call(arguments);
      if (!args.length || args.some((val) => val == null)) {
        return;
      }
      return fn.apply(this, args);
    };
  };

  if (typeof exports !== 'undefined') {
    if (typeof module !== 'undefined' && module.exports) {
      exports = module.exports = xo; // jshint ignore:line
    }
    exports.xo = xo;
  } else {
    root.xo = xo;
  }

}).call(this);
<|endoftext|>


<commit_before>//
//  ____  _                           _
// / ___|| |_ __ _ _ __   ___  ___   (_)___  (*)
// \___ \| __/ _` | '_ \ / _ \/ __|  | / __|
//  ___) | || (_| | |_) |  __/\__ \_ | \__ \
// |____/ \__\__,_| .__/ \___||___(_)/ |___/
//              |_|              |__/
//
// (*) a (really) tiny Javascript MVC microframework
//
// (c) Hay Kranen < hay@bykr.org >
// Released under the terms of the MIT license
// < http://en.wikipedia.org/wiki/MIT_License >
//
// Stapes.js : http://hay.github.com/stapes

(function() {
    'use strict';

    var VERSION = "0.7.0-pre";

    // Global counter for all events in all modules (including mixed in objects)
    var guid = 1;

    // Makes _.create() faster
    if (!Object.create) {
        var CachedFunction = function(){};
    }

    // So we can use slice.call for arguments later on
    var slice = Array.prototype.slice;

    // Private attributes and helper functions, stored in an object so they
    // are overwritable by plugins
    var _ = {
        // Properties
        attributes : {},

        eventHandlers : {
            "-1" : {} // '-1' is used for the global event handling
        },

        guid : -1,

        // Methods
        addEvent : function(event) {
            // If we don't have any handlers for this type of event, add a new
            // array we can use to push new handlers
            if (!_.eventHandlers[event.guid][event.type]) {
                _.eventHandlers[event.guid][event.type] = [];
            }

            // Push an event object
            _.eventHandlers[event.guid][event.type].push({
                "guid" : event.guid,
                "handler" : event.handler,
                "scope" : event.scope,
                "type" : event.type
            });
        },

        addEventHandler : function(argTypeOrMap, argHandlerOrScope, argScope) {
            var eventMap = {},
                scope;

            if (typeof argTypeOrMap === "string") {
                scope = argScope || false;
                eventMap[ argTypeOrMap ] = argHandlerOrScope;
            } else {
                scope = argHandlerOrScope || false;
                eventMap = argTypeOrMap;
            }

            for (var eventString in eventMap) {
                var handler = eventMap[eventString];
                var events = eventString.split(" ");

                for (var i = 0, l = events.length; i < l; i++) {
                    var eventType = events[i];
                    _.addEvent.call(this, {
                        "guid" : this._guid || this._.guid,
                        "handler" : handler,
                        "scope" : scope,
                        "type" : eventType
                    });
                }
            }
        },

        addGuid : function(object, forceGuid) {
            if (object._guid && !forceGuid) return;

            object._guid = guid++;

            _.attributes[object._guid] = {};
            _.eventHandlers[object._guid] = {};
        },

        // This is a really small utility function to save typing and produce
        // better optimized code
        attr : function(guid) {
            return _.attributes[guid];
        },

        clone : function(obj) {
            return _.extend({}, obj);
        },

        create : function(proto) {
            if (Object.create) {
                return Object.create(proto);
            } else {
                CachedFunction.prototype = obj;
                return new CachedFunction();
            }
        },

        createSubclass : function(props, mixinEvents) {
            mixinEvents = mixinEvents || false;
            props = props || {};
            var superclass = props.superclass.prototype;
            // Objects always have a constructor, so we need to be sure this is
            // a property instead of something from the prototype
            var realConstructor = props.hasOwnProperty('constructor') ? props.constructor : function(){};
            var constructor = function() {
                _.addGuid( this, true );
                realConstructor.apply(this, arguments);
            };

            if (mixinEvents) {
                _.extend(superclass, Events);
            }

            constructor.prototype = _.create(superclass);
            constructor.prototype.constructor = constructor;
            _.extend(constructor, {
                extend : function() {
                    return _.extendThis.apply(this, arguments);
                },

                proto : function() {
                    return _.extendThis.apply(this.prototype, arguments);
                },

                subclass : function(obj) {
                    obj = obj || {};
                    obj.superclass = this;
                    return _.createSubclass(obj);
                },

                super : superclass,
            });

            // Copy all props given in the definition to the prototype
            for (var key in props) {
                if (key !== 'constructor') {
                    constructor.prototype[key] = props[key];
                }
            }

            return constructor;
        },

        emitEvents : function(type, data, explicitType, explicitGuid) {
            explicitType = explicitType || false;
            explicitGuid = explicitGuid || this._guid;

            var handlers = _.eventHandlers[explicitGuid][type];

            for (var i = 0, l = handlers.length; i < l; i++) {
                // Clone the event to prevent issue #19
                var event = _.extend({}, handlers[i]);
                var scope = (event.scope) ? event.scope : this;

                if (explicitType) {
                    event.type = explicitType;
                }

                event.scope = scope;
                event.handler.call(event.scope, data, event);
            }
        },

        // Extend an object with more objects
        extend : function() {
            var args = slice.call(arguments);
            var object = args.shift();

            for (var i = 0, l = args.length; i < l; i++) {
                var props = args[i];
                for (var key in props) {
                    object[key] = props[key];
                }
            }

            return object;
        },

        // The same as extend, but uses the this value as the scope
        extendThis : function() {
            var args = slice.call(arguments);
            args.unshift(this);
            return _.extend.apply(this, args);
        },

        // from http://stackoverflow.com/a/2117523/152809
        makeUuid : function() {
            return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
                var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
                return v.toString(16);
            });
        },

        removeAttribute : function(keys, silent) {
            silent = silent || false;

            // Split the key, maybe we want to remove more than one item
            var attributes = _.trim(keys).split(" ");

            // Actually delete the item
            for (var i = 0, l = attributes.length; i < l; i++) {
                var key = _.trim(attributes[i]);

                if (key) {
                    delete _.attr(this._guid)[key];

                    // If 'silent' is set, do not throw any events
                    if (!silent) {
                        this.emit('change', key);
                        this.emit('change:' + key);
                        this.emit('remove', key);
                        this.emit('remove:' + key);
                    }
                }
            }
        },

        removeEventHandler : function(type, handler) {
            var handlers = _.eventHandlers[this._guid];

            if (type && handler) {
                // Remove a specific handler
                handlers = handlers[type];
                if (!handlers) return;

                for (var i = 0, l = handlers.length, h; i < l; i++) {
                    h = handlers[i].handler;
                    if (h && h === handler) {
                        handlers.splice(i--, 1);
                        l--;
                    }
                }
            } else if (type) {
                // Remove all handlers for a specific type
                delete handlers[type];
            } else {
                // Remove all handlers for this module
                _.eventHandlers[this._guid] = {};
            }
        },

        setAttribute : function(key, value, silent) {
            silent = silent || false;

            // We need to do this before we actually add the item :)
            var itemExists = this.has(key);
            var oldValue = _.attr(this._guid)[key];

            // Is the value different than the oldValue? If not, ignore this call
            if (value === oldValue) {
                return;
            }

            // Actually add the item to the attributes
            _.attr(this._guid)[key] = value;

            // If 'silent' flag is set, do not throw any events
            if (silent) {
                return;
            }

            // Throw a generic event
            this.emit('change', key);

            // And a namespaced event as well, NOTE that we pass value instead of
            // key here!
            this.emit('change:' + key, value);

            // Throw namespaced and non-namespaced 'mutate' events as well with
            // the old value data as well and some extra metadata such as the key
            var mutateData = {
                "key" : key,
                "newValue" : value,
                "oldValue" : oldValue || null
            };

            this.emit('mutate', mutateData);
            this.emit('mutate:' + key, mutateData);

            // Also throw a specific event for this type of set
            var specificEvent = itemExists ? 'update' : 'create';

            this.emit(specificEvent, key);

            // And a namespaced event as well, NOTE that we pass value instead of key
            this.emit(specificEvent + ':' + key, value);
        },

        trim : function(str) {
            return str.replace(/^\s\s*/, '').replace(/\s\s*$/, '');
        },

        typeOf : function(val) {
            if (val === null || typeof val === "undefined") {
                return String(val);
            } else {
                return Object.prototype.toString.call(val).replace(/\[object |\]/g, '').toLowerCase();
            }
        },

        updateAttribute : function(key, fn, silent) {
            var item = this.get(key);
            var newValue = fn.call(this, _.clone(item), key);
            _.setAttribute.call(this, key, newValue, silent || false);
        }
    };

    // Can be mixed in later using Stapes.mixinEvents(object);
    var Events = {
        emit : function(types, data) {
            data = (typeof data === "undefined") ? null : data;

            var splittedTypes = types.split(" ");

            for (var i = 0, l = splittedTypes.length; i < l; i++) {
                var type = splittedTypes[i];

                // First 'all' type events: is there an 'all' handler in the
                // global stack?
                if (_.eventHandlers[-1].all) {
                    _.emitEvents.call(this, "all", data, type, -1);
                }

                // Catch all events for this type?
                if (_.eventHandlers[-1][type]) {
                    _.emitEvents.call(this, type, data, type, -1);
                }

                if (typeof this._guid === 'number') {
                    // 'all' event for this specific module?
                    if (_.eventHandlers[this._guid].all) {
                        _.emitEvents.call(this, "all", data, type);
                    }

                    // Finally, normal events :)
                    if (_.eventHandlers[this._guid][type]) {
                        _.emitEvents.call(this, type, data);
                    }
                }
            }
        },

        off : function() {
            _.removeEventHandler.apply(this, arguments);
        },

        on : function() {
            _.addEventHandler.apply(this, arguments);
        }
    };

    _.Module = function(){};
    _.Module.prototype = {
        // create() is deprecated from 0.8.0
        create : function() {
            throw new Error(''.concat(
                'create() on Stapes modules no longer works from 0.8.0. ',
                'Check the docs.'
            ));
        },

        each : function(fn, ctx) {
            var attr = _.attr(this._guid);
            for (var key in attr) {
                var value = attr[key];
                fn.call(ctx || this, value, key);
            }
        },

        extend : function() {
            return _.extendThis.apply(this, arguments);
        },

        filter : function(fn) {
            var filtered = [];
            var attributes = _.attr(this._guid);

            for (var key in attributes) {
                if ( fn.call(this, attributes[key], key)) {
                    filtered.push( attributes[key] );
                }
            }

            return filtered;
        },

        get : function(input) {
            if (typeof input === "string") {
                return this.has(input) ? _.attr(this._guid)[input] : null;
            } else if (typeof input === "function") {
                var items = this.filter(input);
                return (items.length) ? items[0] : null;
            }
        },

        getAll : function() {
            return _.clone( _.attr(this._guid) );
        },

        getAllAsArray : function() {
            var arr = [];
            var attributes = _.attr(this._guid);

            for (var key in attributes) {
                var value = attributes[key];

                if (_.typeOf(value) === "object" && !value.id) {
                    value.id = key;
                }

                arr.push(value);
            }

            return arr;
        },

        has : function(key) {
            return (typeof _.attr(this._guid)[key] !== "undefined");
        },

        map : function(fn, ctx) {
            var mapped = [];
            this.each(function(value, key) {
                mapped.push( fn.call(ctx || this, value, key) );
            }, ctx || this);
            return mapped;
        },

        // Akin to set(), but makes a unique id
        push : function(input, silent) {
            if (_.typeOf(input) === "array") {
                for (var i = 0, l = input.length; i < l; i++) {
                    _.setAttribute.call(this, _.makeUuid(), input[i]);
                }
            } else {
                _.setAttribute.call(this, _.makeUuid(), input, silent || false);
            }

            return this;
        },

        remove : function(input, silent) {
            if (typeof input === "function") {
                this.each(function(item, key) {
                    if (input(item)) {
                        _.removeAttribute.call(this, key, silent);
                    }
                });
            } else {
                // nb: checking for exists happens in removeAttribute
                _.removeAttribute.call(this, input, silent || false);
            }

            return this;
        },

        set : function(objOrKey, value, silent) {
            if (typeof objOrKey === "object") {
                for (var key in objOrKey) {
                    _.setAttribute.call(this, key, objOrKey[key]);
                }
            } else {
                _.setAttribute.call(this, objOrKey, value, silent || false);
            }

            return this;
        },

        size : function() {
            return Object.keys(_.attributes[this._guid]).length;
        },

        update : function(keyOrFn, fn, silent) {
            if (typeof keyOrFn === "string") {
                _.updateAttribute.call(this, keyOrFn, fn, silent || false);
            } else if (typeof keyOrFn === "function") {
                this.each(function(value, key) {
                    _.updateAttribute.call(this, key, keyOrFn);
                });
            }

            return this;
        }
    };

    var Stapes = {
        "_" : _, // private helper functions and properties

        // Compatiblity option, this method is deprecated
        "create" : function() {
            var instance = _.create( _.Module.prototype );
            _.addGuid( instance, true );

            // Mixin events
            Stapes.mixinEvents( instance );

            return instance;
        },

        "extend" : function() {
            return _.extendThis.apply(_.Moduel, arguments);
        },

        "mixinEvents" : function(obj) {
            obj = obj || {};

            _.addGuid(obj);

            return _.extend(obj, Events);
        },

        "on" : function() {
            _.addEventHandler.apply(this, arguments);
        },

        "subclass" : function(obj) {
            obj = obj || {};
            obj.superclass = _.Module;
            return _.createSubclass(obj, true);
        },

        "version" : VERSION
    };

    // This library can be used as an AMD module, a Node.js module, or an
    // old fashioned global
    if (typeof exports !== "undefined") {
        // Server
        if (typeof module !== "undefined" && module.exports) {
            exports = module.exports = Stapes;
        }
        exports.Stapes = Stapes;
    } else if (typeof define === "function" && define.amd) {
        // AMD
        define(function() {
            return Stapes;
        });
    } else {
        // Global scope
        window.Stapes = Stapes;
    }
})();<commit_msg>adding classOnly support, and fixed a bug where 'superclass' property would be set on the instance as well<commit_after>//
//  ____  _                           _
// / ___|| |_ __ _ _ __   ___  ___   (_)___  (*)
// \___ \| __/ _` | '_ \ / _ \/ __|  | / __|
//  ___) | || (_| | |_) |  __/\__ \_ | \__ \
// |____/ \__\__,_| .__/ \___||___(_)/ |___/
//              |_|              |__/
//
// (*) a (really) tiny Javascript MVC microframework
//
// (c) Hay Kranen < hay@bykr.org >
// Released under the terms of the MIT license
// < http://en.wikipedia.org/wiki/MIT_License >
//
// Stapes.js : http://hay.github.com/stapes

(function() {
    'use strict';

    var VERSION = "0.7.0-pre";

    // Global counter for all events in all modules (including mixed in objects)
    var guid = 1;

    // Makes _.create() faster
    if (!Object.create) {
        var CachedFunction = function(){};
    }

    // So we can use slice.call for arguments later on
    var slice = Array.prototype.slice;

    // Private attributes and helper functions, stored in an object so they
    // are overwritable by plugins
    var _ = {
        // Properties
        attributes : {},

        eventHandlers : {
            "-1" : {} // '-1' is used for the global event handling
        },

        guid : -1,

        // Methods
        addEvent : function(event) {
            // If we don't have any handlers for this type of event, add a new
            // array we can use to push new handlers
            if (!_.eventHandlers[event.guid][event.type]) {
                _.eventHandlers[event.guid][event.type] = [];
            }

            // Push an event object
            _.eventHandlers[event.guid][event.type].push({
                "guid" : event.guid,
                "handler" : event.handler,
                "scope" : event.scope,
                "type" : event.type
            });
        },

        addEventHandler : function(argTypeOrMap, argHandlerOrScope, argScope) {
            var eventMap = {},
                scope;

            if (typeof argTypeOrMap === "string") {
                scope = argScope || false;
                eventMap[ argTypeOrMap ] = argHandlerOrScope;
            } else {
                scope = argHandlerOrScope || false;
                eventMap = argTypeOrMap;
            }

            for (var eventString in eventMap) {
                var handler = eventMap[eventString];
                var events = eventString.split(" ");

                for (var i = 0, l = events.length; i < l; i++) {
                    var eventType = events[i];
                    _.addEvent.call(this, {
                        "guid" : this._guid || this._.guid,
                        "handler" : handler,
                        "scope" : scope,
                        "type" : eventType
                    });
                }
            }
        },

        addGuid : function(object, forceGuid) {
            if (object._guid && !forceGuid) return;

            object._guid = guid++;

            _.attributes[object._guid] = {};
            _.eventHandlers[object._guid] = {};
        },

        // This is a really small utility function to save typing and produce
        // better optimized code
        attr : function(guid) {
            return _.attributes[guid];
        },

        clone : function(obj) {
            return _.extend({}, obj);
        },

        create : function(proto) {
            if (Object.create) {
                return Object.create(proto);
            } else {
                CachedFunction.prototype = obj;
                return new CachedFunction();
            }
        },

        createSubclass : function(props, mixinEvents, includeEvents) {
            mixinEvents = mixinEvents || false;
            props = props || {};
            var superclass = props.superclass.prototype;
            // Objects always have a constructor, so we need to be sure this is
            // a property instead of something from the prototype
            var realConstructor = props.hasOwnProperty('constructor') ? props.constructor : function(){};
            var constructor = function() {
                if (includeEvents) {
                    _.addGuid( this, true );
                }

                realConstructor.apply(this, arguments);
            };

            if (mixinEvents) {
                _.extend(superclass, Events);
            }

            constructor.prototype = _.create(superclass);
            constructor.prototype.constructor = constructor;
            _.extend(constructor, {
                extend : function() {
                    return _.extendThis.apply(this, arguments);
                },

                proto : function() {
                    return _.extendThis.apply(this.prototype, arguments);
                },

                subclass : function(obj) {
                    obj = obj || {};
                    obj.superclass = this;
                    return _.createSubclass(obj);
                },

                super : superclass,
            });

            // Copy all props given in the definition to the prototype
            for (var key in props) {
                if (key !== 'constructor' && key !== 'superclass') {
                    constructor.prototype[key] = props[key];
                }
            }

            return constructor;
        },

        emitEvents : function(type, data, explicitType, explicitGuid) {
            explicitType = explicitType || false;
            explicitGuid = explicitGuid || this._guid;

            var handlers = _.eventHandlers[explicitGuid][type];

            for (var i = 0, l = handlers.length; i < l; i++) {
                // Clone the event to prevent issue #19
                var event = _.extend({}, handlers[i]);
                var scope = (event.scope) ? event.scope : this;

                if (explicitType) {
                    event.type = explicitType;
                }

                event.scope = scope;
                event.handler.call(event.scope, data, event);
            }
        },

        // Extend an object with more objects
        extend : function() {
            var args = slice.call(arguments);
            var object = args.shift();

            for (var i = 0, l = args.length; i < l; i++) {
                var props = args[i];
                for (var key in props) {
                    object[key] = props[key];
                }
            }

            return object;
        },

        // The same as extend, but uses the this value as the scope
        extendThis : function() {
            var args = slice.call(arguments);
            args.unshift(this);
            return _.extend.apply(this, args);
        },

        // from http://stackoverflow.com/a/2117523/152809
        makeUuid : function() {
            return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
                var r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
                return v.toString(16);
            });
        },

        removeAttribute : function(keys, silent) {
            silent = silent || false;

            // Split the key, maybe we want to remove more than one item
            var attributes = _.trim(keys).split(" ");

            // Actually delete the item
            for (var i = 0, l = attributes.length; i < l; i++) {
                var key = _.trim(attributes[i]);

                if (key) {
                    delete _.attr(this._guid)[key];

                    // If 'silent' is set, do not throw any events
                    if (!silent) {
                        this.emit('change', key);
                        this.emit('change:' + key);
                        this.emit('remove', key);
                        this.emit('remove:' + key);
                    }
                }
            }
        },

        removeEventHandler : function(type, handler) {
            var handlers = _.eventHandlers[this._guid];

            if (type && handler) {
                // Remove a specific handler
                handlers = handlers[type];
                if (!handlers) return;

                for (var i = 0, l = handlers.length, h; i < l; i++) {
                    h = handlers[i].handler;
                    if (h && h === handler) {
                        handlers.splice(i--, 1);
                        l--;
                    }
                }
            } else if (type) {
                // Remove all handlers for a specific type
                delete handlers[type];
            } else {
                // Remove all handlers for this module
                _.eventHandlers[this._guid] = {};
            }
        },

        setAttribute : function(key, value, silent) {
            silent = silent || false;

            // We need to do this before we actually add the item :)
            var itemExists = this.has(key);
            var oldValue = _.attr(this._guid)[key];

            // Is the value different than the oldValue? If not, ignore this call
            if (value === oldValue) {
                return;
            }

            // Actually add the item to the attributes
            _.attr(this._guid)[key] = value;

            // If 'silent' flag is set, do not throw any events
            if (silent) {
                return;
            }

            // Throw a generic event
            this.emit('change', key);

            // And a namespaced event as well, NOTE that we pass value instead of
            // key here!
            this.emit('change:' + key, value);

            // Throw namespaced and non-namespaced 'mutate' events as well with
            // the old value data as well and some extra metadata such as the key
            var mutateData = {
                "key" : key,
                "newValue" : value,
                "oldValue" : oldValue || null
            };

            this.emit('mutate', mutateData);
            this.emit('mutate:' + key, mutateData);

            // Also throw a specific event for this type of set
            var specificEvent = itemExists ? 'update' : 'create';

            this.emit(specificEvent, key);

            // And a namespaced event as well, NOTE that we pass value instead of key
            this.emit(specificEvent + ':' + key, value);
        },

        trim : function(str) {
            return str.replace(/^\s\s*/, '').replace(/\s\s*$/, '');
        },

        typeOf : function(val) {
            if (val === null || typeof val === "undefined") {
                return String(val);
            } else {
                return Object.prototype.toString.call(val).replace(/\[object |\]/g, '').toLowerCase();
            }
        },

        updateAttribute : function(key, fn, silent) {
            var item = this.get(key);
            var newValue = fn.call(this, _.clone(item), key);
            _.setAttribute.call(this, key, newValue, silent || false);
        }
    };

    // Can be mixed in later using Stapes.mixinEvents(object);
    var Events = {
        emit : function(types, data) {
            data = (typeof data === "undefined") ? null : data;

            var splittedTypes = types.split(" ");

            for (var i = 0, l = splittedTypes.length; i < l; i++) {
                var type = splittedTypes[i];

                // First 'all' type events: is there an 'all' handler in the
                // global stack?
                if (_.eventHandlers[-1].all) {
                    _.emitEvents.call(this, "all", data, type, -1);
                }

                // Catch all events for this type?
                if (_.eventHandlers[-1][type]) {
                    _.emitEvents.call(this, type, data, type, -1);
                }

                if (typeof this._guid === 'number') {
                    // 'all' event for this specific module?
                    if (_.eventHandlers[this._guid].all) {
                        _.emitEvents.call(this, "all", data, type);
                    }

                    // Finally, normal events :)
                    if (_.eventHandlers[this._guid][type]) {
                        _.emitEvents.call(this, type, data);
                    }
                }
            }
        },

        off : function() {
            _.removeEventHandler.apply(this, arguments);
        },

        on : function() {
            _.addEventHandler.apply(this, arguments);
        }
    };

    _.Module = function(){};
    _.Module.prototype = {
        // create() is deprecated from 0.8.0
        create : function() {
            throw new Error(''.concat(
                'create() on Stapes modules no longer works from 0.8.0. ',
                'Check the docs.'
            ));
        },

        each : function(fn, ctx) {
            var attr = _.attr(this._guid);
            for (var key in attr) {
                var value = attr[key];
                fn.call(ctx || this, value, key);
            }
        },

        extend : function() {
            return _.extendThis.apply(this, arguments);
        },

        filter : function(fn) {
            var filtered = [];
            var attributes = _.attr(this._guid);

            for (var key in attributes) {
                if ( fn.call(this, attributes[key], key)) {
                    filtered.push( attributes[key] );
                }
            }

            return filtered;
        },

        get : function(input) {
            if (typeof input === "string") {
                return this.has(input) ? _.attr(this._guid)[input] : null;
            } else if (typeof input === "function") {
                var items = this.filter(input);
                return (items.length) ? items[0] : null;
            }
        },

        getAll : function() {
            return _.clone( _.attr(this._guid) );
        },

        getAllAsArray : function() {
            var arr = [];
            var attributes = _.attr(this._guid);

            for (var key in attributes) {
                var value = attributes[key];

                if (_.typeOf(value) === "object" && !value.id) {
                    value.id = key;
                }

                arr.push(value);
            }

            return arr;
        },

        has : function(key) {
            return (typeof _.attr(this._guid)[key] !== "undefined");
        },

        map : function(fn, ctx) {
            var mapped = [];
            this.each(function(value, key) {
                mapped.push( fn.call(ctx || this, value, key) );
            }, ctx || this);
            return mapped;
        },

        // Akin to set(), but makes a unique id
        push : function(input, silent) {
            if (_.typeOf(input) === "array") {
                for (var i = 0, l = input.length; i < l; i++) {
                    _.setAttribute.call(this, _.makeUuid(), input[i]);
                }
            } else {
                _.setAttribute.call(this, _.makeUuid(), input, silent || false);
            }

            return this;
        },

        remove : function(input, silent) {
            if (typeof input === "function") {
                this.each(function(item, key) {
                    if (input(item)) {
                        _.removeAttribute.call(this, key, silent);
                    }
                });
            } else {
                // nb: checking for exists happens in removeAttribute
                _.removeAttribute.call(this, input, silent || false);
            }

            return this;
        },

        set : function(objOrKey, value, silent) {
            if (typeof objOrKey === "object") {
                for (var key in objOrKey) {
                    _.setAttribute.call(this, key, objOrKey[key]);
                }
            } else {
                _.setAttribute.call(this, objOrKey, value, silent || false);
            }

            return this;
        },

        size : function() {
            return Object.keys(_.attributes[this._guid]).length;
        },

        update : function(keyOrFn, fn, silent) {
            if (typeof keyOrFn === "string") {
                _.updateAttribute.call(this, keyOrFn, fn, silent || false);
            } else if (typeof keyOrFn === "function") {
                this.each(function(value, key) {
                    _.updateAttribute.call(this, key, keyOrFn);
                });
            }

            return this;
        }
    };

    var Stapes = {
        "_" : _, // private helper functions and properties

        // Compatiblity option, this method is deprecated
        "create" : function() {
            var instance = _.create( _.Module.prototype );
            _.addGuid( instance, true );

            // Mixin events
            Stapes.mixinEvents( instance );

            return instance;
        },

        "extend" : function() {
            return _.extendThis.apply(_.Moduel, arguments);
        },

        "mixinEvents" : function(obj) {
            obj = obj || {};

            _.addGuid(obj);

            return _.extend(obj, Events);
        },

        "on" : function() {
            _.addEventHandler.apply(this, arguments);
        },

        "subclass" : function(obj, classOnly) {
            classOnly = classOnly || false;
            obj = obj || {};
            obj.superclass = classOnly ? function(){} : _.Module;
            return _.createSubclass(obj, !classOnly, !classOnly);
        },

        "version" : VERSION
    };

    // This library can be used as an AMD module, a Node.js module, or an
    // old fashioned global
    if (typeof exports !== "undefined") {
        // Server
        if (typeof module !== "undefined" && module.exports) {
            exports = module.exports = Stapes;
        }
        exports.Stapes = Stapes;
    } else if (typeof define === "function" && define.amd) {
        // AMD
        define(function() {
            return Stapes;
        });
    } else {
        // Global scope
        window.Stapes = Stapes;
    }
})();<|endoftext|>


<commit_before>var _ = require("lodash");
var MODULES_ARRAY = require("./modules.json");

var STRING_KEYS = [
  "ModuleCode", "ModuleTitle", "Department", "ModuleDescription",
  "ModuleCredit", "Workload", "Preclusion", "ExamDate", "Prerequisite",
  "CrossModule", "Corequisite"
];

// Returns an object that looks like this:
// {
//    ModuleCode: {
//      module1string: number of times module1string appears,
//      module2string: number of times module2string appears,
//      ...
//    },
//    ModuleTitle: {
//      title1: number of times title1 appears,
//      title2: number of times title2 appears,
//      ...
//    },
//    ...
//    Corequisite: {
//      coreq1: number of times coreq1 appears,
//      coreq2: number of times coreq2 appears
//      ...
//    }
// }
var compute_value_frequencies = function(keyList, modArray) {
  var valuesHash = {};
  _(keyList).forEach(function(key) {
    valuesHash[key] = {};
  });
  _(modArray).forEach(function(mod) {
    _(keyList).forEach(function(key) {
      var val;
      if (_.has(mod, key)) {
        val = mod[key];
        if (val.trim() !== val) {
          console.log("value for key \"" + key + "\" needs trimming");
        }
        if (valuesHash[key][val]) {
          valuesHash[key][val] += 1;
        } else {
          valuesHash[key][val] = 1;
        }
      }
    });
  });
  return valuesHash;
};

// all values for the key are unique
var FREQ_ALL_UNIQUE = 1;
// not all values for the key are unique
var FREQ_MORE_ANALYSIS = 2;

// computes some numbers on the values of string keys from the return value of
// the `compute_value_frequencies_for_string_keys` function
var compute_stats_from_value_frequencies = function(keyList,
    valuesHash) {
  var retHash = {};
  _(keyList).forEach(function(key) {
    var countObj = valuesHash[key];
    var freqHash = {};
    var bytesForStoringAllStrings;
    var sumLengthOfUniqueStrings;
    var bucketStatsArray = [];
    var totalNrUniqueStrings = 0;
    // for each possible value for the current string key, compute an obj with:
    //   number of occurrences --> array of integers. Each integer in the array
    //   is the lengh of a unique string which is present that many times.
    _.forOwn(countObj, function(nrOccurrences, uniqueString) {
      if (!_.has(freqHash, nrOccurrences)) {
        freqHash[nrOccurrences] = [];
      }
      freqHash[nrOccurrences].push(uniqueString.length);
    });
    if (_.keys(freqHash).length === 1 && _.has(freqHash, 1)) {
      // all values are unique
      retHash[key] = { result: FREQ_ALL_UNIQUE };
    } else {
      // compute array of BucketStats object along some other numbers.
      // Each BucketStats object has the following properties:
      // - nrOccurrences
      // - stringLengthArray (optional)
      // - count
      // - minLen
      // - maxLen
      // - medianLen
      // - avgLen
      bytesForStoringAllStrings = 0;
      sumLengthOfUniqueStrings = 0;
      _.forOwn(freqHash, function(stringLengthArray, nrOccurrences) {
        var nrUniqueStrings = stringLengthArray.length;
        var lengthArray = _.reduce(stringLengthArray,
          function(arr, stringLength) {
            _.times(nrOccurrences, function() { arr.push(stringLength); });
            return arr;
          }, []).sort();
        var len = lengthArray.length;
        var sumLengths = _.reduce(stringLengthArray,
          function(sumLengths, currentLength) {
              return sumLengths + nrOccurrences * currentLength;
          }, 0);
        var bytesForStoringIntegerIndices;
        bytesForStoringAllStrings += sumLengths;
        sumLengthOfUniqueStrings += sumLengths / nrOccurrences;
        totalNrUniqueStrings += nrUniqueStrings;
        bucketStatsArray.push({
          nrOccurrences: nrOccurrences,
          stringLengthArray: stringLengthArray,
          count: nrUniqueStrings,
          minLen: lengthArray[0],
          maxLen: lengthArray[len-1],
          medianLen: ((len % 2 === 1) ? lengthArray[Math.floor(len/2)] :
            (lengthArray[len/2 - 1] + lengthArray[len/2]) / 2),
          avgLen: sumLengths / len
        });
      });
      bucketStatsArray.sort(function(a, b) {
        return b.nrOccurrences - a.nrOccurrences
      });
      bytesForStoringIntegerIndices =
        compute_storage_for_integer_indices(bucketStatsArray);
      retHash[key] = {
        result: FREQ_MORE_ANALYSIS,
        bucketStatsArray: _.map(bucketStatsArray, function(bucketStats) {
          return _.omit(bucketStats, 'stringLengthArray');
        }),
        bytesForStoringAllStrings: bytesForStoringAllStrings,
        // '[', ']', commas, quotes, strings themselves
        bytesForStoringUniqueStringsInNewArray: 2 + (totalNrUniqueStrings - 1) +
          totalNrUniqueStrings * 2 + sumLengthOfUniqueStrings,
        bytesForStoringIntegerIndices: bytesForStoringIntegerIndices
      };
    }
  });

  return retHash;
};

// Given an array of BucketStats objects with the `stringLengthArray` property,
// where the array is sorted by descending `nrOccurrences` property, compute
// the space required for storing integer indices in a JSON file.
// Indices will start from 0 for high `nrOccurrences`.
var compute_storage_for_integer_indices = function(bucketStatsArray) {
  var compute_storage_for_one_non_negative_integer = function(x) {
    var nrDigits = 0;
    if (x === 0) {
      return 1;
    }
    while (x > 0) {
      nrDigits += 1;
      x = Math.floor(x / 10);
    }
    return nrDigits;
  };

  return _.reduce(bucketStatsArray, function(accObj, bucketStats) {
    var currentIndex = accObj.currentIndex;
    var totalSpace = accObj.totalSpace;
    var nrOccurrences = bucketStats.nrOccurrences;
    _.forEach(bucketStats.stringLengthArray, function(stringLength) {
      totalSpace += nrOccurrences *
        compute_storage_for_one_non_negative_integer(currentIndex);
      currentIndex += 1;
    });
    return {
      currentIndex: currentIndex,
      totalSpace: totalSpace
    };
  }, { currentIndex: 0, totalSpace: 0 }).totalSpace;
};

// Displays the results from the
// `compute_stats_for_string_keys_from_value_frequencies` function
var show_results = function(keyList, results) {
  _.forEach(keyList, function(key, idx) {
    var keyResult = results[key];
    if (idx > 0) {
      console.log("\n");
    }
    console.log("Key \"" + key + "\"");
    if (keyResult.result === FREQ_ALL_UNIQUE) {
      console.log("    All values are unique");
    } else {
      _.forEach(keyResult.bucketStatsArray, function(bucketStats) {
        console.log("    Values occurring " + bucketStats.nrOccurrences +
          " times: " + bucketStats.count + " unique strings [min length: " +
          bucketStats.minLen + ", max length: " + bucketStats.maxLen +
          ", avg length: " + bucketStats.avgLen + ", median length: " +
          bucketStats.medianLen + "]"
        );
      });
      console.log("Bytes for storing all strings: " +
        keyResult.bytesForStoringAllStrings
      );
      console.log("Bytes for storing unique strings in separate array: " +
        keyResult.bytesForStoringUniqueStringsInNewArray
      );
      console.log("Bytes for storing integer indices: " +
        keyResult.bytesForStoringIntegerIndices
      );
      console.log("Storing all strings vs. storing unique strings + " +
        "integer indices: " + keyResult.bytesForStoringAllStrings + " vs. " +
        (keyResult.bytesForStoringUniqueStringsInNewArray+
         keyResult.bytesForStoringIntegerIndices)
      );
      if (keyResult.bytesForStoringAllStrings <=
          keyResult.bytesForStoringUniqueStringsInNewArray+
            keyResult.bytesForStoringIntegerIndices) {
        console.log("It is more efficient in space to store all the strings.");
      } else {
        console.log("It is more efficient in space to store unique strings " +
          "with integer indices."
        );
      }
    }
  });
};

show_results(
  STRING_KEYS,
  compute_stats_from_value_frequencies(
    STRING_KEYS,
    compute_value_frequencies(STRING_KEYS, MODULES_ARRAY)
  )
);
<commit_msg>value_frequency.js - analyze `Types` and `Lecturers` keys<commit_after>var _ = require("lodash");
var MODULES_ARRAY = require("./modules.json");

var STRING_KEYS = [
  "ModuleCode", "ModuleTitle", "Department", "ModuleDescription",
  "ModuleCredit", "Workload", "Preclusion", "ExamDate", "Prerequisite",
  "CrossModule", "Corequisite"
];

var ARRAY_OF_STRINGS_KEYS = [
  "Types", "Lecturers"
];

// Returns an object that looks like this:
// {
//    ModuleCode: {
//      module1string: number of times module1string appears,
//      module2string: number of times module2string appears,
//      ...
//    },
//    ModuleTitle: {
//      title1: number of times title1 appears,
//      title2: number of times title2 appears,
//      ...
//    },
//    ...
//    Corequisite: {
//      coreq1: number of times coreq1 appears,
//      coreq2: number of times coreq2 appears
//      ...
//    }
// }
var compute_value_frequencies = function(keyList, modArray) {
  var valuesHash = {};
  _(keyList).forEach(function(key) {
    valuesHash[key] = {};
  });
  _(modArray).forEach(function(mod) {
    _(keyList).forEach(function(key) {
      var val;
      if (_.has(mod, key)) {
        val = mod[key];
        if (val.trim() !== val) {
          console.log("value for key \"" + key + "\" needs trimming");
        }
        if (valuesHash[key][val]) {
          valuesHash[key][val] += 1;
        } else {
          valuesHash[key][val] = 1;
        }
      }
    });
  });
  return valuesHash;
};

// all values for the key are unique
var FREQ_ALL_UNIQUE = 1;
// not all values for the key are unique
var FREQ_MORE_ANALYSIS = 2;

// computes some numbers on the values of string keys from the return value of
// the `compute_value_frequencies_for_string_keys` function
var compute_stats_from_value_frequencies = function(keyList,
    valuesHash) {
  var retHash = {};
  _(keyList).forEach(function(key) {
    var countObj = valuesHash[key];
    var freqHash = {};
    var bytesForStoringAllStrings;
    var sumLengthOfUniqueStrings;
    var bucketStatsArray = [];
    var totalNrUniqueStrings = 0;
    // for each possible value for the current string key, compute an obj with:
    //   number of occurrences --> array of integers. Each integer in the array
    //   is the lengh of a unique string which is present that many times.
    _.forOwn(countObj, function(nrOccurrences, uniqueString) {
      if (!_.has(freqHash, nrOccurrences)) {
        freqHash[nrOccurrences] = [];
      }
      freqHash[nrOccurrences].push(uniqueString.length);
    });
    if (_.keys(freqHash).length === 1 && _.has(freqHash, 1)) {
      // all values are unique
      retHash[key] = { result: FREQ_ALL_UNIQUE };
    } else {
      // compute array of BucketStats object along some other numbers.
      // Each BucketStats object has the following properties:
      // - nrOccurrences
      // - stringLengthArray (optional)
      // - count
      // - minLen
      // - maxLen
      // - medianLen
      // - avgLen
      bytesForStoringAllStrings = 0;
      sumLengthOfUniqueStrings = 0;
      _.forOwn(freqHash, function(stringLengthArray, nrOccurrences) {
        var nrUniqueStrings = stringLengthArray.length;
        var lengthArray = _.reduce(stringLengthArray,
          function(arr, stringLength) {
            _.times(nrOccurrences, function() { arr.push(stringLength); });
            return arr;
          }, []).sort();
        var len = lengthArray.length;
        var sumLengths = _.reduce(stringLengthArray,
          function(sumLengths, currentLength) {
              return sumLengths + nrOccurrences * currentLength;
          }, 0);
        var bytesForStoringIntegerIndices;
        bytesForStoringAllStrings += sumLengths;
        sumLengthOfUniqueStrings += sumLengths / nrOccurrences;
        totalNrUniqueStrings += nrUniqueStrings;
        bucketStatsArray.push({
          nrOccurrences: nrOccurrences,
          stringLengthArray: stringLengthArray,
          count: nrUniqueStrings,
          minLen: lengthArray[0],
          maxLen: lengthArray[len-1],
          medianLen: ((len % 2 === 1) ? lengthArray[Math.floor(len/2)] :
            (lengthArray[len/2 - 1] + lengthArray[len/2]) / 2),
          avgLen: sumLengths / len
        });
      });
      bucketStatsArray.sort(function(a, b) {
        return b.nrOccurrences - a.nrOccurrences
      });
      bytesForStoringIntegerIndices =
        compute_storage_for_integer_indices(bucketStatsArray);
      retHash[key] = {
        result: FREQ_MORE_ANALYSIS,
        bucketStatsArray: _.map(bucketStatsArray, function(bucketStats) {
          return _.omit(bucketStats, 'stringLengthArray');
        }),
        bytesForStoringAllStrings: bytesForStoringAllStrings,
        // '[', ']', commas, quotes, strings themselves
        bytesForStoringUniqueStringsInNewArray: 2 + (totalNrUniqueStrings - 1) +
          totalNrUniqueStrings * 2 + sumLengthOfUniqueStrings,
        bytesForStoringIntegerIndices: bytesForStoringIntegerIndices
      };
    }
  });

  return retHash;
};

// Given an array of BucketStats objects with the `stringLengthArray` property,
// where the array is sorted by descending `nrOccurrences` property, compute
// the space required for storing integer indices in a JSON file.
// Indices will start from 0 for high `nrOccurrences`.
var compute_storage_for_integer_indices = function(bucketStatsArray) {
  var compute_storage_for_one_non_negative_integer = function(x) {
    var nrDigits = 0;
    if (x === 0) {
      return 1;
    }
    while (x > 0) {
      nrDigits += 1;
      x = Math.floor(x / 10);
    }
    return nrDigits;
  };

  return _.reduce(bucketStatsArray, function(accObj, bucketStats) {
    var currentIndex = accObj.currentIndex;
    var totalSpace = accObj.totalSpace;
    var nrOccurrences = bucketStats.nrOccurrences;
    _.forEach(bucketStats.stringLengthArray, function(stringLength) {
      totalSpace += nrOccurrences *
        compute_storage_for_one_non_negative_integer(currentIndex);
      currentIndex += 1;
    });
    return {
      currentIndex: currentIndex,
      totalSpace: totalSpace
    };
  }, { currentIndex: 0, totalSpace: 0 }).totalSpace;
};

// Displays the results from the
// `compute_stats_for_string_keys_from_value_frequencies` function
var show_results = function(keyList, results) {
  _.forEach(keyList, function(key, idx) {
    var keyResult = results[key];
    if (idx > 0) {
      console.log("\n");
    }
    console.log("Key \"" + key + "\"");
    if (keyResult.result === FREQ_ALL_UNIQUE) {
      console.log("    All values are unique");
    } else {
      _.forEach(keyResult.bucketStatsArray, function(bucketStats) {
        console.log("    Values occurring " + bucketStats.nrOccurrences +
          " times: " + bucketStats.count + " unique strings [min length: " +
          bucketStats.minLen + ", max length: " + bucketStats.maxLen +
          ", avg length: " + bucketStats.avgLen + ", median length: " +
          bucketStats.medianLen + "]"
        );
      });
      console.log("Bytes for storing all strings: " +
        keyResult.bytesForStoringAllStrings
      );
      console.log("Bytes for storing unique strings in separate array: " +
        keyResult.bytesForStoringUniqueStringsInNewArray
      );
      console.log("Bytes for storing integer indices: " +
        keyResult.bytesForStoringIntegerIndices
      );
      console.log("Storing all strings vs. storing unique strings + " +
        "integer indices: " + keyResult.bytesForStoringAllStrings + " vs. " +
        (keyResult.bytesForStoringUniqueStringsInNewArray+
         keyResult.bytesForStoringIntegerIndices)
      );
      if (keyResult.bytesForStoringAllStrings <=
          keyResult.bytesForStoringUniqueStringsInNewArray+
            keyResult.bytesForStoringIntegerIndices) {
        console.log("It is more efficient in space to store all the strings.");
      } else {
        console.log("It is more efficient in space to store unique strings " +
          "with integer indices."
        );
      }
    }
  });
};

show_results(
  STRING_KEYS,
  compute_stats_from_value_frequencies(
    STRING_KEYS,
    compute_value_frequencies(STRING_KEYS, MODULES_ARRAY)
  )
);

(function() {
  console.log("\n\n");
  // preprocess `ARRAY_OF_STRINGS_KEYS` to get an array of objects where each
  // object only has a single property (which is in `ARRAY_OF_STRINGS_KEY`);
  // the value indexed by a key is a single string.
  //
  // This step is necessary due to the input format of the
  // `compute_value_frequencies` function
  var arrayModsWithArrayOfStringsKeys = [];
  _.forEach(MODULES_ARRAY, function(mod) {
    _.forEach(ARRAY_OF_STRINGS_KEYS, function(key) {
      if (_.has(mod, key)) {
        _.forEach(mod[key], function(value) {
          var obj = {};
          obj[key] = value;
          arrayModsWithArrayOfStringsKeys.push(obj);
        });
      }
    });
 });
 show_results(
   ARRAY_OF_STRINGS_KEYS,
   compute_stats_from_value_frequencies(
     ARRAY_OF_STRINGS_KEYS,
     compute_value_frequencies(ARRAY_OF_STRINGS_KEYS,
       arrayModsWithArrayOfStringsKeys
     )
   )
 );
})();
<|endoftext|>