<commit_before>(function(){

  var platformIsRhino = typeof Packages !== "undefined";
  var platformIsNode = !platformIsRhino;

  if (typeof window === "undefined")
    window = this;

  if (platformIsRhino && typeof console === "undefined") {
    console = {
        log: function() {
          var out = java.lang.System.out;
          for (var i = 0; i < arguments.length; i++)
            out.print(arguments[i]);
          out.println();
          out.flush();
        }
    };
    console.debug = console.log;
    console.warn = console.log;
  }
  if (platformIsNode) {
    // Node suppresses output to the "real" console when calling console.debug, it's only shown
    //  in the debugger
    console.debug = function() {
      var args = [].slice.apply(arguments);
      console.log.apply(this, args);
    };
  }

  window.document = document = {
      readyState: "ready",
      createEvent: function() {
        return {
          initCustomEvent: function() {}
        };
      },
      addListener: function() {},
      removeListener: function() {},
      documentElement: {
        style: {}
      }
  };

  if (!this.window)
    window = this;
  window.dispatchEvent = function() {};

  if (!window.navigator) {
    window.navigator = {
        userAgent: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; de-de) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
        product: "",
        cpuClass: ""
    };
  }

  if (!this.qxloadPrefixUrl)
    qxloadPrefixUrl = "";

  if (!window.qx)
    window.qx = {};

  if (!window.qxvariants)
    qxvariants = {};

  qx.$$start = new Date();

  if (!qx.$$environment) {
    qx.$$environment = {};
  }
  var envinfo = %{EnvSettings};
  for (var k in envinfo) {
    qx.$$environment[k] = envinfo[k];
  }

  if (!qx.$$libraries) {
    qx.$$libraries = {};
  }
  var libinfo = %{Libinfo};
  for (var k in libinfo) {
    qx.$$libraries[k] = libinfo[k];
  }

  var isDebug = qx.$$environment["qx.debugLoader"];
  var log = isDebug ? console.log : function() { };
  var load = qx.$$environment["qx.ooLoader"] ? this[qx.$$environment["qx.ooLoader"]] :
    platformIsRhino && typeof this.load == "function" ? this.load : require;
  if (typeof load !== "function")
    throw new Error("Cannot initialise Qooxdoo application - no URI loader method detected");

  qx.$$resources = %{Resources};
  qx.$$translations = %{Translations};
  qx.$$locales = %{Locales};
  qx.$$packageData = {};
  qx.$$g = {}
  qx.$$createdAt = function(obj, filename, lineNumber, column) {
    if (obj !== undefined && obj !== null) {
      Object.defineProperty(obj, "$$createdAt", {
        value: {
          filename: filename,
          lineNumber: lineNumber,
          column: column
        },
        enumerable: false,
        configurable: false,
        writable: false
      });
    }
    return obj;
  };

  qx.$$loader = {
      parts : %{Parts},
      packages : %{Packages},
      urisBefore : %{UrisBefore},
      boot : %{Boot},
      closureParts : %{ClosureParts},
      delayDefer: false,
      transpiledPath: %{TranspiledPath},

      decodeUris : function(compressedUris, pathName) {
        if (!pathName)
          pathName = "sourceUri";
        var libs = qx.$$libraries;
        var uris = [];
        for (var i = 0; i < compressedUris.length; i++) {
          var uri = compressedUris[i].split(":");
          var euri;
          if (uri.length == 2 && uri[0] in libs) {
            var prefix = libs[uri[0]][pathName];
            euri = prefix + "/" + uri[1];
          } else {
            euri = compressedUris[i];
          }
          %{DecodeUrisPlug}
          uris.push(euri);
        }
        return uris;
      },

      init : function() {
        var l = qx.$$loader;
        this.loadScriptList(l.decodeUris(l.urisBefore, "resourceUri"));

        var bootPackageHash=l.parts[l.boot][0];
        this.loadScriptList(l.decodeUris(l.packages[l.parts[l.boot][0]].uris));
        l.importPackageData(qx.$$packageData[bootPackageHash] || {});
        qx.$$domReady = true;
        l.signalStartup();
      },

      loadScriptList: function(list) {
        list.forEach(function(uri) {
          load(uri);
        });
      },

      signalStartup : function() {
        qx.Bootstrap.executePendingDefers();
        qx.$$loader.delayDefer = false;
        qx.$$loader.scriptLoaded = true;
        qx.core.BaseInit.ready();
        qx.$$loader.applicationHandlerReady = true;
      },

      importPackageData : function(dataMap, callback) {
        if (dataMap["resources"]) {
          var resMap = dataMap["resources"];
          for ( var k in resMap)
            qx.$$resources[k] = resMap[k];
        }
        if (dataMap["locales"]) {
          var locMap = dataMap["locales"];
          var qxlocs = qx.$$locales;
          for ( var lang in locMap) {
            if (!qxlocs[lang])
              qxlocs[lang] = locMap[lang];
            else
              for (var k in locMap[lang])
                qxlocs[lang][k] = locMap[lang][k];
          }
        }
        if (dataMap["translations"]) {
          var trMap = dataMap["translations"];
          var qxtrans = qx.$$translations;
          for ( var lang in trMap) {
            if (!qxtrans[lang])
              qxtrans[lang] = trMap[lang];
            else
              for ( var k in trMap[lang])
                qxtrans[lang][k] = trMap[lang][k];
          }
        }
        if (callback) {
          callback(dataMap);
        }
      }
  };

})();

%{BootPart}

qx.$$loader.init();
if (typeof exports != "undefined") {
  for (var key in qx) {
    exports[key] = qx[key];
  }
}

<commit_msg>2. try for global window object. First commit just fixes load dependencies<commit_after>(function(){

  var platformIsRhino = typeof Packages !== "undefined";
  var platformIsNode = !platformIsRhino;

  if (typeof window === "undefined")
    window = this;

  if (platformIsRhino && typeof console === "undefined") {
    console = {
        log: function() {
          var out = java.lang.System.out;
          for (var i = 0; i < arguments.length; i++)
            out.print(arguments[i]);
          out.println();
          out.flush();
        }
    };
    console.debug = console.log;
    console.warn = console.log;
  }
  if (platformIsNode) {
    // Node suppresses output to the "real" console when calling console.debug, it's only shown
    //  in the debugger
    console.debug = function() {
      var args = [].slice.apply(arguments);
      console.log.apply(this, args);
    };
  }

  window.document = document = {
      readyState: "ready",
      createEvent: function() {
        return {
          initCustomEvent: function() {}
        };
      },
      addListener: function() {},
      removeListener: function() {},
      documentElement: {
        style: {}
      }
  };

  if (!this.window)
    window = this;
  window.dispatchEvent = function() {};

  if (!window.navigator) {
    window.navigator = {
        userAgent: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; de-de) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8",
        product: "",
        cpuClass: ""
    };
  }

  if (!this.qxloadPrefixUrl)
    qxloadPrefixUrl = "";

  if (!window.qx)
    window.qx = {};

  if (!window.qxvariants)
    qxvariants = {};

  qx.$$start = new Date();

  if (!qx.$$environment) {
    qx.$$environment = {};
  }
  var envinfo = %{EnvSettings};
  for (var k in envinfo) {
    qx.$$environment[k] = envinfo[k];
  }

  if (!qx.$$libraries) {
    qx.$$libraries = {};
  }
  var libinfo = %{Libinfo};
  for (var k in libinfo) {
    qx.$$libraries[k] = libinfo[k];
  }

  var isDebug = qx.$$environment["qx.debugLoader"];
  var log = isDebug ? console.log : function() { };
  var load = qx.$$environment["qx.ooLoader"] ? this[qx.$$environment["qx.ooLoader"]] :
    platformIsRhino && typeof this.load == "function" ? this.load : require;
  if (typeof load !== "function")
    throw new Error("Cannot initialise Qooxdoo application - no URI loader method detected");

  qx.$$resources = %{Resources};
  qx.$$translations = %{Translations};
  qx.$$locales = %{Locales};
  qx.$$packageData = {};
  qx.$$g = {}
  qx.$$createdAt = function(obj, filename, lineNumber, column) {
    if (obj !== undefined && obj !== null) {
      Object.defineProperty(obj, "$$createdAt", {
        value: {
          filename: filename,
          lineNumber: lineNumber,
          column: column
        },
        enumerable: false,
        configurable: false,
        writable: false
      });
    }
    return obj;
  };

  qx.$$loader = {
      parts : %{Parts},
      packages : %{Packages},
      urisBefore : %{UrisBefore},
      boot : %{Boot},
      closureParts : %{ClosureParts},
      delayDefer: false,
      transpiledPath: %{TranspiledPath},

      decodeUris : function(compressedUris, pathName) {
        if (!pathName)
          pathName = "sourceUri";
        var libs = qx.$$libraries;
        var uris = [];
        for (var i = 0; i < compressedUris.length; i++) {
          var uri = compressedUris[i].split(":");
          var euri;
          if (uri.length == 2 && uri[0] in libs) {
            var prefix = libs[uri[0]][pathName];
            euri = prefix + "/" + uri[1];
          } else {
            euri = compressedUris[i];
          }
          %{DecodeUrisPlug}
          uris.push(euri);
        }
        return uris;
      },

      init : function() {
        var l = qx.$$loader;
        this.loadScriptList(l.decodeUris(l.urisBefore, "resourceUri"));

        var bootPackageHash=l.parts[l.boot][0];
        this.loadScriptList(l.decodeUris(l.packages[l.parts[l.boot][0]].uris));
        l.importPackageData(qx.$$packageData[bootPackageHash] || {});
        qx.$$domReady = true;
        l.signalStartup();
      },

      loadScriptList: function(list) {
        list.forEach(function(uri) {
          let f = load(uri);
		  if (typeof f === "function") {
			  window[f.name] = f;
		  }
        });
      },

      signalStartup : function() {
        qx.Bootstrap.executePendingDefers();
        qx.$$loader.delayDefer = false;
        qx.$$loader.scriptLoaded = true;
        qx.core.BaseInit.ready();
        qx.$$loader.applicationHandlerReady = true;
      },

      importPackageData : function(dataMap, callback) {
        if (dataMap["resources"]) {
          var resMap = dataMap["resources"];
          for ( var k in resMap)
            qx.$$resources[k] = resMap[k];
        }
        if (dataMap["locales"]) {
          var locMap = dataMap["locales"];
          var qxlocs = qx.$$locales;
          for ( var lang in locMap) {
            if (!qxlocs[lang])
              qxlocs[lang] = locMap[lang];
            else
              for (var k in locMap[lang])
                qxlocs[lang][k] = locMap[lang][k];
          }
        }
        if (dataMap["translations"]) {
          var trMap = dataMap["translations"];
          var qxtrans = qx.$$translations;
          for ( var lang in trMap) {
            if (!qxtrans[lang])
              qxtrans[lang] = trMap[lang];
            else
              for ( var k in trMap[lang])
                qxtrans[lang][k] = trMap[lang][k];
          }
        }
        if (callback) {
          callback(dataMap);
        }
      }
  };

})();

%{BootPart}

qx.$$loader.init();
if (typeof exports != "undefined") {
  for (var key in qx) {
    exports[key] = qx[key];
  }
}

<|endoftext|>


<commit_before>import * as utils from 'base/utils';
import Promise from 'base/promise';
import Reader from 'base/reader';

var FILE_CACHED = {}; //caches files from this reader
var FILE_REQUESTED = {}; //caches files from this reader

var CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} reader_info Information about the reader
   */
  init: function(reader_info) {
    this._name = 'csv';
    this._data = [];
    this._basepath = reader_info.path;
    this._parsers = reader_info.parsers;
    if(!this._basepath) {
      utils.error("Missing base path for csv reader");
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @param {String} language language
   * @returns a promise that will be resolved when data is read
   */
  read: function(query, language) {
    var _this = this;
    var p = new Promise();

    //this specific reader has support for the tag {{LANGUAGE}}
    this.path = this._basepath.replace("{{LANGUAGE}}", language);

    //replace conditional tags {{<any conditional>}}
    this.path = this.path.replace(/{{(.*?)}}/g, function(match, capture) {
      capture = capture.toLowerCase();
      if(utils.isArray(query.where[capture])) {
        return query.where[capture].sort().join('-');
      }
      return query.where[capture];
    });

    //if only one year, files ending in "-YYYY.csv"
    var loadPath = this.path;
    if(query.where.time && query.where.time[0].length === 1) {
      loadPath = loadPath.replace(".csv", "-" + query.where.time[0][0] + ".csv");
    }

    _this._data = [];

    (function(query, p) {

      // load and then read from the cache when loaded
      var loadPromise = _this.load(loadPath, parse);
      loadPromise.then(function() {
        parse(FILE_CACHED[loadPath]);
      })

      function parse(res) {

        var data = res;

        //rename geo.category to geo.cat
        var where = query.where;
        if(where['geo.category']) {
          where['geo.cat'] = utils.clone(where['geo.category']);
          where['geo.category'] = void 0;
        }

        // load (join) any properties if necessary
        var propertiesLoadPromise = _this.loadProperties(data, query);

        // once done, continue parsing
        propertiesLoadPromise.then(function() {

          //make sure conditions don't contain invalid conditions
          var validConditions = [];
          utils.forEach(where, function(v, p) {
            for(var i = 0, s = data.length; i < s; i++) {
              if(data[i].hasOwnProperty(p)) {
                validConditions.push(p);
                return true;
              }
            };
          });

          // only use valid conditions
          where = utils.clone(utils.deepClone(where), validConditions);

          //
          where = utils.mapRows([where], _this._parsers)[0];

          //filter any rows that match where condition
          data = utils.filterAny(data, where);

          //warn if filtering returns empty array
          if(data.length == 0) {
            p.reject("data reader returns empty array, that's bad");
            return;
          }

          //only selected items get returned
          data = data.map(function(row) {
            return utils.clone(row, query.select);
          });

          // grouping
          data = _this.groupData(data, query);

          // sorting
          // one column, one direction (ascending) for now
          if(query.orderBy && data[0]) {
            if (data[0][query.orderBy]) {
              data.sort(function(a, b) {
                return a[query.orderBy] - b[query.orderBy];
              });
            } else {
              p.reject("Cannot sort by " + query.orderBy + ". Column does not exist in result.");
            }
          }

          _this._data = data;
          p.resolve();

        })

      }

    })(query, p);

    return p;
  },

  /**
   * Gets the data
   * @returns all data
   */
  getData: function() {
    return this._data;
  },

  format: function(res) {

    //make category an array
    res = res.map(function(row) {
      if(row['geo.cat']) {
        row['geo.cat'] = [row['geo.cat']];
      }
      return row;
    });

    //format data
    res = utils.mapRows(res, this._parsers);

    return res;
  },

  load: function(path) {
    var _this = this;

    //if not yet cached or request, start a request
    if(!FILE_CACHED.hasOwnProperty(path) && !FILE_REQUESTED.hasOwnProperty(path)) {
      // load the csv
      d3.csv(path, function(error, res) {

        if(!res) {
          utils.error("No permissions or empty file: " + path, error);
          return;
        }

        if(error) {
          utils.error("Error Happened While Loading CSV File: " + path, error);
          return;
        }

        //fix CSV response
        res = _this.format(res);

        //cache and resolve
        FILE_CACHED[path] = res;
        FILE_REQUESTED[path].resolve();
        // commented this out because the promise needs to stay for future requests, indicating it is already in the cache
        // FILE_REQUESTED[path] = void 0;

      });
      FILE_REQUESTED[path] = new Promise();
    }
    // always return a promise, even if it is already in the cache
    return FILE_REQUESTED[path];
  },

  loadProperties: function(data, query) {
      var _this = this;

      // see if there are any properties used in the query and load them
      // At the moment properties are loaded and added to the data-set only when required but for every query. Maybe loading and adding them to the data-set once is better?
      var propertiesPromises = [];
      var propertiesByKey = {};

      // check both select and where for columns that actually refer to properties
      utils.forEach(query.select, function(column) {
        checkForProperty(column);
      });
      utils.forEach(query.where, function(values, column) {
        checkForProperty(column);
      });

      // load properties for each column referring to property in the dataset

      // The below process O(n*m*o) but both n and o are typically small: n = number of property-sets, m = size of data-set, o = number of columns in property-set
      // for each requested property-set
      utils.forEach(propertiesByKey, function(properties, key) {
        properties[key] = true; // also retrieve the key-column
        propertiesPromises.push(loadProperties(properties, key));
      });

      return propertiesPromises.length ? Promise.all(propertiesPromises) : new Promise.resolve();

      function checkForProperty(column) {
        var split = column.split('.');
        if (split.length == 2) {
          propertiesByKey[split[0]] = propertiesByKey[split[0]] || [];
          propertiesByKey[split[0]].push(column);
        }
      }

      function loadProperties(queriedProperties, keyColumn) {

        /*
         * Code below is for a path to a file when properties are shared between datasets
         *

        // parse the url of the original csv
        var parser = document.createElement('a');
        parser.href = path;

        // construct the path of the file with properties of the key column
        var newpathname = parser.pathname.substr(0, parser.pathname.lastIndexOf('/') + 1) + key + "-properties.csv";
        var propertiesPath = parser.protocol + '//' + parser.host + newpathname + parser.search + parser.hash;
        */

        // get path of properties that are specific for the current data-set
        var propertiesPath = _this.path.replace(".csv", "-" + keyColumn + "-properties.csv");

        // load the file and return the promise for loading
        var processedPromise = new Promise();
        var loadPromise = _this.load(propertiesPath);
        loadPromise.then(function() {

          var properties = {};

          // load all the properties in a map with the keyColumn-value as keyColumn (e.g. properties['swe']['geo.name'] = 'Sweden')
          // this map is readable in O(1)
          utils.forEach(FILE_CACHED[propertiesPath], function(object) {
            properties[object[keyColumn]] = object;
          });

          // go through each row of data
          utils.forEach(data, function(row, index) { // e.g. row = { geo: se, pop: 1000, gdp: 5 }
            // copy each property that was queried to the matching data-row (matching = same keyColumn)
            utils.forEach(queriedProperties, function(property) {

                // check if row exists in properties
                if(properties[row[keyColumn]]){
                    row[property] = properties[row[keyColumn]][property];
                }else{
                    // if not, then complain
                    utils.warn(row[keyColumn] + " is missing from GEO-PROPERTIES.CSV");
                }

            })
          });

          processedPromise.resolve();

        });

        return processedPromise;
      }
  },

  groupData: function(data, query) {

    // nested object which will be used to find the right group for each datarow. Each leaf will contain a reference to a data-object for aggregration.
    var grouping_map = {};

    var filtered = data.filter(function(val, index) {

      var keep;
      var leaf = grouping_map; // start at the base

      // find the grouping-index for each grouping property (i.e. entity)
      var keys = Object.keys(query.grouping);
      var n = keys.length;
      for (var i = 0; i < n; i++) {
        var grouping = query.grouping[keys[i]];
        var entity = keys[i];

        var group_index;

        // TO-DO: only age is grouped together for now, should be more generic
        if (entity == 'age') {

          var group_by = grouping;
          var group_offset = 0;

          var group_nr = Math.floor((val[entity] - group_offset) / group_by); // group number
          var group_start = group_nr * group_by + group_offset; // number at which the group starts

          // if the group falls outside the where filter, make the group smaller
          if (group_start < query.where[entity][0][0])
            group_start = query.where[entity][0][0];

          group_index = group_start;
          val[entity] = group_index;
        }

        // if this is not the last grouping property
        if (i < (n-1)) {

          // create if next grouping level doesn't exist yet
          if (!leaf[val[entity]])
            leaf[val[entity]] = {};
          // set leaf to next level to enable recursion
          leaf = leaf[val[entity]];

        } else {

          // if last grouping property: we are at the leaf and can aggegrate

          if (!leaf[val[entity]]) {

            // if the final leaf isn't set yet, start it by letting it refer to the current row in the data. We will keep this row in the data-set.
            leaf[val[entity]] = val;
            keep = true;

          } else {

            // if the final leaf was already set, aggregrate!
            leaf = leaf[val[entity]];
            // if the leaf already had values, apply the aggregrate functions for each property
            utils.forEach(query.select, function(property, key) {
              if(keys.indexOf(property) != -1) return;
              // aggregrate the un-grouped data (now only sum population)
              // leaf[property] = parseFloat(leaf[property]) + parseFloat(val[property]);
              leaf[property] = leaf[property] + val[property];
            });
            keep = false;

          }

        }

      }

      // if this row will function as place for aggregration, keep it, otherwise, discard it through the filter.
      return keep;

    });

    return filtered;

  }

});

export default CSVReader;<commit_msg>Patch erroneous string aggregation<commit_after>import * as utils from 'base/utils';
import Promise from 'base/promise';
import Reader from 'base/reader';

var FILE_CACHED = {}; //caches files from this reader
var FILE_REQUESTED = {}; //caches files from this reader

var CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} reader_info Information about the reader
   */
  init: function(reader_info) {
    this._name = 'csv';
    this._data = [];
    this._basepath = reader_info.path;
    this._parsers = reader_info.parsers;
    if(!this._basepath) {
      utils.error("Missing base path for csv reader");
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @param {String} language language
   * @returns a promise that will be resolved when data is read
   */
  read: function(query, language) {
    var _this = this;
    var p = new Promise();

    //this specific reader has support for the tag {{LANGUAGE}}
    this.path = this._basepath.replace("{{LANGUAGE}}", language);

    //replace conditional tags {{<any conditional>}}
    this.path = this.path.replace(/{{(.*?)}}/g, function(match, capture) {
      capture = capture.toLowerCase();
      if(utils.isArray(query.where[capture])) {
        return query.where[capture].sort().join('-');
      }
      return query.where[capture];
    });

    //if only one year, files ending in "-YYYY.csv"
    var loadPath = this.path;
    if(query.where.time && query.where.time[0].length === 1) {
      loadPath = loadPath.replace(".csv", "-" + query.where.time[0][0] + ".csv");
    }

    _this._data = [];

    (function(query, p) {

      // load and then read from the cache when loaded
      var loadPromise = _this.load(loadPath, parse);
      loadPromise.then(function() {
        parse(FILE_CACHED[loadPath]);
      })

      function parse(res) {

        var data = res;

        //rename geo.category to geo.cat
        var where = query.where;
        if(where['geo.category']) {
          where['geo.cat'] = utils.clone(where['geo.category']);
          where['geo.category'] = void 0;
        }

        // load (join) any properties if necessary
        var propertiesLoadPromise = _this.loadProperties(data, query);

        // once done, continue parsing
        propertiesLoadPromise.then(function() {

          //make sure conditions don't contain invalid conditions
          var validConditions = [];
          utils.forEach(where, function(v, p) {
            for(var i = 0, s = data.length; i < s; i++) {
              if(data[i].hasOwnProperty(p)) {
                validConditions.push(p);
                return true;
              }
            };
          });

          // only use valid conditions
          where = utils.clone(utils.deepClone(where), validConditions);

          //
          where = utils.mapRows([where], _this._parsers)[0];

          //filter any rows that match where condition
          data = utils.filterAny(data, where);

          //warn if filtering returns empty array
          if(data.length == 0) {
            p.reject("data reader returns empty array, that's bad");
            return;
          }

          //only selected items get returned
          data = data.map(function(row) {
            return utils.clone(row, query.select);
          });

          // grouping
          data = _this.groupData(data, query);

          // sorting
          // one column, one direction (ascending) for now
          if(query.orderBy && data[0]) {
            if (data[0][query.orderBy]) {
              data.sort(function(a, b) {
                return a[query.orderBy] - b[query.orderBy];
              });
            } else {
              p.reject("Cannot sort by " + query.orderBy + ". Column does not exist in result.");
            }
          }

          _this._data = data;
          p.resolve();

        })

      }

    })(query, p);

    return p;
  },

  /**
   * Gets the data
   * @returns all data
   */
  getData: function() {
    return this._data;
  },

  format: function(res) {

    //make category an array
    res = res.map(function(row) {
      if(row['geo.cat']) {
        row['geo.cat'] = [row['geo.cat']];
      }
      return row;
    });

    //format data
    res = utils.mapRows(res, this._parsers);

    return res;
  },

  load: function(path) {
    var _this = this;

    //if not yet cached or request, start a request
    if(!FILE_CACHED.hasOwnProperty(path) && !FILE_REQUESTED.hasOwnProperty(path)) {
      // load the csv
      d3.csv(path, function(error, res) {

        if(!res) {
          utils.error("No permissions or empty file: " + path, error);
          return;
        }

        if(error) {
          utils.error("Error Happened While Loading CSV File: " + path, error);
          return;
        }

        //fix CSV response
        res = _this.format(res);

        //cache and resolve
        FILE_CACHED[path] = res;
        FILE_REQUESTED[path].resolve();
        // commented this out because the promise needs to stay for future requests, indicating it is already in the cache
        // FILE_REQUESTED[path] = void 0;

      });
      FILE_REQUESTED[path] = new Promise();
    }
    // always return a promise, even if it is already in the cache
    return FILE_REQUESTED[path];
  },

  loadProperties: function(data, query) {
      var _this = this;

      // see if there are any properties used in the query and load them
      // At the moment properties are loaded and added to the data-set only when required but for every query. Maybe loading and adding them to the data-set once is better?
      var propertiesPromises = [];
      var propertiesByKey = {};

      // check both select and where for columns that actually refer to properties
      utils.forEach(query.select, function(column) {
        checkForProperty(column);
      });
      utils.forEach(query.where, function(values, column) {
        checkForProperty(column);
      });

      // load properties for each column referring to property in the dataset

      // The below process O(n*m*o) but both n and o are typically small: n = number of property-sets, m = size of data-set, o = number of columns in property-set
      // for each requested property-set
      utils.forEach(propertiesByKey, function(properties, key) {
        properties[key] = true; // also retrieve the key-column
        propertiesPromises.push(loadProperties(properties, key));
      });

      return propertiesPromises.length ? Promise.all(propertiesPromises) : new Promise.resolve();

      function checkForProperty(column) {
        var split = column.split('.');
        if (split.length == 2) {
          propertiesByKey[split[0]] = propertiesByKey[split[0]] || [];
          propertiesByKey[split[0]].push(column);
        }
      }

      function loadProperties(queriedProperties, keyColumn) {

        /*
         * Code below is for a path to a file when properties are shared between datasets
         *

        // parse the url of the original csv
        var parser = document.createElement('a');
        parser.href = path;

        // construct the path of the file with properties of the key column
        var newpathname = parser.pathname.substr(0, parser.pathname.lastIndexOf('/') + 1) + key + "-properties.csv";
        var propertiesPath = parser.protocol + '//' + parser.host + newpathname + parser.search + parser.hash;
        */

        // get path of properties that are specific for the current data-set
        var propertiesPath = _this.path.replace(".csv", "-" + keyColumn + "-properties.csv");

        // load the file and return the promise for loading
        var processedPromise = new Promise();
        var loadPromise = _this.load(propertiesPath);
        loadPromise.then(function() {

          var properties = {};

          // load all the properties in a map with the keyColumn-value as keyColumn (e.g. properties['swe']['geo.name'] = 'Sweden')
          // this map is readable in O(1)
          utils.forEach(FILE_CACHED[propertiesPath], function(object) {
            properties[object[keyColumn]] = object;
          });

          // go through each row of data
          utils.forEach(data, function(row, index) { // e.g. row = { geo: se, pop: 1000, gdp: 5 }
            // copy each property that was queried to the matching data-row (matching = same keyColumn)
            utils.forEach(queriedProperties, function(property) {

                // check if row exists in properties
                if(properties[row[keyColumn]]){
                    row[property] = properties[row[keyColumn]][property];
                }else{
                    // if not, then complain
                    utils.warn(row[keyColumn] + " is missing from GEO-PROPERTIES.CSV");
                }

            })
          });

          processedPromise.resolve();

        });

        return processedPromise;
      }
  },

  groupData: function(data, query) {

    // nested object which will be used to find the right group for each datarow. Each leaf will contain a reference to a data-object for aggregration.
    var grouping_map = {};

    var filtered = data.filter(function(val, index) {

      var keep;
      var leaf = grouping_map; // start at the base

      // find the grouping-index for each grouping property (i.e. entity)
      var keys = Object.keys(query.grouping);
      var n = keys.length;
      for (var i = 0; i < n; i++) {
        var grouping = query.grouping[keys[i]];
        var entity = keys[i];

        var group_index;

        // TO-DO: only age is grouped together for now, should be more generic
        if (entity == 'age') {

          var group_by = grouping;
          var group_offset = 0;

          var group_nr = Math.floor((val[entity] - group_offset) / group_by); // group number
          var group_start = group_nr * group_by + group_offset; // number at which the group starts

          // if the group falls outside the where filter, make the group smaller
          if (group_start < query.where[entity][0][0])
            group_start = query.where[entity][0][0];

          group_index = group_start;
          val[entity] = group_index;
        }

        // if this is not the last grouping property
        if (i < (n-1)) {

          // create if next grouping level doesn't exist yet
          if (!leaf[val[entity]])
            leaf[val[entity]] = {};
          // set leaf to next level to enable recursion
          leaf = leaf[val[entity]];

        } else {

          // if last grouping property: we are at the leaf and can aggegrate

          if (!leaf[val[entity]]) {

            // if the final leaf isn't set yet, start it by letting it refer to the current row in the data. We will keep this row in the data-set.
            leaf[val[entity]] = val;
            keep = true;

          } else {

            // if the final leaf was already set, aggregrate!
            leaf = leaf[val[entity]];
            // if the leaf already had values, apply the aggregrate functions for each property
            utils.forEach(query.select, function(property, key) {
              if(keys.indexOf(property) != -1) return;
              // aggregrate the un-grouped data (now only sum population)
              // leaf[property] = parseFloat(leaf[property]) + parseFloat(val[property]);

              //never aggregate strings!
              if(val[property]!=="" && !+val[property] && +val[property]!==0) return;
              leaf[property] = leaf[property] + val[property];
            });
            keep = false;

          }

        }

      }

      // if this row will function as place for aggregration, keep it, otherwise, discard it through the filter.
      return keep;

    });

    return filtered;

  }

});

export default CSVReader;<|endoftext|>


<commit_before>import * as utils from 'base/utils';
import Reader from 'base/reader';

const cached = {};

const QUERY_FROM_CONCEPTS = 'concepts';
const QUERY_FROM_DATAPOINTS = 'datapoints';
const QUERY_FROM_ENTITIES = 'entities';
const DATA_QUERIES = [QUERY_FROM_DATAPOINTS, QUERY_FROM_ENTITIES];

const CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} readerInfo Information about the reader
   */
  init(readerInfo) {
    this._name = 'csv';
    this._data = [];
    this._basepath = readerInfo.path;
    this._parsers = readerInfo.parsers;

    // TODO: remove
    this._parsers.Year = year => new Date(new Date().setYear(year));
    this._parsers['Lines of code'] = lines => +lines;

    if (!this._basepath) {
      utils.error('Missing base path for csv reader');
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @returns a promise that will be resolved when data is read
   */
  read(query) {
    const {
      select: {
        key: keys = []
      },
      from
    } = query;

    return this.load().then(data => {
      if (from === QUERY_FROM_CONCEPTS) {
        this._data = this._getConcepts(data[0]);
      } else if (DATA_QUERIES.includes(from) && keys.length) {
        this._data = data.reduce(this._reduceData(query), []);
      } else {
        this._data = [];
      }
    }).catch(utils.warn);
  },

  _getConcepts(firstRow) {
    return Object.keys(firstRow)
      .map(concept => ({ concept }));
  },

  _reduceData(query) {
    const {
      select: {
        value: values,
        key: keys
      },
      from
    } = query;

    const [key] = query.select.key;
    const uniqueKeys = [];

    return (result, row) => {
      if (from === QUERY_FROM_ENTITIES) {
        const unique = row[key];

        if (uniqueKeys.includes(unique)) {
          return result;
        }

        uniqueKeys.push(unique);
      }

      const rowFilteredByKeys = Object.keys(row)
        .reduce((resultRow, rowKey) => {
          if (keys.includes(rowKey) || values.includes(rowKey)) {
            resultRow[rowKey] = row[rowKey];
          }

          return resultRow;
        }, {});

      result.push(rowFilteredByKeys);

      return result;
    };
  },

  /**
   * Gets the data
   * @returns all data
   */
  getData() {
    return this._data;
  },

  load(path = this._basepath) {
    return cached[path] ?
      Promise.resolve(cached[path]) :
      new Promise((resolve, reject) => {
        d3.csv(path, (error, result) => {
          if (!result) {
            return reject(`No permissions or empty file: ${path}. ${error}`);
          }

          if (error) {
            return reject(`Error happened while loading csv file: ${path}. ${error}`);
          }

          resolve(cached[path] = this.format(result));
        });
      });
  },

  format(data) {
    return data.map(row =>
      Object.keys(row)
        .reduce((result, key) => {
          result[key] = this._parsers[key] ?
            this._parsers[key](row[key]) :
            row[key];

          return result;
        }, {})
    );
  }

});

export default CSVReader;
<commit_msg>Add where conditions support<commit_after>import * as utils from 'base/utils';
import Reader from 'base/reader';

const cached = {};

const QUERY_FROM_CONCEPTS = 'concepts';
const QUERY_FROM_DATAPOINTS = 'datapoints';
const QUERY_FROM_ENTITIES = 'entities';
const DATA_QUERIES = [QUERY_FROM_DATAPOINTS, QUERY_FROM_ENTITIES];
const CONDITION_CALLBACKS = {
  $gt: (a, b) => b > a,
  $gte: (a, b) => b >= a,
  $lt: (a, b) => b < a,
  $lte: (a, b) => b <= a,
};

const CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} readerInfo Information about the reader
   */
  init(readerInfo) {
    this._name = 'csv';
    this._data = [];
    this._basepath = readerInfo.path;
    this._parsers = readerInfo.parsers;

    // TODO: remove
    this._parsers.Year = year => new Date(new Date().setYear(year));
    this._parsers['Lines of code'] = lines => +lines;

    if (!this._basepath) {
      utils.error('Missing base path for csv reader');
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @returns a promise that will be resolved when data is read
   */
  read(query) {
    const {
      select: {
        key = []
      },
      from,
      order_by = []
    } = query;

    const [orderBy] = order_by;

    return this.load()
      .then(data => {
        switch (true) {
          case from === QUERY_FROM_CONCEPTS:
            this._data = this._getConcepts(data[0]);
            break;

          case DATA_QUERIES.includes(from) && key.length > 0:
            this._data = data
              .reduce(this._reduceData(query), [])
              .sort((prev, next) => prev[orderBy] - next[orderBy]);
            break;

          default:
            this._data = [];
        }

        this._data = this.format(this._data);
      })
      .catch(utils.warn);
  },

  _getConcepts(firstRow) {
    return Object.keys(firstRow)
      .map(concept => ({ concept }));
  },

  _reduceData(query) {
    const {
      select: {
        value,
        key
      },
      from,
      join,
      where: {
        $and
      }
    } = query;

    const [uniqueKey] = key;
    const uniqueValues = [];

    return (result, row) => {
      const isSuitable = $and.every(binding =>
        Object.keys(binding).every(conditionKey => {
          const conditionValue = join[binding[conditionKey]].where[conditionKey];

          return typeof conditionValue === 'object' ?
            Object.keys(conditionValue).every(compareKey =>
              CONDITION_CALLBACKS[compareKey](conditionValue[compareKey], row[conditionKey])
            ) :
          row[conditionKey] === conditionValue;
        })
      );

      if (isSuitable) {
        const rowFilteredByKeys = Object.keys(row)
          .reduce((resultRow, rowKey) => {
            if (key.includes(rowKey) || value.includes(rowKey)) {
              resultRow[rowKey] = row[rowKey];
            }

            return resultRow;
          }, {});

        result.push(rowFilteredByKeys);
      }

      const unique = row[uniqueKey];
      if (
        from === QUERY_FROM_ENTITIES
        && !uniqueValues.includes(unique)
      ) {
        uniqueValues.push(unique);
      }

      return result;
    };
  },

  /**
   * Gets the data
   * @returns all data
   */
  getData() {
    return this._data;
  },

  load(path = this._basepath) {
    return cached[path] ?
      Promise.resolve(cached[path]) :
      new Promise((resolve, reject) => {
        d3.csv(path, (error, result) => {
          if (!result) {
            return reject(`No permissions or empty file: ${path}. ${error}`);
          }

          if (error) {
            return reject(`Error happened while loading csv file: ${path}. ${error}`);
          }

          resolve(cached[path] = result);
        });
      });
  },

  format(data) {
    return data.map(row =>
      Object.keys(row)
        .reduce((result, key) => {
          result[key] = this._parsers[key] ?
            this._parsers[key](row[key]) :
            row[key];

          return result;
        }, {})
    );
  }

});

export default CSVReader;
<|endoftext|>


<commit_before>/* eslint-env browser */
import React, { Component } from 'react';
import { connect } from 'react-redux';
import Value from './Value';
import CheckboxQuestionDefinition from '../../models/survey/questions/CheckboxQuestionDefinition';
import RadioQuestionDefinition from '../../models/survey/questions/RadioQuestionDefinition';
import MultiNumberQuestionDefinition from '../../models/survey/questions/MultiNumberQuestionDefinition';
import MatrixQuestionDefinition from '../../models/survey/questions/MatrixQuestionDefinition';
import VisibilityConditionDefinition from '../../models/survey/questions/internal/VisibilityConditionDefinition';
import * as ItemVisibility from '../../../constants/ItemVisibility';
import { OPERATORS } from '../../../constants/NumberValidationRuleConstants';

/** プレビューのためのコンテナ */
class QuestionDetail extends Component {
  createOptions() {
    const { question, random, subItemsRandom, matrixReverse } = this.props;

    if (!random && !subItemsRandom && !matrixReverse) return null;

    return (

        表示制御

            {
              random ?

                  {random === true ? 'ランダム' : random}
                  {question.isRandom() ? '◯' : null}

 : null
            }
            {
              subItemsRandom ?

                  列ランダム
                  {question.isSubItemsRandom() ? '◯' : null}

 : null
            }
            {
              matrixReverse ?

                  列でグルーピング
                  {question.isMatrixReverse() ? '◯' : null}

 : null
            }

    );
  }

  createNumberValidations() {
    const { survey, question } = this.props;

    const replacer = survey.getReplacer();
    const uniquedNumberValidationRules = question.getUniquedNumberValidationRules();

    if (uniquedNumberValidationRules.size === 0) return null;

    const tableRows = uniquedNumberValidationRules.map((numberValidationRule) => {
      const numberValidations = numberValidationRule.getNumberValidations();
      return numberValidations.map((numberValidation, index) => {
        const isReference = replacer.containsReferenceIdIn(numberValidation.getValue());
        const valueType = isReference ? '回答値' : '固定値';
        const value = isReference ? replacer.findReferenceLabelIn(numberValidation.getValue()) : numberValidation.getValue();
        const operatorLabel = OPERATORS[numberValidation.getOperator()];

        return (

            {index === 0 ? 数値制限{numberValidationRule.getValidationTypeInQuestion()}	 : null}
            {valueType}
            {value}
            {operatorLabel}

        );
      });
    }).toArray();
    return (

        数値制限

            {tableRows}

    );
  }

  createCheckCountValidations() {
    const { question, minCheckCount, maxCheckCount } = this.props;
    if (!minCheckCount && !maxCheckCount) return null;

    return (

        チェック数制限

            {
              minCheckCount ?

                  最低選択数
                  {question.getMinCheckCount()}

 : null
            }
            {
              maxCheckCount ?

                  最大選択数
                  {question.getMaxCheckCount()}

 : null
            }

    );
  }

  createValidations() {
    const { question, minCheckCount, maxCheckCount } = this.props;
    if (!(minCheckCount || maxCheckCount)) return null;
    return (

        {this.createCheckCountValidations()}
        {this.createNumberValidations()}

    );
  }

  createItemVisibilityRow(item) {
    const { survey } = this.props;
    const vc = item.getVisibilityCondition();
    const od = survey.findOutputDefinition(vc.getOutputDefinitionId());
    const comparisonType = ItemVisibility.COMPARISON_TYPE_OPTIONS[vc.getComparisonType()];
    const replacer = survey.getReplacer();
    const value = od.isOutputTypeSingleChoice() || vc.getComparisonType() === 'answerValue' ? replacer.findReferenceLabelIn(vc.getValue()) : vc.getValue();
    const operator = VisibilityConditionDefinition.findOperatorSelectOptions(od)[vc.getOperator()];
    const visibilityType = ItemVisibility.VISIBLITY_TYPE_OPTIONS[vc.getVisibilityType()];
    return (

        {item.getIndex() + 1}
        {item.getLabel()}
        {od.getLabelForCondition(true)}
        {comparisonType}
        {value}
        {operator}
        {visibilityType}

    );
  }

  /** Itemの表示条件を表示する */
  createItemVisibilitiesTable(itemLabel, items) {
    if (items.size === 0) return null;

    return (

        {itemLabel}の動的な表示・非表示制御

              {itemLabel}番号
              対象選択肢
              条件設問
              比較値タイプ
              比較値
              比較方法
              動作種別

            { items.map(item => this.createItemVisibilityRow(item))}

    );
  }

  createItemVisibilities() {
    const { question } = this.props;

    const className = 'item-visibilities';
    if (question instanceof CheckboxQuestionDefinition || question instanceof RadioQuestionDefinition) {
      const tableEl = this.createItemVisibilitiesTable('選択肢', this.filterItems(question.getItems()));
      if (tableEl === null) return null;
      return
{tableEl}
;
    } else if (question instanceof MultiNumberQuestionDefinition) {
      const tableEl = this.createItemVisibilitiesTable('項目', this.filterItems(question.getItems()));
      if (tableEl === null) return null;
      return
{tableEl}
;
    } else if (question instanceof MatrixQuestionDefinition) {
      return (

          {this.createItemVisibilitiesTable('行', this.filterItems(question.getItems()))}
          {this.createItemVisibilitiesTable('列', this.filterItems(question.getSubItems()))}

      );
    }
    return null;
  }

  filterItems(items) {
    return items.filter(item => !!item.getVisibilityCondition());
  }

  render() {
    return (

          {this.createOptions()}
          {this.createCheckCountValidations()}

{this.createNumberValidations()}

{this.createItemVisibilities()}

    );
  }
}

const stateToProps = state => ({
  survey: state.getSurvey(),
  runtime: state.getRuntime(),
  view: state.getViewSetting(),
  options: state.getOptions(),
});

export default connect(
  stateToProps,
)(QuestionDetail);
<commit_msg>表示制御に不正な参照があった場合詳細プレビューが開けない不具合を修正<commit_after>/* eslint-env browser */
import React, { Component } from 'react';
import { connect } from 'react-redux';
import Value from './Value';
import CheckboxQuestionDefinition from '../../models/survey/questions/CheckboxQuestionDefinition';
import RadioQuestionDefinition from '../../models/survey/questions/RadioQuestionDefinition';
import MultiNumberQuestionDefinition from '../../models/survey/questions/MultiNumberQuestionDefinition';
import MatrixQuestionDefinition from '../../models/survey/questions/MatrixQuestionDefinition';
import VisibilityConditionDefinition from '../../models/survey/questions/internal/VisibilityConditionDefinition';
import * as ItemVisibility from '../../../constants/ItemVisibility';
import { OPERATORS } from '../../../constants/NumberValidationRuleConstants';

/** プレビューのためのコンテナ */
class QuestionDetail extends Component {
  createOptions() {
    const { question, random, subItemsRandom, matrixReverse } = this.props;

    if (!random && !subItemsRandom && !matrixReverse) return null;

    return (

        表示制御

            {
              random ?

                  {random === true ? 'ランダム' : random}
                  {question.isRandom() ? '◯' : null}

 : null
            }
            {
              subItemsRandom ?

                  列ランダム
                  {question.isSubItemsRandom() ? '◯' : null}

 : null
            }
            {
              matrixReverse ?

                  列でグルーピング
                  {question.isMatrixReverse() ? '◯' : null}

 : null
            }

    );
  }

  createNumberValidations() {
    const { survey, question } = this.props;

    const replacer = survey.getReplacer();
    const uniquedNumberValidationRules = question.getUniquedNumberValidationRules();

    if (uniquedNumberValidationRules.size === 0) return null;

    const tableRows = uniquedNumberValidationRules.map((numberValidationRule) => {
      const numberValidations = numberValidationRule.getNumberValidations();
      return numberValidations.map((numberValidation, index) => {
        const isReference = replacer.containsReferenceIdIn(numberValidation.getValue());
        const valueType = isReference ? '回答値' : '固定値';
        const value = isReference ? replacer.findReferenceLabelIn(numberValidation.getValue()) : numberValidation.getValue();
        const operatorLabel = OPERATORS[numberValidation.getOperator()];

        return (

            {index === 0 ? 数値制限{numberValidationRule.getValidationTypeInQuestion()}	 : null}
            {valueType}
            {value}
            {operatorLabel}

        );
      });
    }).toArray();
    return (

        数値制限

            {tableRows}

    );
  }

  createCheckCountValidations() {
    const { question, minCheckCount, maxCheckCount } = this.props;
    if (!minCheckCount && !maxCheckCount) return null;

    return (

        チェック数制限

            {
              minCheckCount ?

                  最低選択数
                  {question.getMinCheckCount()}

 : null
            }
            {
              maxCheckCount ?

                  最大選択数
                  {question.getMaxCheckCount()}

 : null
            }

    );
  }

  createValidations() {
    const { question, minCheckCount, maxCheckCount } = this.props;
    if (!(minCheckCount || maxCheckCount)) return null;
    return (

        {this.createCheckCountValidations()}
        {this.createNumberValidations()}

    );
  }

  createItemVisibilityRow(item) {
    const { survey } = this.props;
    const vc = item.getVisibilityCondition();
    const od = survey.findOutputDefinition(vc.getOutputDefinitionId());
    const comparisonType = ItemVisibility.COMPARISON_TYPE_OPTIONS[vc.getComparisonType()];
    const replacer = survey.getReplacer();
    const value = (od && od.isOutputTypeSingleChoice()) || vc.getComparisonType() === 'answerValue' ? replacer.findReferenceLabelIn(vc.getValue()) : vc.getValue();
    const operator = VisibilityConditionDefinition.findOperatorSelectOptions(od)[vc.getOperator()];
    const visibilityType = ItemVisibility.VISIBLITY_TYPE_OPTIONS[vc.getVisibilityType()];
    return (

        {item.getIndex() + 1}
        {item.getLabel()}
        {od ? od.getLabelForCondition(true) : '不正な参照です'}
        {comparisonType}
        {value}
        {operator}
        {visibilityType}

    );
  }

  /** Itemの表示条件を表示する */
  createItemVisibilitiesTable(itemLabel, items) {
    if (items.size === 0) return null;

    return (

        {itemLabel}の動的な表示・非表示制御

              {itemLabel}番号
              対象選択肢
              条件設問
              比較値タイプ
              比較値
              比較方法
              動作種別

            { items.map(item => this.createItemVisibilityRow(item))}

    );
  }

  createItemVisibilities() {
    const { question } = this.props;

    const className = 'item-visibilities';
    if (question instanceof CheckboxQuestionDefinition || question instanceof RadioQuestionDefinition) {
      const tableEl = this.createItemVisibilitiesTable('選択肢', this.filterItems(question.getItems()));
      if (tableEl === null) return null;
      return
{tableEl}
;
    } else if (question instanceof MultiNumberQuestionDefinition) {
      const tableEl = this.createItemVisibilitiesTable('項目', this.filterItems(question.getItems()));
      if (tableEl === null) return null;
      return
{tableEl}
;
    } else if (question instanceof MatrixQuestionDefinition) {
      return (

          {this.createItemVisibilitiesTable('行', this.filterItems(question.getItems()))}
          {this.createItemVisibilitiesTable('列', this.filterItems(question.getSubItems()))}

      );
    }
    return null;
  }

  filterItems(items) {
    return items.filter(item => !!item.getVisibilityCondition());
  }

  render() {
    return (

          {this.createOptions()}
          {this.createCheckCountValidations()}

{this.createNumberValidations()}

{this.createItemVisibilities()}

    );
  }
}

const stateToProps = state => ({
  survey: state.getSurvey(),
  runtime: state.getRuntime(),
  view: state.getViewSetting(),
  options: state.getOptions(),
});

export default connect(
  stateToProps,
)(QuestionDetail);
<|endoftext|>


<commit_before>import { reduce } from "lodash";

const toArray = (exts) => {
  return reduce(exts, (acc, loaders, ext) => {
    const test = new RegExp(`${ext.replace(".", "\\.")}$`);

    return [
      ...acc,
      {
        test,
        loaders,
      },
    ];
  }, []);
};

export default function moduleReducer(state) {
  const { loader, preLoader } = state;

  return {
    loaders: toArray(loader),
    preLoaders: toArray(preLoader),
  };
}
<commit_msg>module.loaders are flattened down<commit_after>import { reduce } from "lodash";

const toArray = (exts) => {
  return reduce(exts, (acc, loaders, ext) => {
    const test = new RegExp(`${ext.replace(".", "\\.")}$`);

    return [
      ...acc,
      ...loaders.map((loader) => ({
        test,
        ...loader,
      })),
    ];
  }, []);
};

export default function moduleReducer(state) {
  const { loader, preLoader } = state;

  return {
    loaders: toArray(loader),
    preLoaders: toArray(preLoader),
  };
}
<|endoftext|>


<commit_before>import React, {Component} from 'react';
import gql from 'graphql-tag';
import {graphql} from 'react-apollo';

class RegisterPlayer extends Component {
  constructor(props) {
    super(props);
    // console.log('RegisterPlayer component: ', this.props);
    this.state = {player: null};
    let mutationPromise = props.mutate({variables: {playerId: "" + Date.now(), gameBoardId: this.props.gameBoardId}});
    mutationPromise.then((data)=> {
      this.setState({player: {id: data.data.registerPlayer.id, value: data.data.registerPlayer.value}});
      this.props.onRegistration(data.data.registerPlayer.id);
    });
  }

  displayUser() {
    if (this.state.player) {
      return
Welcome Player 1

    }
  }

  render() {
    return (

        {this.displayUser()}

)
  }
}

export default graphql(gql`
  mutation registerPlayer($playerId: String!, $gameBoardId:Int!) {
    registerPlayer(playerId: $playerId, gameBoardId:$gameBoardId) {
      id
      status
      value
    }
  }
`)(RegisterPlayer);
<commit_msg>Removed unused component<commit_after><|endoftext|>


<commit_before>/**
Generic template helper definitions for HackMyResume / FluentCV.
@license MIT. See LICENSE.md for details.
@module generic-helpers.js
*/

(function() {

  var MD = require('marked')
    , H2W = require('../utils/html-to-wpml')
    , XML = require('xml-escape')
    , moment = require('moment')
    , _ = require('underscore');

  /**
  Generic template helper function definitions.
  @class GenericHelpers
  */
  var GenericHelpers = module.exports = {

    /**
    Convert the input date to a specified format through Moment.js.
    If date is invalid, will return the time provided by the user,
    or default to the fallback param or 'Present' if that is set to true
    @method formatDate
    */
    formatDate: function(datetime, format, fallback) {
      if (moment) {
        var momentDate = moment( datetime );

        if (momentDate.isValid()) {
          return moment.format(format);
        }
      }

      return datetime || (typeof fallback == 'string' ? fallback : (fallback === true ? 'Present' : null));
    },

    /**
    Convert inline Markdown to inline WordProcessingML.
    @method wpml
    */
    wpml: function( txt, inline ) {
      if(!txt) return '';
      inline = (inline && !inline.hash) || false;
      txt = XML(txt.trim());
      txt = inline ?
        MD(txt).replace(/^\s*
|<\/p>\s*$/gi, '') :
        MD(txt);
      txt = H2W( txt );
      return txt;
    },

    /**
    Emit a conditional link.
    @method link
    */
    link: function( text, url ) {
      return url && url.trim() ?
        ('' + text + '') : text;
    },

    /**
    Return the last word of the specified text.
    @method lastWord
    */
    lastWord: function( txt ) {
      return txt && txt.trim() ? _.last( txt.split(' ') ) : '';
    },

    /**
    Convert a skill level to an RGB color triplet.
    @method skillColor
    @param lvl Input skill level. Skill level can be expressed as a string
    ("beginner", "intermediate", etc.), as an integer (1,5,etc), as a string
    integer ("1", "5", etc.), or as an RRGGBB color triplet ('#C00000',
    '#FFFFAA').
    */
    skillColor: function( lvl ) {
      var idx = skillLevelToIndex( lvl );
      var skillColors = (this.theme && this.theme.palette &&
        this.theme.palette.skillLevels) ||
        [ '#FFFFFF', '#5CB85C', '#F1C40F', '#428BCA', '#C00000' ];
      return skillColors[idx];
    },

    /**
    Return an appropriate height.
    @method lastWord
    */
    skillHeight: function( lvl ) {
      var idx = skillLevelToIndex( lvl );
      return ['38.25', '30', '16', '8', '0'][idx];
    },

    /**
    Return all but the last word of the input text.
    @method initialWords
    */
    initialWords: function( txt ) {
      return txt && txt.trim() ? _.initial( txt.split(' ') ).join(' ') : '';
    },

    /**
    Trim the protocol (http or https) from a URL/
    @method trimURL
    */
    trimURL: function( url ) {
      return url && url.trim() ? url.trim().replace(/^https?:\/\//i, '') : '';
    },

    /**
    Convert text to lowercase.
    @method toLower
    */
    toLower: function( txt ) {
      return txt && txt.trim() ? txt.toLowerCase() : '';
    },

    /**
    Return true if either value is truthy.
    @method either
    */
    either: function( lhs, rhs, options ) {
      if (lhs || rhs) return options.fn(this);
    },

    /**
    Conditional stylesheet link. Either display the link or embed the stylesheet
    via  tag.
    */
    styleSheet: function( file, options ) {
      return ( this.opts.css === 'link') ?
        '<link href="' + file + '" rel="stylesheet" type="text/css">' :
        '';
    },

    /**
    Perform a generic comparison.
    See: http://doginthehat.com.au/2012/02/comparison-block-helper-for-handlebars-templates
    @method compare
    */
    compare: function(lvalue, rvalue, options) {
      if (arguments.length < 3)
        throw new Error("Handlerbars Helper 'compare' needs 2 parameters");
      var operator = options.hash.operator || "==";
      var operators = {
          '==':       function(l,r) { return l == r; },
          '===':      function(l,r) { return l === r; },
          '!=':       function(l,r) { return l != r; },
          '<':        function(l,r) { return l < r; },
          '>':        function(l,r) { return l > r; },
          '<=':       function(l,r) { return l <= r; },
          '>=':       function(l,r) { return l >= r; },
          'typeof':   function(l,r) { return typeof l == r; }
      };
      if (!operators[operator])
        throw new Error("Handlerbars Helper 'compare' doesn't know the operator "+operator);
      var result = operators[operator](lvalue,rvalue);
      return result ? options.fn(this) : options.inverse(this);
    }

  };

  function skillLevelToIndex( lvl ) {
    var idx = 0;
    if( String.is( lvl ) ) {
      lvl = lvl.trim().toLowerCase();
      var intVal = parseInt( lvl );
      if( isNaN( intVal ) ) {
        switch( lvl ) {
          case 'beginner': idx = 1; break;
          case 'intermediate': idx = 2; break;
          case 'advanced': idx = 3; break;
          case 'master': idx = 4; break;
        }
      }
      else {
        idx = Math.min( intVal / 2, 4 );
        idx = Math.max( 0, idx );
      }
    }
    else {
      idx = Math.min( lvl / 2, 4 );
      idx = Math.max( 0, idx );
    }
    return idx;
  }

}());
<commit_msg>Fix: formatDate helper references the moment method, not the momentDate object<commit_after>/**
Generic template helper definitions for HackMyResume / FluentCV.
@license MIT. See LICENSE.md for details.
@module generic-helpers.js
*/

(function() {

  var MD = require('marked')
    , H2W = require('../utils/html-to-wpml')
    , XML = require('xml-escape')
    , moment = require('moment')
    , _ = require('underscore');

  /**
  Generic template helper function definitions.
  @class GenericHelpers
  */
  var GenericHelpers = module.exports = {

    /**
    Convert the input date to a specified format through Moment.js.
    If date is invalid, will return the time provided by the user,
    or default to the fallback param or 'Present' if that is set to true
    @method formatDate
    */
    formatDate: function(datetime, format, fallback) {
      if (moment) {
        var momentDate = moment( datetime );
        if (momentDate.isValid()) return momentDate.format(format);
      }

      return datetime || (typeof fallback == 'string' ? fallback : (fallback === true ? 'Present' : null));
    },

    /**
    Convert inline Markdown to inline WordProcessingML.
    @method wpml
    */
    wpml: function( txt, inline ) {
      if(!txt) return '';
      inline = (inline && !inline.hash) || false;
      txt = XML(txt.trim());
      txt = inline ?
        MD(txt).replace(/^\s*
|<\/p>\s*$/gi, '') :
        MD(txt);
      txt = H2W( txt );
      return txt;
    },

    /**
    Emit a conditional link.
    @method link
    */
    link: function( text, url ) {
      return url && url.trim() ?
        ('' + text + '') : text;
    },

    /**
    Return the last word of the specified text.
    @method lastWord
    */
    lastWord: function( txt ) {
      return txt && txt.trim() ? _.last( txt.split(' ') ) : '';
    },

    /**
    Convert a skill level to an RGB color triplet.
    @method skillColor
    @param lvl Input skill level. Skill level can be expressed as a string
    ("beginner", "intermediate", etc.), as an integer (1,5,etc), as a string
    integer ("1", "5", etc.), or as an RRGGBB color triplet ('#C00000',
    '#FFFFAA').
    */
    skillColor: function( lvl ) {
      var idx = skillLevelToIndex( lvl );
      var skillColors = (this.theme && this.theme.palette &&
        this.theme.palette.skillLevels) ||
        [ '#FFFFFF', '#5CB85C', '#F1C40F', '#428BCA', '#C00000' ];
      return skillColors[idx];
    },

    /**
    Return an appropriate height.
    @method lastWord
    */
    skillHeight: function( lvl ) {
      var idx = skillLevelToIndex( lvl );
      return ['38.25', '30', '16', '8', '0'][idx];
    },

    /**
    Return all but the last word of the input text.
    @method initialWords
    */
    initialWords: function( txt ) {
      return txt && txt.trim() ? _.initial( txt.split(' ') ).join(' ') : '';
    },

    /**
    Trim the protocol (http or https) from a URL/
    @method trimURL
    */
    trimURL: function( url ) {
      return url && url.trim() ? url.trim().replace(/^https?:\/\//i, '') : '';
    },

    /**
    Convert text to lowercase.
    @method toLower
    */
    toLower: function( txt ) {
      return txt && txt.trim() ? txt.toLowerCase() : '';
    },

    /**
    Return true if either value is truthy.
    @method either
    */
    either: function( lhs, rhs, options ) {
      if (lhs || rhs) return options.fn(this);
    },

    /**
    Conditional stylesheet link. Either display the link or embed the stylesheet
    via  tag.
    */
    styleSheet: function( file, options ) {
      return ( this.opts.css === 'link') ?
        '<link href="' + file + '" rel="stylesheet" type="text/css">' :
        '';
    },

    /**
    Perform a generic comparison.
    See: http://doginthehat.com.au/2012/02/comparison-block-helper-for-handlebars-templates
    @method compare
    */
    compare: function(lvalue, rvalue, options) {
      if (arguments.length < 3)
        throw new Error("Handlerbars Helper 'compare' needs 2 parameters");
      var operator = options.hash.operator || "==";
      var operators = {
          '==':       function(l,r) { return l == r; },
          '===':      function(l,r) { return l === r; },
          '!=':       function(l,r) { return l != r; },
          '<':        function(l,r) { return l < r; },
          '>':        function(l,r) { return l > r; },
          '<=':       function(l,r) { return l <= r; },
          '>=':       function(l,r) { return l >= r; },
          'typeof':   function(l,r) { return typeof l == r; }
      };
      if (!operators[operator])
        throw new Error("Handlerbars Helper 'compare' doesn't know the operator "+operator);
      var result = operators[operator](lvalue,rvalue);
      return result ? options.fn(this) : options.inverse(this);
    }

  };

  function skillLevelToIndex( lvl ) {
    var idx = 0;
    if( String.is( lvl ) ) {
      lvl = lvl.trim().toLowerCase();
      var intVal = parseInt( lvl );
      if( isNaN( intVal ) ) {
        switch( lvl ) {
          case 'beginner': idx = 1; break;
          case 'intermediate': idx = 2; break;
          case 'advanced': idx = 3; break;
          case 'master': idx = 4; break;
        }
      }
      else {
        idx = Math.min( intVal / 2, 4 );
        idx = Math.max( 0, idx );
      }
    }
    else {
      idx = Math.min( lvl / 2, 4 );
      idx = Math.max( 0, idx );
    }
    return idx;
  }

}());
<|endoftext|>


<commit_before>import React, { Component, PropTypes } from 'react'
import { withRouter } from 'react-router-dom'
import { connect } from 'react-redux'

import DataChannelService from '../../services/webRTCDataChannel'
import ApiService from '../../services/apiService'
import { actions } from '../'
import MemberList from './memberList'
import Chat from '../../chat'

class Room extends Component {
  constructor(props) {
    super(props)
    this.renderMembers = this.renderMembers.bind(this)
    this.exitRoom = this.exitRoom.bind(this)
    this.makePeerConnections = this.makePeerConnections.bind(this)
  }
  componentDidMount() {
    const { currentRoomId, joinRoom, roomError, match } = this.props
    if (!currentRoomId) {
      ApiService.joinRoom(match.params.roomId)
        .then(room => joinRoom(room.id))
        .catch(() => {
          this.props.history.replace('/rooms')
          roomError()
        })
    }
  }

  exitRoom() {
    this.props.history.replace('/rooms')
    this.props.exitRoom(this.props.match.params.roomId)
  }

  makePeerConnections() {
    const { currentRoomId, availableRooms } = this.props
    Object.keys(availableRooms[currentRoomId].members)
      .forEach((peerId) => {
        if (parseInt(peerId, 10) !== this.props.userId) {
          DataChannelService.requestNewPeerConnection(peerId)
        }
      })
  }

  renderMembers() {
    const { currentRoomId, availableRooms } = this.props
    return Object.keys(availableRooms[currentRoomId].members)
      .filter(key => availableRooms[currentRoomId].members[key].active)
      .map(key => (

          {availableRooms[currentRoomId].members[key].username}

      ))
  }

  render() {
    const { availableRooms, currentRoomId, socketIsOpen, peerConnections } = this.props
    return (

Room {currentRoomId}

              <button
                type="button"
                className="btn btn-danger"
                onClick={this.exitRoom}
              >
                exit room
              </button>

              SOCKET IS {socketIsOpen ? 'OPEN' : 'CLOSED'}

            {availableRooms && currentRoomId && !!availableRooms[currentRoomId] ?
              (
                <MemberList
                  members={availableRooms[currentRoomId].members}
                  peerConnections={peerConnections}
                />
              ) : ''
            }
            <button
              className="form-control"
              onClick={this.makePeerConnections}
            >
              connect
            </button>
            <Chat />

    )
  }
}

Room.propTypes = {
  match: PropTypes.shape({
    url: PropTypes.string.isRequired,
    params: PropTypes.shape({ roomId: PropTypes.string.isRequired }).isRequired,
  }).isRequired,
  currentRoomId: PropTypes.number,
  joinRoom: PropTypes.func.isRequired,
  /* eslint-disable */
  availableRooms: PropTypes.object, // eslint-disable-line react/forbid-prop-types
  peerConnections: PropTypes.object, // eslint-disable-line react/forbid-prop-types
  /* eslint-enable */
  socketIsOpen: PropTypes.bool.isRequired,
  exitRoom: PropTypes.func.isRequired,
  roomError: PropTypes.func.isRequired,
  history: PropTypes.shape({
    push: PropTypes.func.isRequired,
    replace: PropTypes.func.isRequired,
  }).isRequired,
  userId: PropTypes.number,
}

Room.defaultProps = {
  peerConnections: null,
  currentRoomId: null,
  availableRooms: null,
  userId: null,
}

const mapStoreToProps = store => ({
  peerConnections: store.dataChannel.peerConnections,
  currentRoomId: store.rooms.currentRoomId,
  availableRooms: store.rooms.availableRooms,
  socketIsOpen: store.socketService.isOpen,
  userId: store.auth.userId,
})

const mapDispatchToProps = dispatch => ({
  joinRoom: id => dispatch(actions.joinRoom(id)),
  exitRoom: id => dispatch(actions.exitRoom(id)),
  roomError: () => dispatch(actions.roomError()),
})

export default connect(mapStoreToProps, mapDispatchToProps)(withRouter(Room))
<commit_msg>🎛 remove unused function, filter out inactive room members for peer connections<commit_after>import React, { Component, PropTypes } from 'react'
import { withRouter } from 'react-router-dom'
import { connect } from 'react-redux'

import DataChannelService from '../../services/webRTCDataChannel'
import ApiService from '../../services/apiService'
import { actions } from '../'
import MemberList from './memberList'
import Chat from '../../chat'

class Room extends Component {
  constructor(props) {
    super(props)
    this.exitRoom = this.exitRoom.bind(this)
    this.makePeerConnections = this.makePeerConnections.bind(this)
  }

  componentDidMount() {
    const { currentRoomId, joinRoom, roomError, match } = this.props
    if (!currentRoomId) {
      ApiService.joinRoom(match.params.roomId)
        .then(room => joinRoom(room.id))
        .catch(() => {
          this.props.history.replace('/rooms')
          roomError()
        })
    }
  }

  exitRoom() {
    this.props.history.replace('/rooms')
    this.props.exitRoom(this.props.match.params.roomId)
  }

  makePeerConnections() {
    const { currentRoomId, availableRooms } = this.props
    Object.keys(availableRooms[currentRoomId].members)
      .filter(peerId => availableRooms[currentRoomId].members[peerId].active)
      .forEach((peerId) => {
        if (parseInt(peerId, 10) !== this.props.userId) {
          DataChannelService.requestNewPeerConnection(peerId)
        }
      })
  }

  render() {
    const { availableRooms, currentRoomId, socketIsOpen, peerConnections } = this.props
    return (

Room {currentRoomId}

              <button
                type="button"
                className="btn btn-danger"
                onClick={this.exitRoom}
              >
                exit room
              </button>

              SOCKET IS {socketIsOpen ? 'OPEN' : 'CLOSED'}

            {availableRooms && currentRoomId && !!availableRooms[currentRoomId] ?
              (
                <MemberList
                  members={availableRooms[currentRoomId].members}
                  peerConnections={peerConnections}
                />
              ) : ''
            }
            <button
              className="form-control"
              onClick={this.makePeerConnections}
            >
              connect
            </button>
            <Chat />

    )
  }
}

Room.propTypes = {
  match: PropTypes.shape({
    url: PropTypes.string.isRequired,
    params: PropTypes.shape({ roomId: PropTypes.string.isRequired }).isRequired,
  }).isRequired,
  currentRoomId: PropTypes.number,
  joinRoom: PropTypes.func.isRequired,
  /* eslint-disable */
  availableRooms: PropTypes.object, // eslint-disable-line react/forbid-prop-types
  peerConnections: PropTypes.object, // eslint-disable-line react/forbid-prop-types
  /* eslint-enable */
  socketIsOpen: PropTypes.bool.isRequired,
  exitRoom: PropTypes.func.isRequired,
  roomError: PropTypes.func.isRequired,
  history: PropTypes.shape({
    push: PropTypes.func.isRequired,
    replace: PropTypes.func.isRequired,
  }).isRequired,
  userId: PropTypes.number,
}

Room.defaultProps = {
  peerConnections: null,
  currentRoomId: null,
  availableRooms: null,
  userId: null,
}

const mapStoreToProps = store => ({
  peerConnections: store.dataChannel.peerConnections,
  currentRoomId: store.rooms.currentRoomId,
  availableRooms: store.rooms.availableRooms,
  socketIsOpen: store.socketService.isOpen,
  userId: store.auth.userId,
})

const mapDispatchToProps = dispatch => ({
  joinRoom: id => dispatch(actions.joinRoom(id)),
  exitRoom: id => dispatch(actions.exitRoom(id)),
  roomError: () => dispatch(actions.roomError()),
})

export default connect(mapStoreToProps, mapDispatchToProps)(withRouter(Room))
<|endoftext|>


<commit_before>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const sem = require('../sensor/SensorEventManager.js').getInstance();

const bone = require('../lib/Bone.js');

const CategoryUpdater = require('../control/CategoryUpdater.js');
const categoryUpdater = new CategoryUpdater();

const categoryHashsetMapping = {
  "games": "app.gaming",
  "social": "app.social",
  // "video": "app.video",
  "porn": "app.porn"  // dnsmasq redirect to blue hole if porn
}

const securityHashMapping = {
  "default_c": "blockset:default:consumer"
}

class CategoryUpdateSensor extends Sensor {
  constructor() {
    super();
    this.config.refreshInterval = 8 * 3600 * 1000; // refresh every 8 hours
  }

  async job() {
    const categories = Object.keys(categoryHashsetMapping)
    for (let i = 0; i < categories.length; i++) {
      const category = categories[i];
      await this.updateCategory(category);
    }

    const securityCategories = Object.keys(securityHashMapping)
    for (let i = 0; i < securityCategories.length; i++) {
      const category = securityCategories[i]
      await this.updateSecurityCategory(category)
    }
  }

  async updateCategory(category) {
    const domains = await this.loadCategoryFromBone(category);
    await categoryUpdater.flushDefaultDomains(category);
    return categoryUpdater.addDefaultDomains(category,domains);
  }

  async updateSecurityCategory(category) {
    const info = await this.loadSecurityInfoFromBone(category);

    const domains = info.domain
    const ipv4Addresses = info["ip4"]
    const ipv6Addresses = info["ip6"]

    // if(domains) {
    //   await categoryUpdater.flushDefaultDomains(category);
    //   await categoryUpdater.addDefaultDomains(category,domains);
    // }

    if(ipv4Addresses) {
      await categoryUpdater.flushIPv4Addresses(category)
      await categoryUpdater.addIPv4Addresses(category, ipv4Addresses)
    }

    if(ipv6Addresses) {
      await categoryUpdater.flushIPv6Addresses(category)
      await categoryUpdater.addIPv6Addresses(category, ipv6Addresses)
    }
  }

  run() {
    this.job()

    setInterval(() => {
      this.job()
    }, this.config.refreshInterval)
  }

  async loadCategoryFromBone(category) {
    const hashset = this.getCategoryHashset(category)

    if(hashset) {
      log.info(`Loading domains for ${category} from cloud`);
      const data = await bone.hashsetAsync(hashset)
      const list = JSON.parse(data)
      log.info(`category ${category} has ${list.length} domains`)
      return list
    } else {
      return []
    }
  }

  async loadSecurityInfoFromBone(category) {
    const hashset = securityHashMapping[category]

    if(hashset) {
      log.info(`Loading domains for ${category} from cloud`);
      const data = await bone.hashsetAsync(hashset)
      const list = JSON.parse(data)
      const ip4List = list["ip4"] || [];
      const ip6List = list["ip6"] || [];
      const domain = list.domain || [];
      log.info(`category ${category} has ${ip4List.length} ipv4, ${ip6List.length} ipv6, ${domain.length} domains`)
      return list
    } else {
      return []
    }
  }

  getCategoryHashset(category) {
    return categoryHashsetMapping[category]
  }
}

module.exports = CategoryUpdateSensor;
<commit_msg>set static hostnames into av category<commit_after>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const sem = require('../sensor/SensorEventManager.js').getInstance();

const bone = require('../lib/Bone.js');

const CategoryUpdater = require('../control/CategoryUpdater.js');
const categoryUpdater = new CategoryUpdater();

const categoryHashsetMapping = {
  "games": "app.gaming",
  "social": "app.social",
  "av": "app.video",
  "porn": "app.porn"  // dnsmasq redirect to blue hole if porn
}

const securityHashMapping = {
  "default_c": "blockset:default:consumer"
}

class CategoryUpdateSensor extends Sensor {
  constructor() {
    super();
    this.config.refreshInterval = 8 * 3600 * 1000; // refresh every 8 hours
  }

  async job() {
    const categories = Object.keys(categoryHashsetMapping)
    for (let i = 0; i < categories.length; i++) {
      const category = categories[i];
      await this.updateCategory(category);
    }

    const securityCategories = Object.keys(securityHashMapping)
    for (let i = 0; i < securityCategories.length; i++) {
      const category = securityCategories[i]
      await this.updateSecurityCategory(category)
    }
  }

  async updateCategory(category) {
    const domains = await this.loadCategoryFromBone(category);
    await categoryUpdater.flushDefaultDomains(category);
    return categoryUpdater.addDefaultDomains(category,domains);
  }

  async updateSecurityCategory(category) {
    const info = await this.loadSecurityInfoFromBone(category);

    const domains = info.domain
    const ipv4Addresses = info["ip4"]
    const ipv6Addresses = info["ip6"]

    // if(domains) {
    //   await categoryUpdater.flushDefaultDomains(category);
    //   await categoryUpdater.addDefaultDomains(category,domains);
    // }

    if(ipv4Addresses) {
      await categoryUpdater.flushIPv4Addresses(category)
      await categoryUpdater.addIPv4Addresses(category, ipv4Addresses)
    }

    if(ipv6Addresses) {
      await categoryUpdater.flushIPv6Addresses(category)
      await categoryUpdater.addIPv6Addresses(category, ipv6Addresses)
    }
  }

  run() {
    this.job()

    setInterval(() => {
      this.job()
    }, this.config.refreshInterval)
  }

  async loadCategoryFromBone(category) {
    const hashset = this.getCategoryHashset(category)

    if(hashset) {
      log.info(`Loading domains for ${category} from cloud`);
      const data = await bone.hashsetAsync(hashset)
      const list = JSON.parse(data)
      log.info(`category ${category} has ${list.length} domains`)
      return list
    } else {
      return []
    }
  }

  async loadSecurityInfoFromBone(category) {
    const hashset = securityHashMapping[category]

    if(hashset) {
      log.info(`Loading domains for ${category} from cloud`);
      const data = await bone.hashsetAsync(hashset)
      const list = JSON.parse(data)
      const ip4List = list["ip4"] || [];
      const ip6List = list["ip6"] || [];
      const domain = list.domain || [];
      log.info(`category ${category} has ${ip4List.length} ipv4, ${ip6List.length} ipv6, ${domain.length} domains`)
      return list
    } else {
      return []
    }
  }

  getCategoryHashset(category) {
    return categoryHashsetMapping[category]
  }
}

module.exports = CategoryUpdateSensor;
<|endoftext|>


<commit_before>import Vue from 'vue'
import runtime from 'offline-plugin/runtime'
import { Notification, Button } from 'element-ui'

const createNotifier = () => new Vue({
  data: {
    updating: false
  },
  created() {
    const h = this.$createElement
    this.notification = Notification({ // eslint-disable-line new-cap
      title: 'Detected an update for CodePan, apply updates will reload the tab automatically!',
      type: 'info',
      message: h('el-button', {
        on: { click: this.update },
        props: {
          icon: 'check',
          type: 'success',
          loading: this.updating
        }
      }, [this.updating ? 'Updating' : 'Apply Updates']),
      duration: 0
    })
  },
  methods: {
    update() {
      this.updating = true
      runtime.applyUpdate()
    }
  },
  components: {
    'el-button': Button
  }
})

let notifier

runtime.install({
  onUpdateReady() {
    console.log('update ready')
    notifier = createNotifier()
  },

  onUpdated() {
    console.log('updated')
    notifier.updating = false
    notifier.notification.close()
    location.reload()
  }
})
<commit_msg>optimize updater<commit_after>import runtime from 'offline-plugin/runtime'
import { Notification } from 'element-ui'

runtime.install({
  onUpdateReady() {
    console.log('update ready')
    runtime.applyUpdate()
  },

  onUpdated() {
    Notification({ // eslint-disable-line new-cap
      title: 'New version is ready',
      type: 'info',
      message: 'CodePan has been updated in the background, you can refresh the page to apply updates!',
      duration: 8000
    })
  }
})
<|endoftext|>