<commit_before>const path = require('path');
const glob = require('glob');
const fs = require('fs-extra');
const utils = require('@jupyterlab/buildutils/lib/utils');

// yarn add chai && yarn add --dev @types/chai
// Remove expect.js from package.json
const source = path.resolve('./test-apputils');
const pkgSrc = utils.readJSONFile(path.join(source, 'package.json'));
const chaiVer = pkgSrc.dependencies['chai'];
const chaiTypeVer = pkgSrc.devDependencies['@types/chai'];

const target = path.resolve(process.argv[2]);
if (!target) {
  console.error('Specify a target dir');
  process.exit(1);
}

const targetPkg = utils.readJSONFile(path.join(target, 'package.json'));
delete targetPkg.dependencies['expect.js'];
targetPkg.dependencies['chai'] = chaiVer;
targetPkg.devDependencies['@types/chai'] = chaiTypeVer;

utils.writePackageData(path.join(target, 'package.json'), targetPkg);

glob.sync(path.join(target, 'src', '*.ts*')).forEach(function(filePath) {
  let src = fs.readFileSync(filePath, 'utf8');
  src = src
    .split("import expect = require('expect.js')")
    .join("import { expect } from 'chai'");
  src = src.split('to.be(').join('to.equal(');
  src = src.split('to.not.be(').join('to.not.equal(');
  src = src.split('to.eql(').join('to.deep.equal(');
  src = src.split('to.be.a(').join('to.be.an.instanceof(');
  src = src.split('to.be.an(').join('to.be.an.instanceof(');
  src = src.split(').to.be.empty()').join('.length).to.equal(0)');
  fs.writeFileSync(filePath, src, 'utf8');
});
<commit_msg>More conversion cleanup<commit_after>const path = require('path');
const glob = require('glob');
const fs = require('fs-extra');
const utils = require('@jupyterlab/buildutils/lib/utils');

// yarn add chai && yarn add --dev @types/chai
// Remove expect.js from package.json
const source = path.resolve('./test-apputils');
const pkgSrc = utils.readJSONFile(path.join(source, 'package.json'));
const chaiVer = pkgSrc.dependencies['chai'];
const chaiTypeVer = pkgSrc.devDependencies['@types/chai'];

const target = path.resolve(process.argv[2]);
if (!target) {
  console.error('Specify a target dir');
  process.exit(1);
}

const targetPkg = utils.readJSONFile(path.join(target, 'package.json'));
delete targetPkg.dependencies['expect.js'];
targetPkg.dependencies['chai'] = chaiVer;
targetPkg.devDependencies['@types/chai'] = chaiTypeVer;

utils.writePackageData(path.join(target, 'package.json'), targetPkg);

glob.sync(path.join(target, 'src', '*.ts*')).forEach(function(filePath) {
  let src = fs.readFileSync(filePath, 'utf8');
  src = src
    .split("import expect = require('expect.js')")
    .join("import { expect } from 'chai'");
  src = src.split('to.be(undefined)').join('to.be.undefined');
  src = src.split('to.be(void 0)').join('to.be.undefined');
  src = src.split('to.be(null').join('to.be.null');
  src = src.split('to.be(').join('to.equal(');
  src = src.split('to.not.be(').join('to.not.equal(');
  src = src.split('to.be.ok()').join('to.be.ok');
  src = src.split('to.be.empty(').join('to.be.empty');
  src = src.split('to.eql(').join('to.deep.equal(');
  src = src.split('to.be.a(').join('to.be.an.instanceof(');
  src = src.split('to.be.an(').join('to.be.an.instanceof(');
  src = src.split(').to.be.empty()').join('.length).to.equal(0)');
  fs.writeFileSync(filePath, src, 'utf8');
});
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2015 Julian Knight (Totally Information)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **/

// Node for Node-Red that lists all files from a given start folder
// with an optionally specified file extension.
//var fs = require('fs')
var path = require('path')
var readdirp = require('readdirp')

// Module name must match this nodes html file
var moduleName = 'fs-file-lister'

module.exports = function(RED) {
	'use strict'

	// The main node definition - most things happen in here
	function nodeGo(config) {
		// Create a RED node
		RED.nodes.createNode(this, config)

		// copy "this" object in case we need it in context of callbacks of other functions.
		var node = this

		// Store local copies of the node configuration (as defined in the .html)
		node.topic    = config.topic
		node.name     = config.name
		node.start    = config.start
		node.pattern  = config.pattern
		node.path     = config.path
		node.single   = config.single
		node.depth    = config.depth
		node.stat     = config.stat

		// Make sure the parameters are strings
		if ( (typeof node.start !== 'string') || (typeof node.pattern !== 'string') ) {
			node.error('Either Folder or Ext is not a string, cannot process. Folder: ' + node.start + ', Pattern: ' + node.ext)
			return
		}

		// respond to inputs....
		node.on('input', function(msg) {

			// Keep the original topic and payload
			msg.topic = msg.topic
			msg.origPayload = msg.payload
			// Output the nodes config
			msg.config = {
				'start': node.start,
				'pattern': node.pattern,
				'path': node.path,
				'single': node.single,
				'depth': node.depth,
				'stat': node.stat
			}

			// override config if passed suitable payload
			if ( (typeof msg.payload === 'object') && ('start' in msg.payload) ) {
				if ( validFolderName(msg.payload.start) ) {
					node.start = msg.payload.start
				}
			}
			if ( (typeof msg.payload === 'object') && ('pattern' in msg.payload) ) {
				if ( (typeof msg.payload.pattern === 'string') && (msg.payload.pattern.length < 1024) ) {
					node.pattern = msg.payload.pattern
				}
			}

			// Keep count of files found
			var totalFiles = 0

			var options = {}
			options.root = path.join(node.start),
			options.fileFilter = node.pattern
			options.entryType = 'files'
			if ( node.depth > -1 ) {
				options.depth = Number(node.depth)
			}

			var arrayOut = []

			// Recursively read the folder using the stream API
			readdirp(options)
				// Called if a found entry cannot be accessed (e.g. permissions)
				.on('warn', function (err) {
					node.warn('File could not be accessed', err)
				})
				// Fatal error
				.on('error', function (err) {
					node.error('Error processing folder listing', err)
				})
				// called for each entry found
				.on('data', function (entry) {
					var file = entry.name
					// Do we want just the file name output or the full path?
					if ( node.path ) {
						file = entry.fullPath
					}
					// Do we want to include file stats?
					if ( node.stat ) {
						file = {name: file}
						file.stat = {
							size: entry.stat.size,
							created: entry.stat.ctime,
							changed: entry.stat.mtime,
							accessed: entry.stat.atime,
							uid: entry.stat.uid,
							gid: entry.stat.gid,
							mode: '0x' + entry.stat.mode.toString(16)
						}
					}
					// If only single output
					if ( node.single ) {
						// accumulate the output array (send later on end)
						arrayOut.push(file)
					} else {
						// or send each file as separate msg
						node.status({fill:'green',shape:'dot',text:'# files ' + ++totalFiles})
						msg.payload = file
						//msg.debug = entry
						node.send( msg )
					}
				})
				// called when all entries have been output
				.on('end', function () {
					// Send a single msg if arrayOut contains anything
					if ( arrayOut.length > 0 ) {
						node.status({fill:'blue',shape:'dot',text:'# files ' + arrayOut.length})
						msg.payload = arrayOut
						node.send(msg)
					}
				})

			/*
			// Async read the folder
			fs.readdir(node.start, function (err, files) {
				if (err) {
					node.error('Oops! Folder does not exist: ' + node.start, err)
					return
				}

				totalFiles = files.length
				node.status({fill:'blue',shape:'dot',text:filteredFiles + '/' + totalFiles})

				var arrayOut = []

				// Any error in here will crash Node-Red!! So catch them instead
				try {
					files
						// calls the given callback func for each file, will only pass names where fn returns true
						.filter( function (file) {
							var len = node.ext.length
							if (len === 0) return true
							// Returns TRUE only if the file name ends in the given extension
							return file.substr(0-len) === node.ext
						})
						.forEach( function (file) {
							if ( node.path ) {
								file = path.join(node.start, file)
							}
							// If only single output
							if ( node.single ) {
								// accumulate the output array (send later)
								arrayOut.push(file)
							} else {
								// or send each file as separate msg
								msg.payload = file
								node.send( msg )
							}
						})
				} catch (err) {
					node.error('Ouch! Something went badly wrong processing the files list', err)
					return
				}

				// Send a single msg if arrayOut contains anything
				if ( arrayOut.length > 0 ) {
					msg.payload = arrayOut
					node.send(msg)
				}

			}) // --- End of readdir callback --- //
			*/

		}) // --- End of node input fn --- //

		// Tidy up if we need to
			//node.on("close", function() {
			// Called when the node is shutdown - eg on redeploy.
			// Allows ports to be closed, connections dropped etc.
			// eg: node.client.disconnect();
		//});

	} // --- End of fnFileLister --- //

	// Check if a folder name is safe
	function validFolderName(folder) {
		var isValid = true
		if ( typeof folder !== 'string' ) {
			isValid = false
		} else {
			if ( folder.length > 1023 ) {
				isValid = false
			}
		}
		return isValid
	} // --- End of validFolderName function --- //

	// Register the node by name. This must be called before overriding any of the
	// Node functions.
	RED.nodes.registerType(moduleName, nodeGo)
} // ---- End of Mobule ---- //
<commit_msg>fixes #9 by sending new messages rather than overwriting the incoming message on every send.<commit_after>/**
 * Copyright (c) 2015 Julian Knight (Totally Information)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 **/

// Node for Node-Red that lists all files from a given start folder
// with an optionally specified file extension.
//var fs = require('fs')
var path = require('path')
var readdirp = require('readdirp')

// Module name must match this nodes html file
var moduleName = 'fs-file-lister'

module.exports = function(RED) {
	'use strict'

	// The main node definition - most things happen in here
	function nodeGo(config) {
		// Create a RED node
		RED.nodes.createNode(this, config)

		// copy "this" object in case we need it in context of callbacks of other functions.
		var node = this

		// Store local copies of the node configuration (as defined in the .html)
		node.topic    = config.topic
		node.name     = config.name
		node.start    = config.start
		node.pattern  = config.pattern
		node.path     = config.path
		node.single   = config.single
		node.depth    = config.depth
		node.stat     = config.stat

		// Make sure the parameters are strings
		if ( (typeof node.start !== 'string') || (typeof node.pattern !== 'string') ) {
			node.error('Either Folder or Ext is not a string, cannot process. Folder: ' + node.start + ', Pattern: ' + node.ext)
			return
		}

		// respond to inputs....
		node.on('input', function(msg) {

			var clonedMsg = RED.util.cloneMessage(msg);
			// Keep the original topic and payload
			clonedMsg.topic = msg.topic
			clonedMsg.origPayload = msg.payload
			// Output the nodes config
			clonedMsg.config = {
				'start': node.start,
				'pattern': node.pattern,
				'path': node.path,
				'single': node.single,
				'depth': node.depth,
				'stat': node.stat
			}

			// override config if passed suitable payload
			if ( (typeof msg.payload === 'object') && ('start' in msg.payload) ) {
				if ( validFolderName(msg.payload.start) ) {
					node.start = msg.payload.start
				}
			}
			if ( (typeof msg.payload === 'object') && ('pattern' in msg.payload) ) {
				if ( (typeof msg.payload.pattern === 'string') && (msg.payload.pattern.length < 1024) ) {
					node.pattern = msg.payload.pattern
				}
			}

			// Keep count of files found
			var totalFiles = 0

			var options = {}
			options.root = path.join(node.start),
			options.fileFilter = node.pattern
			options.entryType = 'files'
			if ( node.depth > -1 ) {
				options.depth = Number(node.depth)
			}

			var arrayOut = []

			// Recursively read the folder using the stream API
			readdirp(options)
				// Called if a found entry cannot be accessed (e.g. permissions)
				.on('warn', function (err) {
					node.warn('File could not be accessed', err)
				})
				// Fatal error
				.on('error', function (err) {
					node.error('Error processing folder listing', err)
				})
				// called for each entry found
				.on('data', function (entry) {
					var file = entry.name
					// Do we want just the file name output or the full path?
					if ( node.path ) {
						file = entry.fullPath
					}
					// Do we want to include file stats?
					if ( node.stat ) {
						file = {name: file}
						file.stat = {
							size: entry.stat.size,
							created: entry.stat.ctime,
							changed: entry.stat.mtime,
							accessed: entry.stat.atime,
							uid: entry.stat.uid,
							gid: entry.stat.gid,
							mode: '0x' + entry.stat.mode.toString(16)
						}
					}
					// If only single output
					if ( node.single ) {
						// accumulate the output array (send later on end)
						arrayOut.push(file)
					} else {
						// or send each file as separate msg
						node.status({fill:'green',shape:'dot',text:'# files ' + ++totalFiles})
						node.send({
							topic: clonedMsg.topic,
							origPayload: clonedMsg.origPayload,
							config: clonedMsg.config,
							payload: file,
							// debug: entry
						})
					}
				})
				// called when all entries have been output
				.on('end', function () {
					// Send a single msg if arrayOut contains anything
					if ( arrayOut.length > 0 ) {
						node.status({fill:'blue',shape:'dot',text:'# files ' + arrayOut.length})
						clonedMsg.payload = arrayOut
						node.send(clonedMsg)
					}
				})

			/*
			// Async read the folder
			fs.readdir(node.start, function (err, files) {
				if (err) {
					node.error('Oops! Folder does not exist: ' + node.start, err)
					return
				}

				totalFiles = files.length
				node.status({fill:'blue',shape:'dot',text:filteredFiles + '/' + totalFiles})

				var arrayOut = []

				// Any error in here will crash Node-Red!! So catch them instead
				try {
					files
						// calls the given callback func for each file, will only pass names where fn returns true
						.filter( function (file) {
							var len = node.ext.length
							if (len === 0) return true
							// Returns TRUE only if the file name ends in the given extension
							return file.substr(0-len) === node.ext
						})
						.forEach( function (file) {
							if ( node.path ) {
								file = path.join(node.start, file)
							}
							// If only single output
							if ( node.single ) {
								// accumulate the output array (send later)
								arrayOut.push(file)
							} else {
								// or send each file as separate msg
								msg.payload = file
								node.send( msg )
							}
						})
				} catch (err) {
					node.error('Ouch! Something went badly wrong processing the files list', err)
					return
				}

				// Send a single msg if arrayOut contains anything
				if ( arrayOut.length > 0 ) {
					msg.payload = arrayOut
					node.send(msg)
				}

			}) // --- End of readdir callback --- //
			*/

		}) // --- End of node input fn --- //

		// Tidy up if we need to
			//node.on("close", function() {
			// Called when the node is shutdown - eg on redeploy.
			// Allows ports to be closed, connections dropped etc.
			// eg: node.client.disconnect();
		//});

	} // --- End of fnFileLister --- //

	// Check if a folder name is safe
	function validFolderName(folder) {
		var isValid = true
		if ( typeof folder !== 'string' ) {
			isValid = false
		} else {
			if ( folder.length > 1023 ) {
				isValid = false
			}
		}
		return isValid
	} // --- End of validFolderName function --- //

	// Register the node by name. This must be called before overriding any of the
	// Node functions.
	RED.nodes.registerType(moduleName, nodeGo)
} // ---- End of Mobule ---- //
<|endoftext|>


<commit_before>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require("../net2/logger.js")(__filename);

const rclient = require('../util/redis_manager.js').getRedisClient()

const firewalla = require("../net2/Firewalla.js");

const DNSTool = require('../net2/DNSTool.js')
const dnsTool = new DNSTool()

const CategoryUpdaterBase = require('./CategoryUpdaterBase.js');
const domainBlock = require('../control/DomainBlock.js')();

const exec = require('child-process-promise').exec

const sem = require('../sensor/SensorEventManager.js').getInstance();

const _ = require('lodash');

let instance = null

const EXPIRE_TIME = 60 * 60 * 48 // one hour

class CategoryUpdater extends CategoryUpdaterBase {

  constructor() {
    if (instance == null) {
      super()
      this.inited = false;
      instance = this

      this.activeCategories = {
        "games": 1,
        "social": 1,
        "porn": 1,
        "shopping": 1,
        "av": 1,
        "default_c": 1,
        "p2p": 1,
        "gamble": 1
      };

      this.excludedDomains = {
        "av": [
          "www.google.com",
          "forcesafesearch.google.com",
          "docs.google.com",
          "*.itunes.apple.com",
          "itunes.apple.com"
        ]
      };

      // only run refresh category records for fire main process
      sem.once('IPTABLES_READY', async () => {
        this.inited = true;
        if (firewalla.isMain()) {
          setInterval(() => {
            this.refreshAllCategoryRecords()
          }, 60 * 60 * 1000) // update records every hour

          await this.refreshAllCategoryRecords()

          sem.on('UPDATE_CATEGORY_DYNAMIC_DOMAIN', (event) => {
            if(event.category) {
              this.recycleIPSet(event.category)
            }
          });
        }
      })
    }

    return instance
  }

  async getDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.zrangeAsync(this.getCategoryKey(category), 0, -1)
  }

  async getDefaultDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getDefaultCategoryKey(category))
  }

  async addDefaultDomains(category, domains) {
    if(!this.isActivated(category))
      return []

    if(domains.length === 0) {
      return []
    }

    let commands = [this.getDefaultCategoryKey(category)]

    commands.push.apply(commands, domains)
    return rclient.saddAsync(commands)
  }

  async flushDefaultDomains(category) {
    if(!this.isActivated(category))
      return [];

    return rclient.delAsync(this.getDefaultCategoryKey(category));
  }

  async getIncludedDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getIncludeCategoryKey(category))
  }

  async addIncludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.saddAsync(this.getIncludeCategoryKey(category), domain)
  }

  async removeIncludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.sremAsync(this.getIncludeCategoryKey(category), domain)
  }

  async getExcludedDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getExcludeCategoryKey(category))
  }

  async addExcludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.saddAsync(this.getExcludeCategoryKey(category), domain)
  }

  async removeExcludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.sremAsync(this.getExcludeCategoryKey(category), domain)
  }

  async includeDomainExists(category, domain) {
    if(!this.isActivated(category))
      return false

    return rclient.sismemberAsync(this.getIncludeCategoryKey(category), domain)
  }

  async excludeDomainExists(category, domain) {
    if(!this.isActivated(category))
      return false

    return rclient.sismemberAsync(this.getExcludeCategoryKey(category), domain)
  }

  async getDomainsWithExpireTime(category) {
    const key = this.getCategoryKey(category)

    const domainAndScores = await rclient.zrevrangebyscoreAsync(key, '+inf', 0, 'withscores')
    const results = []

    for(let i = 0; i < domainAndScores.length; i++) {
      if(i % 2 === 1) {
        const domain = domainAndScores[i-1]
        const score = Number(domainAndScores[i])
        const expireDate = score + EXPIRE_TIME

        results.push({domain: domain, expire: expireDate})
      }
    }

    return results
  }

  async updateDomain(category, domain, isPattern) {

    if(!category || !domain) {
      return;
    }

    if(!this.isActivated(category)) {
      return
    }

    const now = Math.floor(new Date() / 1000)
    const key = this.getCategoryKey(category)

    let d = domain
    if(isPattern) {
      d = `*.${domain}`
    }

    const included = await this.includeDomainExists(category, d);

    if(!included) {
      const excluded = await this.excludeDomainExists(category, d);

      if(excluded) {
        return;
      }
    }

    log.debug(`Found a ${category} domain: ${d}`)

    await rclient.zaddAsync(key, now, d) // use current time as score for zset, it will be used to know when it should be expired out
    await this.updateIPSetByDomain(category, d)
    await this.filterIPSetByDomain(category);
  }

  getDomainMapping(domain) {
    return `rdns:domain:${domain}`
  }

  async getDomainMappingsByDomainPattern(domainPattern) {
    const keys = await rclient.keysAsync(this.getDomainMapping(domainPattern))
    keys.push(this.getDomainMapping(domainPattern.substring(2)))
    return keys
  }

  getSummedDomainMapping(domain) {
    let d = domain
    if(d.startsWith("*.")) {
      d = d.substring(2)
    }

    return `srdns:pattern:${d}`
  }

  // use "ipset restore" to add rdns entries to corresponding ipset
  async updateIPSetByDomain(category, domain, options) {
    if (!this.inited) return
    log.debug(`About to update category ${category} with domain ${domain}, options: ${JSON.stringify(options)}`)

    const mapping = this.getDomainMapping(domain)
    let ipsetName = this.getIPSetName(category)
    let ipset6Name = this.getIPSetNameForIPV6(category)

    if(options && options.useTemp) {
      ipsetName = this.getTempIPSetName(category)
      ipset6Name = this.getTempIPSetNameForIPV6(category)
    }

    if(domain.startsWith("*.")) {
      return this.updateIPSetByDomainPattern(category, domain, options)
    }

    const hasAny = await rclient.zcountAsync(mapping, '-inf', '+inf')

    if(hasAny) {
      let cmd4 = `redis-cli zrange ${mapping} 0 -1 | egrep -v ".*:.*" | sed 's=^=add ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${mapping} 0 -1 | egrep ".*:.*" | sed 's=^=add ${ipset6Name} = ' | sudo ipset restore -!`
      await exec(cmd4).catch((err) => {
        log.error(`Failed to update ipset by category ${category} domain ${domain}, err: ${err}`)
      })
      await exec(cmd6).catch((err) => {
        log.error(`Failed to update ipset6 by category ${category} domain ${domain}, err: ${err}`)
      })
    }

  }

  async filterIPSetByDomain(category, options) {
    if (!this.inited) return

    options = options || {}

    const list = this.excludedDomains && this.excludedDomains[category];

    if(!_.isEmpty(list)) {
      for(const domain of list) {
        if(domain.startsWith("*.")) {
          await this._filterIPSetByDomainPattern(category, domain, options).catch((err) => {
            log.error("Got error when filter ip set for domain pattern", domain, "with err", err);
          });
        } else {
          await this._filterIPSetByDomain(category, domain, options).catch((err) => {
            log.error("Got error when filter ip set for domain", domain, "with err", err);
          })
        }
      }
    }
  }

  async _filterIPSetByDomain(category, domain, options) {
    options = options || {}

    const mapping = this.getDomainMapping(domain)
    let ipsetName = this.getIPSetName(category)
    let ipset6Name = this.getIPSetNameForIPV6(category)

    if(options && options.useTemp) {
      ipsetName = this.getTempIPSetName(category)
      ipset6Name = this.getTempIPSetNameForIPV6(category)
    }

    const hasAny = await rclient.zcountAsync(mapping, '-inf', '+inf')

    if(hasAny) {
      let cmd4 = `redis-cli zrange ${mapping} 0 -1 | egrep -v ".*:.*" | sed 's=^=del ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${mapping} 0 -1 | egrep ".*:.*" | sed 's=^=del ${ipset6Name} = ' | sudo ipset restore -!`
      await exec(cmd4).catch((err) => {
        log.error(`Failed to delete ipset by category ${category} domain ${domain}, err: ${err}`)
      })
      await exec(cmd6).catch((err) => {
        log.error(`Failed to delete ipset6 by category ${category} domain ${domain}, err: ${err}`)
      })
    }
  }

  async _filterIPSetByDomainPattern(category, domain, options) {
    if(!domain.startsWith("*.")) {
      return
    }

    const mappings = await this.getDomainMappingsByDomainPattern(domain)

    if(mappings.length > 0) {
      const smappings = this.getSummedDomainMapping(domain)
      let array = [smappings, mappings.length]

      array.push.apply(array, mappings)

      await rclient.zunionstoreAsync(array)

      const exists = await rclient.typeAsync(smappings);
      if(exists === "none") {
        return; // if smapping doesn't exist, meaning no ip found for this domain, sometimes true for pre-provided domain list
      }

      await rclient.expireAsync(smappings, 600) // auto expire in 10 minutes

      let ipsetName = this.getIPSetName(category)
      let ipset6Name = this.getIPSetNameForIPV6(category)

      if(options && options.useTemp) {
        ipsetName = this.getTempIPSetName(category)
        ipset6Name = this.getTempIPSetNameForIPV6(category)
      }

      let cmd4 = `redis-cli zrange ${smappings} 0 -1 | egrep -v ".*:.*" | sed 's=^=del ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${smappings} 0 -1 | egrep ".*:.*" | sed 's=^=del ${ipset6Name} = ' | sudo ipset restore -!`
      try {
        await exec(cmd4);
        await exec(cmd6);
      } catch(err) {
        log.error(`Failed to filter ipset by category ${category} domain pattern ${domain}, err: ${err}`)
      }
    }
  }

  async updateIPSetByDomainPattern(category, domain, options) {
    if(!domain.startsWith("*.")) {
      return
    }

    log.debug(`About to update category ${category} with domain pattern ${domain}, options: ${JSON.stringify(options)}`)

    const mappings = await this.getDomainMappingsByDomainPattern(domain)

    if(mappings.length > 0) {
      const smappings = this.getSummedDomainMapping(domain)
      let array = [smappings, mappings.length]

      array.push.apply(array, mappings)

      await rclient.zunionstoreAsync(array)

      const exists = await rclient.typeAsync(smappings);
      if(exists === "none") {
        return; // if smapping doesn't exist, meaning no ip found for this domain, sometimes true for pre-provided domain list
      }

      await rclient.expireAsync(smappings, 600) // auto expire in 10 minutes

      let ipsetName = this.getIPSetName(category)
      let ipset6Name = this.getIPSetNameForIPV6(category)

      if(options && options.useTemp) {
        ipsetName = this.getTempIPSetName(category)
        ipset6Name = this.getTempIPSetNameForIPV6(category)
      }

      let cmd4 = `redis-cli zrange ${smappings} 0 -1 | egrep -v ".*:.*" | sed 's=^=add ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${smappings} 0 -1 | egrep ".*:.*" | sed 's=^=add ${ipset6Name} = ' | sudo ipset restore -!`
      try {
        await exec(cmd4)
        await exec(cmd6)
      } catch(err) {
        log.error(`Failed to update ipset by category ${category} domain pattern ${domain}, err: ${err}`)
      }
    }
  }

  // rebuild category ipset
  async recycleIPSet(category) {

    await this.updatePersistentIPSets(category, {useTemp: true});

    const domains = await this.getDomains(category)
    const includedDomains = await this.getIncludedDomains(category);
    const defaultDomains = await this.getDefaultDomains(category);
    const excludeDomains = await this.getExcludedDomains(category);

    let dd = _.union(domains, defaultDomains)
    dd = _.difference(dd, excludeDomains)
    dd = _.union(dd, includedDomains)

    for (const domain of dd) {

      let domainSuffix = domain
      if(domainSuffix.startsWith("*.")) {
        domainSuffix = domainSuffix.substring(2);
      }

      const existing = await dnsTool.reverseDNSKeyExists(domainSuffix)
      if(!existing) { // a new domain
        log.info(`Found a new domain with new rdns: ${domainSuffix}`)
        await domainBlock.resolveDomain(domainSuffix)
      }

      await this.updateIPSetByDomain(category, domain, {useTemp: true}).catch((err) => {
        log.error(`Failed to update ipset for domain ${domain}, err: ${err}`)
      })

      await this.filterIPSetByDomain(category, {useTemp: true}).catch((err) => {
        log.error(`Failed to filter ipset for domain ${domain}, err: ${err}`)
      })
    }

    await this.swapIpset(category);

    log.info(`Successfully recycled ipset for category ${category}`)
  }

  async refreshCategoryRecord(category) {
    const key = this.getCategoryKey(category)
    const date = Math.floor(new Date() / 1000) - EXPIRE_TIME

    return rclient.zremrangebyscoreAsync(key, '-inf', date)
  }
}

module.exports = CategoryUpdater
<commit_msg>Move default_c's priority<commit_after>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require("../net2/logger.js")(__filename);

const rclient = require('../util/redis_manager.js').getRedisClient()

const firewalla = require("../net2/Firewalla.js");

const DNSTool = require('../net2/DNSTool.js')
const dnsTool = new DNSTool()

const CategoryUpdaterBase = require('./CategoryUpdaterBase.js');
const domainBlock = require('../control/DomainBlock.js')();

const exec = require('child-process-promise').exec

const sem = require('../sensor/SensorEventManager.js').getInstance();

const _ = require('lodash');

let instance = null

const EXPIRE_TIME = 60 * 60 * 48 // one hour

class CategoryUpdater extends CategoryUpdaterBase {

  constructor() {
    if (instance == null) {
      super()
      this.inited = false;
      instance = this

      this.activeCategories = {
        "default_c": 1,
        "games": 1,
        "social": 1,
        "porn": 1,
        "shopping": 1,
        "av": 1,
        "p2p": 1,
        "gamble": 1
      };

      this.excludedDomains = {
        "av": [
          "www.google.com",
          "forcesafesearch.google.com",
          "docs.google.com",
          "*.itunes.apple.com",
          "itunes.apple.com"
        ]
      };

      // only run refresh category records for fire main process
      sem.once('IPTABLES_READY', async () => {
        this.inited = true;
        if (firewalla.isMain()) {
          setInterval(() => {
            this.refreshAllCategoryRecords()
          }, 60 * 60 * 1000) // update records every hour

          await this.refreshAllCategoryRecords()

          sem.on('UPDATE_CATEGORY_DYNAMIC_DOMAIN', (event) => {
            if(event.category) {
              this.recycleIPSet(event.category)
            }
          });
        }
      })
    }

    return instance
  }

  async getDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.zrangeAsync(this.getCategoryKey(category), 0, -1)
  }

  async getDefaultDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getDefaultCategoryKey(category))
  }

  async addDefaultDomains(category, domains) {
    if(!this.isActivated(category))
      return []

    if(domains.length === 0) {
      return []
    }

    let commands = [this.getDefaultCategoryKey(category)]

    commands.push.apply(commands, domains)
    return rclient.saddAsync(commands)
  }

  async flushDefaultDomains(category) {
    if(!this.isActivated(category))
      return [];

    return rclient.delAsync(this.getDefaultCategoryKey(category));
  }

  async getIncludedDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getIncludeCategoryKey(category))
  }

  async addIncludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.saddAsync(this.getIncludeCategoryKey(category), domain)
  }

  async removeIncludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.sremAsync(this.getIncludeCategoryKey(category), domain)
  }

  async getExcludedDomains(category) {
    if(!this.isActivated(category))
      return []

    return rclient.smembersAsync(this.getExcludeCategoryKey(category))
  }

  async addExcludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.saddAsync(this.getExcludeCategoryKey(category), domain)
  }

  async removeExcludedDomain(category, domain) {
    if(!this.isActivated(category))
      return

    return rclient.sremAsync(this.getExcludeCategoryKey(category), domain)
  }

  async includeDomainExists(category, domain) {
    if(!this.isActivated(category))
      return false

    return rclient.sismemberAsync(this.getIncludeCategoryKey(category), domain)
  }

  async excludeDomainExists(category, domain) {
    if(!this.isActivated(category))
      return false

    return rclient.sismemberAsync(this.getExcludeCategoryKey(category), domain)
  }

  async getDomainsWithExpireTime(category) {
    const key = this.getCategoryKey(category)

    const domainAndScores = await rclient.zrevrangebyscoreAsync(key, '+inf', 0, 'withscores')
    const results = []

    for(let i = 0; i < domainAndScores.length; i++) {
      if(i % 2 === 1) {
        const domain = domainAndScores[i-1]
        const score = Number(domainAndScores[i])
        const expireDate = score + EXPIRE_TIME

        results.push({domain: domain, expire: expireDate})
      }
    }

    return results
  }

  async updateDomain(category, domain, isPattern) {

    if(!category || !domain) {
      return;
    }

    if(!this.isActivated(category)) {
      return
    }

    const now = Math.floor(new Date() / 1000)
    const key = this.getCategoryKey(category)

    let d = domain
    if(isPattern) {
      d = `*.${domain}`
    }

    const included = await this.includeDomainExists(category, d);

    if(!included) {
      const excluded = await this.excludeDomainExists(category, d);

      if(excluded) {
        return;
      }
    }

    log.debug(`Found a ${category} domain: ${d}`)

    await rclient.zaddAsync(key, now, d) // use current time as score for zset, it will be used to know when it should be expired out
    await this.updateIPSetByDomain(category, d)
    await this.filterIPSetByDomain(category);
  }

  getDomainMapping(domain) {
    return `rdns:domain:${domain}`
  }

  async getDomainMappingsByDomainPattern(domainPattern) {
    const keys = await rclient.keysAsync(this.getDomainMapping(domainPattern))
    keys.push(this.getDomainMapping(domainPattern.substring(2)))
    return keys
  }

  getSummedDomainMapping(domain) {
    let d = domain
    if(d.startsWith("*.")) {
      d = d.substring(2)
    }

    return `srdns:pattern:${d}`
  }

  // use "ipset restore" to add rdns entries to corresponding ipset
  async updateIPSetByDomain(category, domain, options) {
    if (!this.inited) return
    log.debug(`About to update category ${category} with domain ${domain}, options: ${JSON.stringify(options)}`)

    const mapping = this.getDomainMapping(domain)
    let ipsetName = this.getIPSetName(category)
    let ipset6Name = this.getIPSetNameForIPV6(category)

    if(options && options.useTemp) {
      ipsetName = this.getTempIPSetName(category)
      ipset6Name = this.getTempIPSetNameForIPV6(category)
    }

    if(domain.startsWith("*.")) {
      return this.updateIPSetByDomainPattern(category, domain, options)
    }

    const hasAny = await rclient.zcountAsync(mapping, '-inf', '+inf')

    if(hasAny) {
      let cmd4 = `redis-cli zrange ${mapping} 0 -1 | egrep -v ".*:.*" | sed 's=^=add ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${mapping} 0 -1 | egrep ".*:.*" | sed 's=^=add ${ipset6Name} = ' | sudo ipset restore -!`
      await exec(cmd4).catch((err) => {
        log.error(`Failed to update ipset by category ${category} domain ${domain}, err: ${err}`)
      })
      await exec(cmd6).catch((err) => {
        log.error(`Failed to update ipset6 by category ${category} domain ${domain}, err: ${err}`)
      })
    }

  }

  async filterIPSetByDomain(category, options) {
    if (!this.inited) return

    options = options || {}

    const list = this.excludedDomains && this.excludedDomains[category];

    if(!_.isEmpty(list)) {
      for(const domain of list) {
        if(domain.startsWith("*.")) {
          await this._filterIPSetByDomainPattern(category, domain, options).catch((err) => {
            log.error("Got error when filter ip set for domain pattern", domain, "with err", err);
          });
        } else {
          await this._filterIPSetByDomain(category, domain, options).catch((err) => {
            log.error("Got error when filter ip set for domain", domain, "with err", err);
          })
        }
      }
    }
  }

  async _filterIPSetByDomain(category, domain, options) {
    options = options || {}

    const mapping = this.getDomainMapping(domain)
    let ipsetName = this.getIPSetName(category)
    let ipset6Name = this.getIPSetNameForIPV6(category)

    if(options && options.useTemp) {
      ipsetName = this.getTempIPSetName(category)
      ipset6Name = this.getTempIPSetNameForIPV6(category)
    }

    const hasAny = await rclient.zcountAsync(mapping, '-inf', '+inf')

    if(hasAny) {
      let cmd4 = `redis-cli zrange ${mapping} 0 -1 | egrep -v ".*:.*" | sed 's=^=del ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${mapping} 0 -1 | egrep ".*:.*" | sed 's=^=del ${ipset6Name} = ' | sudo ipset restore -!`
      await exec(cmd4).catch((err) => {
        log.error(`Failed to delete ipset by category ${category} domain ${domain}, err: ${err}`)
      })
      await exec(cmd6).catch((err) => {
        log.error(`Failed to delete ipset6 by category ${category} domain ${domain}, err: ${err}`)
      })
    }
  }

  async _filterIPSetByDomainPattern(category, domain, options) {
    if(!domain.startsWith("*.")) {
      return
    }

    const mappings = await this.getDomainMappingsByDomainPattern(domain)

    if(mappings.length > 0) {
      const smappings = this.getSummedDomainMapping(domain)
      let array = [smappings, mappings.length]

      array.push.apply(array, mappings)

      await rclient.zunionstoreAsync(array)

      const exists = await rclient.typeAsync(smappings);
      if(exists === "none") {
        return; // if smapping doesn't exist, meaning no ip found for this domain, sometimes true for pre-provided domain list
      }

      await rclient.expireAsync(smappings, 600) // auto expire in 10 minutes

      let ipsetName = this.getIPSetName(category)
      let ipset6Name = this.getIPSetNameForIPV6(category)

      if(options && options.useTemp) {
        ipsetName = this.getTempIPSetName(category)
        ipset6Name = this.getTempIPSetNameForIPV6(category)
      }

      let cmd4 = `redis-cli zrange ${smappings} 0 -1 | egrep -v ".*:.*" | sed 's=^=del ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${smappings} 0 -1 | egrep ".*:.*" | sed 's=^=del ${ipset6Name} = ' | sudo ipset restore -!`
      try {
        await exec(cmd4);
        await exec(cmd6);
      } catch(err) {
        log.error(`Failed to filter ipset by category ${category} domain pattern ${domain}, err: ${err}`)
      }
    }
  }

  async updateIPSetByDomainPattern(category, domain, options) {
    if(!domain.startsWith("*.")) {
      return
    }

    log.debug(`About to update category ${category} with domain pattern ${domain}, options: ${JSON.stringify(options)}`)

    const mappings = await this.getDomainMappingsByDomainPattern(domain)

    if(mappings.length > 0) {
      const smappings = this.getSummedDomainMapping(domain)
      let array = [smappings, mappings.length]

      array.push.apply(array, mappings)

      await rclient.zunionstoreAsync(array)

      const exists = await rclient.typeAsync(smappings);
      if(exists === "none") {
        return; // if smapping doesn't exist, meaning no ip found for this domain, sometimes true for pre-provided domain list
      }

      await rclient.expireAsync(smappings, 600) // auto expire in 10 minutes

      let ipsetName = this.getIPSetName(category)
      let ipset6Name = this.getIPSetNameForIPV6(category)

      if(options && options.useTemp) {
        ipsetName = this.getTempIPSetName(category)
        ipset6Name = this.getTempIPSetNameForIPV6(category)
      }

      let cmd4 = `redis-cli zrange ${smappings} 0 -1 | egrep -v ".*:.*" | sed 's=^=add ${ipsetName} = ' | sudo ipset restore -!`
      let cmd6 = `redis-cli zrange ${smappings} 0 -1 | egrep ".*:.*" | sed 's=^=add ${ipset6Name} = ' | sudo ipset restore -!`
      try {
        await exec(cmd4)
        await exec(cmd6)
      } catch(err) {
        log.error(`Failed to update ipset by category ${category} domain pattern ${domain}, err: ${err}`)
      }
    }
  }

  // rebuild category ipset
  async recycleIPSet(category) {

    await this.updatePersistentIPSets(category, {useTemp: true});

    const domains = await this.getDomains(category)
    const includedDomains = await this.getIncludedDomains(category);
    const defaultDomains = await this.getDefaultDomains(category);
    const excludeDomains = await this.getExcludedDomains(category);

    let dd = _.union(domains, defaultDomains)
    dd = _.difference(dd, excludeDomains)
    dd = _.union(dd, includedDomains)

    for (const domain of dd) {

      let domainSuffix = domain
      if(domainSuffix.startsWith("*.")) {
        domainSuffix = domainSuffix.substring(2);
      }

      const existing = await dnsTool.reverseDNSKeyExists(domainSuffix)
      if(!existing) { // a new domain
        log.info(`Found a new domain with new rdns: ${domainSuffix}`)
        await domainBlock.resolveDomain(domainSuffix)
      }

      await this.updateIPSetByDomain(category, domain, {useTemp: true}).catch((err) => {
        log.error(`Failed to update ipset for domain ${domain}, err: ${err}`)
      })

      await this.filterIPSetByDomain(category, {useTemp: true}).catch((err) => {
        log.error(`Failed to filter ipset for domain ${domain}, err: ${err}`)
      })
    }

    await this.swapIpset(category);

    log.info(`Successfully recycled ipset for category ${category}`)
  }

  async refreshCategoryRecord(category) {
    const key = this.getCategoryKey(category)
    const date = Math.floor(new Date() / 1000) - EXPIRE_TIME

    return rclient.zremrangebyscoreAsync(key, '-inf', date)
  }
}

module.exports = CategoryUpdater
<|endoftext|>


<commit_before>// mocha.js tests for the functions in Curious
(function () {
'use strict';

var expect = require('chai').expect;
var curious = require('../curious2.js');

// HELPER METHODS

/**
 * Generate sample data as would be returned by a Curious server in JSON.
 *
 * @return {Object} The response data
 */
function _sampleResponse() {

  // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
  return {
    result: {
      computed_on: '2015-05-21 14:36:07.478769',
      last_model: 'Reaction',
      results: [
        {
          model: 'Experiment',
          join_index: -1,
          tree: null,
          objects: [
            [403, null],
          ],
        }, {
          model: 'Reaction',
          join_index: 0,
          tree: null,
          objects: [
            [23063, 403],
            [23064, 403],
            [23057, 403],
          ],
        },
      ],
      data: [
        { // Experiments
          fields: [
            'id',
            'created_by_id',
            'created_on',
            'updated_by_id',
            'updated_on',
            'assay_id',
            'name',
            'description',
            'completed',
            'ignore',
            'temperature',
          ],
          objects: [
              [
                403,
                22,
                '2015-01-16 14:54:13+00:00',
                22,
                '2015-05-14 16:03:13+00:00',
                1,
                'MS2 for x401 and x402',
                '',
                true,
                false,
                null,
              ],
          ],
          urls: [
            'http://example.com/experiment/403/'
          ],
        }, {
          fields: [
            'id',
            'created_by_id',
            'created_on',
            'updated_by_id',
            'updated_on',
            'experiment_id',
            'is_blank',
            'is_control',
            'notes',
            'sample_id',
          ],
          objects: [
            [
              23057,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              false,
              false,
              '',
              454565,
            ], [
              23063,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              true,
              false,
              '',
              null,
            ], [
              23064,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              true,
              false,
              '',
              null,
            ],
          ],
          urls: [
             'http://example.com/experiment/403/',
             'http://example.com/experiment/403/',
             'http://example.com/experiment/403/',
          ],
        },
      ],
    }
  };
  // jscs:enable
}

/**
 * Generate sample data as would be returned by a Curious server in JSON.
 *
 * @return {Object} The response data
 */
function _expectedObjects() {
  var exp;
  var rxns;

  // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
  exp = {
    id: 403,
    created_by_id: 22,
    created_on: '2015-01-16 14:54:13+00:00',
    updated_by_id: 22,
    updated_on: '2015-05-14 16:03:13+00:00',
    assay_id: 1,
    name: 'MS2 for x401 and x402',
    description: '',
    completed: true,
    ignore: false,
    temperature: null,
    __url: 'http://example.com/experiment/403/',
    __model: 'Experiment',
    __dirty: false,
  };

  rxns = [
    {
      id: 23063,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: true,
      is_control: false,
      notes: '',
      sample_id: null,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    }, {
      id: 23064,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: true,
      is_control: false,
      notes: '',
      sample_id: null,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    }, {
      id: 23057,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: false,
      is_control: false,
      notes: '',
      sample_id: 454565,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    },
  ];

  // Link foregin keys
  exp.reactions = rxns;
  rxns.forEach(function (rxn) { rxn.experiments = [exp]; });

  return {
    experiments: {
      403: exp
    },
    reactions: {
      23063: rxns[0],
      23064: rxns[1],
      23057: rxns[2],
    },
  };
  // jscs:enable
}

// TESTS

describe('CuriousObjects', function () {
  describe('#values', function () {
    var obj;

    beforeEach(function () {
      obj = {
        a: 1,
        b: 'two',
        c: ['3'],
        d: {'4': true},
        e: function () { return 5; },
      };
    });

    it('should return all of the values of an object', function () {
      expect(curious.CuriousObjects.values(obj)).to.have.deep.members(
        [obj.a, obj.b, obj.c, obj.d, obj.e]
      );
    });

    it('should return values in the same order as "for-in"', function () {
      var key;
      var valuesInOrder = [];

      for (key in obj) {
        if (obj.hasOwnProperty(key)) {
          valuesInOrder.push(obj[key]);
        }
      }
      expect(curious.CuriousObjects.values(obj)).to.deep.equal(valuesInOrder);
    });

    it('should maintain only complex references', function () {
      var vals = curious.CuriousObjects.values(obj);

      vals.forEach(function (value, ix) {
        // Simple reference
        if (value === 1) {
          // Use vals[ix] instead of value here to make sure we mutate vals
          // and not a shallow copy
          vals[ix] = 'moop';
        }

        // Complex reference
        if (JSON.stringify(value) === JSON.stringify({'4': true})) {
          vals[ix][4] = 'something';
        }
      });

      // Simple references do not change
      expect(obj.a).to.equal(1);
      // Complex references do change
      expect(obj.d['4']).to.equal('something');
    });
  });

  describe('#groupObjectsByID', function () {
    var objs;

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should return an object', function () {
      expect(curious.CuriousObjects.groupObjectsByID(objs)).to.be.an('object');
    });

    it('should contain all of the IDs', function () {
      expect(curious.CuriousObjects.groupObjectsByID(objs)).to.have.all.keys(
        {'1': objs[0], '2': objs[1], '3': objs[2]}
      );
    });

    it('should maintain only complex references', function () {
      var objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      // Simple reference
      objsByID[1] = 'moop'; // objs[0]
      // Complex reference
      objsByID[2].name = 'broop'; // objs[1]

      // Simple references do not change
      expect(objs[0]).to.have.all.keys({name: 'a', id: 1});
      // Complex references do change
      expect(objs[1]).to.have.all.keys({name: 'broop', id: 2});
    });

    it('should take the last object in the arary in case of duplicate IDs', function () {
      var objsByID;

      objs.push({name: 'b2', id: 2});
      objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      expect(objsByID[2]).to.have.property('name', 'b2');
    });

    it('should ignore objects without IDs', function () {
      var objsByID;
      var objsByIDWithNoID;

      objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      objs.push({name: 'noID'});

      objsByIDWithNoID = curious.CuriousObjects.groupObjectsByID(objs);

      expect(objsByID).to.deep.equal(objsByIDWithNoID);
    });

  });

  describe('#idList', function () {
    var objs;

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should contain all of the IDs', function () {
      expect(curious.CuriousObjects.idList(objs)).to.have.members([1, 2, 3]);
    });

    it('should not contain duplicates', function () {
      objs.push({name: 'b2', id: 2});

      expect(curious.CuriousObjects.idList(objs)).to.have.members([1, 2, 3]);
    });

    it('should maintain the order of IDs and keep the first unique one', function () {

      // This object with id = 2 will be ignored, but the first one will be kept.
      objs.push({name: 'b2', id: 2});
      objs.push({name: 'd', id: 4});
      objs.push({name: 'e', id: 0});
      // This object with id = 2 will also be ignored
      objs.push({name: 'b3', id: 2});
      objs.push({name: 'f', id: 23});

      expect(curious.CuriousObjects.idList(objs)).to.deep.equal([1, 2, 3, 4, 0, 23]);
    });
  });

  describe('#idString', function () {
    var objs;

    function _parseToNumbers(idString) {
      return idString.split(',').map(function (id) {return Number(id);});
    }

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should be equivalent to idList', function () {
      expect(_parseToNumbers(curious.CuriousObjects.idString(objs))).to.deep.equal(
        curious.CuriousObjects.idList(objs)
      );

      // This object with id = 2 will be ignored, but the first one will be kept.
      objs.push({name: 'b2', id: 2});
      objs.push({name: 'd', id: 4});
      objs.push({name: 'e', id: 0});
      // This object with id = 2 will also be ignored
      objs.push({name: 'b3', id: 2});
      objs.push({name: 'f', id: 23});

      expect(_parseToNumbers(curious.CuriousObjects.idString(objs))).to.deep.equal(
        curious.CuriousObjects.idList(objs)
      );
    });

    it('should not escape commas', function () {
      objs.push({name: 'commas', id: '4,5,6'});

      expect(
        // 1, 2, 3, 4, 5, 6, converted to numbers
        _parseToNumbers(curious.CuriousObjects.idString(objs))
      ).to.deep.equal(
        // Remove the element we added ('4,5,6') and add 4, 5, 6 to the end
        curious.CuriousObjects.idList(objs).slice(0, -1).concat([4, 5, 6])
      );
    });
  });

  describe('#parse', function () {
    var queryJSONResponse;

    beforeEach(function () {
      queryJSONResponse = _sampleResponse().result;
    });

    it('should correctly parse the output', function () {
      var expectedObjects = _expectedObjects();

      expect(curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        null,
        queryJSONResponse
      )).to.deep.equal(
        {
          trees: [null, null],
          objects: [
            expectedObjects.experiments,
            expectedObjects.reactions,
          ],
        }
      );
    });

    it('should construct objects correctly with custom constructors', function () {
      var constructors;
      var parsedData;

      // Fake constructors
      function Experiment() {
        // Do nothing
      }

      function Reaction() {
        // Do nothing
      }

      constructors = [Experiment, Reaction];

      parsedData = curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        constructors,
        queryJSONResponse
      );

      parsedData.objects.forEach(function (objectsByID, ix) {
        var objects = curious.CuriousObjects.values(objectsByID);

        objects.forEach(function (object) {
          expect(object).to.be.an.instanceof(constructors[ix]);
        });
      });

    });

    it('should combine new objects with existing ones', function () {
      var parsedData;
      // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
      var existingObjects = [
        null,
        {
          23063: {
            id: 23063,
            created_by_id: 12345,
            created_on: 'another date',
            updated_by_id: 67890,
            updated_on: 'another time',
            experiment_id: 403,
            is_blank: false,
            is_control: false,
            notes: '',
            sample_id: 124816,
            __url: 'http://example.com/experiment/403/',
            __model: 'Reaction',
            __dirty: false,
          }
        },
      ];
      // jscs:enable

      parsedData = curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        null,
        queryJSONResponse,
        existingObjects
      );

      expect(parsedData.objects[1][23063]).to.contain.keys(existingObjects[1][23063]);
    });

  });
});

}());
<commit_msg>Update header comment for curious_objects.js<commit_after>// mocha.js tests for functions dealing with Curious objects
(function () {
'use strict';

var expect = require('chai').expect;
var curious = require('../curious2.js');

// HELPER METHODS

/**
 * Generate sample data as would be returned by a Curious server in JSON.
 *
 * @return {Object} The response data
 */
function _sampleResponse() {

  // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
  return {
    result: {
      computed_on: '2015-05-21 14:36:07.478769',
      last_model: 'Reaction',
      results: [
        {
          model: 'Experiment',
          join_index: -1,
          tree: null,
          objects: [
            [403, null],
          ],
        }, {
          model: 'Reaction',
          join_index: 0,
          tree: null,
          objects: [
            [23063, 403],
            [23064, 403],
            [23057, 403],
          ],
        },
      ],
      data: [
        { // Experiments
          fields: [
            'id',
            'created_by_id',
            'created_on',
            'updated_by_id',
            'updated_on',
            'assay_id',
            'name',
            'description',
            'completed',
            'ignore',
            'temperature',
          ],
          objects: [
              [
                403,
                22,
                '2015-01-16 14:54:13+00:00',
                22,
                '2015-05-14 16:03:13+00:00',
                1,
                'MS2 for x401 and x402',
                '',
                true,
                false,
                null,
              ],
          ],
          urls: [
            'http://example.com/experiment/403/'
          ],
        }, {
          fields: [
            'id',
            'created_by_id',
            'created_on',
            'updated_by_id',
            'updated_on',
            'experiment_id',
            'is_blank',
            'is_control',
            'notes',
            'sample_id',
          ],
          objects: [
            [
              23057,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              false,
              false,
              '',
              454565,
            ], [
              23063,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              true,
              false,
              '',
              null,
            ], [
              23064,
              null,
              '2015-01-16 14:54:27+00:00',
              null,
              '2015-01-16 14:54:27+00:00',
              403,
              true,
              false,
              '',
              null,
            ],
          ],
          urls: [
             'http://example.com/experiment/403/',
             'http://example.com/experiment/403/',
             'http://example.com/experiment/403/',
          ],
        },
      ],
    }
  };
  // jscs:enable
}

/**
 * Generate sample data as would be returned by a Curious server in JSON.
 *
 * @return {Object} The response data
 */
function _expectedObjects() {
  var exp;
  var rxns;

  // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
  exp = {
    id: 403,
    created_by_id: 22,
    created_on: '2015-01-16 14:54:13+00:00',
    updated_by_id: 22,
    updated_on: '2015-05-14 16:03:13+00:00',
    assay_id: 1,
    name: 'MS2 for x401 and x402',
    description: '',
    completed: true,
    ignore: false,
    temperature: null,
    __url: 'http://example.com/experiment/403/',
    __model: 'Experiment',
    __dirty: false,
  };

  rxns = [
    {
      id: 23063,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: true,
      is_control: false,
      notes: '',
      sample_id: null,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    }, {
      id: 23064,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: true,
      is_control: false,
      notes: '',
      sample_id: null,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    }, {
      id: 23057,
      created_by_id: null,
      created_on: '2015-01-16 14:54:27+00:00',
      updated_by_id: null,
      updated_on: '2015-01-16 14:54:27+00:00',
      experiment_id: 403,
      is_blank: false,
      is_control: false,
      notes: '',
      sample_id: 454565,
      __url: 'http://example.com/experiment/403/',
      __model: 'Reaction',
      __dirty: false,
    },
  ];

  // Link foregin keys
  exp.reactions = rxns;
  rxns.forEach(function (rxn) { rxn.experiments = [exp]; });

  return {
    experiments: {
      403: exp
    },
    reactions: {
      23063: rxns[0],
      23064: rxns[1],
      23057: rxns[2],
    },
  };
  // jscs:enable
}

// TESTS

describe('CuriousObjects', function () {
  describe('#values', function () {
    var obj;

    beforeEach(function () {
      obj = {
        a: 1,
        b: 'two',
        c: ['3'],
        d: {'4': true},
        e: function () { return 5; },
      };
    });

    it('should return all of the values of an object', function () {
      expect(curious.CuriousObjects.values(obj)).to.have.deep.members(
        [obj.a, obj.b, obj.c, obj.d, obj.e]
      );
    });

    it('should return values in the same order as "for-in"', function () {
      var key;
      var valuesInOrder = [];

      for (key in obj) {
        if (obj.hasOwnProperty(key)) {
          valuesInOrder.push(obj[key]);
        }
      }
      expect(curious.CuriousObjects.values(obj)).to.deep.equal(valuesInOrder);
    });

    it('should maintain only complex references', function () {
      var vals = curious.CuriousObjects.values(obj);

      vals.forEach(function (value, ix) {
        // Simple reference
        if (value === 1) {
          // Use vals[ix] instead of value here to make sure we mutate vals
          // and not a shallow copy
          vals[ix] = 'moop';
        }

        // Complex reference
        if (JSON.stringify(value) === JSON.stringify({'4': true})) {
          vals[ix][4] = 'something';
        }
      });

      // Simple references do not change
      expect(obj.a).to.equal(1);
      // Complex references do change
      expect(obj.d['4']).to.equal('something');
    });
  });

  describe('#groupObjectsByID', function () {
    var objs;

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should return an object', function () {
      expect(curious.CuriousObjects.groupObjectsByID(objs)).to.be.an('object');
    });

    it('should contain all of the IDs', function () {
      expect(curious.CuriousObjects.groupObjectsByID(objs)).to.have.all.keys(
        {'1': objs[0], '2': objs[1], '3': objs[2]}
      );
    });

    it('should maintain only complex references', function () {
      var objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      // Simple reference
      objsByID[1] = 'moop'; // objs[0]
      // Complex reference
      objsByID[2].name = 'broop'; // objs[1]

      // Simple references do not change
      expect(objs[0]).to.have.all.keys({name: 'a', id: 1});
      // Complex references do change
      expect(objs[1]).to.have.all.keys({name: 'broop', id: 2});
    });

    it('should take the last object in the arary in case of duplicate IDs', function () {
      var objsByID;

      objs.push({name: 'b2', id: 2});
      objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      expect(objsByID[2]).to.have.property('name', 'b2');
    });

    it('should ignore objects without IDs', function () {
      var objsByID;
      var objsByIDWithNoID;

      objsByID = curious.CuriousObjects.groupObjectsByID(objs);

      objs.push({name: 'noID'});

      objsByIDWithNoID = curious.CuriousObjects.groupObjectsByID(objs);

      expect(objsByID).to.deep.equal(objsByIDWithNoID);
    });

  });

  describe('#idList', function () {
    var objs;

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should contain all of the IDs', function () {
      expect(curious.CuriousObjects.idList(objs)).to.have.members([1, 2, 3]);
    });

    it('should not contain duplicates', function () {
      objs.push({name: 'b2', id: 2});

      expect(curious.CuriousObjects.idList(objs)).to.have.members([1, 2, 3]);
    });

    it('should maintain the order of IDs and keep the first unique one', function () {

      // This object with id = 2 will be ignored, but the first one will be kept.
      objs.push({name: 'b2', id: 2});
      objs.push({name: 'd', id: 4});
      objs.push({name: 'e', id: 0});
      // This object with id = 2 will also be ignored
      objs.push({name: 'b3', id: 2});
      objs.push({name: 'f', id: 23});

      expect(curious.CuriousObjects.idList(objs)).to.deep.equal([1, 2, 3, 4, 0, 23]);
    });
  });

  describe('#idString', function () {
    var objs;

    function _parseToNumbers(idString) {
      return idString.split(',').map(function (id) {return Number(id);});
    }

    beforeEach(function () {
      objs = [
        {name: 'a', id: 1},
        {name: 'b', id: 2},
        {name: 'c', id: 3},
      ];
    });

    it('should be equivalent to idList', function () {
      expect(_parseToNumbers(curious.CuriousObjects.idString(objs))).to.deep.equal(
        curious.CuriousObjects.idList(objs)
      );

      // This object with id = 2 will be ignored, but the first one will be kept.
      objs.push({name: 'b2', id: 2});
      objs.push({name: 'd', id: 4});
      objs.push({name: 'e', id: 0});
      // This object with id = 2 will also be ignored
      objs.push({name: 'b3', id: 2});
      objs.push({name: 'f', id: 23});

      expect(_parseToNumbers(curious.CuriousObjects.idString(objs))).to.deep.equal(
        curious.CuriousObjects.idList(objs)
      );
    });

    it('should not escape commas', function () {
      objs.push({name: 'commas', id: '4,5,6'});

      expect(
        // 1, 2, 3, 4, 5, 6, converted to numbers
        _parseToNumbers(curious.CuriousObjects.idString(objs))
      ).to.deep.equal(
        // Remove the element we added ('4,5,6') and add 4, 5, 6 to the end
        curious.CuriousObjects.idList(objs).slice(0, -1).concat([4, 5, 6])
      );
    });
  });

  describe('#parse', function () {
    var queryJSONResponse;

    beforeEach(function () {
      queryJSONResponse = _sampleResponse().result;
    });

    it('should correctly parse the output', function () {
      var expectedObjects = _expectedObjects();

      expect(curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        null,
        queryJSONResponse
      )).to.deep.equal(
        {
          trees: [null, null],
          objects: [
            expectedObjects.experiments,
            expectedObjects.reactions,
          ],
        }
      );
    });

    it('should construct objects correctly with custom constructors', function () {
      var constructors;
      var parsedData;

      // Fake constructors
      function Experiment() {
        // Do nothing
      }

      function Reaction() {
        // Do nothing
      }

      constructors = [Experiment, Reaction];

      parsedData = curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        constructors,
        queryJSONResponse
      );

      parsedData.objects.forEach(function (objectsByID, ix) {
        var objects = curious.CuriousObjects.values(objectsByID);

        objects.forEach(function (object) {
          expect(object).to.be.an.instanceof(constructors[ix]);
        });
      });

    });

    it('should combine new objects with existing ones', function () {
      var parsedData;
      // jscs:disable requireCamelCaseOrUpperCaseIdentifiers
      var existingObjects = [
        null,
        {
          23063: {
            id: 23063,
            created_by_id: 12345,
            created_on: 'another date',
            updated_by_id: 67890,
            updated_on: 'another time',
            experiment_id: 403,
            is_blank: false,
            is_control: false,
            notes: '',
            sample_id: 124816,
            __url: 'http://example.com/experiment/403/',
            __model: 'Reaction',
            __dirty: false,
          }
        },
      ];
      // jscs:enable

      parsedData = curious.CuriousObjects.parse(
        ['experiments', 'reactions'],
        null,
        queryJSONResponse,
        existingObjects
      );

      expect(parsedData.objects[1][23063]).to.contain.keys(existingObjects[1][23063]);
    });

  });
});

}());
<|endoftext|>


<commit_before>(function() {
	function requestingrService($q, $http) {
		return function(method, url, headers, data) {
			var deferred = $q.defer();

			$http({
				method: method,
				url: url,
				headers: headers,
				data: data
			})
				.success(deferred.resolve)
				.error(deferred.reject);

			return deferred.promise;
		}
	}

	angular
		.module('ITracker')
		.factory('requestingrService', ['$q', '$http', requestingrService]);
} ());<commit_msg>fix error naming<commit_after>(function() {
	function requestingService($q, $http) {
		return function(method, url, headers, data) {
			var deferred = $q.defer();

			$http({
				method: method,
				url: url,
				headers: headers,
				data: data
			})
				.success(deferred.resolve)
				.error(deferred.reject);

			return deferred.promise;
		}
	}

	angular
		.module('ITracker')
		.factory('requestingService', ['$q', '$http', requestingService]);
} ());<|endoftext|>


<commit_before>var
    resolves = new WeakMap()
    , promises = new WeakMap();

function default_merge_fn(object, data, bypassFields) {
    // check if given data is Array of Objects (with field :Id)
    if( object instanceof Array && data instanceof Array) {
        var shouldBeInArray = [];
        for(var i = 0; i < data.length; i++) {
            var currentObject = object.filter(function(it) {
                return (it.Id    && it.Id   == data[i].Id)
            })[0];
            if(currentObject) {
                default_merge_fn(currentObject, data[i], bypassFields);
            } else {
                object.push(data[i]);
            }
            shouldBeInArray.push(currentObject || data[i]);
        }
        while(object.length) {
            object.shift();
        }
        shouldBeInArray.map(function(it) {
            object.push(it);
        });
    } else {
        var fields = []
            .concat(Object.keys(object))
            .concat(Object.keys(data))
            .filter(function(it, i, arr) { return arr.indexOf(it) == i;});

        fields.map(function(field) {
            if(bypassFields && bypassFields.indexOf(field) > -1 || field[0] == '$') return;

            if(object[field] && data[field] && typeof object[field] == 'object' && typeof data[field] == 'object') {
                default_merge_fn(object[field], data[field], bypassFields);
            } else if(!data.hasOwnProperty(field)) {
                delete object[field];
            } else {
                object[field] = data[field];
            }
        });
    }
}

/**
 * Async Data Object
 * @constructor
 */
AsyncData = function(custom_merge_fn) {
    Object.defineProperty(this, '_loaded'       , { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_updating'     , { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_changed'      , { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_old_value'    , { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_error'        , { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_error_message', { enumerable: false, value: false, writable: true });
    Object.defineProperty(this, '_merge_fn'     , { enumerable: false, value: custom_merge_fn || default_merge_fn, writable: true });
    //Object.defineProperty(this, '_stack'        , { enumerable: false, value: new Error().stack });
};

/**
 * getPromise
 * @function
 *
 * @returns {Promise} promise object which you can use to know if the data has been loaded/updated from the server
 */
AsyncData.prototype.getPromise = function() {
    var _this = this;
    if(!promises.get(_this)) {
        promises.set(_this, new Promise(function(resolve, reject) {
            resolves.set(_this, function() {
                resolve(true);
            });
        }));
        promises.get(_this).then(function() {
            promises.delete(_this);
            resolves.delete(_this);
        });
    }

    return promises.get(_this)
};

/**
 * setUpdating
 * @function
 *
 * calls resolve when data is received from the server, merges data
 */
AsyncData.prototype.setUpdating = AsyncData.prototype._update = function(cb) {
    if(!(cb instanceof Function)) {
        if(cb) {
            this._old_value = cb;
            this._merge_fn(this, cb);
            return;
        } else {
            return {
                diff: diff.bind(this)
            }
        }
    }

    this._updating = true;

    var apply = function(new_data) {
        this._loaded = true;
        this._old_value = new_data;
        this._updating = false;
        this._error = false;
        this._error_message = false;
        if(!new_data) {
            (function clone(to, from){
                Object.keys(from).map(function(key) {
                    if(from[key] && typeof from[key] == 'object') {
                        clone(to[key] = from[key] instanceof Array ? [] : {}, from[key])
                    } else {
                        to[key] = from[key]
                    }
                })
            })(this._old_value = {}, this);
        } else {
            this._merge_fn(this, new_data);
        }
        resolves.get(this) && resolves.get(this)(this);
    }.bind(this);

    apply.diff = diff.bind(this);

    function diff(to, from) {
        if(!from) {
            throw Error("second argument to diff(to, from) should be object");
        }
        ([].concat(Object.keys(to)).concat(Object.keys[from])).map(function(key) {
            if(typeof to[key] == 'undefined') {
                delete this._old_value[key];
                delete this[key];
            } else {

                if (typeof from[key] == 'undefined') {
                    if (to[key] && typeof to[key] == 'object') {
                        this[key] = to[key] instanceof Array ? [] : {};
                        this._old_value[key] = to[key] instanceof Array ? [] : {};
                    }
                }

                if (to[key] && typeof to[key] == 'object') {
                    this._merge_fn(this[key], to[key]);
                    this._old_value[key] = to[key];
                } else {
                    this[key] = to[key];
                    this._old_value[key] = to[key];
                }
            }
        }.bind(this));

        this._loaded = true;
        this._updating = false;
        this._error = false;
        this._error_message = false;
        resolves.get(this) && resolves.get(this)(this);
    }

    var error = function(err_message) {
        this._updating = false;
        this._error = true;
        this._error_message = err_message;
    }.bind(this);

    var reject = function() {
        this._updating = false;
    }.bind(this);

    cb(apply, error, reject)
};

/**
 * isChanged
 * @function
 * Check if the object has been changed by user
 */
AsyncData.prototype.isChanged = function() {
    // compare this and this._old_value
    return (function cmp(obj, old) {
        var keys = [], key;
        for(key in obj) {
            if(obj.hasOwnProperty(key) && obj[key] && !obj[key].Id) {
                keys.push(key);
            }
        }
        for(key in old) {
            if(old.hasOwnProperty(key) && key.indexOf('$$') < 0 && keys.indexOf(key) === -1 && old[key] && !old[key].Id) {
                return true;
            }
        }
        for(var i = 0; i < keys.length; i++) {
            key = keys[i];
            if(typeof obj[key] == 'object' && typeof old[key] == 'object') {
                if(!cmp(obj[key], old[key])) {
                    return true;
                }
            }
            else if(obj[key] != old[key] && key.indexOf('$$') < 0) {
                return true;
            }
        }
        return false;
    })(this, this._old_value || {});
};

/**
 * getChangedFields
 * @function
 *
 * Get fields that have been changed locally
 */
AsyncData.prototype.getChangedFields = function() {
    return (function(obj, old) {
        var keys = [], key, changed = {};
        for(key in obj) {
            if(obj.hasOwnProperty(key) && obj[key] && !obj[key].Id) {
                keys.push(key);
            }
        }
        for(key in old) {
            if(old.hasOwnProperty(key) && keys.indexOf(key) === -1 && old[key] && !old[key].Id) {
                keys.push(key);
            }
        }
        for(var i = 0; i < keys.length; i++) {
            key = keys[i];
            if(old[key] != obj[key]) {
                changed[key] = obj[key];
            }
        }
        return changed;
    })(this, this._old_value || {})
};

/**
 * revert
 * @function
 *
 * Revert old value
 */
AsyncData.prototype.revert = function() {
    this._merge_fn(this, this._old_value);
};<commit_msg>added ability to export<commit_after>(function() {
    var
          resolves = new WeakMap()
        , promises = new WeakMap();

    function default_merge_fn(object, data, bypassFields) {
        // check if given data is Array of Objects (with field :Id)
        if( object instanceof Array && data instanceof Array) {
            var shouldBeInArray = [];
            for(var i = 0; i < data.length; i++) {
                var currentObject = object.filter(function(it) {
                    return (it.Id    && it.Id   == data[i].Id)
                })[0];
                if(currentObject) {
                    default_merge_fn(currentObject, data[i], bypassFields);
                } else {
                    object.push(data[i]);
                }
                shouldBeInArray.push(currentObject || data[i]);
            }
            while(object.length) {
                object.shift();
            }
            shouldBeInArray.map(function(it) {
                object.push(it);
            });
        } else {
            var fields = []
                .concat(Object.keys(object))
                .concat(Object.keys(data))
                .filter(function(it, i, arr) { return arr.indexOf(it) == i;});

            fields.map(function(field) {
                if(bypassFields && bypassFields.indexOf(field) > -1 || field[0] == '$') return;

                if(object[field] && data[field] && typeof object[field] == 'object' && typeof data[field] == 'object') {
                    default_merge_fn(object[field], data[field], bypassFields);
                } else if(!data.hasOwnProperty(field)) {
                    delete object[field];
                } else {
                    object[field] = data[field];
                }
            });
        }
    }

    /**
     * Async Data Object
     * @constructor
     */
    AsyncData = function(custom_merge_fn) {
        Object.defineProperty(this, '_loaded'       , { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_updating'     , { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_changed'      , { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_old_value'    , { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_error'        , { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_error_message', { enumerable: false, value: false, writable: true });
        Object.defineProperty(this, '_merge_fn'     , { enumerable: false, value: custom_merge_fn || default_merge_fn, writable: true });
        //Object.defineProperty(this, '_stack'        , { enumerable: false, value: new Error().stack });
    };

    /**
     * getPromise
     * @function
     *
     * @returns {Promise} promise object which you can use to know if the data has been loaded/updated from the server
     */
    AsyncData.prototype.getPromise = function() {
        var _this = this;
        if(!promises.get(_this)) {
            promises.set(_this, new Promise(function(resolve, reject) {
                resolves.set(_this, function() {
                    resolve(true);
                });
            }));
            promises.get(_this).then(function() {
                promises.delete(_this);
                resolves.delete(_this);
            });
        }

        return promises.get(_this)
    };

    /**
     * setUpdating
     * @function
     *
     * calls resolve when data is received from the server, merges data
     */
    AsyncData.prototype.setUpdating = AsyncData.prototype._update = function(cb) {
        if(!(cb instanceof Function)) {
            if(cb) {
                this._old_value = cb;
                this._merge_fn(this, cb);
                return;
            } else {
                return {
                    diff: diff.bind(this)
                }
            }
        }

        this._updating = true;

        var apply = function(new_data) {
            this._loaded = true;
            this._old_value = new_data;
            this._updating = false;
            this._error = false;
            this._error_message = false;
            if(!new_data) {
                (function clone(to, from){
                    Object.keys(from).map(function(key) {
                        if(from[key] && typeof from[key] == 'object') {
                            clone(to[key] = from[key] instanceof Array ? [] : {}, from[key])
                        } else {
                            to[key] = from[key]
                        }
                    })
                })(this._old_value = {}, this);
            } else {
                this._merge_fn(this, new_data);
            }
            resolves.get(this) && resolves.get(this)(this);
        }.bind(this);

        apply.diff = diff.bind(this);

        function diff(to, from) {
            if(!from) {
                throw Error("second argument to diff(to, from) should be object");
            }
            ([].concat(Object.keys(to)).concat(Object.keys[from])).map(function(key) {
                if(typeof to[key] == 'undefined') {
                    delete this._old_value[key];
                    delete this[key];
                } else {

                    if (typeof from[key] == 'undefined') {
                        if (to[key] && typeof to[key] == 'object') {
                            this[key] = to[key] instanceof Array ? [] : {};
                            this._old_value[key] = to[key] instanceof Array ? [] : {};
                        }
                    }

                    if (to[key] && typeof to[key] == 'object') {
                        this._merge_fn(this[key], to[key]);
                        this._old_value[key] = to[key];
                    } else {
                        this[key] = to[key];
                        this._old_value[key] = to[key];
                    }
                }
            }.bind(this));

            this._loaded = true;
            this._updating = false;
            this._error = false;
            this._error_message = false;
            resolves.get(this) && resolves.get(this)(this);
        }

        var error = function(err_message) {
            this._updating = false;
            this._error = true;
            this._error_message = err_message;
        }.bind(this);

        var reject = function() {
            this._updating = false;
        }.bind(this);

        cb(apply, error, reject)
    };

    /**
     * isChanged
     * @function
     * Check if the object has been changed by user
     */
    AsyncData.prototype.isChanged = function() {
        // compare this and this._old_value
        return (function cmp(obj, old) {
            var keys = [], key;
            for(key in obj) {
                if(obj.hasOwnProperty(key) && obj[key] && !obj[key].Id) {
                    keys.push(key);
                }
            }
            for(key in old) {
                if(old.hasOwnProperty(key) && key.indexOf('$$') < 0 && keys.indexOf(key) === -1 && old[key] && !old[key].Id) {
                    return true;
                }
            }
            for(var i = 0; i < keys.length; i++) {
                key = keys[i];
                if(typeof obj[key] == 'object' && typeof old[key] == 'object') {
                    if(!cmp(obj[key], old[key])) {
                        return true;
                    }
                }
                else if(obj[key] != old[key] && key.indexOf('$$') < 0) {
                    return true;
                }
            }
            return false;
        })(this, this._old_value || {});
    };

    /**
     * getChangedFields
     * @function
     *
     * Get fields that have been changed locally
     */
    AsyncData.prototype.getChangedFields = function() {
        return (function(obj, old) {
            var keys = [], key, changed = {};
            for(key in obj) {
                if(obj.hasOwnProperty(key) && obj[key] && !obj[key].Id) {
                    keys.push(key);
                }
            }
            for(key in old) {
                if(old.hasOwnProperty(key) && keys.indexOf(key) === -1 && old[key] && !old[key].Id) {
                    keys.push(key);
                }
            }
            for(var i = 0; i < keys.length; i++) {
                key = keys[i];
                if(old[key] != obj[key]) {
                    changed[key] = obj[key];
                }
            }
            return changed;
        })(this, this._old_value || {})
    };

    /**
     * revert
     * @function
     *
     * Revert old value
     */
    AsyncData.prototype.revert = function() {
        this._merge_fn(this, this._old_value);
    };

    if (typeof module !== 'undefined' && typeof module.exports !== 'undefined') {
        module.exports = AsyncData;
    } else if (typeof window !== 'undefined') {
        window.AsyncData = AsyncData;
    } else {
        throw new Error('Unknown environment. Cannot be loaded.');
    }
})();<|endoftext|>


<commit_before>(function(app) {
  // Create function to attach everything to and pass any injected service.
  function ActivityComponent(dataService) {
    // Invoke service and save to components name space
    this.name = dataService.getActivity();
    this.activities = [{ name: "zak" }];
    // Had activities in a datasService Prototype for development

    var that = this;
    dataService.getJoinedActivities(function(data) {
      var temp = [];
      for(let item of data) {
        if(item) {
          temp.push(item);
        }
      }
      that.activities = temp;
    });
    //dataService.enterNewActivities();
    this.activities = dataService.getJoinedActivities();
  }
  // Declare components injectables
  ActivityComponent.parameters = [
    app.DataService
  ];
  // Create new DOM Space for Component
  ActivityComponent.annotations = [
    // create new component
    new ng.core.Component({
      // create selector for html
      selector: 'joined-activities',
      templateUrl: 'app/joinedEvents.html'
    })
  ];

  // Name and link component
  app.ActivityJoinedComponent = ActivityComponent;

})(window.app = window.app || {});

//

//
{{activity.eventName}}

//
<commit_msg>Works without workaround, fixed merge conflict<commit_after>(function(app) {
  // Create function to attach everything to and pass any injected service.
  function ActivityComponent(dataService) {
    // Invoke service and save to components name space
    this.name = dataService.getActivity();
    // this.activities = [];

    // Quick and dirty way to pass this to dataservice
    var that = this;
    dataService.getJoinedActivities(function(data) {
      //  var temp = [];
      //  for(let item of data) {
      //   if(item) {
      //     temp.push(item);
      //   }
      // }
      that.activities = data;
    });
  }
  // Declare components injectables
  ActivityComponent.parameters = [
    app.DataService
  ];
  // Create new DOM Space for Component
  ActivityComponent.annotations = [
    // create new component
    new ng.core.Component({
      // create selector for html
      selector: 'joined-activities',
      templateUrl: 'app/joinedEvents.html'
    })
  ];

  // Name and link component
  app.ActivityJoinedComponent = ActivityComponent;

})(window.app = window.app || {});<|endoftext|>


<commit_before>//=========================================================================================================================
// Batch Pin Insertion/Deletion Timeline Library (Mathew Reiss)
//=========================================================================================================================

// Bool for testing in DEBUG mode
var DEBUG = true;

// The timeline public URL root
var API_URL_ROOT = 'https://timeline-api.getpebble.com/';

// Arrays for temporarily storing pins to be inserted and pisn to be deleted
var INSERT_PINS = [], DELETE_PINS = [];

/**
 * Send a request to the Pebble public web timeline API.
 *
 * Iterating through both arrays is placed inside the Pebble.getTimelineToken function,
 * as attempting to process a significant number of pins one-at-a-time can cause
 * issues in behavior (namely, getTimelineToken is asynchronous).
 */
function timelineRequest() {
  Pebble.getTimelineToken(function(token) {
	var pin, url, xhr;

	if(DEBUG) console.log("Inserting " + INSERT_PINS.length + " pin(s); Deleting " + DELETE_PINS.length + " pin(s).");

	for(var i = 0, i_len = INSERT_PINS.length; i < i_len; i++){
		pin = INSERT_PINS[i];
		url = API_URL_ROOT + 'v1/user/pins/' + pin.id;
		xhr = new XMLHttpRequest();

		if(DEBUG) console.log("Insert Pin: " + pin.id + " @ " + pin.time);

		xhr.open('PUT', url);
		xhr.setRequestHeader('Content-Type', 'application/json');
		xhr.setRequestHeader('X-User-Token', '' + token);
		xhr.send(JSON.stringify(pin));
	}
	INSERT_PINS = []; //Clear array

	for(var d = 0, d_len = DELETE_PINS.length; d < d_len; d++){
		pin = DELETE_PINS[d];
		url = API_URL_ROOT + 'v1/user/pins/' + pin.id;
		xhr = new XMLHttpRequest();

		if(DEBUG) console.log("Delete Pin: " + pin.id + " @ " + pin.time);

		xhr.open('DELETE', url);
		xhr.setRequestHeader('Content-Type', 'application/json');
		xhr.setRequestHeader('X-User-Token', '' + token);
		xhr.send(JSON.stringify(pin));
	}
	DELETE_PINS = []; //Clear array

  }, function(error) { console.error('Timeline: Error getting timeline token: ' + error); });
}

/**
 * Add a pin to the INSERT_PINS array, queued for later
 * @param pin The JSON pin to insert.
 */
function insertUserPin(pin) {
	if(DEBUG) console.log("Pin added to Insertion array: " + pin.id);
	INSERT_PINS.push(pin);
}

/**
 * Add a pin to the DELETE_PINS array, queued for later
 * @param pin The JSON pin to delete.
 */
function deleteUserPin(pin, callback) {
	if(DEBUG) console.log("Pin added to Deletion array: " + pin.id);
	DELETE_PINS.push(pin);
}

/**
 * Begin processing pins in both arrays
 */
function releasePins(){
	if(DEBUG) console.log("Pins released...");
	timelineRequest();
}

//=========================================================================================================================
<commit_msg>Spelling<commit_after>//=========================================================================================================================
// Batch Pin Insertion/Deletion Timeline Library (Mathew Reiss)
//=========================================================================================================================

// Bool for testing in DEBUG mode
var DEBUG = true;

// The timeline public URL root
var API_URL_ROOT = 'https://timeline-api.getpebble.com/';

// Arrays for temporarily storing pins to be inserted and pins to be deleted
var INSERT_PINS = [], DELETE_PINS = [];

/**
 * Send a request to the Pebble public web timeline API.
 *
 * Iterating through both arrays is placed inside the Pebble.getTimelineToken function,
 * as attempting to process a significant number of pins one-at-a-time can cause
 * issues in behavior (namely, getTimelineToken is asynchronous).
 */
function timelineRequest() {
  Pebble.getTimelineToken(function(token) {
	var pin, url, xhr;

	if(DEBUG) console.log("Inserting " + INSERT_PINS.length + " pin(s); Deleting " + DELETE_PINS.length + " pin(s).");

	for(var i = 0, i_len = INSERT_PINS.length; i < i_len; i++){
		pin = INSERT_PINS[i];
		url = API_URL_ROOT + 'v1/user/pins/' + pin.id;
		xhr = new XMLHttpRequest();

		if(DEBUG) console.log("Insert Pin: " + pin.id + " @ " + pin.time);

		xhr.open('PUT', url);
		xhr.setRequestHeader('Content-Type', 'application/json');
		xhr.setRequestHeader('X-User-Token', '' + token);
		xhr.send(JSON.stringify(pin));
	}
	INSERT_PINS = []; //Clear array

	for(var d = 0, d_len = DELETE_PINS.length; d < d_len; d++){
		pin = DELETE_PINS[d];
		url = API_URL_ROOT + 'v1/user/pins/' + pin.id;
		xhr = new XMLHttpRequest();

		if(DEBUG) console.log("Delete Pin: " + pin.id + " @ " + pin.time);

		xhr.open('DELETE', url);
		xhr.setRequestHeader('Content-Type', 'application/json');
		xhr.setRequestHeader('X-User-Token', '' + token);
		xhr.send(JSON.stringify(pin));
	}
	DELETE_PINS = []; //Clear array

  }, function(error) { console.error('Timeline: Error getting timeline token: ' + error); });
}

/**
 * Add a pin to the INSERT_PINS array, queued for later
 * @param pin The JSON pin to insert.
 */
function insertUserPin(pin) {
	if(DEBUG) console.log("Pin added to Insertion array: " + pin.id);
	INSERT_PINS.push(pin);
}

/**
 * Add a pin to the DELETE_PINS array, queued for later
 * @param pin The JSON pin to delete.
 */
function deleteUserPin(pin) {
	if(DEBUG) console.log("Pin added to Deletion array: " + pin.id);
	DELETE_PINS.push(pin);
}

/**
 * Begin processing pins in both arrays
 */
function releasePins(){
	if(DEBUG) console.log("Pins released...");
	timelineRequest();
}

//=========================================================================================================================
<|endoftext|>