<commit_before>/* eslint-disable func-names */
import warning from 'warning'
import invariant from 'invariant'
import {
  takeEveryHelper as takeEvery,
  takeLatestHelper as takeLatest,
  throttleHelper as throttle,
} from 'redux-saga/lib/internal/sagaHelpers'
import { delay, CANCEL as CANCEL_DELAY } from 'redux-saga'
import * as sagaEffects from 'redux-saga/effects'
import { CANCEL_EFFECTS, MUTATE } from './constants'
import { prefixType, unfixType, prefixAndValidate } from './utils'
import { getModelActions } from './actions'

function applyOnEffect(handlers, effect, actionType, metadata) {
  return handlers.reduce(
    (_effect, handler) => handler(_effect, sagaEffects, actionType, metadata),
    effect,
  )
}

function* callGenerator(fn, ...args) {
  const ret = yield fn(...args)
  if (ret.then) {
    return yield new Promise((resolve) => {
      ret.then(data => resolve(data))
    })
  }

  return ret
}

function getEffects(model) {
  const { put, take } = sagaEffects
  const { namespace } = model

  function assertAction(type, name) {
    invariant(type, `${name}: action should be a plain Object with type`)
    warning(
      type.indexOf(prefixType(namespace, '')) !== 0,
      `${name}: ${type} should not be prefixed with namespace "${namespace}"`,
    )
  }

  function innerPut(action) {
    const { type } = action
    assertAction(type, 'innerPut')
    return put({ ...action, type: prefixAndValidate(type, model) })
  }

  function innerTake(type) {
    if (typeof type === 'string') {
      assertAction(type, 'innerTake')
      return take(prefixAndValidate(type, model))
    }
    return take(type)
  }

  function mutate(payload, debug) {
    const action = {
      payload,
      type: prefixType(namespace, model.actions[MUTATE]),
    }

    if (process.env.NODE_ENV !== 'production' && debug) {
      action.debug = debug
    }

    return put(action)
  }

  return {
    ...sagaEffects,
    delay,
    CANCEL_DELAY,
    innerPut,
    innerTake,
    [MUTATE]: mutate,
    digest: callGenerator,
  }
}

function getCallbacks(model, actionType) {
  const { put } = sagaEffects
  const { namespace, effects, callbacks } = model
  const callbackEffects = {}
  if (effects[actionType] && callbacks[actionType]) {
    callbacks[actionType].forEach((callback) => {
      callbackEffects[callback] = (payload) => {
        const actionName = unfixType(namespace, actionType)
        const prefixed = prefixType(actionName, callback)
        const fullType = prefixAndValidate(prefixed, model)
        return put({ payload, type: fullType })
      }
    })
  }

  return callbackEffects
}

function getWatcher({ onError, onEffect, app, model, type, effect }) {
  let effectFn = effect
  let effectType = 'takeEvery'
  let ms

  if (Array.isArray(effect)) {
    effectFn = effect[0]
    const options = effect[1]
    if (options && options.type) {
      effectType = options.type
      if (effectType === 'throttle') {
        invariant(
          options.ms,
          'options.ms should be defined if type is throttle',
        )
        ms = options.ms
      }
    }
    invariant(
      ['watcher', 'takeEvery', 'takeLatest', 'throttle'].includes(effectType),
      'effect type should be takeEvery, takeLatest, throttle or watcher',
    )
  }

  const actions = app.actions
  const effects = getEffects(model)
  const callbacks = getCallbacks(model, type)
  const innerActions = getModelActions(model, sagaEffects.put)

  function* sagaWithCatch(...args) {
    const { payload, resolver: { resolve, reject } } = args[0]
    try {
      const ret = yield effectFn(
        payload,
        { ...effects, resolve, reject }, // 支持在业务逻辑中调用 resolve 和 reject
        callbacks,
        innerActions,
        actions,
      )
      // 默认调用 resolve，ret 是在 effect 函数的返回值(return ret)
      // 如果在业务逻辑中已经触发 resolve，那么此处的 resolve 将不会生效
      resolve(ret)
    } catch (err) {
      onError(err)
      reject(err)
    }
  }

  const sagaWithOnEffect = applyOnEffect(onEffect, sagaWithCatch, type, {
    app,
    model: model.namespace,
    actions,
    effects,
  })

  switch (effectType) {
    case 'watcher':
      return sagaWithCatch
    case 'takeLatest':
      return function* () {
        yield takeLatest(type, sagaWithOnEffect)
      }
    case 'throttle':
      return function* () {
        yield throttle(ms, type, sagaWithOnEffect)
      }
    default:
      return function* () {
        yield takeEvery(type, sagaWithOnEffect)
      }
  }
}

function getTask(model, type, watcher) {
  const { namespace } = model
  return function* run() {
    const task = yield sagaEffects.fork(watcher)
    yield sagaEffects.fork(function* () {
      const { cancel } = yield sagaEffects.race({
        cancel: sagaEffects.take(prefixType(type, CANCEL_EFFECTS)),
        eject: sagaEffects.take(prefixType(namespace, CANCEL_EFFECTS)),
      })

      yield sagaEffects.cancel(task)

      if (cancel) {
        yield sagaEffects.fork(run)
      }
    })
  }
}

export default function getSaga(onError, onEffect, app, model) {
  return function* () {
    const { effects } = model
    const keys = Object.keys(effects)
    for (let i = 0, l = keys.length; i < l; i += 1) {
      const type = keys[i]
      const watcher = getWatcher({ onError, onEffect, app, model, type, effect: effects[type] })
      const task = getTask(model, type, watcher)
      yield sagaEffects.fork(task)
    }
  }
}
<commit_msg>digest signature and usage<commit_after>/* eslint-disable func-names */
import warning from 'warning'
import invariant from 'invariant'
import {
  takeEveryHelper as takeEvery,
  takeLatestHelper as takeLatest,
  throttleHelper as throttle,
} from 'redux-saga/lib/internal/sagaHelpers'
import { delay, CANCEL as CANCEL_DELAY } from 'redux-saga'
import * as sagaEffects from 'redux-saga/effects'
import { CANCEL_EFFECTS, MUTATE } from './constants'
import { prefixType, unfixType, prefixAndValidate } from './utils'
import { getModelActions } from './actions'

function applyOnEffect(handlers, effect, actionType, metadata) {
  return handlers.reduce(
    (_effect, handler) => handler(_effect, sagaEffects, actionType, metadata),
    effect,
  )
}

// signature: *effect(payload, {call, digest}, callbacks, innerActions, actions) { }
// usage: const result = yield digest(generator, arg1, arg2, ...)
function* callGenerator(fn, ...args) {
  const ret = yield fn(...args)
  if (ret.then) {
    return yield new Promise((resolve) => {
      ret.then(data => resolve(data))
    })
  }

  return ret
}

function getEffects(model) {
  const { put, take } = sagaEffects
  const { namespace } = model

  function assertAction(type, name) {
    invariant(type, `${name}: action should be a plain Object with type`)
    warning(
      type.indexOf(prefixType(namespace, '')) !== 0,
      `${name}: ${type} should not be prefixed with namespace "${namespace}"`,
    )
  }

  function innerPut(action) {
    const { type } = action
    assertAction(type, 'innerPut')
    return put({ ...action, type: prefixAndValidate(type, model) })
  }

  function innerTake(type) {
    if (typeof type === 'string') {
      assertAction(type, 'innerTake')
      return take(prefixAndValidate(type, model))
    }
    return take(type)
  }

  function mutate(payload, debug) {
    const action = {
      payload,
      type: prefixType(namespace, model.actions[MUTATE]),
    }

    if (process.env.NODE_ENV !== 'production' && debug) {
      action.debug = debug
    }

    return put(action)
  }

  return {
    ...sagaEffects,
    delay,
    CANCEL_DELAY,
    innerPut,
    innerTake,
    [MUTATE]: mutate,
    digest: callGenerator,
  }
}

function getCallbacks(model, actionType) {
  const { put } = sagaEffects
  const { namespace, effects, callbacks } = model
  const callbackEffects = {}
  if (effects[actionType] && callbacks[actionType]) {
    callbacks[actionType].forEach((callback) => {
      callbackEffects[callback] = (payload) => {
        const actionName = unfixType(namespace, actionType)
        const prefixed = prefixType(actionName, callback)
        const fullType = prefixAndValidate(prefixed, model)
        return put({ payload, type: fullType })
      }
    })
  }

  return callbackEffects
}

function getWatcher({ onError, onEffect, app, model, type, effect }) {
  let effectFn = effect
  let effectType = 'takeEvery'
  let ms

  if (Array.isArray(effect)) {
    effectFn = effect[0]
    const options = effect[1]
    if (options && options.type) {
      effectType = options.type
      if (effectType === 'throttle') {
        invariant(
          options.ms,
          'options.ms should be defined if type is throttle',
        )
        ms = options.ms
      }
    }
    invariant(
      ['watcher', 'takeEvery', 'takeLatest', 'throttle'].includes(effectType),
      'effect type should be takeEvery, takeLatest, throttle or watcher',
    )
  }

  const actions = app.actions
  const effects = getEffects(model)
  const callbacks = getCallbacks(model, type)
  const innerActions = getModelActions(model, sagaEffects.put)

  function* sagaWithCatch(...args) {
    const { payload, resolver: { resolve, reject } } = args[0]
    try {
      const ret = yield effectFn(
        payload,
        { ...effects, resolve, reject }, // 支持在业务逻辑中调用 resolve 和 reject
        callbacks,
        innerActions,
        actions,
      )
      // 默认调用 resolve，ret 是在 effect 函数的返回值(return ret)
      // 如果在业务逻辑中已经触发 resolve，那么此处的 resolve 将不会生效
      resolve(ret)
    } catch (err) {
      onError(err)
      reject(err)
    }
  }

  const sagaWithOnEffect = applyOnEffect(onEffect, sagaWithCatch, type, {
    app,
    model: model.namespace,
    actions,
    effects,
  })

  switch (effectType) {
    case 'watcher':
      return sagaWithCatch
    case 'takeLatest':
      return function* () {
        yield takeLatest(type, sagaWithOnEffect)
      }
    case 'throttle':
      return function* () {
        yield throttle(ms, type, sagaWithOnEffect)
      }
    default:
      return function* () {
        yield takeEvery(type, sagaWithOnEffect)
      }
  }
}

function getTask(model, type, watcher) {
  const { namespace } = model
  return function* run() {
    const task = yield sagaEffects.fork(watcher)
    yield sagaEffects.fork(function* () {
      const { cancel } = yield sagaEffects.race({
        cancel: sagaEffects.take(prefixType(type, CANCEL_EFFECTS)),
        eject: sagaEffects.take(prefixType(namespace, CANCEL_EFFECTS)),
      })

      yield sagaEffects.cancel(task)

      if (cancel) {
        yield sagaEffects.fork(run)
      }
    })
  }
}

export default function getSaga(onError, onEffect, app, model) {
  return function* () {
    const { effects } = model
    const keys = Object.keys(effects)
    for (let i = 0, l = keys.length; i < l; i += 1) {
      const type = keys[i]
      const watcher = getWatcher({ onError, onEffect, app, model, type, effect: effects[type] })
      const task = getTask(model, type, watcher)
      yield sagaEffects.fork(task)
    }
  }
}
<|endoftext|>


<commit_before>module.exports = function(robot){

	var thankYouResponses = [
		"You're welcome!",
		"No problem",
		"Not a problem",
		"It's no bother",
		"No problem at all",
		"It's my pleasure",
		"My pleasure",
		"It's nothing",
		"No, no.  Thank you!",
		"Sure thing"
	];

	var beerResponses = [
		"I think I've had too many",
		"I'm trashed"
	];

	robot.hear(/thank(s| you|s to) bocbot/i, function(res){
		res.random(thankYouResponses);
	});

	robot.hear(/I like pie/i, function(res){
		robot.send('I like pie too');
	});

	robot.respond(/open (.*) door/i, function(res){
		var doorType = res.match[1];
		if (doorType == 'pod bay')
			res.reply("I'm afraid I can't let you do that.");
		else
			res.reply('Opening ' + doorType + ' doors.');
	});

	robot.respond(/you('re| are)(.*) slow/i, function(res){
		setTimeout(function(){
			res.reply('Who you callin\' slow?');
		}, 1000 * 15);
	});

	robot.respond(/(have|drink|consume)(.*) beer/i, function(res){
		var beersHad = robot.brain.get('totalBeersHad') || 0;
		if (beersHad > 4){
			var lastBeerFrom = robot.brain.get('lastBeerFrom');
			var initialResponse = robot.util.random(beerResponses);
			res.reply(initialResponse + ".  " + lastBeerFrom + " got me too drunk.  I need to sleep it off first.");
		}
		else{
			res.reply("Sure thing!  _chugs beer_");
			robot.brain.set('totalBeersHad', beersHad + 1);
			robot.brain.set('lastBeerFrom', res.message.user.name);
		}
	});

	robot.respond(/sleep it off/i, function(res){
		robot.brain.set('totalBeersHad', 0);
		res.reply('zzzzz');
	});

	robot.respond(/who[ ]*am[ ]*i/i, function(res){
		res.reply('You\'re ' + res.message.user.slack.profile.first_name + '.  @' + res.message.user.name + '.  ID: ' + res.message.user.id);
	});
}<commit_msg>Fix error in eastereggs.js<commit_after>module.exports = function(robot){

	var thankYouResponses = [
		"You're welcome!",
		"No problem",
		"Not a problem",
		"It's no bother",
		"No problem at all",
		"It's my pleasure",
		"My pleasure",
		"It's nothing",
		"No, no.  Thank you!",
		"Sure thing"
	];

	var beerResponses = [
		"I think I've had too many",
		"I'm trashed"
	];

	robot.hear(/thank(s| you|s to) bocbot/i, function(res){
		res.random(thankYouResponses);
	});

	robot.hear(/I like pie/i, function(res){
		res.send('I like pie too');
	});

	robot.respond(/open (.*) door/i, function(res){
		var doorType = res.match[1];
		if (doorType.replace('the ') == 'pod bay')
			res.reply("I'm afraid I can't let you do that.");
		else
			res.reply('Opening ' + doorType + ' doors.');
	});

	robot.respond(/you('re| are)(.*) slow/i, function(res){
		setTimeout(function(){
			res.reply('Who you callin\' slow?');
		}, 1000 * 15);
	});

	robot.respond(/(have|drink|consume)(.*) beer/i, function(res){
		var beersHad = robot.brain.get('totalBeersHad') || 0;
		if (beersHad > 4){
			var lastBeerFrom = robot.brain.get('lastBeerFrom');
			var initialResponse = robot.util.random(beerResponses);
			res.reply(initialResponse + ".  " + lastBeerFrom + " got me too drunk.  I need to sleep it off first.");
		}
		else{
			res.reply("Sure thing!  _chugs beer_");
			robot.brain.set('totalBeersHad', beersHad + 1);
			robot.brain.set('lastBeerFrom', res.message.user.name);
		}
	});

	robot.respond(/sleep it off/i, function(res){
		robot.brain.set('totalBeersHad', 0);
		res.reply('zzzzz');
	});

	robot.respond(/who[ ]*am[ ]*i/i, function(res){
		res.reply('You\'re ' + res.message.user.slack.profile.first_name + '.  @' + res.message.user.name + '.  ID: ' + res.message.user.id);
	});
}<|endoftext|>


<commit_before>/* Hides 'my-profile-box by default */

console.log("saldjflsdjfklsjdlf");

/* This is the angular for the main window */
(function () {
  angular.module('MainWindow',[]).controller('MainCtrl',function($scope) {
    /* EDIT THE FOLLOWING FOR REAL DATA */

    console.log("b");

    /* The defaults for the home page.
     * The only thing that should show is the sidebar and the fb login.
     */
    $scope.user = false;
    $scope.friend = false;
    $scope.search = false;
    $scope.map = false;

    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);

  });
}());
<commit_msg>please work<commit_after>/* Hides 'my-profile-box by default */

console.log("saldjflsdjfklsjdlf");

/* This is the angular for the main window */
function () {
  angular.module('MainWindow',[]).controller('MainCtrl',function($scope) {
    /* EDIT THE FOLLOWING FOR REAL DATA */

    console.log("b");

    /* The defaults for the home page.
     * The only thing that should show is the sidebar and the fb login.
     */
    $scope.user = false;
    $scope.friend = false;
    $scope.search = false;
    $scope.map = false;

    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);
    console.log($scope.user);

  });
}();
<|endoftext|>


<commit_before>"use strict"
let redis = require('redis');
let log = global.logger.create('REDIS');
let helpers = require('../helpers');
let soother = () => {};

let retry_strategy = function(options){
	if(options.error && options.error.code === 'ECONNREFUSED'){
		// End reconnecting on a specific error and flush all commands with a individual error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return new Error('The server refused the connection');
	}

	if(options.total_retry_time > 1000 * 60 * 60){
		// End reconnecting after a specific timeout and flush all commands with a individual error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return new Error('Retry time exhausted');
	}

	if(options.times_connected > 10){
		// End reconnecting with built in error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return undefined;
	}

	// reconnect after
	return Math.max(options.attempt * 100, 3000);
};

let DAL = {
	connections: [],
	opened: [],
	getOrCreate: (mainCb) => {
		let lazy = lazyDefine.get()
		let cb = (...args) => {
			args.push(lazy);
			mainCb(...args);
			cb = soother;
		}
		if(DAL.connections.length){
			let connection = DAL.connections.shift();
			connection.lastOpened = Date.now();
			DAL.opened.push(connection);
			DAL._checkConnections();
			log.d('connection', connection.id, 'opened');
			return cb(null, connection.redis);
		}

		let connection = {
			id: helpers.generateId(),
			redis: {}
		};
		let config = {
			retry_strategy: retry_strategy,
			emit_error: err => connection && connection.redis.emit('error', err)
		};
		if(DAL.config && DAL.config){
			for(let i in DAL.config){
				if(!DAL.config.hasOwnProperty(i)){
					continue;
				}

				config[i] = DAL.config[i];
			}
		}

		connection.redis = redis.createClient(config);
		connection.redis.on('error', err => {
			log.e('Error in redis exports.connection:', connection.id, err);
			if(!connection){
				return;
			}

			connection.redis.quit();

			let conn = DAL._closeConnection(connection.id);
			if(conn){
				conn = undefined;
				log.d('connection', connection.id, 'deleted from opened by error');
			}

			connection = null;

			return cb(err);
		});
		connection.redis.on('ready', () => {
			connection.lastOpened = Date.now();
			DAL.opened.push(connection);
			log.d('connection', connection.id, 'created and added to opened');
			return cb(null, connection.redis);
		});
		connection.redis.on('requestEnded', () => {
			if(!connection){
				return;
			}

			let conn = DAL._closeConnection(connection.id);
			if(!conn){
				return log.e('No opened id', id);
			}

			DAL.connections.push(conn[0]);

			log.d('connection', connection.id, 'moved to waited');
		});
		connection.redis.on('end', () => {
			if(!connection){
				return;
			}

			let conn = DAL._closeConnection(connection.id);
			if(conn){
				conn = undefined;
				log.d('connection', connection.id, 'deleted from opened by end connection');
			}

			connection = undefined;

			return cb('No connection');
		});

		DAL._checkConnections();
		return lazy;
	},
	_checkConnections: () => {
		DAL.opened = DAL.opened.reduce((res, conn) =>{
			if(Date.now() - conn.lastOpened > 14400000){
				conn.redis.quit();
				delete conn.redis;
				conn = null;
			}
			else{
				res.push(conn);
			}
			return res;
		}, [])
		DAL.connections = DAL.connections.reduce((res, conn) => {
			if(Date.now() - conn.lastOpened > 14400000){
				conn.redis.quit();
				delete conn.redis;
				conn = null;
			}
			else{
				res.push(conn);
			}
			return res;
		}, []);
	},
	_closeConnection: (id) => {
		let sid;
		let conn;
		for(let i in DAL.opened){
			if(DAL.opened[i].id == id){
				sid = i;
				break;
			}
		}
		if(sid){
			return DAL.opened.splice(sid, 1);
		}
		for(let i in DAL.connections){
			if(DAL.connections[i].id == id){
				sid = i;
				break;
			}
		}
		if(sid){
			return DAL.connections.splice(sid, 1);
		}
	}
};

exports.init = (config, dalConfig) =>{
	DAL.config = dalConfig;
}
let lazyDefine = {
	methods: {},
	get: ()=>Object.assign({}, this.methods, {list: []})
}
exports.methods = {};
for(let name in redis.RedisClient.prototype){// eslint-disable-line guard-for-in
	wrapMethod(name);
}
function wrapMethod(name){
	exports.methods[name] = (...args) => {
		let conn;
		let originalCb = soother;
		let cb = (...resargs) => {
			if(conn){
				conn.emit('requestEnded');
			}

			originalCb(...resargs);
			conn = undefined;
		};

		if(typeof args[args.length -1] == 'function' && args[args.length -1] instanceof Function){
			originalCb = args[args.length -1];
			args[args.length -1] = cb;
		}
		else{
			args.push(cb);
		}

		return DAL.getOrCreate((err, connection, lazy) => {
			if(err){
				return cb(err);
			}

			conn = connection;
			if(!lazy.list.length){
				return connection[name](...args);
			}
			else{
				lazy.list.unshift({cmd: name, args: args});
				lazy.list.map(i=>{
					conn[i.cmd](...i.args)
				});
				setTimeout(cb, 2000);
			}
		});
	};
	Object.defineProperty(lazyDefine.methods, name, {get: function(...args){
		this.list.push({cmd: name, args: args})
		return this;
	}}
}
<commit_msg>light fi<commit_after>"use strict"
let redis = require('redis');
let log = global.logger.create('REDIS');
let helpers = require('../helpers');
let soother = () => {};

let retry_strategy = function(options){
	if(options.error && options.error.code === 'ECONNREFUSED'){
		// End reconnecting on a specific error and flush all commands with a individual error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return new Error('The server refused the connection');
	}

	if(options.total_retry_time > 1000 * 60 * 60){
		// End reconnecting after a specific timeout and flush all commands with a individual error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return new Error('Retry time exhausted');
	}

	if(options.times_connected > 10){
		// End reconnecting with built in error
		if(this.emit_error){
			this.emit_error('No connection');
		}
		return undefined;
	}

	// reconnect after
	return Math.max(options.attempt * 100, 3000);
};

let DAL = {
	connections: [],
	opened: [],
	getOrCreate: (mainCb) => {
		let lazy = lazyDefine.get()
		let cb = (...args) => {
			args.push(lazy);
			mainCb(...args);
			cb = soother;
		}
		if(DAL.connections.length){
			let connection = DAL.connections.shift();
			connection.lastOpened = Date.now();
			DAL.opened.push(connection);
			DAL._checkConnections();
			log.d('connection', connection.id, 'opened');
			return cb(null, connection.redis);
		}

		let connection = {
			id: helpers.generateId(),
			redis: {}
		};
		let config = {
			retry_strategy: retry_strategy,
			emit_error: err => connection && connection.redis.emit('error', err)
		};
		if(DAL.config && DAL.config){
			for(let i in DAL.config){
				if(!DAL.config.hasOwnProperty(i)){
					continue;
				}

				config[i] = DAL.config[i];
			}
		}

		connection.redis = redis.createClient(config);
		connection.redis.on('error', err => {
			log.e('Error in redis exports.connection:', connection.id, err);
			if(!connection){
				return;
			}

			connection.redis.quit();

			let conn = DAL._closeConnection(connection.id);
			if(conn){
				conn = undefined;
				log.d('connection', connection.id, 'deleted from opened by error');
			}

			connection = null;

			return cb(err);
		});
		connection.redis.on('ready', () => {
			connection.lastOpened = Date.now();
			DAL.opened.push(connection);
			log.d('connection', connection.id, 'created and added to opened');
			return cb(null, connection.redis);
		});
		connection.redis.on('requestEnded', () => {
			if(!connection){
				return;
			}

			let conn = DAL._closeConnection(connection.id);
			if(!conn){
				return log.e('No opened id', id);
			}

			DAL.connections.push(conn[0]);

			log.d('connection', connection.id, 'moved to waited');
		});
		connection.redis.on('end', () => {
			if(!connection){
				return;
			}

			let conn = DAL._closeConnection(connection.id);
			if(conn){
				conn = undefined;
				log.d('connection', connection.id, 'deleted from opened by end connection');
			}

			connection = undefined;

			return cb('No connection');
		});

		DAL._checkConnections();
		return lazy;
	},
	_checkConnections: () => {
		DAL.opened = DAL.opened.reduce((res, conn) =>{
			if(Date.now() - conn.lastOpened > 14400000){
				conn.redis.quit();
				delete conn.redis;
				conn = null;
			}
			else{
				res.push(conn);
			}
			return res;
		}, [])
		DAL.connections = DAL.connections.reduce((res, conn) => {
			if(Date.now() - conn.lastOpened > 14400000){
				conn.redis.quit();
				delete conn.redis;
				conn = null;
			}
			else{
				res.push(conn);
			}
			return res;
		}, []);
	},
	_closeConnection: (id) => {
		let sid;
		let conn;
		for(let i in DAL.opened){
			if(DAL.opened[i].id == id){
				sid = i;
				break;
			}
		}
		if(sid){
			return DAL.opened.splice(sid, 1);
		}
		for(let i in DAL.connections){
			if(DAL.connections[i].id == id){
				sid = i;
				break;
			}
		}
		if(sid){
			return DAL.connections.splice(sid, 1);
		}
	}
};

exports.init = (config, dalConfig) =>{
	DAL.config = dalConfig;
}
let lazyDefine = {
	methods: {},
	get: ()=>Object.assign({}, this.methods, {list: []})
}
exports.methods = {};
for(let name in redis.RedisClient.prototype){// eslint-disable-line guard-for-in
	wrapMethod(name);
}
function wrapMethod(name){
	exports.methods[name] = (...args) => {
		let conn;
		let originalCb = soother;
		let cb = (...resargs) => {
			if(conn){
				conn.emit('requestEnded');
			}

			originalCb(...resargs);
			conn = undefined;
		};

		if(typeof args[args.length -1] == 'function' && args[args.length -1] instanceof Function){
			originalCb = args[args.length -1];
			args[args.length -1] = cb;
		}
		else{
			args.push(cb);
		}

		return DAL.getOrCreate((err, connection, lazy) => {
			if(err){
				return cb(err);
			}

			conn = connection;
			if(!lazy.list.length){
				return connection[name](...args);
			}
			else{
				lazy.list.unshift({cmd: name, args: args});
				lazy.list.map(i=>{
					conn[i.cmd](...i.args)
				});
				setTimeout(cb, 2000);
			}
		});
	};
	Object.defineProperty(lazyDefine.methods, name, {get: function(...args){
		this.list.push({cmd: name, args: args})
		return this;
	}});
}
<|endoftext|>


<commit_before>var inherits = require('util').inherits;
var StreamStack = require('stream-stack').StreamStack;

// On the meta-length byte, you must multiply/divide by META_BLOCK_SIZE
// to get the real metadata byte-length. We export is as a 'getter' only
// to ensure that it remains Read-Only.
const META_BLOCK_SIZE = 16;
exports.__defineGetter__('META_BLOCK_SIZE', function() { return META_BLOCK_SIZE; });

// Usually, there will be no metadata event to inject (on the WriteStack), so
// a single byte 0 is sent indicating 0 bytes of metadata.
const NO_METADATA_BYTE = new Buffer(1);
NO_METADATA_BYTE[0] = 0;

/**
 * A StreamStack meant for ReadStreams, or more specifically, a `net.Stream`
 * connected to and receiving audio data from an Icecast radio stream.
 * This stack removes the 'metadata' bytes and parses them into a "metadata"
 * event that your Node script can do something useful with.
 */
function IcecastReadStack(stream, metaint, retainMetadata) {

  StreamStack.call(this, stream, {
    data: function(chunk) {
      this._currentOnData(chunk);
    }
  });

  // The amount of audio bytes to read before a metadata byte gets sent.
  this.metaint = metaint;

  // Not currently used:
  // TODO: If this is true, emit the metadata bytes as well
  this.retainMetadata = retainMetadata || false;

  // The counter used to keep track of count of the audio/metadata bytes parsed.
  this.counter = 0;

  this._currentOnData = this.onData;
}
exports.IcecastReadStack = IcecastReadStack;
inherits(IcecastReadStack, StreamStack);

exports.appendBuffer = function(a, b) {
  var temp = new Buffer(a.length + b.length);
  a.copy(temp, 0, 0);
  b.copy(temp, a.length, 0);
  return temp;
}

// Called when the underlying Stream emits a "data" event.
// Emits 'data' events passing 'chunk' until "metaint" bytes have
// been sent, then it sets 'onMetaLengthByte' for 'data' events.
IcecastReadStack.prototype.onData = function(chunk) {
  if (this.metaint && this.counter == this.metaint) {
    this.counter = 0;
    this._currentOnData = this.onMetaLengthByte;
    this._currentOnData(chunk);

  } else if (this.metaint && this.counter + chunk.length >= this.metaint) {
    var audioEnd = this.metaint - this.counter;
    var audioChunk = chunk.slice(0, audioEnd);
    this.emit("data", audioChunk);
    this.counter += audioChunk.length;

    // There's still remaining data! It should be metadata!
    if (chunk.length != audioChunk.length) {
      var metadata = chunk.slice(audioEnd);
      this._currentOnData(metadata);
    }

  } else if (chunk.length) {
    this.emit("data", chunk);
    this.counter += chunk.length;
  }
}

// Called when the underlying Stream emits a "data" event.
// This 'data' parser is used when the radio stream is sending a 'metadata' event.
IcecastReadStack.prototype.onMetaData = function(chunk) {

  if (this.counter + chunk.length >= this.metaLength) {
    var metaEnd = this.metaLength - this.counter - 1;
    this.counter += metaEnd;
    var metaChunk = chunk.slice(0, metaEnd);

    if (this.metaBuffer) {
      this.metaBuffer = exports.appendBuffer(this.metaBuffer, metaChunk);
    } else {
      this.metaBuffer = metaChunk;
    }
    this.emit("metadata", this.metaBuffer);
    //console.error("Meta Bytes Recieved: " + this.counter + ", " + this.metaBuffer.length);
    this.metaBuffer = null;
    this.metaLength = null;

    this.counter = 0;
    this._currentOnData = this.onData;
    if (metaEnd+1 < chunk.length) {
      var remainder = chunk.slice(metaEnd+1, chunk.length);
      //console.error(remainder.slice(0, Math.min(5, remainder.length)));
      this._currentOnData(remainder);
    }
  } else {
    if (this.metaBuffer) {
      this.metaBuffer = exports.appendBuffer(this.metaBuffer, chunk);
    } else {
      this.metaBuffer = chunk;
    }
    this.counter += chunk.length;
  }
}

// Called when the underlying "net" Stream emits a "data" event.
// This 'data' handler checks to see if there's any metadata in
// the upcoming bytes.
// 'chunk' is guaranteed to be at least 1 byte long.
IcecastReadStack.prototype.onMetaLengthByte = function(chunk) {
  var metaByte = chunk[0];
  //console.error("MetaByte: " + metaByte);
  this.metaLength = metaByte * META_BLOCK_SIZE;
  //console.error("MetaData Length: " + this.metaLength);
  this.counter = 0;
  this._currentOnData = this.metaLength > 0 ? this.onMetaData : this.onData;
  if (chunk.length > 1) {
    this._currentOnData(chunk.slice(1));
  }
}

/**
 * A StreamStack for WriteStreams (specifically `http.ServerResponse`,
 * `http.ClientRequest`, and `net.Stream`). It offers a 'queueMetadata(d)'
 * function that will inject the passed metadata into the audio stream
 * at the next given (metaint) interval.
 */
function IcecastWriteStack(stream, metaint) {
  StreamStack.call(this, stream);
  this.metaint = metaint;
  this.counter = 0;
  this._metadataQueue = [];
}
exports.IcecastWriteStack = IcecastWriteStack;
inherits(IcecastWriteStack, StreamStack);

/**
 * The 'queueMetadata' function queues a new 'metadata' event to be injected
 * upstream at the next metadata interval (metaint).
 */
var streamTitleRegExp = /StreamTitle='/;
IcecastWriteStack.prototype.queueMetadata = function(metadata) {
  //console.error(metadata);
  if (typeof(metadata) == 'string') {
    if (!streamTitleRegExp.test(metadata)) {
      // Ensure a 'StreamTitle' is somewhere in the string.
      metadata = "StreamTitle='" + metadata + "';";
    }
  } else if ('StreamTitle' in metadata) {
    // A regular object containing at least a StreamTitle property.
    // Append the 'StreamTitle' part first, to ensure it's part of the metadata.
    var str = "StreamTitle='" + metadata['StreamTitle'] + "';";
    delete metadata['StreamTitle'];
    // Then append the rest of the properies on the object (if any).
    for (var i in metadata) {
      str += i + "='" + metadata[i] + "';";
    }
    metadata = str;
  }
  //console.error(metadata);

  if (!streamTitleRegExp.test(metadata)) {
    throw new Error('"queueMetadata()" expects a String, or an object with a "StreamTitle" property.');
  }

  // Calculate the metalength byte, with a maximum of 255.
  var metaLength = Math.min(255, Math.ceil(metadata.length/META_BLOCK_SIZE));
  // Create a new Buffer that will contain the inital metalength byte, then
  // the metadata itself, and then will pad the rest of the Buffer with null bytes.
  var metaBuf = new Buffer((metaLength * META_BLOCK_SIZE)+1);
  metaBuf[0] = metaLength;
  var end = metaBuf.write(metadata, 1) + 1;
  if (end < metaBuf.length) {
    // Pad with null bytes
    for (var l=metaBuf.length; end<l; end++) {
      metaBuf[end] = 0;
    }
  }
  //console.error(metaBuf);
  //console.error(require('util').inspect(metaBuf.toString()));

  this._metadataQueue.push(metaBuf);
}

/**
 * The overwritten 'write()' function takes care of injecting the next item
 * in the metadataQueue if metaint bytes have been passed
 */
IcecastWriteStack.prototype.write = function(chunk, enc) {
  if (!Buffer.isBuffer(chunk)) {
    chunk = new Buffer(chunk, enc);
  }
  this._writeAudioChunk(chunk);
}

IcecastWriteStack.prototype._writeAudioChunk = function(chunk) {
  if (this.counter + chunk.length > this.metaint) {
    var first = chunk.slice(0, this.metaint - this.counter);
    this._writeAudioChunk(first);

    if (first.length < chunk.length) {
      this._writeAudioChunk(chunk.slice(first.length));
    }
  } else {
    this.stream.write(chunk);
    this.counter += chunk.length;
    //console.error('counter: ' + this.counter);

    if (this.counter == this.metaint) {
      this._injectNextMetadata();
    }
  }
}

/**
 * Checks if there's any entires in the 'metadataQueue' array, and if there are,
 * injects the next item in the queue into the audio write stream.
 */
IcecastWriteStack.prototype._injectNextMetadata = function() {
  //console.error('Injecting metadata!');
  //console.error('counter (metadata): ' + this.counter);
  var metaBuf = NO_METADATA_BYTE;
  if (this._metadataQueue.length > 0) {
    metaBuf = this._metadataQueue.shift();
  }
  this.stream.write(metaBuf);
  this.counter = 0;
}

/**
 * Returns a new ReadStream for the given Internet Radio URL.
 * First arg is the URL to the radio stream. Second arg is a
 * boolean indicating whether or not to include the metadata
 * chunks in the 'data' events. Defaults to 'false' (metadata,
 * is stripped, parsed, and formatted into the 'metadata' event).
 */
function createReadStream(url, headers, retainMetadata) {
  return require('./client').createClient(url, headers, retainMetadata);
}
exports.createReadStream = createReadStream;

// Used to strip 'null' bytes from the metadata event
var nullExp = new RegExp('\0', 'g');
/**
 * Accepts the Buffer passed from the 'metadata' event, and parses it into
 * a JavaScript object.
 */
function parseMetadata(metadata) {
  var rtn = {}, pieces = metadata.toString().replace(nullExp, '').split(";");
  for (var i=0, l=pieces.length; i<l; i++) {
    var piece = pieces[i];
    if (piece.length) {
      piece = piece.split("='");
      rtn[piece[0]] = piece[1].substring(0, piece[1].length-1);
    }
  }
  return rtn;
}
exports.parseMetadata = parseMetadata;
<commit_msg>remove old "icecast-stack.js" file<commit_after><|endoftext|>


<commit_before>/*
 * grunt-contrib-handlebars
 * http://gruntjs.com/
 *
 * Copyright (c) 2012 Tim Branyen, contributors
 * Licensed under the MIT license.
 * https://github.com/gruntjs/grunt-contrib-handlebars/blob/master/LICENSE-MIT
 */

module.exports = function(grunt) {
  "use strict";

  // TODO: ditch this when grunt v0.4 is released
  grunt.util = grunt.util || grunt.utils;

  var _ = grunt.util._;
  var helpers = require('grunt-contrib-lib').init(grunt);

  // filename conversion for templates
  var defaultProcessName = function(name) { return name; };

  // filename conversion for partials
  var defaultProcessPartialName = function(filePath) {
    var pieces = _.last(filePath.split("/")).split(".");
    var name   = _(pieces).without(_.last(pieces)).join("."); // strips file extension
    return name.substr(1, name.length);                       // strips leading _ character
  };

  // full template compilation
  var templateFn = function(source, filepath, namespace) {
    try {
      var output = "Handlebars.template(" + require("handlebars").precompile(source) + ");";
      return namespace + "['" + filepath + "'] = " + output;
    } catch (e) {
      grunt.log.error(e);
      grunt.fail.warn("Handlebars failed to compile.");
    }
  };

  // partial template compilation
  var partialFn = function(source, filepath, namespace) {
    try {
      return "Handlebars.registerPartial('" + filepath + "', " + "Handlebars.template(" + require("handlebars").precompile(source) + "));";
    } catch (e) {
      grunt.log.error(e);
      grunt.fail.warn("Handlebars failed to compile partial.");
    }
  };

  grunt.registerMultiTask("handlebars", "Compile handlebars templates and partials.", function() {
    var options = helpers.options(this, {namespace: "JST"});

    grunt.verbose.writeflags(options, "Options");

    // TODO: ditch this when grunt v0.4 is released
    this.files = this.files || helpers.normalizeMultiTaskFiles(this.data, this.target);

    var srcFiles, src, filename;
    var partials = [];
    var templates = [];
    var output = [];
    var namespace = "this['" + options.namespace + "']";
    var isPartial = options.partialRegex || /^_/;
    var processName = options.processName || defaultProcessName;
    var processPartialName = options.processName || defaultProcessPartialName;

    this.files.forEach(function(files) {

      srcFiles = grunt.file.expandFiles(files.src);

      srcFiles.forEach(function(file) {
        src = grunt.file.read(file);
        console.log(src);

        if(isPartial.test(_.last(file.split("/")))) {
          filename = processPartialName(file);
          partials.push(partialFn(src, filename, namespace));
        } else {
          filename = processName(file);
          templates.push(templateFn(src, filename, namespace));
        }
      });
      output = output.concat(partials, templates);

      if (output.length > 0) {
        output.unshift(namespace + " = " + namespace + " || {};");
        grunt.file.write(files.dest, output.join("\n\n"));
        grunt.log.writeln("File '" + files.dest + "' created.");
      }
    });
  });

};
<commit_msg>commenting<commit_after>/*
 * grunt-contrib-handlebars
 * http://gruntjs.com/
 *
 * Copyright (c) 2012 Tim Branyen, contributors
 * Licensed under the MIT license.
 * https://github.com/gruntjs/grunt-contrib-handlebars/blob/master/LICENSE-MIT
 */

module.exports = function(grunt) {
  "use strict";

  // TODO: ditch this when grunt v0.4 is released
  grunt.util = grunt.util || grunt.utils;

  var _ = grunt.util._;
  var helpers = require('grunt-contrib-lib').init(grunt);

  // filename conversion for templates
  var defaultProcessName = function(name) { return name; };

  // filename conversion for partials
  var defaultProcessPartialName = function(filePath) {
    var pieces = _.last(filePath.split("/")).split(".");
    var name   = _(pieces).without(_.last(pieces)).join("."); // strips file extension
    return name.substr(1, name.length);                       // strips leading _ character
  };

  // full template compilation
  var templateFn = function(source, filepath, namespace) {
    try {
      var output = "Handlebars.template(" + require("handlebars").precompile(source) + ");";
      return namespace + "['" + filepath + "'] = " + output;
    } catch (e) {
      grunt.log.error(e);
      grunt.fail.warn("Handlebars failed to compile.");
    }
  };

  // partial template compilation
  var partialFn = function(source, filepath, namespace) {
    try {
      return "Handlebars.registerPartial('" + filepath + "', " + "Handlebars.template(" + require("handlebars").precompile(source) + "));";
    } catch (e) {
      grunt.log.error(e);
      grunt.fail.warn("Handlebars failed to compile partial.");
    }
  };

  grunt.registerMultiTask("handlebars", "Compile handlebars templates and partials.", function() {
    var options = helpers.options(this, {namespace: "JST"});

    grunt.verbose.writeflags(options, "Options");

    // TODO: ditch this when grunt v0.4 is released
    this.files = this.files || helpers.normalizeMultiTaskFiles(this.data, this.target);

    var srcFiles, src, filename;
    var partials = [];
    var templates = [];
    var output = [];
    var namespace = "this['" + options.namespace + "']";

    // assign regex for partial detection
    var isPartial = options.partialRegex || /^_/;

    // assign filename transformation functions
    var processName = options.processName || defaultProcessName;
    var processPartialName = options.processName || defaultProcessPartialName;

    // iterate files, processing partials and templates separately
    this.files.forEach(function(files) {
      srcFiles = grunt.file.expandFiles(files.src);
      srcFiles.forEach(function(file) {
        src = grunt.file.read(file);

        if(isPartial.test(_.last(file.split("/")))) {
          filename = processPartialName(file);
          partials.push(partialFn(src, filename, namespace));
        } else {
          filename = processName(file);
          templates.push(templateFn(src, filename, namespace));
        }
      });
      output = output.concat(partials, templates);

      if (output.length > 0) {
        output.unshift(namespace + " = " + namespace + " || {};");
        grunt.file.write(files.dest, output.join("\n\n"));
        grunt.log.writeln("File '" + files.dest + "' created.");
      }
    });
  });

};
<|endoftext|>


<commit_before>/**
 * webcrawler.js - PhantomJS based web crawler with support for #! routing.
 */

'use strict';

var phantom = require('node-phantom');
var URI = require('URIjs');
var URIFragment = require('URIjs/src/URI.fragmentURI');
var document;

function Crawler(url, depth) {
    var date = new Date();
    var self = this;

    this.url = url;
    this.depth = depth;
    this.crawledUrls = [];
    this.sitemapUrls = [];
    this.sitemap = null;
    this.options = {};
    this.currDate = date.getFullYear() + '-' + date.getMonth() + '-' + date.getDate();
    this.doneCrawling = false;
    this.urls = {};

    this.createPhantom = function(error, ph) {
        if (error) {
            ph.exit();
        }

        self.crawl(self.url, 0, ph);
    };
}

Crawler.prototype.startCrawling = function() {
    phantom.create(this.createPhantom);

    return this;
};

Crawler.prototype.stopCrawling = function() {
    if (this.ph !== undefined) {
        this.ph.exit();
    }

    return this;
};

Crawler.prototype.setUrl = function(url) {
    this.url = url;

    return this;
};

Crawler.prototype.setDepth = function(depth) {
    this.depth = depth;

    return this;
};

Crawler.prototype.setOptions = function(opts) {
    this.options = opts;

    return this;
};

Crawler.prototype.addUrl = function(url) {
    this.urls[url] = false;

    return this;
};

/**
 * Add a Url to the site map. See http://www.sitemaps.org/protocol.html.
 *
 * @param {string} url  - A valid URL.
 * @param {float} priority  - The priority of the url from 0.0 - 1.0.
 * @param {string} frequency  - The frequency to update: monthly.
 * @returns {Object} this
 */
Crawler.prototype.addUrlToSitemap = function(url, priority, frequency) {
    frequency = typeof frequency === 'undefined' ? 'monthly' : frequency;
    priority = typeof priority === 'undefined' ? 0.5 : priority;

    this.sitemapUrls.push({
        "loc": this.getAbsoluteUrl(url),
        "lastmod": this.currDate,
        "priority": priority,
        "changefreq": frequency
    });

    return this;
};

/**
 * Create an object that can be converted into sitemap.xml by data2xml.
 *
 * I looked at several options that all suck:
 *   - xmlserializer: HTML only..
 *   - xmldom: doesn't support namespaces currently.
 *   - JXON: annoying to read.
 *
 * @returns {Object} data2xml object - see https://github.com/chilts/data2xml.
 */
Crawler.prototype.createSitemap = function() {
    var url, n;
    var doc = {
        _attr: {xmlns: 'https://www.sitemaps.org/schemas/sitemap/0.9'},
        'url': []
    };

    if (this.sitemapUrls !== undefined && this.sitemapUrls.length !== null && this.sitemapUrls.length > 0) {
        for (n in this.sitemapUrls) {
            url = {
                'loc': this.sitemapUrls[n].loc,
                'lastmod': this.sitemapUrls[n].lastmod,
                'priority': this.sitemapUrls[n].priority,
                'changefreq': this.sitemapUrls[n].changefreq
            };
            doc.url.push(url);
        }
    }

    return doc;
};

/**
 * Get internal URLs on a given page.
 *
 * @param {Object} page Phantom.js page object.
 * @param {integer} depth Current crawl depth.
 * @returns {Object} this
 */
Crawler.prototype.getUrls = function(page, depth) {
    var delay, findUrls;
    var self = this;
    var intervalCount = 0;

//    findUrls = function() {
        page.evaluate(function() {
            var nav = document.querySelector('a');

            if (typeof nav !== 'undefined' || nav !== null) {
                return [].map.call(document.querySelectorAll('a'), function(link) {
                    return link.getAttribute('href');
                });
            }
        }, function(err, links) {
//            clearInterval(delay);

            // Filter links.
            links.filter(function(url) {
                if (self.urls[url]) {
                    return false;
                } else if (url === self.url) {
                    return false;
                } else if (self.options.followFragment && url === self.url + '#' + self.options.fragmentPrefix + '/') {
                    return false;
                } else if (self.isRelative(url) || self.isSameOrigin(self.url, url)) {
                    self.addUrl(url).crawl(url, depth + 1, self.ph);
                } else {
                    return false;
                }
            });
        });
/*
        intervalCount++;
        if (intervalCount > 10) {
            clearInterval(delay);
        }
    };

    delay = setInterval(findUrls, 100);
*/
    return this;
};

/**
 * Compares two URL strings to see if latter is the same origin as the former.
 *
 * @param {string} base A valid URL string.
 * @param {string} url A valid URL string to compare against base.
 * @returns {boolean} TRUE if url is the same origin as base.
 */
Crawler.prototype.isSameOrigin = function (base, url) {
    var baseUrl = URI(base);
    var linkUrl = URI(url);

    return baseUrl.normalizeProtocol().protocol() === linkUrl.normalizeProtocol().protocol() &&
           baseUrl.normalizeHostname().host() === linkUrl.normalizeHostname().host();
};

/**
 * Check if a URL string is relative or a hashbang.
 *
 * @param {string} url A valid URL string.
 * @returns {boolean} TRUE if URL is relative.
 */
Crawler.prototype.isRelative = function(url) {
    var linkUrl = URI(url);
    var furi = linkUrl.fragment(this.options.followFragment);

    if (linkUrl.is('relative')) {
        // Only use fragments when defined.
        if (this.options.followFragment) {
            linkUrl.fragmentPrefix(this.options.fragmentPrefix);

            if (furi.pathname() !== '/') {
                return true;
            }
        } else if (linkUrl.pathname() !== '') {
            return true;
        }
    }

    return false;
};

/**
 * Get the absolute URL to crawl.
 *
 * @param {string} url A valid URI including relative or fragment path.
 * @returns {string} A valid URL string.
 */
Crawler.prototype.getAbsoluteUrl = function(url) {
    var linkUrl = URI(url);

    if (linkUrl.is('absolute')) {
        return linkUrl.toString();
    }

    return this.url + url;
};

/**
 * Get the relative path of a Url.
 *
 * @param {string} url A valid URI including relative or fragment path.
 * @returns {string} A relative path to use as file name.
 */
Crawler.prototype.getRelativePath = function(url) {
    var linkUrl = URI(url);
    var furi, path;

    path = linkUrl.pathname(true);

    if (this.options.followFragment) {
        linkUrl.fragmentPrefix(this.options.fragmentPrefix);
        furi = linkUrl.fragment(true);
        path = furi.pathname(true);
    }

    // Add file suffix for path names and directories.
    if (linkUrl.filename() === "") {
        if (url === this.url) {
            path = 'index.html';
        } else {
            path = path.replace(/\/?$/, '.html');
        }
    }

    return path;
};

/**
 * Load the provided url if it is below or equal to the depth limit. And then
 * check for additional urls to recursively crawl.
 *
 * @param {string} url The url to crawl.
 * @param {integer} depth The current depth.
 * @param {Object} ph phantom object created by node-phantom.
 * @returns {Object} this
 */
Crawler.prototype.crawl = function (url, depth, ph) {
    var self = this;
    this.ph = ph;

    if (this.depth >= depth && (self.urls[url] === undefined || !self.urls[url])) {
        self.urls[url] = true;

        this.ph.createPage(function(error, page) {
            var absoluteUrl = self.getAbsoluteUrl(url);
            console.log('Crawling: ' + absoluteUrl);

            page.viewportSize = {
              width: self.options.viewportWidth,
              height: self.options.viewportHeight
            };

            page.onConsoleMessage = function(msg) {
              console.log(msg);
            };

            if (error) {
                console.log('            ....error occurred.');
                return;
            }

            return page.open(absoluteUrl, function(error, status) {
                var wait, onLoad;
                var count = 0;

                if ('fail' !== status) {

                    onLoad = function() {
                        page.evaluate(function(s) {
                            var body = document.querySelector(s);
                            if (body.getAttribute('data-status') === 'ready') {
                                return document.documentElement && document.documentElement.outerHTML ? document.documentElement.outerHTML : "";
                            }
                        }, function(err, documentHtml) {
                            var delay, waitForChildren;

                            // Clear interval.
                            clearInterval(wait);

                            if (self.options.content && documentHtml === null) {
                                return;
                            }

                            // Add content to crawled urls.
                            self.crawledUrls.push({
                                "url": url,
                                "content": self.options.content ? documentHtml : null
                            });

                            // Crawl through current page's children.
                            self.getUrls(page, depth);

                            if (depth === 0) {
                                waitForChildren = function() {
                                    var i, done = true;

                                    for (i in self.urls) {
                                        if (!self.urls[i]) {
                                            done = false;
                                        }
                                    }

                                    if (done) {
                                        clearInterval(delay);
                                        self.doneCrawling = true;
                                    }
                                };

                                delay = setInterval(waitForChildren, 1000);
                            }
                        }, self.options.readySelector);

                        count++;
                        if (count > 10) {
                            clearInterval(wait);
                        }
                    };

                    wait = setInterval(onLoad, 1000);
                } else {
                    console.log('           ...failed to load page.');
                    return;
                }

                if (error) {
                    console.log('           ...failed to load page.');
                    return;
                }
            });
        });
    } else if (depth === 0) {
        console.log('           ...finished crawling.');
        self.doneCrawling = true;
    }

    return this;
};

module.exports = Crawler;
<commit_msg>Fix bug with static content saving.<commit_after>/**
 * webcrawler.js - PhantomJS based web crawler with support for #! routing.
 */

'use strict';

var phantom = require('node-phantom');
var URI = require('URIjs');
var URIFragment = require('URIjs/src/URI.fragmentURI');
var document;

function Crawler(url, depth) {
    var date = new Date();
    var self = this;

    this.url = url;
    this.depth = depth;
    this.crawledUrls = [];
    this.sitemapUrls = [];
    this.sitemap = null;
    this.options = {};
    this.currDate = date.getFullYear() + '-' + date.getMonth() + '-' + date.getDate();
    this.doneCrawling = false;
    this.urls = {};

    this.createPhantom = function(error, ph) {
        if (error) {
            ph.exit();
        }

        self.crawl(self.url, 0, ph);
    };
}

Crawler.prototype.startCrawling = function() {
    phantom.create(this.createPhantom);

    return this;
};

Crawler.prototype.stopCrawling = function() {
    if (this.ph !== undefined) {
        this.ph.exit();
    }

    return this;
};

Crawler.prototype.setUrl = function(url) {
    this.url = url;

    return this;
};

Crawler.prototype.setDepth = function(depth) {
    this.depth = depth;

    return this;
};

Crawler.prototype.setOptions = function(opts) {
    this.options = opts;

    return this;
};

Crawler.prototype.addUrl = function(url) {
    this.urls[url] = false;

    return this;
};

/**
 * Add a Url to the site map. See http://www.sitemaps.org/protocol.html.
 *
 * @param {string} url  - A valid URL.
 * @param {float} priority  - The priority of the url from 0.0 - 1.0.
 * @param {string} frequency  - The frequency to update: monthly.
 * @returns {Object} this
 */
Crawler.prototype.addUrlToSitemap = function(url, priority, frequency) {
    frequency = typeof frequency === 'undefined' ? 'monthly' : frequency;
    priority = typeof priority === 'undefined' ? 0.5 : priority;

    this.sitemapUrls.push({
        "loc": this.getAbsoluteUrl(url),
        "lastmod": this.currDate,
        "priority": priority,
        "changefreq": frequency
    });

    return this;
};

/**
 * Create an object that can be converted into sitemap.xml by data2xml.
 *
 * I looked at several options that all suck:
 *   - xmlserializer: HTML only..
 *   - xmldom: doesn't support namespaces currently.
 *   - JXON: annoying to read.
 *
 * @returns {Object} data2xml object - see https://github.com/chilts/data2xml.
 */
Crawler.prototype.createSitemap = function() {
    var url, n;
    var doc = {
        _attr: {xmlns: 'https://www.sitemaps.org/schemas/sitemap/0.9'},
        'url': []
    };

    if (this.sitemapUrls !== undefined && this.sitemapUrls.length !== null && this.sitemapUrls.length > 0) {
        for (n in this.sitemapUrls) {
            url = {
                'loc': this.sitemapUrls[n].loc,
                'lastmod': this.sitemapUrls[n].lastmod,
                'priority': this.sitemapUrls[n].priority,
                'changefreq': this.sitemapUrls[n].changefreq
            };
            doc.url.push(url);
        }
    }

    return doc;
};

/**
 * Get internal URLs on a given page.
 *
 * @param {Object} page Phantom.js page object.
 * @param {integer} depth Current crawl depth.
 * @returns {Object} this
 */
Crawler.prototype.getUrls = function(page, depth) {
    var delay, findUrls;
    var self = this;
    var intervalCount = 0;

//    findUrls = function() {
        page.evaluate(function() {
            var nav = document.querySelector('a');

            if (typeof nav !== 'undefined' || nav !== null) {
                return [].map.call(document.querySelectorAll('a'), function(link) {
                    return link.getAttribute('href');
                });
            }
        }, function(err, links) {
//            clearInterval(delay);

            // Filter links.
            links.filter(function(url) {
                if (self.urls[url]) {
                    return false;
                } else if (url === self.url) {
                    return false;
                } else if (self.options.followFragment && url === self.url + '#' + self.options.fragmentPrefix + '/') {
                    return false;
                } else if (self.isRelative(url) || self.isSameOrigin(self.url, url)) {
                    self.addUrl(url).crawl(url, depth + 1, self.ph);
                } else {
                    return false;
                }
            });
        });
/*
        intervalCount++;
        if (intervalCount > 10) {
            clearInterval(delay);
        }
    };

    delay = setInterval(findUrls, 100);
*/
    return this;
};

/**
 * Compares two URL strings to see if latter is the same origin as the former.
 *
 * @param {string} base A valid URL string.
 * @param {string} url A valid URL string to compare against base.
 * @returns {boolean} TRUE if url is the same origin as base.
 */
Crawler.prototype.isSameOrigin = function (base, url) {
    var baseUrl = URI(base);
    var linkUrl = URI(url);

    return baseUrl.normalizeProtocol().protocol() === linkUrl.normalizeProtocol().protocol() &&
           baseUrl.normalizeHostname().host() === linkUrl.normalizeHostname().host();
};

/**
 * Check if a URL string is relative or a hashbang.
 *
 * @param {string} url A valid URL string.
 * @returns {boolean} TRUE if URL is relative.
 */
Crawler.prototype.isRelative = function(url) {
    var linkUrl = URI(url);
    var furi = linkUrl.fragment(this.options.followFragment);

    if (linkUrl.is('relative')) {
        // Only use fragments when defined.
        if (this.options.followFragment) {
            linkUrl.fragmentPrefix(this.options.fragmentPrefix);

            if (furi.pathname() !== '/') {
                return true;
            }
        } else if (linkUrl.pathname() !== '') {
            return true;
        }
    }

    return false;
};

/**
 * Get the absolute URL to crawl.
 *
 * @param {string} url A valid URI including relative or fragment path.
 * @returns {string} A valid URL string.
 */
Crawler.prototype.getAbsoluteUrl = function(url) {
    var linkUrl = URI(url);

    if (linkUrl.is('absolute')) {
        return linkUrl.toString();
    }

    return this.url + url;
};

/**
 * Get the relative path of a Url.
 *
 * @param {string} url A valid URI including relative or fragment path.
 * @returns {string} A relative path to use as file name.
 */
Crawler.prototype.getRelativePath = function(url) {
    var linkUrl = URI(url);
    var furi, path;

    path = linkUrl.pathname(true);

    if (this.options.followFragment) {
        linkUrl.fragmentPrefix(this.options.fragmentPrefix);
        furi = linkUrl.fragment(true);
        path = furi.pathname(true);
    }

    // Add file suffix for path names and directories.
    if (linkUrl.filename() === "") {
        if (url === this.url) {
            path = '/index.html';
        } else {
            path = path.replace(/\/?$/, '.html');
        }
    }

    return path;
};

/**
 * Load the provided url if it is below or equal to the depth limit. And then
 * check for additional urls to recursively crawl.
 *
 * @param {string} url The url to crawl.
 * @param {integer} depth The current depth.
 * @param {Object} ph phantom object created by node-phantom.
 * @returns {Object} this
 */
Crawler.prototype.crawl = function (url, depth, ph) {
    var self = this;
    this.ph = ph;

    if (this.depth >= depth && (self.urls[url] === undefined || !self.urls[url])) {
        self.urls[url] = true;

        this.ph.createPage(function(error, page) {
            var absoluteUrl = self.getAbsoluteUrl(url);
            console.log('Crawling: ' + absoluteUrl);

            page.viewportSize = {
              width: self.options.viewportWidth,
              height: self.options.viewportHeight
            };

            page.onConsoleMessage = function(msg) {
              console.log(msg);
            };

            if (error) {
                console.log('            ....error occurred.');
                return;
            }

            return page.open(absoluteUrl, function(error, status) {
                var wait, onLoad;
                var count = 0;

                if ('fail' !== status) {

                    onLoad = function() {
                        page.evaluate(function(s) {
                            var body = document.querySelector(s);
                            if (body.getAttribute('data-status') === 'ready') {
                                return document.documentElement && document.documentElement.outerHTML ? document.documentElement.outerHTML : "";
                            }
                        }, function(err, documentHtml) {
                            var delay, waitForChildren;

                            // Clear interval.
                            clearInterval(wait);

                            if (self.options.content && documentHtml === null) {
                                return;
                            }

                            // Add content to crawled urls.
                            self.crawledUrls.push({
                                "url": url,
                                "content": self.options.content ? documentHtml : null
                            });

                            // Crawl through current page's children.
                            self.getUrls(page, depth);

                            if (depth === 0) {
                                waitForChildren = function() {
                                    var i, done = true;

                                    for (i in self.urls) {
                                        if (!self.urls[i]) {
                                            done = false;
                                        }
                                    }

                                    if (done) {
                                        clearInterval(delay);
                                        self.doneCrawling = true;
                                    }
                                };

                                delay = setInterval(waitForChildren, 1000);
                            }
                        }, self.options.readySelector);

                        count++;
                        if (count > 10) {
                            clearInterval(wait);
                        }
                    };

                    wait = setInterval(onLoad, 1000);
                } else {
                    console.log('           ...failed to load page.');
                    return;
                }

                if (error) {
                    console.log('           ...failed to load page.');
                    return;
                }
            });
        });
    } else if (depth === 0) {
        console.log('           ...finished crawling.');
        self.doneCrawling = true;
    }

    return this;
};

module.exports = Crawler;
<|endoftext|>


<commit_before>#!/usr/bin/env node

const fs = require('fs')
const path = require('path')
const { promisify } = require('util')

const mime = require('mime-types')
const readDir = require('recursive-readdir')
const { S3, CloudFront } = require('aws-sdk')

const { version: PKG_VERSION } = require('../lerna.json')

const PRODUCTION_VERSION = 'v6'
const DISTRIBUTION_ID = process.env.DISTRIBUTION_ID
const BUCKET_NAME = 'webdriver.io'
const BUILD_DIR = path.resolve(__dirname, '..', 'website', 'build')
const UPLOAD_OPTIONS = { partSize: 10 * 1024 * 1024, queueSize: 1 }
const IGNORE_FILE_SUFFIX = ['*.rb']

/* eslint-disable no-console */
;(async () => {
    const timestamp = Date.now()
    const s3 = new S3()
    const files = await readDir(BUILD_DIR, IGNORE_FILE_SUFFIX)

    const version = `v${PKG_VERSION.split('.')[0]}`
    const bucketName = version === PRODUCTION_VERSION ? BUCKET_NAME : `${version}.${BUCKET_NAME}`

    /**
     * upload assets
     */
    console.log(`Uploading ${BUILD_DIR} to S3 bucket ${bucketName}`)
    await Promise.all(files.map((file) => new Promise((resolve, reject) => s3.upload({
        Bucket: bucketName,
        Key: file.replace(BUILD_DIR + '/', ''),
        Body: fs.createReadStream(file),
        ContentType: mime.lookup(file),
        ACL: 'public-read'
    }, UPLOAD_OPTIONS, (err, res) => {
        if (err) {
            console.error(`Couldn't upload file ${file}: ${err.stack}`)
            return reject(err)
        }

        console.log(`${file} uploaded`)
        return resolve(res)
    }))))

    /**
     * invalidate distribution
     */
    const distributionId = version === PRODUCTION_VERSION
        ? DISTRIBUTION_ID
        : process.env[`DISTRIBUTION_ID_${version.toUpperCase()}`]
    console.log(`Invalidate objects from distribution ${distributionId}`)
    if (distributionId) {
        const cloudfront = new CloudFront()
        const { Invalidation } = await promisify(cloudfront.createInvalidation.bind(cloudfront))({
            DistributionId: distributionId,
            InvalidationBatch: {
                CallerReference: `${timestamp}`,
                Paths: { Quantity: 1, Items: ['/*'] }
            }
        })
        console.log(`Created new invalidation with ID ${Invalidation.Id}`)
    }

    /**
     * delete old assets
     */
    const objects = await promisify(s3.listObjects.bind(s3))({
        Bucket: bucketName
    })
    const objectsToDelete = objects.Contents.filter((obj) => (
        (obj.LastModified)).getTime() < timestamp)
    console.log(`Found ${objectsToDelete.length} outdated objects to remove...`)

    await Promise.all(objectsToDelete.map((obj) => (
        promisify(s3.deleteObject.bind(s3))({
            Bucket: bucketName,
            Key: obj.Key
        })
    )))
    console.log('Deleted obsolete items successfully')

    return
})().then(
    () => console.log('Successfully updated webdriver.io docs'),
    (err) => console.error(`Error uploading docs: ${err.stack}`)
)
/* eslint-enable no-console */
<commit_msg>log only if distribution id is available<commit_after>#!/usr/bin/env node

const fs = require('fs')
const path = require('path')
const { promisify } = require('util')

const mime = require('mime-types')
const readDir = require('recursive-readdir')
const { S3, CloudFront } = require('aws-sdk')

const { version: PKG_VERSION } = require('../lerna.json')

const PRODUCTION_VERSION = 'v6'
const DISTRIBUTION_ID = process.env.DISTRIBUTION_ID
const BUCKET_NAME = 'webdriver.io'
const BUILD_DIR = path.resolve(__dirname, '..', 'website', 'build')
const UPLOAD_OPTIONS = { partSize: 10 * 1024 * 1024, queueSize: 1 }
const IGNORE_FILE_SUFFIX = ['*.rb']

/* eslint-disable no-console */
;(async () => {
    const timestamp = Date.now()
    const s3 = new S3()
    const files = await readDir(BUILD_DIR, IGNORE_FILE_SUFFIX)

    const version = `v${PKG_VERSION.split('.')[0]}`
    const bucketName = version === PRODUCTION_VERSION ? BUCKET_NAME : `${version}.${BUCKET_NAME}`

    /**
     * upload assets
     */
    console.log(`Uploading ${BUILD_DIR} to S3 bucket ${bucketName}`)
    await Promise.all(files.map((file) => new Promise((resolve, reject) => s3.upload({
        Bucket: bucketName,
        Key: file.replace(BUILD_DIR + '/', ''),
        Body: fs.createReadStream(file),
        ContentType: mime.lookup(file),
        ACL: 'public-read'
    }, UPLOAD_OPTIONS, (err, res) => {
        if (err) {
            console.error(`Couldn't upload file ${file}: ${err.stack}`)
            return reject(err)
        }

        console.log(`${file} uploaded`)
        return resolve(res)
    }))))

    /**
     * invalidate distribution
     */
    const distributionId = version === PRODUCTION_VERSION
        ? DISTRIBUTION_ID
        : process.env[`DISTRIBUTION_ID_${version.toUpperCase()}`]
    if (distributionId) {
        console.log(`Invalidate objects from distribution ${distributionId}`)
        const cloudfront = new CloudFront()
        const { Invalidation } = await promisify(cloudfront.createInvalidation.bind(cloudfront))({
            DistributionId: distributionId,
            InvalidationBatch: {
                CallerReference: `${timestamp}`,
                Paths: { Quantity: 1, Items: ['/*'] }
            }
        })
        console.log(`Created new invalidation with ID ${Invalidation.Id}`)
    }

    /**
     * delete old assets
     */
    const objects = await promisify(s3.listObjects.bind(s3))({
        Bucket: bucketName
    })
    const objectsToDelete = objects.Contents.filter((obj) => (
        (obj.LastModified)).getTime() < timestamp)
    console.log(`Found ${objectsToDelete.length} outdated objects to remove...`)

    await Promise.all(objectsToDelete.map((obj) => (
        promisify(s3.deleteObject.bind(s3))({
            Bucket: bucketName,
            Key: obj.Key
        })
    )))
    console.log('Deleted obsolete items successfully')

    return
})().then(
    () => console.log('Successfully updated webdriver.io docs'),
    (err) => console.error(`Error uploading docs: ${err.stack}`)
)
/* eslint-enable no-console */
<|endoftext|>


<commit_before>(function(module) {

    var login = {};

    login.callPortfolio = function(token) {
        $.ajax({
            headers: {
                authorization: `Bearer ${token}`
            },
            url: '/portfolios',
            type: 'GET',
            contentType: 'application/json',
            error: function(error) {
                console.log(error);
            }
        })
        .done(data => {
            stockTrans.renderCashValue(data)
        });
    };

    $(function() {
        if (localStorage.token) {
            $.ajax({
                url: '/users/validate',
                type: 'POST',
                contentType: 'application/json',
                headers: {
                    'Authorization': 'Bearer ' + localStorage.getItem()
                },
                error: function(error) {
                    console.log(error);
                }
            })
            .done(function(username) {
                login.loggedIn(username.username);
            });
        }
    });

    login.clearAll = function() {
        $('#stock-data').empty();
        $('#canvas-holder').hide();
        $('#portfolio-details').empty();
        $('#signin-form').show().trigger('reset');
        $('#signup-form').show().trigger('reset');
        $('#stock-search-input').trigger('reset');
        $('input').trigger('reset');
        $('.error-msg').empty();
        $('.msg').empty();
    };

    login.newUser = function() {
        $('#signup-button').on('click touchstart', function(event) {
            event.preventDefault();
            $('.error-msg').empty();
            $('#logged-in').empty();
            let data = {};
            data.username = $('#new-user-name').val();
            data.password = $('#new-user-pw').val();
            $.ajax({
                url: '/users/signup',
                type: 'POST',
                contentType: 'application/json',
                data: JSON.stringify(data),
                error: function(error) {
                    console.log(error);
                    $('#signup-form').prepend('Error: ' + error.responseJSON.error + '');
                }
            })
            .done(function(user) {
                localStorage.setItem('token', user.token);
                login.loggedIn(user.username);
                login.callPortfolio(user.token);
            });
        });
    };

    login.existingUser = function() {
        $('#signin-button').on('click touchstart', function(event) {
            event.preventDefault();
            $('.error-msg').empty();
            $('#logged-in').empty();
            let data = {};
            data.username = $('#user-name').val();
            data.password = $('#user-pw').val();
            $.ajax({
                url: '/users/signin',
                type: 'POST',
                contentType: 'application/json',
                data: JSON.stringify(data),
                error: function(error) {
                    console.log(error);
                    $('#signin-form').prepend('Error: ' + error.responseJSON.error + '');
                }
            })
            .done(function(user) {
                console.log('user logged in as: ', user);
                localStorage.setItem('token', user.token);
                login.loggedIn(user.username);
                login.callPortfolio(user.token);
            });
        });
    };

    login.loggedIn = function(user) {
        $('#signin-form').hide();
        $('#signup-form').hide();
        $('#logged-in').hide();
        $('input').trigger('reset');
        $('#login')
            .prepend('
Welcome! You are logged in as ' + user + '.<button type="button" id="logout-button">Log Out</button>
');
        login.logOut();
    };

    login.logOut = function() {
        $('#logout-button').on('click touchstart', function(event) {
            event.preventDefault();
            login.clearAll();
            localStorage.removeItem('token');
            $('#logged-in').html('You have logged out. See you next time!');
        });
    };

    login.logOut();
    login.newUser();
    login.existingUser();

    module.login = login;

})(window);
<commit_msg>added token to getItem from localStorage for authorization during login anonymous function in loging in<commit_after>(function(module) {

    var login = {};

    login.callPortfolio = function(token) {
        $.ajax({
            headers: {
                authorization: `Bearer ${token}`
            },
            url: '/portfolios',
            type: 'GET',
            contentType: 'application/json',
            error: function(error) {
                console.log(error);
            }
        })
        .done(data => {
            stockTrans.renderCashValue(data)
        });
    };

    $(function() {
        if (localStorage.token) {
            $.ajax({
                url: '/users/validate',
                type: 'POST',
                contentType: 'application/json',
                headers: {
                    'Authorization': 'Bearer ' + localStorage.getItem('token')
                },
                error: function(error) {
                    console.log(error);
                }
            })
            .done(function(username) {
                login.loggedIn(username.username);
            });
        }
    });

    login.clearAll = function() {
        $('#stock-data').empty();
        $('#canvas-holder').hide();
        $('#portfolio-details').empty();
        $('#signin-form').show().trigger('reset');
        $('#signup-form').show().trigger('reset');
        $('#stock-search-input').trigger('reset');
        $('input').trigger('reset');
        $('.error-msg').empty();
        $('.msg').empty();
    };

    login.newUser = function() {
        $('#signup-button').on('click touchstart', function(event) {
            event.preventDefault();
            $('.error-msg').empty();
            $('#logged-in').empty();
            let data = {};
            data.username = $('#new-user-name').val();
            data.password = $('#new-user-pw').val();
            $.ajax({
                url: '/users/signup',
                type: 'POST',
                contentType: 'application/json',
                data: JSON.stringify(data),
                error: function(error) {
                    console.log(error);
                    $('#signup-form').prepend('Error: ' + error.responseJSON.error + '');
                }
            })
            .done(function(user) {
                localStorage.setItem('token', user.token);
                login.loggedIn(user.username);
                login.callPortfolio(user.token);
            });
        });
    };

    login.existingUser = function() {
        $('#signin-button').on('click touchstart', function(event) {
            event.preventDefault();
            $('.error-msg').empty();
            $('#logged-in').empty();
            let data = {};
            data.username = $('#user-name').val();
            data.password = $('#user-pw').val();
            $.ajax({
                url: '/users/signin',
                type: 'POST',
                contentType: 'application/json',
                data: JSON.stringify(data),
                error: function(error) {
                    console.log(error);
                    $('#signin-form').prepend('Error: ' + error.responseJSON.error + '');
                }
            })
            .done(function(user) {
                console.log('user logged in as: ', user);
                localStorage.setItem('token', user.token);
                login.loggedIn(user.username);
                login.callPortfolio(user.token);
            });
        });
    };

    login.loggedIn = function(user) {
        $('#signin-form').hide();
        $('#signup-form').hide();
        $('#logged-in').hide();
        $('input').trigger('reset');
        $('#login')
            .prepend('
Welcome! You are logged in as ' + user + '.<button type="button" id="logout-button">Log Out</button>
');
        login.logOut();
    };

    login.logOut = function() {
        $('#logout-button').on('click touchstart', function(event) {
            event.preventDefault();
            login.clearAll();
            localStorage.removeItem('token');
            $('#logged-in').html('You have logged out. See you next time!');
        });
    };

    login.logOut();
    login.newUser();
    login.existingUser();

    module.login = login;

})(window);
<|endoftext|>


<commit_before>// batchSize 500,
// tableName users_working

var Q = require('q');
var elasticsearch = require('elasticsearch');
var pg = require('pg').native;

var SearchIndexService = function(options){

    var esClient = elasticsearch.Client({
      hosts: [
        options.elasticsearchEndpoint
      ]
    });

    var BATCH_SIZE              = options.batchSize,
        TABLE_NAME              = options.tableName,
        DB_CONNECTION_STRING    = options.dbConnectionString;

    var iterationCount = 0;

    var createIndex = function(deferred){
        var deferred = deferred || Q.defer();

        var sql = 'SELECT * FROM ' + TABLE_NAME + ' WHERE ("user"->>\'_ninya_synced_v4\')::boolean is null LIMIT ' + BATCH_SIZE;

        var rejectWithError = function(err){
            console.log('ElasticSearchSync: error ->' + err);
            deferred.reject(err);
        };

        console.log('ElasticSearchSync: connecting to postgres...');

        pg.connect(DB_CONNECTION_STRING, function(err, client, done) {
            if(err) {
                console.log('ElasticSearchSync: could not connect to postgres');
                deferred.reject(err);
                return;
            }

            console.log('ElasticSearchSync: connected to postgres');
            client.query(sql, function(err, result) {
                if(err) {
                    console.log('ElasticSearchSync: could not retrieve batch from postgres');
                    return deferred.reject(err);
                }

                console.log('ElasticSearchSync: retrieved batch from postgres');

                if (result.rows.length === 0){
                    console.log('ElasticSearchSync: sync finished');
                    deferred.resolve('finished');
                    return;
                }

                var tasks = result.rows.map(function(obj){

                    obj.user._ninya_location_lowercase = obj.user.location && obj.user.location.toLowerCase();
                    obj.user._ninya_location = obj.user.location;
                    obj.user._ninya_site = options.stackexchangeSite;

                    var esId = options.stackexchangeSite + '_' + obj.user.user_id;

                    return esClient.index({
                        index: options.elasticsearchIndex,
                        type: 'user',
                        id: esId,
                        body: obj.user
                    });
                });

                Q.all(tasks)
                 .then(function(res){
                    console.log('ElasticSearchSync: added batch');
                    //mark batch as synced
                    return Q.all(result.rows.map(function(obj){
                        var user = obj.user;
                        user._ninya_synced_v4 = true;
                        return client.query('UPDATE ' + TABLE_NAME + ' SET "user" = $1 WHERE "user"->>\'user_id\' = \'' + user.user_id  + '\'', [user], function(err, result) {
                            if (err){
                                console.log(err);
                            }
                        });
                    }))
                    .then(function(){
                        iterationCount++;
                        done();
                        console.log('ElasticSearchSync: flagged batch as processed');
                        console.log('ElasticSearchSync: finished ' + iterationCount + ' iteration(s)');
                        //start over
                        createIndex(deferred);
                    });
                 }, rejectWithError);
            });
        });

        return deferred.promise;
    };

    return {
        createIndex: createIndex
    }
};

module.exports = SearchIndexService;<commit_msg>chore(searchIndexService): increment sync property<commit_after>// batchSize 500,
// tableName users_working

var Q = require('q');
var elasticsearch = require('elasticsearch');
var pg = require('pg').native;

var SearchIndexService = function(options){

    var esClient = elasticsearch.Client({
      hosts: [
        options.elasticsearchEndpoint
      ]
    });

    var BATCH_SIZE              = options.batchSize,
        TABLE_NAME              = options.tableName,
        DB_CONNECTION_STRING    = options.dbConnectionString;

    var iterationCount = 0;

    var createIndex = function(deferred){
        var deferred = deferred || Q.defer();

        var sql = 'SELECT * FROM ' + TABLE_NAME + ' WHERE ("user"->>\'_ninya_synced_v5\')::boolean is null LIMIT ' + BATCH_SIZE;

        var rejectWithError = function(err){
            console.log('ElasticSearchSync: error ->' + err);
            deferred.reject(err);
        };

        console.log('ElasticSearchSync: connecting to postgres...');

        pg.connect(DB_CONNECTION_STRING, function(err, client, done) {
            if(err) {
                console.log('ElasticSearchSync: could not connect to postgres');
                deferred.reject(err);
                return;
            }

            console.log('ElasticSearchSync: connected to postgres');
            client.query(sql, function(err, result) {
                if(err) {
                    console.log('ElasticSearchSync: could not retrieve batch from postgres');
                    return deferred.reject(err);
                }

                console.log('ElasticSearchSync: retrieved batch from postgres');

                if (result.rows.length === 0){
                    console.log('ElasticSearchSync: sync finished');
                    deferred.resolve('finished');
                    return;
                }

                var tasks = result.rows.map(function(obj){

                    obj.user._ninya_location_lowercase = obj.user.location && obj.user.location.toLowerCase();
                    obj.user._ninya_location = obj.user.location;
                    obj.user._ninya_site = options.stackexchangeSite;

                    var esId = options.stackexchangeSite + '_' + obj.user.user_id;

                    return esClient.index({
                        index: options.elasticsearchIndex,
                        type: 'user',
                        id: esId,
                        body: obj.user
                    });
                });

                Q.all(tasks)
                 .then(function(res){
                    console.log('ElasticSearchSync: added batch');
                    //mark batch as synced
                    return Q.all(result.rows.map(function(obj){
                        var user = obj.user;
                        user._ninya_synced_v5 = true;
                        return client.query('UPDATE ' + TABLE_NAME + ' SET "user" = $1 WHERE "user"->>\'user_id\' = \'' + user.user_id  + '\'', [user], function(err, result) {
                            if (err){
                                console.log(err);
                            }
                        });
                    }))
                    .then(function(){
                        iterationCount++;
                        done();
                        console.log('ElasticSearchSync: flagged batch as processed');
                        console.log('ElasticSearchSync: finished ' + iterationCount + ' iteration(s)');
                        //start over
                        createIndex(deferred);
                    });
                 }, rejectWithError);
            });
        });

        return deferred.promise;
    };

    return {
        createIndex: createIndex
    }
};

module.exports = SearchIndexService;<|endoftext|>


<commit_before>var gulp = require('gulp'),
    jshint = require('gulp-jshint'),
    uglify = require('gulp-uglify'),
    rename = require('gulp-rename'),
    concat = require('gulp-concat'),
    gulpif = require('gulp-if'),
    rjs  = require('gulp-requirejs'),
    qunit  = require('gulp-qunit'),
    handlebars = require('gulp-handlebars'),
    defineModule = require('gulp-define-module'),
    declare = require('gulp-declare'),

    requirejs_config = require('./src/require.config.json'),

    errorCode = 0;

/*****************************
 * HELPERS
 *****************************/
function skipLibDir(file) {
    return file.path.search("lib/") >= 0 ? false : true;
}

/*****************************
 * TASKS
 *****************************/
gulp.task('test', function () {
    gulp.src('./tests/test_runner.html')
        .pipe(qunit())
        .on('error', function(err) {
            errorCode = 1;
            process.emit('exit');
        });;
});

gulp.task('jshint', function (){
    return gulp.src('src/**/*.js')
        .pipe(gulpif(skipLibDir, jshint('.jshintrc')))
        .pipe(jshint.reporter('default'));
});

gulp.task('templates', function () {
    var stream = gulp.src(['src/templates/*.handlebars'])
        .pipe(handlebars())
        .pipe(defineModule('plain'))
        .pipe(declare({
          namespace: 'Aptivate.data.templates'
        }))
        .pipe(concat('src/lib/templates.js'))
        .pipe(gulp.dest('.'));
    return stream;
});

gulp.task('default', ['templates', 'jshint'], function(t){
    requirejs_config.baseUrl = "./src/";
    requirejs_config.out = "logframe.js";
    requirejs_config.name = "main";
    requirejs_config.findNestedDependencies = true;

    return rjs(requirejs_config)
        .pipe(gulp.dest('dist'))
        .pipe(rename({suffix: '.min'}))
        .pipe(uglify())
        .pipe(gulp.dest('dist'));
});

/*****************************
 * WATCHERS
 *****************************/
gulp.task('watch', function () {
    gulp.watch('src/templates/**/*.handlebars', ['templates']);
    gulp.watch('src/**/*.js', ['jshint']);
});

/*
 * Report error codes for tests
 */
process.on('exit', function () {
      process.exit(errorCode);
});
<commit_msg>Remove gulpfile.js<commit_after><|endoftext|>


<commit_before>/*
 * grouter
 * https://github.com/goliatone/grouter
 * Created with gbase.
 * Copyright (c) 2014 goliatone
 * Licensed under the MIT license.
 */
/* jshint strict: false, plusplus: true */
/*global define: false, require: false, module: false, exports: false */
(function(root, name, deps, factory) {
    "use strict";
    // Node
    if (typeof deps === 'function') {
        factory = deps;
        deps = [];
    }

    if (typeof exports === 'object') {
        module.exports = factory.apply(root, deps.map(require));
    } else if (typeof define === 'function' && 'amd' in define) {
        //require js, here we assume the file is named as the lower
        //case module name.
        define(name.toLowerCase(), deps, factory);
    } else {
        // Browser
        var d, i = 0,
            global = root,
            old = global[name],
            mod;
        while ((d = deps[i]) !== undefined) deps[i++] = root[d];
        global[name] = mod = factory.apply(global, deps);
        //Export no 'conflict module', aliases the module.
        mod.noConflict = function() {
            global[name] = old;
            return mod;
        };
    }
}(this, "GRouter", ['extend'], function(extend) {

    /*
     *
     */
    var _extend = extend;

    /**
     * Shim console, make sure that if no console
     * available calls do not generate errors.
     * @return {Object} Console shim.
     */
    var _shimConsole = function(con) {

        if (con) return con;

        con = {};
        var empty = {},
            noop = function() {},
            properties = 'memory'.split(','),
            methods = ('assert,clear,count,debug,dir,dirxml,error,exception,group,' +
                'groupCollapsed,groupEnd,info,log,markTimeline,profile,profileEnd,' +
                'table,time,timeEnd,timeStamp,trace,warn').split(','),
            prop,
            method;

        while (method = methods.pop()) con[method] = noop;
        while (prop = properties.pop()) con[prop] = empty;

        return con;
    };

    ///////////////////////////////////////////////////
    // CONSTRUCTOR
    ///////////////////////////////////////////////////

    var options = {
        initialRoute: 'home'
    };

    /**
     * GRouter constructor
     *
     * @param  {object} config Configuration object.
     */
    var GRouter = function(config) {
        config = config || {};

        config = _extend({}, this.constructor.DEFAULTS, config);

        this.init(config);
    };

    GRouter.name = GRouter.prototype.name = 'GRouter';

    GRouter.VERSION = '0.0.0';

    /**
     * Make default options available so we
     * can override.
     */
    GRouter.DEFAULTS = options;

    ///////////////////////////////////////////////////
    // PRIVATE METHODS
    ///////////////////////////////////////////////////

    GRouter.prototype.init = function(config) {
        if (this.initialized) return;
        this.initialized = true;

        this.logger.log('GRouter Initializing', config);

        config = _extend(config, this.constructor.DEFAULTS);
        _extend(this, config);

        this.currentId = this.currentId || this.initialRoute;

        return this;
    };

    GRouter.prototype.handler = function(e) {
        var data = e.data;
        this.logger.warn('=============\nhandler\n=============', data);

        //Store the current payload.
        this.event = e;
        this.data = e.data;

        this.emit('route.' + data.scene, data);

        if (data.payload && data.payload.subscene) {
            var etype = ['route', data.scene, data.payload.subscene[0]].join('.');
            this.emit(etype, data);
        }

        this.emit('route', data);
    };

    GRouter.prototype.goto = function(id, options) {

        var oldId = this.currentId,
            isTransitioning = oldId !== id;

        /*
         * Event type fired before we exit a route.
         * Default route event is:
         * exit.home
         * goto.update or goto.change
         * goto.<SCENE_ID>.<update | change>
         */
        var exitType = isTransitioning ? 'exit.' + oldId : null,
            actionType = isTransitioning ? 'change' : 'update',
            actionSceneType = 'goto.' + id + '.' + actionType;

        this.currentId = id;

        [exitType, 'goto', 'goto.' + actionType, actionSceneType].forEach(function(type) {
            if (!type) return;

            // this.logger.info('EMIT GOTO:', type);
            this.emit(type, {
                id: id,
                old: oldId,
                isTransitioning: isTransitioning,
                //TODO: Should we rename options to payload to make it
                //consistent?!!!
                options: options
            });
        }, this);
    };

    GRouter.prototype.emit = function() {
        this.logger.warn('emit not implemented', arguments);
    };

    /**
     * Logger method, meant to be implemented by
     * mixin. As a placeholder, we use console if available
     * or a shim if not present.
     */
    GRouter.prototype.logger = _shimConsole(console);

    return GRouter;
}));<commit_msg>SRC: Refactor default handler, triggers route events.<commit_after>/*
 * grouter
 * https://github.com/goliatone/grouter
 * Created with gbase.
 * Copyright (c) 2014 goliatone
 * Licensed under the MIT license.
 */
/* jshint strict: false, plusplus: true */
/*global define: false, require: false, module: false, exports: false */
(function(root, name, deps, factory) {
    "use strict";
    // Node
    if (typeof deps === 'function') {
        factory = deps;
        deps = [];
    }

    if (typeof exports === 'object') {
        module.exports = factory.apply(root, deps.map(require));
    } else if (typeof define === 'function' && 'amd' in define) {
        //require js, here we assume the file is named as the lower
        //case module name.
        define(name.toLowerCase(), deps, factory);
    } else {
        // Browser
        var d, i = 0,
            global = root,
            old = global[name],
            mod;
        while ((d = deps[i]) !== undefined) deps[i++] = root[d];
        global[name] = mod = factory.apply(global, deps);
        //Export no 'conflict module', aliases the module.
        mod.noConflict = function() {
            global[name] = old;
            return mod;
        };
    }
}(this, "GRouter", ['extend'], function(extend) {

    /*
     *
     */
    var _extend = extend;

    /**
     * Shim console, make sure that if no console
     * available calls do not generate errors.
     * @return {Object} Console shim.
     */
    var _shimConsole = function(con) {

        if (con) return con;

        con = {};
        var empty = {},
            noop = function() {},
            properties = 'memory'.split(','),
            methods = ('assert,clear,count,debug,dir,dirxml,error,exception,group,' +
                'groupCollapsed,groupEnd,info,log,markTimeline,profile,profileEnd,' +
                'table,time,timeEnd,timeStamp,trace,warn').split(','),
            prop,
            method;

        while (method = methods.pop()) con[method] = noop;
        while (prop = properties.pop()) con[prop] = empty;

        return con;
    };

    ///////////////////////////////////////////////////
    // CONSTRUCTOR
    ///////////////////////////////////////////////////

    var OPTIONS = {
        initialRoute: 'home'
    };

    /**
     * GRouter constructor
     *
     * @param  {object} config Configuration object.
     */
    var GRouter = function(config) {
        config = config || {};

        config = _extend({}, this.constructor.DEFAULTS, config);

        this.init(config);
    };

    GRouter.name = GRouter.prototype.name = 'GRouter';

    GRouter.VERSION = '0.0.1';

    /**
     * Make default options available so we
     * can override.
     */
    GRouter.DEFAULTS = OPTIONS;

    ///////////////////////////////////////////////////
    // PRIVATE METHODS
    ///////////////////////////////////////////////////

    GRouter.prototype.init = function(config) {
        if (this.initialized) return;
        this.initialized = true;

        this.logger.log('GRouter Initializing', config);

        config = _extend(config, this.constructor.DEFAULTS);
        _extend(this, config);

        this.currentId = this.currentId || this.initialRoute;

        return this;
    };

    GRouter.prototype.handler = function(e) {
        var data = e.data;
        this.logger.warn('=============\nhandler\n=============', data);

        //Store the current payload.
        this.event = e;
        this.data = data;

        var subscene = Keypath.get(data, 'payload.subscene', undefined);
        subscene && (subscene = ['route', data.scene, data.payload.subscene[0]].join('.'));

        ['route', 'route.' + data.scene, subscene].forEach(function(type) {
            if (!type) return;
            this.logger.info('EMIT ROUTE:', type);
            this.emit(type, data);
        }, this);
    };

    GRouter.prototype.goto = function(id, options) {

        var oldId = this.currentId,
            isTransitioning = oldId !== id;

        /*
         * Event type fired before we exit a route.
         * Default route event is:
         * exit.home
         * goto.update or goto.change
         * goto.<SCENE_ID>.<update | change>
         */
        var exitType = isTransitioning ? 'exit.' + oldId : null,
            actionType = isTransitioning ? 'change' : 'update',
            actionSceneType = 'goto.' + id + '.' + actionType;

        this.currentId = id;

        [exitType, 'goto', 'goto.' + actionType, actionSceneType].forEach(function(type) {
            if (!type) return;

            // this.logger.info('EMIT GOTO:', type);
            this.emit(type, {
                id: id,
                old: oldId,
                isTransitioning: isTransitioning,
                //TODO: Should we rename options to payload to make it
                //consistent?!!!
                options: options
            });
        }, this);
    };

    GRouter.prototype.emit = function() {
        this.logger.warn('emit not implemented', arguments);
    };

    /**
     * Logger method, meant to be implemented by
     * mixin. As a placeholder, we use console if available
     * or a shim if not present.
     */
    GRouter.prototype.logger = _shimConsole(console);

    return GRouter;
}));<|endoftext|>