<commit_before>import _            from "underscore";
import $            from "jquery";
import {Collection} from "backbone"
import context      from "context-utils";
import BaseView     from "../BaseView";

/*
new SelectField({
	fieldId:       'colorField',
	field:         'color',
	collection:    Colors,
	selectedValue: 'blue',
	empty:         'No color',
	disabled:      false,
	invalid:       false,
	firstOptionEmpty: false,
	groupBy:       function
});
*/
export default class SelectField extends BaseView {

	className() { return 'ui-select-field' }

	constructor(options) {
		super(options);

		this.addEvents({
			'touchstart':    'onTouchStart',
			'focus select':  'onFocus',
			'blur select':   'onBlur',
			'change select': 'onChange',
			'click':         'onClick'
		});

		if (!options)
			throw new Error('No options provided');

		if (!options.field)
			throw new Error('The field option is required');

		this.setDefaultsOptions({
			empty:            null,
			selectedValue:    null,
			disabled:         null,
			autoFocus:        false,
			label:            null,
			invalid:          false,
			accessoryBar:     true,
			addClass:         null,
			firstOptionEmpty: false,
			groupBy:          null
		});

		this.fieldAttributes = {
			'data-field': options.field
		};

		if (!this.options.fieldId && this.options.field) {
			this.options.fieldId = this.options.field;
		}

		if (options.fieldId) {
			this.fieldAttributes['id'] = options.fieldId;
		}

		if (this.options.addClass) {
			this.$el.addClass(this.options.addClass);
		}
	}

	setCollection(newCollection, newSelectedValue) {
		this.collection = newCollection;
		this.options.selectedValue = newSelectedValue;
		this.render();
	}

	onRender(rendered) {
		this.undelegateEvents();
		if ( !rendered ){
			this.$el.empty();

			if(this.options.label)
				this.$el.append($('<label>').attr('for',this.fieldAttributes['id']).append(this.options.label));

			this.cache.$field = $('<select>').attr(this.fieldAttributes);
			this.$el.append(this.cache.$field);

			if (this.options.invalid)
				this.$el.append($('<div>').addClass('ui-input-invalid').attr('data-invalid-field', this.options.field));
		}

		let $field = this.cache.$field;
		$field.empty();

		if (this.options.empty) {
			this.cache.$field.append($('<option value="">').addClass('empty').text(this.options.empty));
		}

		if ( this.options.firstOptionEmpty ){
			let labelEmpty = '';
			if ( _.isString(this.options.firstOptionEmpty) )
				labelEmpty = this.options.firstOptionEmpty;
			$field.append( $('<option>').attr('value', '').text( labelEmpty ) );
		}

		if (this.collection instanceof Collection) {
			this.renderCollection()
		} else {
			this.renderArray();
		}

		setTimeout(() => {
			if (this.options.selectedValue)
				$field.val(this.options.selectedValue);
		}, 100);

		this.rendered = true;
		this.disabled(this.options.disabled);
		this.delegateEvents();
		return this;
	}

	renderCollection() {
		let $field = this.cache.$field;
		let lastGroupBy = null;
		let aGroupBy;
		let $aGroupByElement;
		this.collection.forEach((anElement, index) => {
			if (this.options.groupBy) {
				if (typeof this.options.groupBy === 'function') {
					aGroupBy = this.options.groupBy(anElement);
				} else {
					aGroupBy = anElement.get(this.options.groupBy);
				}
				if (aGroupBy != lastGroupBy) {
					$aGroupByElement = $('<optgroup>').attr('label', aGroupBy);
					$field.append($aGroupByElement);
				}
			}

			let disabled  = _.result(anElement, 'isDisabled') || false;
			let $anOption = $('<option>').attr('value', anElement.id).text(anElement.toString());
			if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement.id) {
				$anOption.attr('selected', 'selected');
			}
			if (disabled) {
				$anOption.attr('disabled', 'disabled');
			}

			if (this.options.groupBy) {
				$aGroupByElement.append($anOption);
				lastGroupBy = aGroupBy;
			} else {
				$field.append($anOption);
			}
		});
	}

	renderArray() {
		let $field = this.cache.$field;
		let lastGroupBy = null;
		let aGroupBy;
		let $aGroupByElement;
		this.collection.forEach((anElement) => {
			if (this.options.groupBy) {
				if (typeof this.options.groupBy === 'function') {
					aGroupBy = this.options.groupBy(anElement);
				} else {
					aGroupBy = anElement[this.options.groupBy];
				}
				if (aGroupBy != lastGroupBy) {
					$aGroupByElement = $('<optgroup>').attr('label', aGroupBy);
					$field.append($aGroupByElement);
				}
			}

			let $anOption;
			if (_.isObject(anElement)) {
				$anOption = $('<option>').attr('value', anElement.value).text(anElement.label);
				if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement.value) {
					$anOption.attr('selected', 'selected');
				}
			}
			else {
				$anOption = $('<option>').attr('value', anElement).text(anElement);
				if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement) {
					$anOption.attr('selected', 'selected');
				}
			}

			if (this.options.groupBy) {
				$aGroupByElement.append($anOption);
				lastGroupBy = aGroupBy;
			} else {
				$field.append($anOption);
			}
		});
	}

	onChange(e) {
		let model = this.getValue();
		if (!model) model = null;
		this.trigger('change', model);
	}

	disabled(value) {
		this.options.disabled = value;
		if (this.rendered) {
			if (value) {
				this.$el.addClass('disabled');
				this.cache.$field.attr('disabled', 'disabled');
			} else {
				this.$el.removeClass('disabled');
				this.cache.$field.removeAttr('disabled');
			}
		}
	}

	setSelectedValue(value) {
		this.options.selectedValue = value;
	}

	onClick(e) {
		if (this.options.autoFocus)
			this.cache.$field.focus();
	}

	getValue() {
		let value;
		if (this.collection instanceof Collection )
			value = this.collection.get(this.cache.$field.val());
		else
			value = this.cache.$field.val();
		return value;
	}

	onTouchStart() {
		let hideFormAccessoryBar;
		if (typeof cordova !== 'undefined' && cordova.plugins && cordova.plugins.Keyboard && cordova.plugins.Keyboard.hideKeyboardAccessoryBar ) {
			hideFormAccessoryBar = cordova.plugins.Keyboard.hideKeyboardAccessoryBar;
		}else if ( window.Keyboard && window.Keyboard.hideFormAccessoryBar ){
			hideFormAccessoryBar = window.Keyboard.hideFormAccessoryBar;
		}
		if (!_.isFunction(hideFormAccessoryBar))
			hideFormAccessoryBar = ()=>{};
		hideFormAccessoryBar(!this.options.accessoryBar);
	}

	onFocus() {
		this.trigger('focus', this);
	}

	onBlur() {
		if (this.options.accessoryBar == false && typeof cordova !== 'undefined' && cordova.plugins && cordova.plugins.Keyboard) {
			cordova.plugins.Keyboard.hideKeyboardAccessoryBar(true);
		}
		this.trigger('focus', this);
	}

};
<commit_msg>Add documentation to the SelectField<commit_after>import _            from "underscore";
import $            from "jquery";
import {Collection} from "backbone"
import context      from "context-utils";
import BaseView     from "../BaseView";

/*
new SelectField({
	fieldId:       'colorField',
	field:         'color',
	collection:    Colors,
	selectedValue: 'blue',
	empty:         'No color',
	disabled:      false,
	invalid:       false,
	firstOptionEmpty: false,
	groupBy:       function
});
*/
/**
 * It rapresents an HTML select field
 * @extends BaseView
 * @version 2.0.0
 * @example
 * const argumentSelectField = new SelectField({
 *   field: 'argument',
 *   fieldId: 'argument',
 *   label: 'Argument',
 *   collection: context.collections.arguments,
 *   firstOptionEmpty: true,
 *   autoFocus: false,
 *   selectedValue: this.model.get('argument')
 * });
 * @example
 * const argumentSelectField = new SelectField({
 *   field: 'argument',
 *   fieldId: 'argument',
 *   label: 'Argument',
 *   collection: [
 *     { value: 'arg1', label: 'Foo' },
 *     { value: 'arg2', label: 'Bar' }
 *   ],
 *   firstOptionEmpty: false,
 *   selectedValue: this.model.get('argument')
 * });
 */
export default class SelectField extends BaseView {

	className() { return 'ui-select-field' }

	constructor(options) {
		super(options);

		this.addEvents({
			'touchstart':    'onTouchStart',
			'focus select':  'onFocus',
			'blur select':   'onBlur',
			'change select': 'onChange',
			'click':         'onClick'
		});

		if (!options)
			throw new Error('No options provided');

		if (!options.field)
			throw new Error('The field option is required');

		this.setDefaultsOptions({
			empty:            null,
			selectedValue:    null,
			disabled:         null,
			autoFocus:        false,
			label:            null,
			invalid:          false,
			accessoryBar:     true,
			addClass:         null,
			firstOptionEmpty: false,
			groupBy:          null
		});

		this.fieldAttributes = {
			'data-field': options.field
		};

		if (!this.options.fieldId && this.options.field) {
			this.options.fieldId = this.options.field;
		}

		if (options.fieldId) {
			this.fieldAttributes['id'] = options.fieldId;
		}

		if (this.options.addClass) {
			this.$el.addClass(this.options.addClass);
		}
	}

	setCollection(newCollection, newSelectedValue) {
		this.collection = newCollection;
		this.options.selectedValue = newSelectedValue;
		this.render();
	}

	onRender(rendered) {
		this.undelegateEvents();
		if ( !rendered ){
			this.$el.empty();

			if(this.options.label)
				this.$el.append($('<label>').attr('for',this.fieldAttributes['id']).append(this.options.label));

			this.cache.$field = $('<select>').attr(this.fieldAttributes);
			this.$el.append(this.cache.$field);

			if (this.options.invalid)
				this.$el.append($('<div>').addClass('ui-input-invalid').attr('data-invalid-field', this.options.field));
		}

		let $field = this.cache.$field;
		$field.empty();

		if (this.options.empty) {
			this.cache.$field.append($('<option value="">').addClass('empty').text(this.options.empty));
		}

		if ( this.options.firstOptionEmpty ){
			let labelEmpty = '';
			if ( _.isString(this.options.firstOptionEmpty) )
				labelEmpty = this.options.firstOptionEmpty;
			$field.append( $('<option>').attr('value', '').text( labelEmpty ) );
		}

		if (this.collection instanceof Collection) {
			this.renderCollection()
		} else {
			this.renderArray();
		}

		setTimeout(() => {
			if (this.options.selectedValue)
				$field.val(this.options.selectedValue);
		}, 100);

		this.rendered = true;
		this.disabled(this.options.disabled);
		this.delegateEvents();
		return this;
	}

	renderCollection() {
		let $field = this.cache.$field;
		let lastGroupBy = null;
		let aGroupBy;
		let $aGroupByElement;
		this.collection.forEach((anElement, index) => {
			if (this.options.groupBy) {
				if (typeof this.options.groupBy === 'function') {
					aGroupBy = this.options.groupBy(anElement);
				} else {
					aGroupBy = anElement.get(this.options.groupBy);
				}
				if (aGroupBy != lastGroupBy) {
					$aGroupByElement = $('<optgroup>').attr('label', aGroupBy);
					$field.append($aGroupByElement);
				}
			}

			let disabled  = _.result(anElement, 'isDisabled') || false;
			let $anOption = $('<option>').attr('value', anElement.id).text(anElement.toString());
			if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement.id) {
				$anOption.attr('selected', 'selected');
			}
			if (disabled) {
				$anOption.attr('disabled', 'disabled');
			}

			if (this.options.groupBy) {
				$aGroupByElement.append($anOption);
				lastGroupBy = aGroupBy;
			} else {
				$field.append($anOption);
			}
		});
	}

	renderArray() {
		let $field = this.cache.$field;
		let lastGroupBy = null;
		let aGroupBy;
		let $aGroupByElement;
		this.collection.forEach((anElement) => {
			if (this.options.groupBy) {
				if (typeof this.options.groupBy === 'function') {
					aGroupBy = this.options.groupBy(anElement);
				} else {
					aGroupBy = anElement[this.options.groupBy];
				}
				if (aGroupBy != lastGroupBy) {
					$aGroupByElement = $('<optgroup>').attr('label', aGroupBy);
					$field.append($aGroupByElement);
				}
			}

			let $anOption;
			if (_.isObject(anElement)) {
				$anOption = $('<option>').attr('value', anElement.value).text(anElement.label);
				if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement.value) {
					$anOption.attr('selected', 'selected');
				}
			}
			else {
				$anOption = $('<option>').attr('value', anElement).text(anElement);
				if (this.options.selectedValue != void 0 && this.options.selectedValue == anElement) {
					$anOption.attr('selected', 'selected');
				}
			}

			if (this.options.groupBy) {
				$aGroupByElement.append($anOption);
				lastGroupBy = aGroupBy;
			} else {
				$field.append($anOption);
			}
		});
	}

	onChange(e) {
		let model = this.getValue();
		if (!model) model = null;
		this.trigger('change', model);
	}

	disabled(value) {
		this.options.disabled = value;
		if (this.rendered) {
			if (value) {
				this.$el.addClass('disabled');
				this.cache.$field.attr('disabled', 'disabled');
			} else {
				this.$el.removeClass('disabled');
				this.cache.$field.removeAttr('disabled');
			}
		}
	}

	setSelectedValue(value) {
		this.options.selectedValue = value;
	}

	onClick(e) {
		if (this.options.autoFocus)
			this.cache.$field.focus();
	}

	getValue() {
		let value;
		if (this.collection instanceof Collection )
			value = this.collection.get(this.cache.$field.val());
		else
			value = this.cache.$field.val();
		return value;
	}

	onTouchStart() {
		let hideFormAccessoryBar;
		if (typeof cordova !== 'undefined' && cordova.plugins && cordova.plugins.Keyboard && cordova.plugins.Keyboard.hideKeyboardAccessoryBar ) {
			hideFormAccessoryBar = cordova.plugins.Keyboard.hideKeyboardAccessoryBar;
		}else if ( window.Keyboard && window.Keyboard.hideFormAccessoryBar ){
			hideFormAccessoryBar = window.Keyboard.hideFormAccessoryBar;
		}
		if (!_.isFunction(hideFormAccessoryBar))
			hideFormAccessoryBar = ()=>{};
		hideFormAccessoryBar(!this.options.accessoryBar);
	}

	onFocus() {
		this.trigger('focus', this);
	}

	onBlur() {
		if (this.options.accessoryBar == false && typeof cordova !== 'undefined' && cordova.plugins && cordova.plugins.Keyboard) {
			cordova.plugins.Keyboard.hideKeyboardAccessoryBar(true);
		}
		this.trigger('focus', this);
	}

};
<|endoftext|>


<commit_before>var config = require(__dirname + '/../../config.js');
var utils  = require(__dirname + '/../../utils.js');

var pair = function() {
   return {
      'internId'  : 0,
      'aid'       : 0,
      'pair'      : '',
      'mid'       : 0,
      'timestamp' : 0,
      'last'      : 0,
      'lowestAsk' : 0,
      'highestBid': 0,
      'baseVolume': 0,
      'quoteVolume':0,
      'high24h'   : 0,
      'low24h'    : 0,
      'percentChange':0
   }
};
exports.pair = pair;

var splitPair = function(pair) {
   var p = pair.split('_');
   return {
      name: p[0].toUpperCase() + '_' + p[1].toUpperCase(),
      base: p[0].toUpperCase(),
      counter: p[1].toUpperCase()
   };
};
exports.splitPair = splitPair;

var mirrorPair = function(pair) {
   var tPair = pair;
   tPair.base = pair.counter;
   tPair.counter = pair.base;
   return tPair;
};
exports.mirrorPair = mirrorPair;

var getAssetId = function(pair, mysqlconnection, callback){
   var query = '';
   query += 'SELECT * FROM assets WHERE ' +
            'base LIKE \'' + pair.base + '\' AND ' +
            'counter LIKE \'' + pair.counter + '\' ' +
            'LIMIT 0,1';
   if (config.dev) utils.log(query);
   mysqlconnection.query(query, function(err, rowsStats) {
      if(err) {
         utils.log(err)
         callback(err);
      }
      else if (!rowsStats.length) {
         // Create new asset if non existent
         mysqlconnection.query('INSERT INTO assets SET ?', pair, function(e, results, fields){
            if (e) callback(e);
            if (config.dev) console.log(e);
            if (config.dev) console.log(results);
            callback(null, [{aid:results.insertId}]);
         });
      }
      else {
         utils.log(rowsStats);
         callback(null, rowsStats);
      }

   });
};
exports.getAssetId = getAssetId;<commit_msg>usd_Btc fix<commit_after>var config = require(__dirname + '/../../config.js');
var utils  = require(__dirname + '/../../utils.js');

var pair = function() {
   return {
      'internId'  : 0,
      'aid'       : 0,
      'pair'      : '',
      'mid'       : 0,
      'timestamp' : 0,
      'last'      : 0,
      'lowestAsk' : 0,
      'highestBid': 0,
      'baseVolume': 0,
      'quoteVolume':0,
      'high24h'   : 0,
      'low24h'    : 0,
      'percentChange':0
   }
};
exports.pair = pair;

var splitPair = function(pair) {
   var p = pair.split('_');
   return {
      name: p[0].toUpperCase() + '_' + p[1].toUpperCase(),
      base: p[0].toUpperCase(),
      counter: p[1].toUpperCase()
   };
};
exports.splitPair = splitPair;

var mirrorPair = function(pair) {
   return {name:pair.name, base:pair.counter, counter:pair.base};
};
exports.mirrorPair = mirrorPair;

var getAssetId = function(pair, mysqlconnection, callback){
   var query = '';
   query += 'SELECT * FROM assets WHERE ' +
            'base LIKE \'' + pair.base + '\' AND ' +
            'counter LIKE \'' + pair.counter + '\' ' +
            'LIMIT 0,1';
   if (config.dev) utils.log(query);
   mysqlconnection.query(query, function(err, rowsStats) {
      if(err) {
         utils.log(err)
         callback(err);
      }
      else if (!rowsStats.length) {
         // Create new asset if non existent
         mysqlconnection.query('INSERT INTO assets SET ?', pair, function(e, results, fields){
            if (e) callback(e);
            if (config.dev) console.log(e);
            if (config.dev) console.log(results);
            callback(null, [{aid:results.insertId}]);
         });
      }
      else {
         utils.log(rowsStats);
         callback(null, rowsStats);
      }

   });
};
exports.getAssetId = getAssetId;<|endoftext|>


<commit_before>/*
 * Copyright 2009-2011 Mozilla Foundation and contributors
 * Licensed under the New BSD license. See LICENSE.txt or:
 * http://opensource.org/licenses/BSD-3-Clause
 */

define(function(require, exports, module) {

var test = require('test/assert');
var helpers = require('gclitest/helpers');
var mockCommands = require('gclitest/mockCommands');

exports.setup = function(options) {
  mockCommands.setup();
  helpers.setup(options);
};

exports.shutdown = function(options) {
  mockCommands.shutdown();
  helpers.shutdown(options);
};

exports.testNode = function(options) {
  var requisition = options.display.requisition;

  helpers.setInput('tse ');
  helpers.check({
    input:  'tse ',
    hints:      '<node> [options]',
    markup: 'VVVV',
    cursor: 4,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: { status: 'INCOMPLETE', message: '' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :');
  helpers.check({
    input:  'tse :',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        arg: ' :',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :root');
  helpers.check({
    input:  'tse :root',
    hints:           ' [options]',
    markup: 'VVVVVVVVV',
    cursor: 9,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :root ');
  helpers.check({
    input:  'tse :root ',
    hints:            '[options]',
    markup: 'VVVVVVVVVV',
    cursor: 10,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root ', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse #gcli-input');
  helpers.check({
    input:  'tse #gcli-input',
    hints:                 ' [options]',
    markup: 'VVVVVVVVVVVVVVV',
    cursor: 15,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' #gcli-input', status: 'VALID', message: '' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.id,
          'gcli-input',
          '#gcli-input id');

  helpers.setInput('tse #gcli-nomatch');
  helpers.check({
    input:  'tse #gcli-nomatch',
    hints:                   ' [options]',
    markup: 'VVVVIIIIIIIIIIIII',
    cursor: 17,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' #gcli-nomatch',
        status: 'INCOMPLETE',
        message: 'No matches'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse #');
  helpers.check({
    input:  'tse #',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' #',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse .');
  helpers.check({
    input:  'tse .',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' .',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse *');
  helpers.check({
    input:  'tse *',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' *',
        status: 'ERROR',
        message: 'Too many matches (128)'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
};

exports.testNodes = function(options) {
  var requisition = options.display.requisition;

  helpers.setInput('tse :root --nodes div');
  helpers.check({
    input:  'tse :root --nodes div',
    hints:                       ' [options]',
    markup: 'VVVVVVVVVVVVVVVVVVVVV',
    cursor: 21,
    current: 'nodes',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { arg: ' --nodes div', status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse :root --nodes2 div');
  helpers.check({
    input:  'tse :root --nodes2 div',
    hints:                       ' [options]',
    markup: 'VVVVVVVVVVVVVVVVVVVVVV',
    cursor: 22,
    current: 'nodes2',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { arg: ' --nodes2 div', status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse --nodes ffff');
  helpers.check({
    input:  'tse --nodes ffff',
    hints:                  ' <node> [options]',
    markup: 'VVVVIIIIIIIVIIII',
    cursor: 16,
    current: 'nodes',
    status: 'ERROR',
    outputState: 'false:default',
    tooltipState: 'true:isError',
    args: {
      command: { name: 'tse' },
      node: { value: undefined, arg: '', status: 'INCOMPLETE', message: '' },
      nodes: { value: undefined, arg: ' --nodes ffff', status: 'INCOMPLETE', message: 'No matches' },
      nodes2: { arg: '', status: 'VALID', message: '' },
    }
  });
  test.is(requisition.getAssignment('nodes2').value.constructor.name,
          'NodeList',
          '#gcli-input id');

  helpers.setInput('tse --nodes2 ffff');
  helpers.check({
    input:  'tse --nodes2 ffff',
    hints:                   ' <node> [options]',
    markup: 'VVVVVVVVVVVVVVVVV',
    cursor: 17,
    current: 'nodes2',
    status: 'ERROR',
    outputState: 'false:default',
    tooltipState: 'false:default',
    args: {
      command: { name: 'tse' },
      node: { value: undefined, arg: '', status: 'INCOMPLETE', message: '' },
      nodes: { arg: '', status: 'VALID', message: '' },
      nodes2: { arg: ' --nodes2 ffff', status: 'VALID', message: '' },
    }
  });
  test.is(requisition.getAssignment('nodes').value.constructor.name,
          'NodeList',
          '#gcli-input id');
  test.is(requisition.getAssignment('nodes2').value.constructor.name,
          'NodeList',
          '#gcli-input id');
};

});
<commit_msg>nohelp-773227: Comment out fragile tests<commit_after>/*
 * Copyright 2009-2011 Mozilla Foundation and contributors
 * Licensed under the New BSD license. See LICENSE.txt or:
 * http://opensource.org/licenses/BSD-3-Clause
 */

define(function(require, exports, module) {

var test = require('test/assert');
var helpers = require('gclitest/helpers');
var mockCommands = require('gclitest/mockCommands');

exports.setup = function(options) {
  mockCommands.setup();
  helpers.setup(options);
};

exports.shutdown = function(options) {
  mockCommands.shutdown();
  helpers.shutdown(options);
};

exports.testNode = function(options) {
  var requisition = options.display.requisition;

  helpers.setInput('tse ');
  helpers.check({
    input:  'tse ',
    hints:      '<node> [options]',
    markup: 'VVVV',
    cursor: 4,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: { status: 'INCOMPLETE', message: '' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :');
  helpers.check({
    input:  'tse :',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        arg: ' :',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :root');
  helpers.check({
    input:  'tse :root',
    hints:           ' [options]',
    markup: 'VVVVVVVVV',
    cursor: 9,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse :root ');
  helpers.check({
    input:  'tse :root ',
    hints:            '[options]',
    markup: 'VVVVVVVVVV',
    cursor: 10,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root ', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse #gcli-input');
  helpers.check({
    input:  'tse #gcli-input',
    hints:                 ' [options]',
    markup: 'VVVVVVVVVVVVVVV',
    cursor: 15,
    current: 'node',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' #gcli-input', status: 'VALID', message: '' },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.id,
          'gcli-input',
          '#gcli-input id');

  helpers.setInput('tse #gcli-nomatch');
  helpers.check({
    input:  'tse #gcli-nomatch',
    hints:                   ' [options]',
    markup: 'VVVVIIIIIIIIIIIII',
    cursor: 17,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' #gcli-nomatch',
        status: 'INCOMPLETE',
        message: 'No matches'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse #');
  helpers.check({
    input:  'tse #',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' #',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse .');
  helpers.check({
    input:  'tse .',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' .',
        status: 'ERROR',
        message: 'Syntax error in CSS query'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });

  helpers.setInput('tse *');
  helpers.check({
    input:  'tse *',
    hints:       ' [options]',
    markup: 'VVVVE',
    cursor: 5,
    current: 'node',
    status: 'ERROR',
    args: {
      command: { name: 'tse' },
      node: {
        value: undefined,
        arg: ' *',
        status: 'ERROR',
        // message: 'Too many matches (128)'
      },
      nodes: { status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
};

exports.testNodes = function(options) {
  var requisition = options.display.requisition;

  helpers.setInput('tse :root --nodes div');
  helpers.check({
    input:  'tse :root --nodes div',
    hints:                       ' [options]',
    markup: 'VVVVVVVVVVVVVVVVVVVVV',
    cursor: 21,
    current: 'nodes',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { arg: ' --nodes div', status: 'VALID' },
      nodes2: { status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse :root --nodes2 div');
  helpers.check({
    input:  'tse :root --nodes2 div',
    hints:                       ' [options]',
    markup: 'VVVVVVVVVVVVVVVVVVVVVV',
    cursor: 22,
    current: 'nodes2',
    status: 'VALID',
    args: {
      command: { name: 'tse' },
      node: { arg: ' :root', status: 'VALID' },
      nodes: { status: 'VALID' },
      nodes2: { arg: ' --nodes2 div', status: 'VALID' }
    }
  });
  test.is(requisition.getAssignment('node').value.tagName,
          'HTML',
          '#gcli-input id');

  helpers.setInput('tse --nodes ffff');
  helpers.check({
    input:  'tse --nodes ffff',
    hints:                  ' <node> [options]',
    markup: 'VVVVIIIIIIIVIIII',
    cursor: 16,
    current: 'nodes',
    status: 'ERROR',
    outputState: 'false:default',
    tooltipState: 'true:isError',
    args: {
      command: { name: 'tse' },
      node: { value: undefined, arg: '', status: 'INCOMPLETE', message: '' },
      nodes: { value: undefined, arg: ' --nodes ffff', status: 'INCOMPLETE', message: 'No matches' },
      nodes2: { arg: '', status: 'VALID', message: '' },
    }
  });
  /*
  test.is(requisition.getAssignment('nodes2').value.constructor.name,
          'NodeList',
          '#gcli-input id');
  */

  helpers.setInput('tse --nodes2 ffff');
  helpers.check({
    input:  'tse --nodes2 ffff',
    hints:                   ' <node> [options]',
    markup: 'VVVVVVVVVVVVVVVVV',
    cursor: 17,
    current: 'nodes2',
    status: 'ERROR',
    outputState: 'false:default',
    tooltipState: 'false:default',
    args: {
      command: { name: 'tse' },
      node: { value: undefined, arg: '', status: 'INCOMPLETE', message: '' },
      nodes: { arg: '', status: 'VALID', message: '' },
      nodes2: { arg: ' --nodes2 ffff', status: 'VALID', message: '' },
    }
  });
  /*
  test.is(requisition.getAssignment('nodes').value.constructor.name,
          'NodeList',
          '#gcli-input id');
  test.is(requisition.getAssignment('nodes2').value.constructor.name,
          'NodeList',
          '#gcli-input id');
  */
};

});
<|endoftext|>


<commit_before>module.exports = function list(collection) {
    'use strict';

    var _ = require('underscore'),
        DEFAULT = {
        PAGE_SIZE: 20,
        PAGE_SKIP_SIZE: 0
    };

    function getListPageSize(options){
        return (options.limit !== null && _.isNumber(options.limit)) ? parseInt(options.limit, 10) : DEFAULT.PAGE_SIZE;
    }

    function getListSkipSize(options){
        return (options.skip !== null && _.isNumber(options.skip)) ? parseInt(options.skip) : DEFAULT.PAGE_SKIP_SIZE;
    }

    return function(kontx, next){

        var _ = require('underscore'),
            async = require('async'),
            db = require('../db'),
            args = kontx.args,
            options = {
                query: ''
            };

        if(!_.isArray(kontx.args)){
            args = [kontx.args];
        }

        if(!_.isUndefined(args[0])){
            options.limit = getListPageSize(args[0]);
            options.skip = getListSkipSize(args[0]);
        }
        else {
            options.limit = DEFAULT.PAGE_SIZE;
            options.skip = DEFAULT.PAGE_SKIP_SIZE;
        }

        async.parallel(
            [
                function(cb){
                    db[collection].list(options).then(function(value){
                            cb(null, value);
                        },
                        function(err){
                            cb(err);
                        }).done();
                },
                function(cb){
                    db.users.describe(options).then(function(value){
                            cb(null, value);
                        },
                        function(err){
                            cb(err);
                        }).done();
                }
            ],function(err, results){
                if(err){
                    next(err);
                }
                else {
                    kontx.payload = {
                        total: results[1][0].count,
                        pagesize: options.limit,
                        results: results[0]
                    };
                    next();
                }
            }
        );
    };
};<commit_msg>Added limit and skip to the response of the list module.<commit_after>module.exports = function list(collection) {
    'use strict';

    var _ = require('underscore'),
        DEFAULT = {
        PAGE_SIZE: 20,
        PAGE_SKIP_SIZE: 0
    };

    function getListPageSize(options){
        return (options.limit !== null && _.isNumber(options.limit)) ? parseInt(options.limit, 10) : DEFAULT.PAGE_SIZE;
    }

    function getListSkipSize(options){
        return (options.skip !== null && _.isNumber(options.skip)) ? parseInt(options.skip) : DEFAULT.PAGE_SKIP_SIZE;
    }

    return function(kontx, next){

        var _ = require('underscore'),
            async = require('async'),
            db = require('../db'),
            args = kontx.args,
            options = {
                query: ''
            };

        if(!_.isArray(kontx.args)){
            args = [kontx.args];
        }

        if(!_.isUndefined(args[0])){
            options.limit = getListPageSize(args[0]);
            options.skip = getListSkipSize(args[0]);
        }
        else {
            options.limit = DEFAULT.PAGE_SIZE;
            options.skip = DEFAULT.PAGE_SKIP_SIZE;
        }

        async.parallel(
            [
                function(cb){
                    db[collection].list(options).then(function(value){
                            cb(null, value);
                        },
                        function(err){
                            cb(err);
                        }).done();
                },
                function(cb){
                    db.users.describe(options).then(function(value){
                            cb(null, value);
                        },
                        function(err){
                            cb(err);
                        }).done();
                }
            ],function(err, results){
                if(err){
                    next(err);
                }
                else {
                    kontx.payload = {
                        total: results[1][0].count,
                        limit: options.limit,
                        skip: options.skip,
                        results: results[0]
                    };
                    next();
                }
            }
        );
    };
};<|endoftext|>


<commit_before>var each = require('lodash/each'),
    bunyan = require('bunyan'),
    isObject = require('lodash/isObject'),
    Bunyan2Loggly = require('bunyan-loggly'),
    jsonStringifySafe = require('json-stringify-safe'),
    GhostPrettyStream = require('./PrettyStream');

function GhostLogger(options) {
    options = options || {};
    var self = this;

    this.env = options.env || 'development';
    this.domain = options.domain || 'localhost';
    this.transports = options.transports || ['stdout'];
    this.level = process.env.LEVEL || options.level || 'info';
    this.mode = process.env.MODE || options.mode || 'short';
    this.path = options.path || process.cwd();
    this.loggly = options.loggly || {};

    // special env variable to enable long mode and level info
    if (process.env.LOIN) {
        this.level = 'info';
        this.mode = 'long';
    }

    // ensure we have a trailing slash
    if (!this.path.match(/\/$|\\$/)) {
        this.path = this.path + '/';
    }

    this.rotation = options.rotation || {
            enabled: false,
            period: '1w',
            count: 100
        };

    this.streams = {};
    this.setSerializers();

    each(this.transports, function (transport) {
        self['set' + transport.slice(0, 1).toUpperCase() + transport.slice(1) + 'Stream']();
    });
}

GhostLogger.prototype.setStdoutStream = function () {
    var prettyStdOut = new GhostPrettyStream({
        mode: this.mode
    });

    prettyStdOut.pipe(process.stdout);

    this.streams.stdout = {
        name: 'stdout',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                type: 'raw',
                stream: prettyStdOut,
                level: this.level
            }],
            serializers: this.serializers
        })
    };
};

GhostLogger.prototype.setLogglyStream = function () {
    var logglyStream = new Bunyan2Loggly({
        token: this.loggly.token,
        subdomain: this.loggly.subdomain,
        tags: this.loggly.tags
    });

    this.streams.loggly = {
        name: 'loggly',
        match: this.loggly.match,
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                type: 'raw',
                stream: logglyStream,
                level: 'error'
            }],
            serializers: this.serializers
        })
    };
};

/**
 * by default we log into two files
 * 1. file-errors: all errors only
 * 2. file-all: everything
 */
GhostLogger.prototype.setFileStream = function () {
    // e.g. http://my-domain.com --> http___my_domain_com
    var sanitizedDomain = this.domain.replace(/[^\w]/gi, '_');

    this.streams['file-errors'] = {
        name: 'file',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                path: this.path + sanitizedDomain + '_' + this.env + '.error.log',
                level: 'error'
            }],
            serializers: this.serializers
        })
    };

    this.streams['file-all'] = {
        name: 'file',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                path: this.path + sanitizedDomain + '_' + this.env + '.log',
                level: this.level
            }],
            serializers: this.serializers
        })
    };

    if (this.rotation.enabled) {
        this.streams['rotation-errors'] = {
            name: 'rotation-errors',
            log: bunyan.createLogger({
                name: 'Log',
                streams: [{
                    type: 'rotating-file',
                    path: this.path + sanitizedDomain + '_' + this.env + '.error.log',
                    period: this.rotation.period,
                    count: this.rotation.count,
                    level: "error"
                }],
                serializers: this.serializers
            })
        };

        this.streams['rotation-all'] = {
            name: 'rotation-all',
            log: bunyan.createLogger({
                name: 'Log',
                streams: [{
                    type: 'rotating-file',
                    path: this.path + sanitizedDomain + '_' + this.env + '.log',
                    period: this.rotation.period,
                    count: this.rotation.count,
                    level: this.level
                }],
                serializers: this.serializers
            })
        };
    }
};

// @TODO: add correlation identifier
// @TODO: res.on('finish') has no access to the response body
GhostLogger.prototype.setSerializers = function setSerializers() {
    var self = this;

    this.serializers = {
        req: function (req) {
            return {
                meta: {
                    requestId: req.requestId,
                    userId: req.userId
                },
                url: req.url,
                method: req.method,
                originalUrl: req.originalUrl,
                params: req.params,
                headers: self.removeSensitiveData(req.headers),
                body: self.removeSensitiveData(req.body),
                query: self.removeSensitiveData(req.query)
            };
        },
        res: function (res) {
            return {
                _headers: self.removeSensitiveData(res._headers),
                statusCode: res.statusCode,
                responseTime: res.responseTime
            };
        },
        err: function (err) {
            return {
                id: err.id,
                domain: self.domain,
                code: err.code,
                name: err.errorType,
                statusCode: err.statusCode,
                level: err.level,
                message: err.message,
                context: err.context,
                help: err.help,
                stack: err.stack,
                hideStack: err.hideStack
            };
        }
    };
};

GhostLogger.prototype.removeSensitiveData = function removeSensitiveData(obj) {
    var newObj = {};

    each(obj, function (value, key) {
        try {
            if (isObject(value)) {
                value = removeSensitiveData(value);
            }

            if (!key.match(/pin|password|authorization|cookie/gi)) {
                newObj[key] = value;
            }
        } catch (err) {
            newObj[key] = value;
        }
    });

    return newObj;
};

/**
 * Because arguments can contain lot's of different things, we prepare the arguments here.
 * This function allows us to use logging very flexible!
 *
 * logging.info('HEY', 'DU') --> is one string
 * logging.info({}, {}) --> is one object
 * logging.error(new Error()) --> is {err: new Error()}
 */
GhostLogger.prototype.log = function log(type, args) {
    var self = this,
        modifiedArguments;

    each(args, function (value) {
        if (value instanceof Error) {
            if (!modifiedArguments) {
                modifiedArguments = {};
            }

            modifiedArguments.err = value;
        } else if (isObject(value)) {
            if (!modifiedArguments) {
                modifiedArguments = {};
            }

            var keys = Object.keys(value);
            each(keys, function (key) {
                modifiedArguments[key] = value[key];
            });
        } else {
            if (!modifiedArguments) {
                modifiedArguments = '';
            }

            modifiedArguments += value;
            modifiedArguments += ' ';
        }
    });

    each(self.streams, function (logger) {
        // only stream the log if regex matches log entry
        // use jsonStringifySafe because req/res can contain circular dependencies
        if (logger.match) {
            if (new RegExp(logger.match).test(jsonStringifySafe(modifiedArguments).replace(/"/g, ''))) {
                logger.log[type](modifiedArguments);
            }
        } else {
            logger.log[type](modifiedArguments);
        }
    });
};

GhostLogger.prototype.info = function info() {
    this.log('info', arguments);
};

GhostLogger.prototype.warn = function warn() {
    this.log('warn', arguments);
};

GhostLogger.prototype.error = function error() {
    this.log('error', arguments);
};

module.exports = GhostLogger;
<commit_msg>🎨  require bunyan-loggly only when used<commit_after>var each = require('lodash/each'),
    bunyan = require('bunyan'),
    isObject = require('lodash/isObject'),
    jsonStringifySafe = require('json-stringify-safe'),
    GhostPrettyStream = require('./PrettyStream');

function GhostLogger(options) {
    options = options || {};
    var self = this;

    this.env = options.env || 'development';
    this.domain = options.domain || 'localhost';
    this.transports = options.transports || ['stdout'];
    this.level = process.env.LEVEL || options.level || 'info';
    this.mode = process.env.MODE || options.mode || 'short';
    this.path = options.path || process.cwd();
    this.loggly = options.loggly || {};

    // special env variable to enable long mode and level info
    if (process.env.LOIN) {
        this.level = 'info';
        this.mode = 'long';
    }

    // ensure we have a trailing slash
    if (!this.path.match(/\/$|\\$/)) {
        this.path = this.path + '/';
    }

    this.rotation = options.rotation || {
            enabled: false,
            period: '1w',
            count: 100
        };

    this.streams = {};
    this.setSerializers();

    each(this.transports, function (transport) {
        self['set' + transport.slice(0, 1).toUpperCase() + transport.slice(1) + 'Stream']();
    });
}

GhostLogger.prototype.setStdoutStream = function () {
    var prettyStdOut = new GhostPrettyStream({
        mode: this.mode
    });

    prettyStdOut.pipe(process.stdout);

    this.streams.stdout = {
        name: 'stdout',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                type: 'raw',
                stream: prettyStdOut,
                level: this.level
            }],
            serializers: this.serializers
        })
    };
};

GhostLogger.prototype.setLogglyStream = function () {
    var Bunyan2Loggly = require('bunyan-loggly');

    var logglyStream = new Bunyan2Loggly({
        token: this.loggly.token,
        subdomain: this.loggly.subdomain,
        tags: this.loggly.tags
    });

    this.streams.loggly = {
        name: 'loggly',
        match: this.loggly.match,
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                type: 'raw',
                stream: logglyStream,
                level: 'error'
            }],
            serializers: this.serializers
        })
    };
};

/**
 * by default we log into two files
 * 1. file-errors: all errors only
 * 2. file-all: everything
 */
GhostLogger.prototype.setFileStream = function () {
    // e.g. http://my-domain.com --> http___my_domain_com
    var sanitizedDomain = this.domain.replace(/[^\w]/gi, '_');

    this.streams['file-errors'] = {
        name: 'file',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                path: this.path + sanitizedDomain + '_' + this.env + '.error.log',
                level: 'error'
            }],
            serializers: this.serializers
        })
    };

    this.streams['file-all'] = {
        name: 'file',
        log: bunyan.createLogger({
            name: 'Log',
            streams: [{
                path: this.path + sanitizedDomain + '_' + this.env + '.log',
                level: this.level
            }],
            serializers: this.serializers
        })
    };

    if (this.rotation.enabled) {
        this.streams['rotation-errors'] = {
            name: 'rotation-errors',
            log: bunyan.createLogger({
                name: 'Log',
                streams: [{
                    type: 'rotating-file',
                    path: this.path + sanitizedDomain + '_' + this.env + '.error.log',
                    period: this.rotation.period,
                    count: this.rotation.count,
                    level: "error"
                }],
                serializers: this.serializers
            })
        };

        this.streams['rotation-all'] = {
            name: 'rotation-all',
            log: bunyan.createLogger({
                name: 'Log',
                streams: [{
                    type: 'rotating-file',
                    path: this.path + sanitizedDomain + '_' + this.env + '.log',
                    period: this.rotation.period,
                    count: this.rotation.count,
                    level: this.level
                }],
                serializers: this.serializers
            })
        };
    }
};

// @TODO: add correlation identifier
// @TODO: res.on('finish') has no access to the response body
GhostLogger.prototype.setSerializers = function setSerializers() {
    var self = this;

    this.serializers = {
        req: function (req) {
            return {
                meta: {
                    requestId: req.requestId,
                    userId: req.userId
                },
                url: req.url,
                method: req.method,
                originalUrl: req.originalUrl,
                params: req.params,
                headers: self.removeSensitiveData(req.headers),
                body: self.removeSensitiveData(req.body),
                query: self.removeSensitiveData(req.query)
            };
        },
        res: function (res) {
            return {
                _headers: self.removeSensitiveData(res._headers),
                statusCode: res.statusCode,
                responseTime: res.responseTime
            };
        },
        err: function (err) {
            return {
                id: err.id,
                domain: self.domain,
                code: err.code,
                name: err.errorType,
                statusCode: err.statusCode,
                level: err.level,
                message: err.message,
                context: err.context,
                help: err.help,
                stack: err.stack,
                hideStack: err.hideStack
            };
        }
    };
};

GhostLogger.prototype.removeSensitiveData = function removeSensitiveData(obj) {
    var newObj = {};

    each(obj, function (value, key) {
        try {
            if (isObject(value)) {
                value = removeSensitiveData(value);
            }

            if (!key.match(/pin|password|authorization|cookie/gi)) {
                newObj[key] = value;
            }
        } catch (err) {
            newObj[key] = value;
        }
    });

    return newObj;
};

/**
 * Because arguments can contain lot's of different things, we prepare the arguments here.
 * This function allows us to use logging very flexible!
 *
 * logging.info('HEY', 'DU') --> is one string
 * logging.info({}, {}) --> is one object
 * logging.error(new Error()) --> is {err: new Error()}
 */
GhostLogger.prototype.log = function log(type, args) {
    var self = this,
        modifiedArguments;

    each(args, function (value) {
        if (value instanceof Error) {
            if (!modifiedArguments) {
                modifiedArguments = {};
            }

            modifiedArguments.err = value;
        } else if (isObject(value)) {
            if (!modifiedArguments) {
                modifiedArguments = {};
            }

            var keys = Object.keys(value);
            each(keys, function (key) {
                modifiedArguments[key] = value[key];
            });
        } else {
            if (!modifiedArguments) {
                modifiedArguments = '';
            }

            modifiedArguments += value;
            modifiedArguments += ' ';
        }
    });

    each(self.streams, function (logger) {
        // only stream the log if regex matches log entry
        // use jsonStringifySafe because req/res can contain circular dependencies
        if (logger.match) {
            if (new RegExp(logger.match).test(jsonStringifySafe(modifiedArguments).replace(/"/g, ''))) {
                logger.log[type](modifiedArguments);
            }
        } else {
            logger.log[type](modifiedArguments);
        }
    });
};

GhostLogger.prototype.info = function info() {
    this.log('info', arguments);
};

GhostLogger.prototype.warn = function warn() {
    this.log('warn', arguments);
};

GhostLogger.prototype.error = function error() {
    this.log('error', arguments);
};

module.exports = GhostLogger;
<|endoftext|>


<commit_before>'use strict';

var util = require('util'),
    _ = require('lodash'),
    events = require('events'),
    Client = require('./client'),
    protocol = require('./protocol'),
    uuid = require('node-uuid'),
    Offset = require('./offset'),
    async = require("async"),
    errors = require('./errors'),
    zk = require('./zookeeper'),
    ZookeeperConsumerMappings = zk.ZookeeperConsumerMappings;

var DEFAULTS = {
    groupId: 'kafka-node-group',
    // Auto commit config
    autoCommit: true,
    autoCommitMsgCount: 100,
    autoCommitIntervalMs: 5000,
    // Fetch message config
    fetchMaxWaitMs: 100,
    paused: false,
    maxNumSegments: 1000,
    fetchMinBytes: 1,
    fetchMaxBytes: 1024 * 1024,
    fromOffset: false
};

var nextId = (function () {
    var id = 0;
    return function () {
        return id++;
    }
})();

var HighLevelConsumer = function (client, topics, options) {
    if (!topics) {
        throw new Error('Must have payloads');
    }
    this.fetchCount = 0;
    this.client = client;
    this.options = _.defaults((options || {}), DEFAULTS);
    this.initialised = false;
    this.ready = false;
    this.paused = options.paused;
    this.rebalancing = false;
    this.id = options.groupId + '_' + uuid.v4();
    this.payloads = this.buildPayloads(topics);
    this.topicPayloads = this.buildTopicPayloads(topics);
    this.connect();
}
util.inherits(HighLevelConsumer, events.EventEmitter);

HighLevelConsumer.prototype.buildPayloads = function (payloads) {
    var self = this;
    return payloads.map(function (p) {
        if (typeof p !== 'object') p = { topic: p };
        p.partition = p.partition || 0;
        p.offset = p.offset || 0;
        p.maxBytes = self.options.fetchMaxBytes;
        p.metadata = 'm'; // metadata can be arbitrary
        return p;
    });
}

// Initially this will always be empty - until after a re-balance
HighLevelConsumer.prototype.buildTopicPayloads = function (topics) {
    var self = this;
    return topics.map(function (j) {
        var k = { topic: j.topic };
        return k;
    });
}

// Provide the topic payloads if requested
HighLevelConsumer.prototype.getTopicPayloads = function () {
    if (!this.rebalancing) return this.topicPayloads;
    return null;
}

HighLevelConsumer.prototype.connect = function () {
    var self = this;
    //Client alreadyexists
    if (this.client.ready)
        this.init();

    this.client.on('ready', function () {
        if (!self.initialised) self.init();

        // Check the topics exist and create a watcher on them
        var topics = self.payloads.map(function (p) {
            return p.topic;
        })

        self.client.topicExists(topics, function (err, topics) {
            if (err) {
                return self.emit('error', new errors.TopicsNotExistError(topics));
            }
            self.initialised = true;
        });
    });

    // Wait for the consumer to be ready
    this.on('ready', function () {
        self.ready = true;

        self.fetchOffset(self.topicPayloads, function (err, topics) {
            if (err) {
                return self.emit('error', err);
            }

            self.updateOffsets(topics, true);
            self.fetch();
        });
    });

    function rebalance() {

        if (!self.rebalancing) {

            self.emit('rebalancing');

            self.rebalancing = true;
            // Nasty hack to retry 3 times to re-balance - TBD fix this
            var oldTopicPayloads = JSON.parse(JSON.stringify(self.topicPayloads));
            self.topicPayloads = [];
            self.rebalanceAttempt(oldTopicPayloads, function (err) {
                if (err) {
                    console.log(err);
                    setTimeout(function () {
                        self.rebalanceAttempt(oldTopicPayloads, function (err) {
                            if (err) {
                                console.log(err);
                                setTimeout(function () {
                                    self.rebalanceAttempt(oldTopicPayloads, function (err) {
                                        if (err) {
                                            self.rebalancing = false;
                                            return self.emit('error', new errors.FailedToRebalanceConsumerError(self));
                                        }
                                        else {
                                            self.rebalancing = false;
                                            self.ready = true;
                                            return self.emit('ready');
                                        }
                                    })
                                }, 1000);
                            }
                            else {
                                self.rebalancing = false;
                                self.ready = true;
                                return self.emit('ready');
                            }
                        })
                    }, 1000);

                }
                else {
                    self.rebalancing = false;
                    self.ready = true;
                    return self.emit('ready');
                }
            });
        }
    }

    // Wait for the consumer to be ready
    this.on('registered', function () {
        rebalance();
    });

    // Register for re-balances (broker or consumer changes)
    this.client.zk.on('consumersChanged', function () {
        if (self.ready) {
            rebalance();
        }
        else console.log("consumersChanged not ready");
    });
    this.client.on('brokersChanged', function () {
        if (self.ready) {
            rebalance();
        }
        else console.log("brokersChanged not ready");
    });

    this.client.on('error', function (err) {
        self.emit('error', err);
    });

    this.client.on('close', function () {
        console.log('close');
    });

    this.client.on('brokersChanged', function () {
        var topicNames = self.topicPayloads.map(function (p) {
            return p.topic;
        });
        this.refreshMetadata(topicNames, 0, function () {
            self.fetch();
        });
    });

    this.on('offsetOutOfRange', function (topic) {
        topic.maxNum = self.options.maxNumSegments;
        topic.metadata = 'm';
        topic.time = Date.now();
        self.offsetRequest([topic], function (err, offsets) {
            if (err) self.emit('error', new errors.InvalidConsumerOffsetError(self));
            else {
                var min = Math.min.apply(null, offsets[topic.topic][topic.partition]);
                // set minimal offset
                self.setOffset(topic.topic, topic.partition, min);
            }
        });
    });

    // 'done' will be emit when a message fetch request complete
    this.on('done', function (topics) {
        self.updateOffsets(topics);
        setImmediate(function () {
            self.fetch();
        });
    });
}

HighLevelConsumer.prototype.rebalanceAttempt = function (oldTopicPayloads, cb) {
    var self = this;
    // Do the rebalance.....
    var consumerPerTopicMap;
    var newTopicPayloads = [];
    console.log("HighLevelConsumer %s is attempting to rebalance", self.id);
    async.series([

            // Stop fetching data and commit offsets
            function (callback) {
                console.log("HighLevelConsumer %s stopping data read during rebalance", self.id);
                self.stop(function () {
                    callback();
                })
            },

            // Assemble the data
            function (callback) {
                console.log("HighLevelConsumer %s assembling data for rebalance", self.id);
                self.client.zk.getConsumersPerTopic(self.options.groupId, function (err, obj) {
                    if (err) {
                        callback(err);
                    }
                    else {
                        consumerPerTopicMap = obj;
                        callback();
                    }
                });
            },

            // Release current partitions
            function (callback) {
                console.log("HighLevelConsumer %s releasing current partitions during rebalance", self.id);
                async.each(oldTopicPayloads, function (tp, cbb) {
                    if (tp['partition'] !== undefined) {
                        self.client.zk.deletePartitionOwnership(self.options.groupId, tp['topic'], tp['partition'], function (err) {
                            if (err) {
                                cbb(err);
                            }
                            else cbb();
                        });
                    }
                    else {
                        cbb();
                    }
                }, function (err) {
                    if (err) callback(err);
                    else callback();
                });
            },

            // Reblannce
            function (callback) {
                console.log("HighLevelConsumer %s determining the partitions to own during rebalance", self.id);
                for (var topic in consumerPerTopicMap.consumerTopicMap[self.id]) {
                    var topicToAdd = consumerPerTopicMap.consumerTopicMap[self.id][topic];
                    var numberOfConsumers = consumerPerTopicMap.topicConsumerMap[topicToAdd].length;
                    var numberOfPartition = consumerPerTopicMap.topicPartitionMap[topicToAdd].length;
                    var partitionsPerConsumer = Math.floor(numberOfPartition / numberOfConsumers);
                    var extraPartitions = numberOfPartition % numberOfConsumers;
                    var currentConsumerIndex;
                    for (var index in consumerPerTopicMap.topicConsumerMap[topicToAdd]) {
                        if (consumerPerTopicMap.topicConsumerMap[topicToAdd][index] === self.id) {
                            currentConsumerIndex = parseInt(index);
                            break;
                        }
                    }
                    var extraBit = currentConsumerIndex;
                    if (currentConsumerIndex > extraPartitions) extraBit = extraPartitions
                    var startPart = partitionsPerConsumer * currentConsumerIndex + extraBit;
                    var extraNParts = 1;
                    if (currentConsumerIndex + 1 > extraPartitions) extraNParts = 0;
                    var nParts = partitionsPerConsumer + extraNParts;

                    for (var i = startPart; i < startPart + nParts; i++) {
                        newTopicPayloads.push({ topic: topicToAdd, partition: consumerPerTopicMap.topicPartitionMap[topicToAdd][i], offset: -1, maxBytes: self.options.fetchMaxBytes, metadata: 'm'});
                    }
                }
                callback();
            },

            // Update ZK with new ownership
            function (callback) {
                if (newTopicPayloads.length) {
                    console.log("HighLevelConsumer %s gaining ownership of partitions during rebalance", self.id);
                    async.each(newTopicPayloads, function (tp, cbb) {
                        if (tp['partition'] !== undefined) {
                            self.client.zk.addPartitionOwnership(self.id, self.options.groupId, tp['topic'], tp['partition'], function (err) {
                                if (err) {
                                    cbb(err);
                                }
                                else cbb();
                            });
                        }
                        else {
                            cbb();
                        }
                    }, function (err) {
                        if (err)
                            callback(err);
                        else
                            callback();
                    });
                }
                else {
                    console.log("HighLevelConsumer %s has been assigned no partitions during rebalance", self.id);
                    callback();
                }
            },

            // Update the new topic offsets
            function (callback) {
                self.topicPayloads = newTopicPayloads;
                callback();
            }],
        function (err) {
            if (err) {
                console.log("HighLevelConsumer %s rebalance attempt failed", self.id);
                cb(err);
            }
            else {
                console.log("HighLevelConsumer %s rebalance attempt was successful", self.id);
                cb();
            }
        });
}

HighLevelConsumer.prototype.init = function () {

    var self = this
        , topics = self.topicPayloads.map(function (p) {
            return p.topic;
        })

    if (!self.topicPayloads.length) {
        return;
    }

    self.registerConsumer(function (err) {
        if (err) {
            return self.emit('error', new errors.FailedToRegisterConsumerError(self));
        }

        // Close the
        return self.emit('registered');
    });

}

/*
 * Update offset info in current payloads
 * @param {Object} Topic-partition-offset
 * @param {Boolean} Don't commit when initing consumer
 */
HighLevelConsumer.prototype.updateOffsets = function (topics, initing) {
    this.topicPayloads.forEach(function (p) {
        if (!_.isEmpty(topics[p.topic]) && topics[p.topic][p.partition] !== undefined)
            p.offset = topics[p.topic][p.partition] + 1;
    });

    if (this.options.autoCommit && !initing) this.autoCommit();
}

function autoCommit(force, cb) {
    if (this.committing && !force) return cb && cb('Offset committing');

    this.committing = true;
    setTimeout(function () {
        this.committing = false;
    }.bind(this), this.options.autoCommitIntervalMs);

    var commits = this.topicPayloads.reduce(function (out, p) {
        if (p.offset !== 0)
            out.push(_.defaults({ offset: p.offset - 1 }, p));
        return out;
    }, []);

    if (commits.length) {
        this.client.sendOffsetCommitRequest(this.options.groupId, commits, cb);
    } else {
        cb && cb();
    }
}
HighLevelConsumer.prototype.commit = HighLevelConsumer.prototype.autoCommit = autoCommit;

HighLevelConsumer.prototype.fetch = function () {
    if (!this.ready) return;

    if (this.rebalancing) return;

    // If paused request 0 data - need to pause in this way otherwise the connection is dropped
    if (this.paused) {

        // Build empty payloads on demand
        var empty_payloads = this.topicPayloads.map(function (j) {
            var k = { topic: j.topic, partition: j.partition || 0, offset: j.offset || 0, maxBytes: 0, metadata: 'm'};
            return k;
        });
        this.client.sendFetchRequest(this, empty_payloads, this.options.fetchMaxWaitMs, 0);
    }
    else {
        this.client.sendFetchRequest(this, this.topicPayloads, this.options.fetchMaxWaitMs, this.options.fetchMinBytes);
    }
}

HighLevelConsumer.prototype.fetchOffset = function (payloads, cb) {
    this.client.sendOffsetFetchRequest(this.options.groupId, payloads, cb);
}

HighLevelConsumer.prototype.offsetRequest = function (payloads, cb) {
    this.client.sendOffsetRequest(payloads, cb);
}

/**
 * Register a consumer against a group
 *
 * @param consumer to register
 *
 * @param {Client~failedToRegisterConsumerCallback} cb A function to call the consumer has been registered
 */
HighLevelConsumer.prototype.registerConsumer = function (cb) {
    this.client.zk.registerConsumer(this.options.groupId, this.id, this.payloads, function (err) {
        if (err) return cb(err);
        cb();
    })
    this.client.zk.listConsumers(this.options.groupId);
}

HighLevelConsumer.prototype.addTopics = function (topics, cb) {
    var self = this;
    if (!this.ready) {
        setTimeout(function () {
                self.addTopics(topics, cb)
            }
            , 100);
        return;
    }
    this.client.addTopics(
        topics,
        function (err, added) {
            if (err) return cb && cb(err, added);

            var payloads = self.buildPayloads(topics);
            // update offset of topics that will be added
            self.fetchOffset(payloads, function (err, offsets) {
                if (err) return cb(err);
                payloads.forEach(function (p) {
                    p.offset = offsets[p.topic][p.partition];
                    self.topicPayloads.push(p);
                });
                cb && cb(null, added);
            });
        }
    );
}

HighLevelConsumer.prototype.removeTopics = function (topics, cb) {
    topics = typeof topics === 'string' ? [topics] : topics;
    this.payloads = this.payloads.filter(function (p) {
        return !~topics.indexOf(p.topic);
    });

    this.client.removeTopicMetadata(topics, cb);
}

HighLevelConsumer.prototype.close = function (force, cb) {
    this.ready = false;
    if (typeof force === 'function') {
        cb = force;
        force = false;
    }

    if (force) {
        this.commit(force, function (err) {
            this.client.close(cb);
        }.bind(this));
    } else {
        this.client.close(cb);
    }
}

HighLevelConsumer.prototype.stop = function (cb) {
    this.ready = false;
    if (!this.options.autoCommit) return cb && cb();
    this.commit(true, function (err) {
        cb && cb();
    }.bind(this));
}

HighLevelConsumer.prototype.setOffset = function (topic, partition, offset) {
    this.topicPayloads.every(function (p) {
        if (p.topic === topic && p.partition == partition) {
            p.offset = offset;
            return false;
        }
        return true;
    });
}

HighLevelConsumer.prototype.pause = function () {
    this.paused = true;
}

HighLevelConsumer.prototype.resume = function () {
    this.paused = false;
}

module.exports = HighLevelConsumer;
<commit_msg>fixing generation of errors<commit_after>'use strict';

var util = require('util'),
    _ = require('lodash'),
    events = require('events'),
    Client = require('./client'),
    protocol = require('./protocol'),
    uuid = require('node-uuid'),
    Offset = require('./offset'),
    async = require("async"),
    errors = require('./errors'),
    zk = require('./zookeeper'),
    ZookeeperConsumerMappings = zk.ZookeeperConsumerMappings;

var DEFAULTS = {
    groupId: 'kafka-node-group',
    // Auto commit config
    autoCommit: true,
    autoCommitMsgCount: 100,
    autoCommitIntervalMs: 5000,
    // Fetch message config
    fetchMaxWaitMs: 100,
    paused: false,
    maxNumSegments: 1000,
    fetchMinBytes: 1,
    fetchMaxBytes: 1024 * 1024,
    fromOffset: false
};

var nextId = (function () {
    var id = 0;
    return function () {
        return id++;
    }
})();

var HighLevelConsumer = function (client, topics, options) {
    if (!topics) {
        throw new Error('Must have payloads');
    }
    this.fetchCount = 0;
    this.client = client;
    this.options = _.defaults((options || {}), DEFAULTS);
    this.initialised = false;
    this.ready = false;
    this.paused = options.paused;
    this.rebalancing = false;
    this.id = options.groupId + '_' + uuid.v4();
    this.payloads = this.buildPayloads(topics);
    this.topicPayloads = this.buildTopicPayloads(topics);
    this.connect();
}
util.inherits(HighLevelConsumer, events.EventEmitter);

HighLevelConsumer.prototype.buildPayloads = function (payloads) {
    var self = this;
    return payloads.map(function (p) {
        if (typeof p !== 'object') p = { topic: p };
        p.partition = p.partition || 0;
        p.offset = p.offset || 0;
        p.maxBytes = self.options.fetchMaxBytes;
        p.metadata = 'm'; // metadata can be arbitrary
        return p;
    });
}

// Initially this will always be empty - until after a re-balance
HighLevelConsumer.prototype.buildTopicPayloads = function (topics) {
    var self = this;
    return topics.map(function (j) {
        var k = { topic: j.topic };
        return k;
    });
}

// Provide the topic payloads if requested
HighLevelConsumer.prototype.getTopicPayloads = function () {
    if (!this.rebalancing) return this.topicPayloads;
    return null;
}

HighLevelConsumer.prototype.connect = function () {
    var self = this;
    //Client alreadyexists
    if (this.client.ready)
        this.init();

    this.client.on('ready', function () {
        if (!self.initialised) self.init();

        // Check the topics exist and create a watcher on them
        var topics = self.payloads.map(function (p) {
            return p.topic;
        })

        self.client.topicExists(topics, function (err, topics) {
            if (err) {
                return self.emit('error', new errors.TopicsNotExistError(topics));
            }
            self.initialised = true;
        });
    });

    // Wait for the consumer to be ready
    this.on('ready', function () {
        self.ready = true;

        self.fetchOffset(self.topicPayloads, function (err, topics) {
            if (err) {
                return self.emit('error', err);
            }

            self.updateOffsets(topics, true);
            self.fetch();
        });
    });

    function rebalance() {

        if (!self.rebalancing) {

            self.emit('rebalancing');

            self.rebalancing = true;
            // Nasty hack to retry 3 times to re-balance - TBD fix this
            var oldTopicPayloads = JSON.parse(JSON.stringify(self.topicPayloads));
            self.topicPayloads = [];
            self.rebalanceAttempt(oldTopicPayloads, function (err) {
                if (err) {
                    console.log(err);
                    setTimeout(function () {
                        self.rebalanceAttempt(oldTopicPayloads, function (err) {
                            if (err) {
                                console.log(err);
                                setTimeout(function () {
                                    self.rebalanceAttempt(oldTopicPayloads, function (err) {
                                        if (err) {
                                            self.rebalancing = false;
                                            return self.emit('error', new errors.FailedToRebalanceConsumerError(err.toString()));
                                        }
                                        else {
                                            self.rebalancing = false;
                                            self.ready = true;
                                            return self.emit('ready');
                                        }
                                    })
                                }, 1000);
                            }
                            else {
                                self.rebalancing = false;
                                self.ready = true;
                                return self.emit('ready');
                            }
                        })
                    }, 1000);

                }
                else {
                    self.rebalancing = false;
                    self.ready = true;
                    return self.emit('ready');
                }
            });
        }
    }

    // Wait for the consumer to be ready
    this.on('registered', function () {
        rebalance();
    });

    // Register for re-balances (broker or consumer changes)
    this.client.zk.on('consumersChanged', function () {
        if (self.ready) {
            rebalance();
        }
        else console.log("consumersChanged not ready");
    });
    this.client.on('brokersChanged', function () {
        if (self.ready) {
            rebalance();
        }
        else console.log("brokersChanged not ready");
    });

    this.client.on('error', function (err) {
        self.emit('error', err);
    });

    this.client.on('close', function () {
        console.log('close');
    });

    this.client.on('brokersChanged', function () {
        var topicNames = self.topicPayloads.map(function (p) {
            return p.topic;
        });
        this.refreshMetadata(topicNames, 0, function () {
            self.fetch();
        });
    });

    this.on('offsetOutOfRange', function (topic) {
        topic.maxNum = self.options.maxNumSegments;
        topic.metadata = 'm';
        topic.time = Date.now();
        self.offsetRequest([topic], function (err, offsets) {
            if (err) self.emit('error', new errors.InvalidConsumerOffsetError(self));
            else {
                var min = Math.min.apply(null, offsets[topic.topic][topic.partition]);
                // set minimal offset
                self.setOffset(topic.topic, topic.partition, min);
            }
        });
    });

    // 'done' will be emit when a message fetch request complete
    this.on('done', function (topics) {
        self.updateOffsets(topics);
        setImmediate(function () {
            self.fetch();
        });
    });
}

HighLevelConsumer.prototype.rebalanceAttempt = function (oldTopicPayloads, cb) {
    var self = this;
    // Do the rebalance.....
    var consumerPerTopicMap;
    var newTopicPayloads = [];
    console.log("HighLevelConsumer %s is attempting to rebalance", self.id);
    async.series([

            // Stop fetching data and commit offsets
            function (callback) {
                console.log("HighLevelConsumer %s stopping data read during rebalance", self.id);
                self.stop(function () {
                    callback();
                })
            },

            // Assemble the data
            function (callback) {
                console.log("HighLevelConsumer %s assembling data for rebalance", self.id);
                self.client.zk.getConsumersPerTopic(self.options.groupId, function (err, obj) {
                    if (err) {
                        callback(err);
                    }
                    else {
                        consumerPerTopicMap = obj;
                        callback();
                    }
                });
            },

            // Release current partitions
            function (callback) {
                console.log("HighLevelConsumer %s releasing current partitions during rebalance", self.id);
                async.each(oldTopicPayloads, function (tp, cbb) {
                    if (tp['partition'] !== undefined) {
                        self.client.zk.deletePartitionOwnership(self.options.groupId, tp['topic'], tp['partition'], function (err) {
                            if (err) {
                                cbb(err);
                            }
                            else cbb();
                        });
                    }
                    else {
                        cbb();
                    }
                }, function (err) {
                    if (err) callback(err);
                    else callback();
                });
            },

            // Reblannce
            function (callback) {
                console.log("HighLevelConsumer %s determining the partitions to own during rebalance", self.id);
                for (var topic in consumerPerTopicMap.consumerTopicMap[self.id]) {
                    var topicToAdd = consumerPerTopicMap.consumerTopicMap[self.id][topic];
                    var numberOfConsumers = consumerPerTopicMap.topicConsumerMap[topicToAdd].length;
                    var numberOfPartition = consumerPerTopicMap.topicPartitionMap[topicToAdd].length;
                    var partitionsPerConsumer = Math.floor(numberOfPartition / numberOfConsumers);
                    var extraPartitions = numberOfPartition % numberOfConsumers;
                    var currentConsumerIndex;
                    for (var index in consumerPerTopicMap.topicConsumerMap[topicToAdd]) {
                        if (consumerPerTopicMap.topicConsumerMap[topicToAdd][index] === self.id) {
                            currentConsumerIndex = parseInt(index);
                            break;
                        }
                    }
                    var extraBit = currentConsumerIndex;
                    if (currentConsumerIndex > extraPartitions) extraBit = extraPartitions
                    var startPart = partitionsPerConsumer * currentConsumerIndex + extraBit;
                    var extraNParts = 1;
                    if (currentConsumerIndex + 1 > extraPartitions) extraNParts = 0;
                    var nParts = partitionsPerConsumer + extraNParts;

                    for (var i = startPart; i < startPart + nParts; i++) {
                        newTopicPayloads.push({ topic: topicToAdd, partition: consumerPerTopicMap.topicPartitionMap[topicToAdd][i], offset: -1, maxBytes: self.options.fetchMaxBytes, metadata: 'm'});
                    }
                }
                callback();
            },

            // Update ZK with new ownership
            function (callback) {
                if (newTopicPayloads.length) {
                    console.log("HighLevelConsumer %s gaining ownership of partitions during rebalance", self.id);
                    async.each(newTopicPayloads, function (tp, cbb) {
                        if (tp['partition'] !== undefined) {
                            self.client.zk.addPartitionOwnership(self.id, self.options.groupId, tp['topic'], tp['partition'], function (err) {
                                if (err) {
                                    cbb(err);
                                }
                                else cbb();
                            });
                        }
                        else {
                            cbb();
                        }
                    }, function (err) {
                        if (err)
                            callback(err);
                        else
                            callback();
                    });
                }
                else {
                    console.log("HighLevelConsumer %s has been assigned no partitions during rebalance", self.id);
                    callback();
                }
            },

            // Update the new topic offsets
            function (callback) {
                self.topicPayloads = newTopicPayloads;
                callback();
            }],
        function (err) {
            if (err) {
                console.log("HighLevelConsumer %s rebalance attempt failed", self.id);
                cb(err);
            }
            else {
                console.log("HighLevelConsumer %s rebalance attempt was successful", self.id);
                cb();
            }
        });
}

HighLevelConsumer.prototype.init = function () {

    var self = this
        , topics = self.topicPayloads.map(function (p) {
            return p.topic;
        })

    if (!self.topicPayloads.length) {
        return;
    }

    self.registerConsumer(function (err) {
        if (err) {
            return self.emit('error', new errors.FailedToRegisterConsumerError(err.toString()));
        }

        // Close the
        return self.emit('registered');
    });

}

/*
 * Update offset info in current payloads
 * @param {Object} Topic-partition-offset
 * @param {Boolean} Don't commit when initing consumer
 */
HighLevelConsumer.prototype.updateOffsets = function (topics, initing) {
    this.topicPayloads.forEach(function (p) {
        if (!_.isEmpty(topics[p.topic]) && topics[p.topic][p.partition] !== undefined)
            p.offset = topics[p.topic][p.partition] + 1;
    });

    if (this.options.autoCommit && !initing) this.autoCommit();
}

function autoCommit(force, cb) {
    if (this.committing && !force) return cb && cb('Offset committing');

    this.committing = true;
    setTimeout(function () {
        this.committing = false;
    }.bind(this), this.options.autoCommitIntervalMs);

    var commits = this.topicPayloads.reduce(function (out, p) {
        if (p.offset !== 0)
            out.push(_.defaults({ offset: p.offset - 1 }, p));
        return out;
    }, []);

    if (commits.length) {
        this.client.sendOffsetCommitRequest(this.options.groupId, commits, cb);
    } else {
        cb && cb();
    }
}
HighLevelConsumer.prototype.commit = HighLevelConsumer.prototype.autoCommit = autoCommit;

HighLevelConsumer.prototype.fetch = function () {
    if (!this.ready) return;

    if (this.rebalancing) return;

    // If paused request 0 data - need to pause in this way otherwise the connection is dropped
    if (this.paused) {

        // Build empty payloads on demand
        var empty_payloads = this.topicPayloads.map(function (j) {
            var k = { topic: j.topic, partition: j.partition || 0, offset: j.offset || 0, maxBytes: 0, metadata: 'm'};
            return k;
        });
        this.client.sendFetchRequest(this, empty_payloads, this.options.fetchMaxWaitMs, 0);
    }
    else {
        this.client.sendFetchRequest(this, this.topicPayloads, this.options.fetchMaxWaitMs, this.options.fetchMinBytes);
    }
}

HighLevelConsumer.prototype.fetchOffset = function (payloads, cb) {
    this.client.sendOffsetFetchRequest(this.options.groupId, payloads, cb);
}

HighLevelConsumer.prototype.offsetRequest = function (payloads, cb) {
    this.client.sendOffsetRequest(payloads, cb);
}

/**
 * Register a consumer against a group
 *
 * @param consumer to register
 *
 * @param {Client~failedToRegisterConsumerCallback} cb A function to call the consumer has been registered
 */
HighLevelConsumer.prototype.registerConsumer = function (cb) {
    this.client.zk.registerConsumer(this.options.groupId, this.id, this.payloads, function (err) {
        if (err) return cb(err);
        cb();
    })
    this.client.zk.listConsumers(this.options.groupId);
}

HighLevelConsumer.prototype.addTopics = function (topics, cb) {
    var self = this;
    if (!this.ready) {
        setTimeout(function () {
                self.addTopics(topics, cb)
            }
            , 100);
        return;
    }
    this.client.addTopics(
        topics,
        function (err, added) {
            if (err) return cb && cb(err, added);

            var payloads = self.buildPayloads(topics);
            // update offset of topics that will be added
            self.fetchOffset(payloads, function (err, offsets) {
                if (err) return cb(err);
                payloads.forEach(function (p) {
                    p.offset = offsets[p.topic][p.partition];
                    self.topicPayloads.push(p);
                });
                cb && cb(null, added);
            });
        }
    );
}

HighLevelConsumer.prototype.removeTopics = function (topics, cb) {
    topics = typeof topics === 'string' ? [topics] : topics;
    this.payloads = this.payloads.filter(function (p) {
        return !~topics.indexOf(p.topic);
    });

    this.client.removeTopicMetadata(topics, cb);
}

HighLevelConsumer.prototype.close = function (force, cb) {
    this.ready = false;
    if (typeof force === 'function') {
        cb = force;
        force = false;
    }

    if (force) {
        this.commit(force, function (err) {
            this.client.close(cb);
        }.bind(this));
    } else {
        this.client.close(cb);
    }
}

HighLevelConsumer.prototype.stop = function (cb) {
    this.ready = false;
    if (!this.options.autoCommit) return cb && cb();
    this.commit(true, function (err) {
        cb && cb();
    }.bind(this));
}

HighLevelConsumer.prototype.setOffset = function (topic, partition, offset) {
    this.topicPayloads.every(function (p) {
        if (p.topic === topic && p.partition == partition) {
            p.offset = offset;
            return false;
        }
        return true;
    });
}

HighLevelConsumer.prototype.pause = function () {
    this.paused = true;
}

HighLevelConsumer.prototype.resume = function () {
    this.paused = false;
}

module.exports = HighLevelConsumer;
<|endoftext|>