<commit_before>(function(root, factory) {
  if (typeof exports !== 'undefined') {
    // Node.js
    module.exports = factory(root);
  } else if (typeof define === 'function' && define.amd) {
    // AMD
    define(function() {
      // If Basbosa is loaded, do not register a global logger
      if (typeof Basbosa === 'undefined') {
        return root.BasbosaValidation = factory(root);
      }
      return factory(root);
    });
  } else {
    // Browser globals
    root.BasbosaValidation = factory(root);
  }
}(this, function(root) {

	var BasbosaValidation = function() {}, instance;
	var _ = require('underscore');
	var Async = require('async');
	function checkArguments(obj, field) {
		if(field === undefined || field ==='function') return false;
		if(obj !== undefined && field !== 'function') return true;
	};
	function finalize (one, second, returned) {
		if(one === 'function') {
			second = one;
		}
		typeof second === 'function' && second(returned);
		return returned;
	};
	BasbosaValidation.prototype.Rules = {
		urlPattern : new RegExp("((http|https)(:\/\/))?([a-zA-Z0-9]+[.]{1}){2}[a-zA-z0-9]+(\/{1}[a-zA-Z0-9]+)*\/?", "i"),
		isObjectId : function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.string(obj, field) && !(obj[field].length != 24) && (/[\da-f]{24}/.test(obj[field])));
			} else {
				return finalize(field, cb , !(obj[field].length != 24) && (/[\da-f]{24}/.test(obj)));
			}
		},
		isPresent: function(obj, field, options, cb) {
			if (typeof options === 'function') {
		    cb = options;
		    options = undefined;
		  }
			typeof cb === 'function' &&	cb(_.has(obj, field));
			return _.has(obj, field);
		},
		string: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && _.isString(obj[field]));
			} else {
				return finalize(field, cb, _.isString(obj));
			}
		},
		numeric: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && _.isNumber(obj[field]));
			} else {
				return finalize(field, cb, (_.isNumber(obj)));
			}
		},
		url: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && BasbosaValidation.prototype.Rules.urlPattern.test(obj[field]));
			} else {
				return finalize(field, cb, BasbosaValidation.prototype.Rules.urlPattern.test(obj));
			}
		},
		maxLength: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (obj[field].length >= options.max));
				} else {
					return finalize(options, cb , (obj.length >= field.max));
				}
			}
		},
		email :  function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i.test(obj[field]));
			} else {
				return finalize(field, cb , /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i.test(obj));
			}
		},
		minLength: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (obj[field].length <= options.min));
				} else {
					return finalize(options, cb , (obj.length <= field.min));
				}
			}
		},
		rang: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , BasbosaValidation.prototype.Rules.minLength(obj, field, {min:	options.max}) && BasbosaValidation.prototype.Rules.maxLength(obj, field, {max:	options.min}));
				}
			}
		},
		isUnique: function(obj, field, options, cb) {

		},
		alphaNumeric: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && (/^[A-Za-z]*$|^[A-Za-z][A-Za-z0-9]*$/i.test(obj[field])));
			} else {
				return finalize(field, cb , /^[A-Za-z]*$|^[A-Za-z][A-Za-z0-9]*$/i.test(obj));
			}
		},
		extension: function(obj, field, options, cb) {
			var partOne = "/\.(", partTwo = ")$/i";
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (partOne + options.extension + partTwo).test(obj[field]));
				} else {
					return finalize(options, cb , (partOne + field.extension + partTwo).test(obj));
				}
			}
		}
	};
	BasbosaValidation.prototype.validate = function(obj, rules, callback) {
		var rulesFunctions = [];
		if(_.isEmpty(rules)) typeof callback === 'function' && callback(null, true);
		_.each(rules, function(validationRules, fieldName) {
			_.each(validationRules, function(rule) {
				rulesFunctions.push(function(cb) {
					rule.rule(obj,fieldName,{min: rule.min, max: rule.max}, function(result) {
						if(!result) cb(rule.message);
						if(result) cb(null, result);
					});
				});
			});
		});
		Async.parallel(rulesFunctions, function(error, result) {
			if(error) {
				typeof callback === 'function' && callback(error);
			} else {
				if(result) typeof callback === 'function' && callback(null, result);
			}

		});
	};
	BasbosaValidation.prototype.validateA = function(validator, msg, callback) {
		if(_.isEmpty(validator)) {
			typeof callback === 'function' && callback(false);
			return false;
		}	else {
			var validation_results = new Array();
			_.each(validator, function(fields) {
				_.each(fields, function(validators, key) {
					_.each(validators, function(obj) {
						if(_.isArray(obj)) {
							var func = _.first(obj);
							var args = _.union( [msg, key], _.rest(obj, 1) );
							if( func.length < _.size(args) ) {
								typeof callback === 'function' && callback(false);
								return false;
							} else if( func.length > _.size( args ) ) {
								typeof callback === 'function' && callback(false);
								return false;
							} else {
								validation_results.push( func.apply(null, args ) );
							}
						} else {
							validation_results.push(obj(msg, key));
						}
					});
				});
			});
			if (_.all(validation_results, _.identity)) {
				typeof callback === 'function' && callback(true);
				return true;
			} else {
				typeof callback === 'function' && callback(false);
				return false;
			}
		}
	};
	if (typeof instance === 'undefined') instance = new BasbosaValidation;
  if (typeof Basbosa !== 'undefined') Basbosa.add('BasbosaValidation', instance);
  return instance;
}));<commit_msg>Remove old Validate function<commit_after>(function(root, factory) {
  if (typeof exports !== 'undefined') {
    // Node.js
    module.exports = factory(root);
  } else if (typeof define === 'function' && define.amd) {
    // AMD
    define(function() {
      // If Basbosa is loaded, do not register a global logger
      if (typeof Basbosa === 'undefined') {
        return root.BasbosaValidation = factory(root);
      }
      return factory(root);
    });
  } else {
    // Browser globals
    root.BasbosaValidation = factory(root);
  }
}(this, function(root) {

	var BasbosaValidation = function() {}, instance;
	var _ = require('underscore');
	var Async = require('async');
	function checkArguments(obj, field) {
		if(field === undefined || field ==='function') return false;
		if(obj !== undefined && field !== 'function') return true;
	};
	function finalize (one, second, returned) {
		if(one === 'function') {
			second = one;
		}
		typeof second === 'function' && second(returned);
		return returned;
	};
	BasbosaValidation.prototype.Rules = {
		urlPattern : new RegExp("((http|https)(:\/\/))?([a-zA-Z0-9]+[.]{1}){2}[a-zA-z0-9]+(\/{1}[a-zA-Z0-9]+)*\/?", "i"),
		isObjectId : function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.string(obj, field) && !(obj[field].length != 24) && (/[\da-f]{24}/.test(obj[field])));
			} else {
				return finalize(field, cb , !(obj[field].length != 24) && (/[\da-f]{24}/.test(obj)));
			}
		},
		isPresent: function(obj, field, options, cb) {
			if (typeof options === 'function') {
		    cb = options;
		    options = undefined;
		  }
			typeof cb === 'function' &&	cb(_.has(obj, field));
			return _.has(obj, field);
		},
		string: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && _.isString(obj[field]));
			} else {
				return finalize(field, cb, _.isString(obj));
			}
		},
		numeric: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && _.isNumber(obj[field]));
			} else {
				return finalize(field, cb, (_.isNumber(obj)));
			}
		},
		url: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && BasbosaValidation.prototype.Rules.urlPattern.test(obj[field]));
			} else {
				return finalize(field, cb, BasbosaValidation.prototype.Rules.urlPattern.test(obj));
			}
		},
		maxLength: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (obj[field].length >= options.max));
				} else {
					return finalize(options, cb , (obj.length >= field.max));
				}
			}
		},
		email :  function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i.test(obj[field]));
			} else {
				return finalize(field, cb , /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i.test(obj));
			}
		},
		minLength: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (obj[field].length <= options.min));
				} else {
					return finalize(options, cb , (obj.length <= field.min));
				}
			}
		},
		rang: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , BasbosaValidation.prototype.Rules.minLength(obj, field, {min:	options.max}) && BasbosaValidation.prototype.Rules.maxLength(obj, field, {max:	options.min}));
				}
			}
		},
		isUnique: function(obj, field, options, cb) {

		},
		alphaNumeric: function(obj, field, options, cb) {
			if(checkArguments(obj, field)) {
				return finalize(options, cb , BasbosaValidation.prototype.Rules.isPresent(obj, field) && (/^[A-Za-z]*$|^[A-Za-z][A-Za-z0-9]*$/i.test(obj[field])));
			} else {
				return finalize(field, cb , /^[A-Za-z]*$|^[A-Za-z][A-Za-z0-9]*$/i.test(obj));
			}
		},
		extension: function(obj, field, options, cb) {
			var partOne = "/\.(", partTwo = ")$/i";
			if(checkArguments(obj, field)) {
				if(BasbosaValidation.prototype.Rules.isPresent(obj, field)) {
					return finalize(options, cb , (partOne + options.extension + partTwo).test(obj[field]));
				} else {
					return finalize(options, cb , (partOne + field.extension + partTwo).test(obj));
				}
			}
		}
	};
	BasbosaValidation.prototype.validate = function(obj, rules, callback) {
		var rulesFunctions = [];
		if(_.isEmpty(rules)) typeof callback === 'function' && callback(null, true);
		_.each(rules, function(validationRules, fieldName) {
			_.each(validationRules, function(rule) {
				rulesFunctions.push(function(cb) {
					rule.rule(obj,fieldName,{min: rule.min, max: rule.max}, function(result) {
						if(!result) cb(rule.message);
						if(result) cb(null, result);
					});
				});
			});
		});
		Async.parallel(rulesFunctions, function(error, result) {
			if(error) {
				typeof callback === 'function' && callback(error);
			} else {
				if(result) typeof callback === 'function' && callback(null, result);
			}

		});
	};
	if (typeof instance === 'undefined') instance = new BasbosaValidation;
  if (typeof Basbosa !== 'undefined') Basbosa.add('BasbosaValidation', instance);
  return instance;
}));<|endoftext|>


<commit_before>import request from 'request';
import cheerio from 'cheerio';

import queryMap from './queryMap.json'

export function sendInitialQuery(params, callback) {
  if (!params.query) {
    return process.nextTick(callback, new Error('params is not provided'));
  }
  const j = request.jar();
  const alisEndpoint = `${params.alisEndpoint}`;
  const firstPageUrl = `/alis/EK/do_searh.php?radiodate=simple&valueINP=${encodeURIComponent(params.query)}&tema=${queryMap.recordType[params.recordType]}&tag=${queryMap.queryType[params.queryType]}`
  const INITIAL_URL = `${alisEndpoint}${firstPageUrl}`;

  request({ url: INITIAL_URL, jar: j }, (err, response, body) => {
    if (err) {
      callback(err);
      return;
    }
    callback(null, { page: body, jar: j });
  });
}

export function getPage(options, callback) {
  request({ url: options.url, jar: options.jar }, (err, response, body) => {
    if (err) {
      return callback(err);
    }
    callback(null, body);
  });
}

export function getNextPageUrl($) {
  const pageLink = $('#Agt');
  const pageUrl = $(pageLink).attr('href');
  return pageUrl;
}

export function getTotal($) {
  const total = $('.listbzstat').first().text().match(/\d+(?=\sзап\.)/)[0];
  return total;
}

export function getItems($) {
  const items = $('.article').map(function (i, el) {
    return {
      id: $(el).attr('id'),
      title: $(el).text().trim(),
    };
  }).toArray();
  return items;
}

export function getItemsId($) {
  const items = $('.article').map(function (i, el) {
    return {
      id: $(el).attr('id'),
    };
  }).toArray();
  return items;

}

export function getNumberedPageUrls($) {
  const pageLinks = $('a[href^=\'do_other\']');
  const relativePageUrls = $(pageLinks).map((i, link) => $(link).attr('href').replace(/\r|\n/g, '')).toArray();
  return relativePageUrls;
}

export function parsePage(body) {
  const $ = cheerio.load(body);
  return $;
}

export function processItems(memo, q, options, callback) {
  if (!options) {
    return process.nextTick(callback, new Error('options is not provided'));
  }

  getPage({ url: options.url, jar: options.jar }, (err, body) => {
    if (err) return callback(err);
    const $ = parsePage(body);

    const items = getItemsId($);

    const nextPageUrl = getNextPageUrl($);
    const remainingQueue = q.slice(1);
    if (q.length === 1) {
      remainingQueue.push(`${nextPageUrl}`);
    }
    items.forEach((item) => {
      memo.push(item);
    });
    callback(null, memo, remainingQueue);
  });
}

export function run(fn, q, memo, options, callback) {
  if (!q) {
    return new Error('q is not provided');
  }

  if (q[0] === 'undefined') {
    return callback(null, memo);
  }
  fn(memo, q, { url: `${options.alisEndpoint}/alis/EK/${q[0]}`, jar: options.jar }, (err, nextMemo, nextQ) => {
    if (err) {
      return err;
    }
    console.log('memo.length: ', memo.length);
    console.log('nextQ.length: ', nextQ.length);
    run(fn, nextQ, memo, options, callback);
  });
}

export function getRecordsByQuery(initParams, callback) {

  sendInitialQuery(initParams, (err, res) => {
    if (err) {
      return new Error(err);
    }
    const options = {
      alisEndpoint: initParams.alisEndpoint,
      jar: res.jar,
    };
    const $ = parsePage(res.page);
    const firstNumberedPageUrls = getNumberedPageUrls($);
    const remainingQueue = firstNumberedPageUrls;
    run(processItems, remainingQueue, [], options, (runErr, memo) => {
      if (err) {
        return callback(err);
      }
      callback(null, memo);
    });
  });
}
<commit_msg>fix some error-handl.<commit_after>import request from 'request';
import cheerio from 'cheerio';

import queryMap from './queryMap.json'

export function sendInitialQuery(params, callback) {
  if (!params) {
    return callback(new Error('params is not provided'));
  }
  if (!params.query) {
    return callback(new Error('query string is not provided'));
  }
  if (!params.queryType) {
    return callback(new Error('queryType string is not provided'));
  }
  if (!params.recordType) {
    return callback(new Error('recordType string is not provided'));
  }
  const j = request.jar();
  const alisEndpoint = `${params.alisEndpoint}`;
  const firstPageUrl = `/alis/EK/do_searh.php?radiodate=simple&valueINP=${encodeURIComponent(params.query)}&tema=${queryMap.recordType[params.recordType]}&tag=${queryMap.queryType[params.queryType]}`
  const INITIAL_URL = `${alisEndpoint}${firstPageUrl}`;

  request({ url: INITIAL_URL, jar: j }, (err, response, body) => {
    if (err) {
      return callback(err);
    }
    callback(null, { page: body, jar: j });
  });
}

export function getPage(options, callback) {
  request({ url: options.url, jar: options.jar }, (err, response, body) => {
    if (err) {
      return callback(err);
    }
    callback(null, body);
  });
}

export function getNextPageUrl($) {
  const pageLink = $('#Agt');
  const pageUrl = $(pageLink).attr('href');
  return pageUrl;
}

export function getTotal($) {
  const total = $('.listbzstat').first().text().match(/\d+(?=\sзап\.)/)[0];
  return total;
}

export function getItems($) {
  const items = $('.article').map(function (i, el) {
    return {
      id: $(el).attr('id'),
      title: $(el).text().trim(),
    };
  }).toArray();
  return items;
}

export function getItemsId($) {
  const items = $('.article').map(function (i, el) {
    return {
      id: $(el).attr('id'),
    };
  }).toArray();
  return items;

}

export function getNumberedPageUrls($) {
  const pageLinks = $('a[href^=\'do_other\']');
  const relativePageUrls = $(pageLinks).map((i, link) => $(link).attr('href').replace(/\r|\n/g, '')).toArray();
  return relativePageUrls;
}

export function parsePage(body) {
  const $ = cheerio.load(body);
  return $;
}

export function processItems(memo, q, options, callback) {
  if (!options) {
    return process.nextTick(callback, new Error('options is not provided'));
  }

  getPage({ url: options.url, jar: options.jar }, (err, body) => {
    if (err) return callback(err);
    const $ = parsePage(body);

    const items = getItemsId($);

    const nextPageUrl = getNextPageUrl($);
    const remainingQueue = q.slice(1);
    if (q.length === 1) {
      remainingQueue.push(`${nextPageUrl}`);
    }
    items.forEach((item) => {
      memo.push(item);
    });
    callback(null, memo, remainingQueue);
  });
}

export function run(fn, q, memo, options, callback) {
  if (!q) {
    return new Error('q is not provided');
  }

  if (q[0] === 'undefined') {
    return callback(null, memo);
  }
  fn(memo, q, { url: `${options.alisEndpoint}/alis/EK/${q[0]}`, jar: options.jar }, (err, nextMemo, nextQ) => {
    if (err) {
      return callback(err);
    }
    console.log('memo.length: ', memo.length);
    console.log('nextQ.length: ', nextQ.length);
    run(fn, nextQ, memo, options, callback);
  });
}

export function getRecordsByQuery(initParams, callback) {

  sendInitialQuery(initParams, (err, res) => {
    if (err) {
      return callback(err);
    }
    const options = {
      alisEndpoint: initParams.alisEndpoint,
      jar: res.jar,
    };
    if(res.page.match('Не результативный поиск')){
      return callback(new Error('no match'));
    }
    const $ = parsePage(res.page);
    const firstNumberedPageUrls = getNumberedPageUrls($);
    const remainingQueue = firstNumberedPageUrls;
    run(processItems, remainingQueue, [], options, (runErr, memo) => {
      if (err) {
        return callback(err);
      }
      callback(null, memo);
    });
  });
}
<|endoftext|>


<commit_before>var fs = require('fs')
var once = require('once')
var split = require('split')
var through = require('through')

var WINDOWS = process.platform === 'win32'
var EOL = WINDOWS
  ? '\r\n'
  : '\n'

exports.HOSTS = WINDOWS
  ? 'C:/Windows/System32/drivers/etc/hosts'
  : '/etc/hosts'

  /**
   * Get a list of the lines that make up the filePath. If the
   * `preserveFormatting` parameter is true, then include comments, blank lines
   * and other non-host entries in the result.
   *
   * @param  {boolean}   preserveFormatting
   * @param  {function(err, lines)=} cb
   */

exports.getFile = function (filePath, preserveFormatting, cb) {
  var lines = []
  if (typeof cb !== 'function') {
    fs.readFileSync(filePath, { encoding: 'utf8' }).split(/\r?\n/).forEach(online)
    return lines
  }

  cb = once(cb)
  fs.createReadStream(filePath, { encoding: 'utf8' })
    .pipe(split())
    .pipe(through(online))
    .on('close', function () {
      cb(null, lines)
    })
    .on('error', cb)

  function online (line) {
    // Remove all comment text from the line
    var lineSansComments = line.replace(/#.*/, '')
    var matches = /^\s*?(.+?)\s+(.+?)$/.exec(lineSansComments)
    if (matches && matches.length === 3) {
      // Found a hosts entry
      var ip = matches[1]
      var host = matches[2]
      lines.push([ip, host])
    } else {
      // Found a comment, blank line, or something else
      if (preserveFormatting) {
        lines.push(line)
      }
    }
  }
}

/**
 * Wrapper of `getFile` for getting a list of lines in the Host file
 *
 * @param  {boolean}   preserveFormatting
 * @param  {function(err, lines)=} cb
 */
exports.get = function (preserveFormatting, cb) {
  return exports.getFile(exports.HOSTS, preserveFormatting, cb)
}

/**
 * Add a rule to /etc/hosts. If the rule already exists, then this does nothing.
 *
 * @param  {string}   ip
 * @param  {string}   host
 * @param  {function(Error)=} cb
 */
exports.set = function (ip, host, cb) {
  var didUpdate = false
  if (typeof cb !== 'function') {
    return _set(exports.get(true))
  }

  exports.get(true, function (err, lines) {
    if (err) return cb(err)
    _set(lines)
  })

  function _set (lines) {
    // Try to update entry, if host already exists in file
    lines = lines.map(mapFunc)

    // If entry did not exist, let's add it
    if (!didUpdate) {
      // If the last line is empty, or just whitespace, then insert the new entry
      // right before it
      var lastLine = lines[lines.length - 1]
      if (typeof lastLine === 'string' && /\s*/.test(lastLine)) {
        lines.splice(lines.length - 1, 0, [ip, host])
      } else {
        lines.push([ip, host])
      }
    }

    exports.writeFile(lines, cb)
  }

  function mapFunc (line) {
    if (Array.isArray(line) && line[1] === host) {
      line[0] = ip
      didUpdate = true
    }
    return line
  }
}

/**
 * Remove a rule from /etc/hosts. If the rule does not exist, then this does
 * nothing.
 *
 * @param  {string}   ip
 * @param  {string}   host
 * @param  {function(Error)=} cb
 */
exports.remove = function (ip, host, cb) {
  if (typeof cb !== 'function') {
    return _remove(exports.get(true))
  }

  exports.get(true, function (err, lines) {
    if (err) return cb(err)
    _remove(lines)
  })

  function _remove (lines) {
    // Try to remove entry, if it exists
    lines = lines.filter(filterFunc)
    return exports.writeFile(lines, cb)
  }

  function filterFunc (line) {
    return !(Array.isArray(line) && line[0] === ip && line[1] === host)
  }
}

/**
 * Write out an array of lines to the host file. Assumes that they're in the
 * format that `get` returns.
 *
 * @param  {Array.<string|Array.<string>>} lines
 * @param  {function(Error)=} cb
 */
exports.writeFile = function (lines, cb) {
  lines = lines.map(function (line, lineNum) {
    if (Array.isArray(line)) {
      line = line[0] + ' ' + line[1]
    }
    return line + (lineNum === lines.length - 1 ? '' : EOL)
  })

  if (typeof cb !== 'function') {
    var stat = fs.statSync(exports.HOSTS)
    fs.writeFileSync(exports.HOSTS, lines.join(''), { mode: stat.mode })
    return true
  }

  cb = once(cb)
  fs.stat(exports.HOSTS, function (err, stat) {
    if (err) {
      return cb(err)
    }
    var s = fs.createWriteStream(exports.HOSTS, { mode: stat.mode })
    s.on('close', cb)
    s.on('error', cb)

    lines.forEach(function (data) {
      s.write(data)
    })
    s.end()
  })
}
<commit_msg>Remove trailing whitespace from parsed domains<commit_after>var fs = require('fs')
var once = require('once')
var split = require('split')
var through = require('through')

var WINDOWS = process.platform === 'win32'
var EOL = WINDOWS
  ? '\r\n'
  : '\n'

exports.HOSTS = WINDOWS
  ? 'C:/Windows/System32/drivers/etc/hosts'
  : '/etc/hosts'

  /**
   * Get a list of the lines that make up the filePath. If the
   * `preserveFormatting` parameter is true, then include comments, blank lines
   * and other non-host entries in the result.
   *
   * @param  {boolean}   preserveFormatting
   * @param  {function(err, lines)=} cb
   */

exports.getFile = function (filePath, preserveFormatting, cb) {
  var lines = []
  if (typeof cb !== 'function') {
    fs.readFileSync(filePath, { encoding: 'utf8' }).split(/\r?\n/).forEach(online)
    return lines
  }

  cb = once(cb)
  fs.createReadStream(filePath, { encoding: 'utf8' })
    .pipe(split())
    .pipe(through(online))
    .on('close', function () {
      cb(null, lines)
    })
    .on('error', cb)

  function online (line) {
    // Remove all comment text from the line
    var lineSansComments = line.replace(/#.*/, '')
    var matches = /^\s*?(.+?)\s+(\S+?)\s*$/.exec(lineSansComments)
    if (matches && matches.length === 3) {
      // Found a hosts entry
      var ip = matches[1]
      var host = matches[2]
      lines.push([ip, host])
    } else {
      // Found a comment, blank line, or something else
      if (preserveFormatting) {
        lines.push(line)
      }
    }
  }
}

/**
 * Wrapper of `getFile` for getting a list of lines in the Host file
 *
 * @param  {boolean}   preserveFormatting
 * @param  {function(err, lines)=} cb
 */
exports.get = function (preserveFormatting, cb) {
  return exports.getFile(exports.HOSTS, preserveFormatting, cb)
}

/**
 * Add a rule to /etc/hosts. If the rule already exists, then this does nothing.
 *
 * @param  {string}   ip
 * @param  {string}   host
 * @param  {function(Error)=} cb
 */
exports.set = function (ip, host, cb) {
  var didUpdate = false
  if (typeof cb !== 'function') {
    return _set(exports.get(true))
  }

  exports.get(true, function (err, lines) {
    if (err) return cb(err)
    _set(lines)
  })

  function _set (lines) {
    // Try to update entry, if host already exists in file
    lines = lines.map(mapFunc)

    // If entry did not exist, let's add it
    if (!didUpdate) {
      // If the last line is empty, or just whitespace, then insert the new entry
      // right before it
      var lastLine = lines[lines.length - 1]
      if (typeof lastLine === 'string' && /\s*/.test(lastLine)) {
        lines.splice(lines.length - 1, 0, [ip, host])
      } else {
        lines.push([ip, host])
      }
    }

    exports.writeFile(lines, cb)
  }

  function mapFunc (line) {
    if (Array.isArray(line) && line[1] === host) {
      line[0] = ip
      didUpdate = true
    }
    return line
  }
}

/**
 * Remove a rule from /etc/hosts. If the rule does not exist, then this does
 * nothing.
 *
 * @param  {string}   ip
 * @param  {string}   host
 * @param  {function(Error)=} cb
 */
exports.remove = function (ip, host, cb) {
  if (typeof cb !== 'function') {
    return _remove(exports.get(true))
  }

  exports.get(true, function (err, lines) {
    if (err) return cb(err)
    _remove(lines)
  })

  function _remove (lines) {
    // Try to remove entry, if it exists
    lines = lines.filter(filterFunc)
    return exports.writeFile(lines, cb)
  }

  function filterFunc (line) {
    return !(Array.isArray(line) && line[0] === ip && line[1] === host)
  }
}

/**
 * Write out an array of lines to the host file. Assumes that they're in the
 * format that `get` returns.
 *
 * @param  {Array.<string|Array.<string>>} lines
 * @param  {function(Error)=} cb
 */
exports.writeFile = function (lines, cb) {
  lines = lines.map(function (line, lineNum) {
    if (Array.isArray(line)) {
      line = line[0] + ' ' + line[1]
    }
    return line + (lineNum === lines.length - 1 ? '' : EOL)
  })

  if (typeof cb !== 'function') {
    var stat = fs.statSync(exports.HOSTS)
    fs.writeFileSync(exports.HOSTS, lines.join(''), { mode: stat.mode })
    return true
  }

  cb = once(cb)
  fs.stat(exports.HOSTS, function (err, stat) {
    if (err) {
      return cb(err)
    }
    var s = fs.createWriteStream(exports.HOSTS, { mode: stat.mode })
    s.on('close', cb)
    s.on('error', cb)

    lines.forEach(function (data) {
      s.write(data)
    })
    s.end()
  })
}
<|endoftext|>


<commit_before>var through = require('through')

module.exports = function BlkDatStream() {
  var buffers = []
  var buffer = new Buffer(0)
  var len = 0
  var needed = 0

  return through(function write(data) {
    len += data.length
    buffers.push(data)

    // do we have a header? and do we have enough data?
    if (needed !== 0 && len < needed) return

    // merge buffers
    buffer = Buffer.concat([buffer].concat(buffers), len)
    buffers = []

    var offset = 0
    var remaining = len

    // do we [still] have enough data?
    while (remaining >= needed) {
      // do we need to parse a magic header?
      if (needed === 0) {
        // do we have enough for a magic header?
        if (remaining < 8) break

        // read the magic header (magic number, block length)
        var magicInt = buffer.readUInt32LE(offset)
        var blockLength = buffer.readUInt32LE(offset + 4)

        if (magicInt !== 0xd9b4bef9) {
          throw new Error('Unexpected data')
        }

        // now, block length is what is needed
        needed = blockLength
        remaining -= 8
        offset += 8

        // and loop
        continue
      }

      // read the block
      var block = buffer.slice(offset, offset + needed)
      offset += needed
      remaining -= needed
      needed = 0

      // update cursor information
      buffer = buffer.slice(offset)
      len = buffer.length
      offset = 0

      // process block
      this.queue(block)
    }

    // reset buffer
    buffer = buffer.slice(offset)
    len = buffer.length
  })
}
<commit_msg>index: group cursor update<commit_after>var through = require('through')

module.exports = function BlkDatStream() {
  var buffers = []
  var buffer = new Buffer(0)
  var len = 0
  var needed = 0

  return through(function write(data) {
    len += data.length
    buffers.push(data)

    // do we have a header? and do we have enough data?
    if (needed !== 0 && len < needed) return

    // merge buffers
    buffer = Buffer.concat([buffer].concat(buffers), len)
    buffers = []

    var offset = 0
    var remaining = len

    // do we [still] have enough data?
    while (remaining >= needed) {
      // do we need to parse a magic header?
      if (needed === 0) {
        // do we have enough for a magic header?
        if (remaining < 8) break

        // read the magic header (magic number, block length)
        var magicInt = buffer.readUInt32LE(offset)
        var blockLength = buffer.readUInt32LE(offset + 4)

        if (magicInt !== 0xd9b4bef9) {
          throw new Error('Unexpected data')
        }

        // now, block length is what is needed
        needed = blockLength
        remaining -= 8
        offset += 8

        // and loop
        continue
      }

      // read the block
      var block = buffer.slice(offset, offset + needed)

      // process block
      this.queue(block)

      // update cursor information
      offset += needed
      remaining -= needed
      needed = 0
      buffer = buffer.slice(offset)
      len = buffer.length
      offset = 0
    }

    // reset buffer
    buffer = buffer.slice(offset)
    len = buffer.length
  })
}
<|endoftext|>


<commit_before>/* global Buffer */
var dgram = require('dgram');
var client = dgram.createSocket("udp4");
var broadcastAddress = "255.255.255.255";
var deviceBroadcastPort = 39169;
client.on("error", function (err) {
  console.log("server error:\n" + err.stack);
  client.close();
});

var commands = {
	deviceDescovery: 0xA5,
	setSound: 0x96,
	dataReceived1: 0xC1,
	dataReceived2: 0xD1,
	reply: 0x69,
	deviceDiscovery: 0xF1
}

var soundType = {
	shortBeepOnce: 0,
	shortBeepTwice: 1,
	shortBeepTriple: 2,
	longBeepOnce: 3,
	longBeepTwice: 4,
	longBeepTriple: 5,
};

function createDiscoveryCommand() {
	return new Buffer([commands.deviceDescovery]);
}

function setSoundCommand(param1, param2) {
	if (param1 < 0 && param1 > 65535) {
		throw new Error("param2 should be within 0 to 65535 range");
	}

	if (param2 < 0 && param2 > 255) {
		throw new Error("param2 should be within 0 to 255 range");
	}

	var command = new Buffer(4);
	command[0] = commands.setSound;
	command.writeUInt16LE(param1, 1);
	command[3] = param2;
	return command;
}

function failOnError(err) {
	if (err) {
		throw err;
	}
}

client.on("listening", function () {
	var address = client.address();
  	console.log("server listening " + address.address + ":" + address.port);
	console.log("Start listening.")
    client.setBroadcast(true);
	var setSound = setSoundCommand(0, soundType.shortBeepOnce);
	client.send(setSound, 0, setSound.length, deviceBroadcastPort, broadcastAddress, failOnError);
	var message = createDiscoveryCommand();
	client.send(message, 0, message.length, deviceBroadcastPort, broadcastAddress, failOnError);
	client.on("message", function (msg, rinfo) {
		var magicByte = msg[0];
		if (magicByte === commands.dataReceived1 || magicByte == commands.dataReceived2)
		{
			processBindingData(msg);
		} else if (magicByte == commands.deviceDiscovery) {
			registerDevice(msg);
		} else {
			console.log("Server got: " + JSON.stringify(msg.toJSON()) + " from " + rinfo.address + ":" + rinfo.port);
		}
	});
});

function readIpAddress(msg, offset, separator) {
	separator = separator || ".";
	return msg[offset].toString() + separator
		 + msg[offset + 1].toString() + separator
		 + msg[offset + 2].toString() + separator
		 + msg[offset + 3].toString();
}

function readDeviceNumber(msg, offset, separator) {
	separator = separator || "-";
	return msg[offset].toString() + separator
		 + msg[offset + 1].toString() + separator
		 + msg[offset + 2].toString() + separator
		 + msg[offset + 3].toString();
}

function processBindingData(msg) {
	var reply = new Buffer(9);
	reply[0] = commands.reply;
	for (var i = 1; i < 9; i++)
	{
		reply[i] = msg[i];
	}

	var readerIpAddressString = readIpAddress(msg, 1);
	console.log("Reader IP string", readerIpAddressString);

	console.log("Send reply", reply);
	//client.setBroadcast(false);
	client.send(reply, 0, reply.length, deviceBroadcastPort, readerIpAddressString, function(err, bytes) {
		client.setBroadcast(true);
		var jihaostr = msg.readUInt16LE(5);
		var packetNumber = msg.readUInt16LE(7);
		console.log("jihao", jihaostr);
		console.log("Packet No:", packetNumber);

		var cardNumber = msg.readUInt32LE(9);
		var cardNumberString = "000000000" + cardNumber;
		cardNumberString = cardNumberString.substr(cardNumberString.length - 10, 10);
		console.log("Card Number", cardNumberString);
	});
}

function registerDevice(msg) {
	var readerIpAddressString = readIpAddress(msg, 1);
	var readerIpAddressMaskString = readIpAddress(msg, 5);
	var readerIpAddressGatewayString = readIpAddress(msg, 9);
	console.log("Reader IP address:", readerIpAddressString);
	console.log("Reader IP mask:", readerIpAddressMaskString);
	console.log("Reader IP gateway:", readerIpAddressGatewayString);
	var jihaostr = msg.readUInt16LE(13);
	var deviceNumber = readDeviceNumber(msg, 15, "-");
	console.log("jihao", jihaostr);
	console.log("Device No:", deviceNumber);
}

client.bind({
	port: deviceBroadcastPort,
	address: "169.254.167.154"
});
<commit_msg>Extract methods which decode the message<commit_after>/* global Buffer */
var dgram = require('dgram');
var client = dgram.createSocket("udp4");
var broadcastAddress = "255.255.255.255";
var deviceBroadcastPort = 39169;

var commands = {
	deviceDescovery: 0xA5,
	setSound: 0x96,
	dataReceived1: 0xC1,
	dataReceived2: 0xD1,
	reply: 0x69,
	deviceDiscovery: 0xF1
}

var soundType = {
	shortBeepOnce: 0,
	shortBeepTwice: 1,
	shortBeepTriple: 2,
	longBeepOnce: 3,
	longBeepTwice: 4,
	longBeepTriple: 5,
};

function createDiscoveryCommand() {
	return new Buffer([commands.deviceDescovery]);
}

function setSoundCommand(param1, param2) {
	if (param1 < 0 && param1 > 65535) {
		throw new Error("param2 should be within 0 to 65535 range");
	}

	if (param2 < 0 && param2 > 255) {
		throw new Error("param2 should be within 0 to 255 range");
	}

	var command = new Buffer(4);
	command[0] = commands.setSound;
	command.writeUInt16LE(param1, 1);
	command[3] = param2;
	return command;
}

function failOnError(err) {
	if (err) {
		throw err;
	}
}

function readIpAddress(msg, offset, separator) {
	separator = separator || ".";
	return msg[offset].toString() + separator
		 + msg[offset + 1].toString() + separator
		 + msg[offset + 2].toString() + separator
		 + msg[offset + 3].toString();
}

function readDeviceNumber(msg, offset, separator) {
	separator = separator || "-";
	return msg[offset].toString() + separator
		 + msg[offset + 1].toString() + separator
		 + msg[offset + 2].toString() + separator
		 + msg[offset + 3].toString();
}

function decodeCardMessage(msg) {
	var readerIpAddressString = readIpAddress(msg, 1);
	var jihaostr = msg.readUInt16LE(5);
	var packetNumber = msg.readUInt16LE(7);

	var cardNumber = msg.readUInt32LE(9);
	var cardNumberString = "000000000" + cardNumber;
	cardNumberString = cardNumberString.substr(cardNumberString.length - 10, 10);

	return {
		readerIpAddress: readerIpAddressString,
		jihao: jihaostr,
		packetNumber: packetNumber,
		cardNumber: cardNumberString
	};
}

function createReply(msg) {
	var reply = new Buffer(9);
	reply[0] = commands.reply;
	for (var i = 1; i < 9; i++)
	{
		reply[i] = msg[i];
	}

	return reply;
}

function decodeRegisterDeviceMessage(msg) {

	var readerIpAddressString = readIpAddress(msg, 1);
	var readerIpAddressMaskString = readIpAddress(msg, 5);
	var readerIpAddressGatewayString = readIpAddress(msg, 9);
	var jihaostr = msg.readUInt16LE(13);
	var deviceNumber = readDeviceNumber(msg, 15, "-");

	return {
		readerIpAddress: readerIpAddressString,
		readerIpAddressMask: readerIpAddressMaskString,
		readerIpAddressGateway: readerIpAddressGatewayString,
		jihao: jihaostr,
		deviceNumber: deviceNumber,
	}
}

function processBindingData(msg) {
	var reply = createReply(msg);
	var messageData = decodeCardMessage(msg);
	console.log("Reader IP string", messageData.readerIpAddress);
	console.log("jihao", messageData.jihao);
	console.log("Packet No:", messageData.packetNumber);
	console.log("Card Number", messageData.cardNumber);

	console.log("Sending reply", reply);
	//client.setBroadcast(false);
	client.send(reply, 0, reply.length, deviceBroadcastPort, messageData.readerIpAddress, function(err, bytes) {
		//client.setBroadcast(true);
	});
}

function registerDevice(msg) {
	var messageData = decodeRegisterDeviceMessage(msg);
	console.log("Reader IP address:", messageData.readerIpAddress);
	console.log("Reader IP mask:", messageData.readerIpAddressMask);
	console.log("Reader IP gateway:", messageData.readerIpAddressGateway);
	console.log("jihao", messageData.jihaostr);
	console.log("Device No:", messageData.deviceNumber);
}

client.on("error", function (err) {
  console.log("server error:\n" + err.stack);
  client.close();
});

client.on("listening", function () {
	var address = client.address();
  	console.log("server listening " + address.address + ":" + address.port);
	console.log("Start listening.")
    client.setBroadcast(true);
	var setSound = setSoundCommand(0, soundType.shortBeepOnce);
	client.send(setSound, 0, setSound.length, deviceBroadcastPort, broadcastAddress, failOnError);
	var message = createDiscoveryCommand();
	client.send(message, 0, message.length, deviceBroadcastPort, broadcastAddress, failOnError);
	client.on("message", function (msg, rinfo) {
		var magicByte = msg[0];
		if (magicByte === commands.dataReceived1 || magicByte == commands.dataReceived2)
		{
			processBindingData(msg);
		} else if (magicByte == commands.deviceDiscovery) {
			registerDevice(msg);
		} else {
			console.log("Server got: " + JSON.stringify(msg.toJSON()) + " from " + rinfo.address + ":" + rinfo.port);
		}
	});
});

client.bind({
	port: deviceBroadcastPort,
	address: "169.254.167.154"
});
<|endoftext|>


<commit_before>var inside = require('turf-inside');

/**
 * Calculates the average value of a field for points
 * within a set of polygons.
 *
 * @module turf/average
 * @param {FeatureCollection} polygons a featurecollection of features with {@link Polygon} geometries
 * @param {FeatureCollection} points a featurecollection of features with {@link Point} geometries
 * @param {string} field the field from which to pull values from the properties
 * of each {@link Feature} in the `points` FeatureCollection
 * @param {string} outputField the field from which to put values from the properties
 * of each {@link Feature} in the `polygons` FeatureCollection
 * @returns {FeatureCollection} a collection of features with {@link Polygon}
 * geometries and the value of `outField` set to an average.
 * @example
 * var poly1 = turf.polygon([[[0,0],[10,0],[10,10], [0,10]]])
 * var poly2 = turf.polygon([[[10,0],[20,10],[20,20], [20,0]]])
 * var polyFC = turf.featurecollection([poly1, poly2])
 * var pt1 = turf.point(5,5, {population: 200})
 * var pt2 = turf.point(1,3, {population: 600})
 * var pt3 = turf.point(14,2, {population: 100})
 * var pt4 = turf.point(13,1, {population: 200})
 * var pt5 = turf.point(19,7, {population: 300})
 * var ptFC = turf.featurecollection([pt1, pt2, pt3, pt4, pt5])
 * var averaged = turf.average(polyFC, ptFC, 'population', 'pop_avg')
 * console.log(averaged.features[0].properties.pop_avg) // 400
 * console.log(averaged.features[1].properties.pop_avg) // 200
 */
module.exports = function(polyFC, ptFC, inField, outField, done){
  polyFC.features.forEach(function(poly){
    if(!poly.properties) poly.properties = {};
    var values = [];
    ptFC.features.forEach(function(pt){
      if (inside(pt, poly)) values.push(pt.properties[inField]);
    });
    poly.properties[outField] = average(values);
  });

  return polyFC;
}

function average(values) {
  var sum = 0;
  for (var i = 0; i < values.length; i++) {
    sum += values[i];
  }
  return sum / values.length;
}
<commit_msg>rpl-ize documentation<commit_after>var inside = require('turf-inside');

/**
 * Calculates the average value of a field for points
 * within a set of polygons.
 *
 * @module turf/average
 * @param {FeatureCollection} polygons a featurecollection of features with {@link Polygon} geometries
 * @param {FeatureCollection} points a featurecollection of features with {@link Point} geometries
 * @param {string} field the field from which to pull values from the properties
 * of each {@link Feature} in the `points` FeatureCollection
 * @param {string} outputField the field from which to put values from the properties
 * of each {@link Feature} in the `polygons` FeatureCollection
 * @returns {FeatureCollection} a collection of features with {@link Polygon}
 * geometries and the value of `outField` set to an average.
 * @example
 * var poly1 = turf.polygon([[[0,0],[10,0],[10,10], [0,10]]]);
 * var poly2 = turf.polygon([[[10,0],[20,10],[20,20], [20,0]]]);
 * var polyFC = turf.featurecollection([poly1, poly2]);
 * var pt1 = turf.point(5,5, {population: 200});
 * var pt2 = turf.point(1,3, {population: 600});
 * var pt3 = turf.point(14,2, {population: 100});
 * var pt4 = turf.point(13,1, {population: 200});
 * var pt5 = turf.point(19,7, {population: 300});
 * var ptFC = turf.featurecollection([pt1, pt2, pt3, pt4, pt5]);
 * var averaged = turf.average(polyFC, ptFC, 'population', 'pop_avg');
 *
 * //=averaged
 */
module.exports = function(polyFC, ptFC, inField, outField, done){
  polyFC.features.forEach(function(poly){
    if(!poly.properties) poly.properties = {};
    var values = [];
    ptFC.features.forEach(function(pt){
      if (inside(pt, poly)) values.push(pt.properties[inField]);
    });
    poly.properties[outField] = average(values);
  });

  return polyFC;
}

function average(values) {
  var sum = 0;
  for (var i = 0; i < values.length; i++) {
    sum += values[i];
  }
  return sum / values.length;
}
<|endoftext|>


<commit_before>const crawl = require("./src/puppeteer_utils.js").crawl;
const http = require("http");
const express = require("express");
const serveStatic = require("serve-static");
// @ts-ignore
const fallback = require("express-history-api-fallback");
const path = require("path");
const fs = require("fs");
const mkdirp = require("mkdirp");
const minify = require("html-minifier").minify;
const url = require("url");
// @ts-ignore https://github.com/peterbe/minimalcss/pull/30
const minimalcss = require("minimalcss");
const CleanCSS = require("clean-css");
const twentyKb = 20 * 1024;

const defaultOptions = {
  //# stable configurations
  port: 45678,
  source: "build",
  destination: null,
  concurrency: 4,
  include: ["/"],
  userAgent: "ReactSnap",
  headless: true,
  puppeteerArgs: [],
  publicPath: "/",
  minifyCSS: {},
  minifyHTML: {
    collapseBooleanAttributes: true,
    collapseWhitespace: true,
    decodeEntities: true,
    keepClosingSlash: true,
    sortAttributes: true,
    sortClassName: true
  },
  viewport: false, // TODO: set to mobile viewport by default?
  //# feature creeps to generate screenshots
  saveAs: "html",
  crawl: true,
  waitFor: false,
  externalServer: false,
  //# workarounds
  fixWebpackChunksIssue: true,
  removeBlobs: true,
  skipThirdPartyRequests: false,
  //# unstable configurations
  preconnectThirdParty: true,
  // Experimental. This config stands for two strategies inline and critical.
  // TODO: inline strategy can contain errors, like, confuse relative urls
  // TODO: critical strategy miss noscript fallback
  inlineCss: false,
  // Experimental. TODO: need to fix issues with sourcemaps
  sourceMaps: false,
  cacheAjaxRequests: false,
  //# even more workarounds
  removeStyleTags: false,
  preloadImages: false,
  // add async true to scripts and move them to the header, to start download earlier
  // can use <link rel="preload"> instead
  asyncJs: false
};

/**
 *
 * @param {{source: ?string, destination: ?string, include: ?Array<string>, sourceMaps: ?boolean, skipThirdPartyRequests: ?boolean }} userOptions
 * @return {*}
 */
const defaults = userOptions => {
  const options = {
    ...defaultOptions,
    ...userOptions
  };
  options.destination = options.destination || options.source;
  if (!options.include || !options.include.length)
    throw new Error("include should be an array");

  if (options.preloadResources) {
    console.log(
      "preloadResources option deprecated. Use preloadImages or cacheAjaxRequests"
    );
    process.exit(1);
  }
  if (options.minifyOptions) {
    console.log("minifyOptions option renamed to minifyHTML");
    process.exit(1);
  }

  if (options.minifyHTML && !options.minifyHTML.minifyCSS) {
    options.minifyHTML.minifyCSS = options.minifyCSS;
  }

  if (!options.publicPath.startsWith("/")) {
    options.publicPath = `/${options.publicPath}`;
  }
  options.publicPath = options.publicPath.replace(/\/$/, "");

  options.include = options.include.map(
    include => options.publicPath + include
  );
  return options;
};

/**
 *
 * @param {{page: Page, basePath: string}} opt
 */
const preloadResources = opt => {
  const {
    page,
    basePath,
    preloadImages,
    cacheAjaxRequests,
    preconnectThirdParty
  } = opt;
  const uniqueResources = new Set();
  page.on("response", async response => {
    const responseUrl = response.url;
    if (/^data:/i.test(responseUrl)) return;
    const ct = response.headers["content-type"] || "";
    const route = responseUrl.replace(basePath, "");
    if (/^http:\/\/localhost/i.test(responseUrl)) {
      if (uniqueResources.has(responseUrl)) return;
      if (preloadImages && /\.(png|jpg|jpeg|webp|gif)$/.test(responseUrl)) {
        await page.evaluate(route => {
          var linkTag = document.createElement("link");
          linkTag.setAttribute("rel", "preload");
          linkTag.setAttribute("as", "image");
          linkTag.setAttribute("href", route);
          document.body.appendChild(linkTag);
        }, route);
      } else if (cacheAjaxRequests && ct.indexOf("json") > -1) {
        const json = await response.json();
        await page.evaluate(
          (route, json) => {
            var scriptTag = document.createElement("script");
            scriptTag.type = "text/javascript";
            scriptTag.text = [
              'window.snapStore = window.snapStore || {}; window.snapStore["',
              route,
              '"] = ',
              JSON.stringify(json),
              ";"
            ].join("");
            document.body.appendChild(scriptTag);
          },
          route,
          json
        );
      }
      uniqueResources.add(responseUrl);
    } else if (preconnectThirdParty) {
      const urlObj = url.parse(responseUrl);
      const domain = `${urlObj.protocol}//${urlObj.host}`;
      if (uniqueResources.has(domain)) return;
      await page.evaluate(route => {
        var linkTag = document.createElement("link");
        linkTag.setAttribute("rel", "preconnect");
        linkTag.setAttribute("href", route);
        document.head.appendChild(linkTag);
      }, domain);
      uniqueResources.add(domain);
    }
  });
};

const removeStyleTags = ({ page }) =>
  page.evaluate(() => {
    var x = Array.from(document.querySelectorAll("style"));
    for (var i = x.length - 1; i >= 0; i--) {
      const ell = x[i];
      ell.parentElement && ell.parentElement.removeChild(ell);
    }
  });

const preloadPolyfill = fs.readFileSync(
  `${__dirname}/vendor/preload_polyfill.min.js`,
  "utf8"
);

/**
 * TODO: do we need to remove blobs for js?
 * @param {{page: Page}} opt
 * @return Promise
 */
const removeBlobs = async opt => {
  const { page } = opt;
  return page.evaluate(() => {
    var stylesheets = Array.from(
      document.querySelectorAll("link[rel=stylesheet]")
    );
    stylesheets.forEach(link => {
      if (link.href && link.href.startsWith("blob:")) {
        link.parentNode && link.parentNode.removeChild(link);
      }
    });
  });
};

/**
 * @param {{page: Page, pageUrl: string, options: {skipThirdPartyRequests: boolean, userAgent: string}, basePath: string, browser: Browser}} opt
 * @return {Promise}
 */
const inlineCss = async opt => {
  const { page, pageUrl, options, basePath, browser } = opt;

  const minimalcssResult = await minimalcss.minimize({
    urls: [pageUrl],
    skippable: request =>
      options.skipThirdPartyRequests && !request.url.startsWith(basePath),
    browser: browser,
    userAgent: options.userAgent
  });
  const criticalCss = minimalcssResult.finalCss;
  const criticalCssSize = Buffer.byteLength(criticalCss, "utf8");

  const result = await page.evaluate(async () => {
    var stylesheets = Array.from(
      document.querySelectorAll("link[rel=stylesheet]")
    );
    const cssArray = await Promise.all(
      stylesheets.map(async link => {
        const response = await fetch(link.href);
        return response.text();
      })
    );
    return cssArray.join("");
  });
  const allCss = new CleanCSS(options.minifyCSS).minify(result).styles;
  const allCssSize = Buffer.byteLength(allCss, "utf8");

  let cssStrategy, cssSize;
  if (criticalCssSize * 2 >= allCssSize) {
    cssStrategy = "inline";
    cssSize = criticalCssSize;
  } else {
    cssStrategy = "critical";
    cssSize = allCssSize;
  }

  if (cssSize > twentyKb) console.log("⚠  inlining CSS more than 20kb");

  if (cssStrategy === "critical") {
    return page.evaluate(
      (criticalCss, preloadPolyfill) => {
        var head = document.head || document.getElementsByTagName("head")[0],
          style = document.createElement("style");
        style.type = "text/css";
        style.appendChild(document.createTextNode(criticalCss));
        head.appendChild(style);

        var stylesheets = Array.from(
          document.querySelectorAll("link[rel=stylesheet]")
        );
        stylesheets.forEach(link => {
          // noscript fallback which doesn't work
          // var wrap = document.createElement('div');
          // wrap.appendChild(link.cloneNode(false));
          // var noscriptTag = document.createElement('noscript');
          // noscriptTag.innerHTML = wrap.innerHTML;
          // document.head.appendChild(noscriptTag);
          link.parentNode && link.parentNode.removeChild(link);
          link.setAttribute("rel", "preload");
          link.setAttribute("as", "style");
          link.setAttribute("onload", "this.rel='stylesheet'");
          document.head.appendChild(link);
        });

        var scriptTag = document.createElement("script");
        scriptTag.type = "text/javascript";
        scriptTag.text = preloadPolyfill;
        // scriptTag.id = "preloadPolyfill";
        document.body.appendChild(scriptTag);
      },
      criticalCss,
      preloadPolyfill
    );
  } else {
    return page.evaluate(allCss => {
      var head = document.head || document.getElementsByTagName("head")[0],
        style = document.createElement("style");
      style.type = "text/css";
      style.appendChild(document.createTextNode(allCss));
      head.appendChild(style);

      var stylesheets = Array.from(
        document.querySelectorAll("link[rel=stylesheet]")
      );
      stylesheets.forEach(link => {
        link.parentNode && link.parentNode.removeChild(link);
      });
    }, allCss);
  }
};

const asyncJs = ({ page }) => {
  return page.evaluate(() => {
    Array.from(document.querySelectorAll("script[src]")).forEach(x => {
      x.parentNode && x.parentNode.removeChild(x);
      x.setAttribute("async", "true");
      document.head.appendChild(x);
    });
  });
};

const fixWebpackChunksIssue = ({ page, basePath }) => {
  return page.evaluate(basePath => {
    const localScripts = Array.from(document.scripts).filter(
      x => x.src && x.src.startsWith(basePath)
    );
    const mainRegexp = /main\.[\w]{8}.js/;
    const mainScript = localScripts.filter(x => mainRegexp.test(x.src))[0];
    const chunkRegexp = /\.[\w]{8}\.chunk\.js/;
    const chunkSripts = localScripts.filter(x => chunkRegexp.test(x.src));
    chunkSripts.forEach(x => {
      if (x.parentElement && mainScript.parentNode) {
        x.parentElement.removeChild(x);
        // this will produce links like http://localhost:45678/static...
        // and will confuse order of load
        // const linkTag = document.createElement("link");
        // linkTag.setAttribute("rel", "preload");
        // linkTag.setAttribute("as", "script");
        // linkTag.setAttribute("href", x.src);
        // mainScript.parentNode.insertBefore(linkTag, mainScript.nextSibling);
      }
    });
  }, basePath);
};

const saveAsHtml = async ({ page, filePath, options, route }) => {
  const content = await page.content();
  const title = await page.title();
  const minifiedContent = options.minifyHTML
    ? minify(content, options.minifyHTML)
    : content;
  filePath = filePath.replace(/\//g, path.sep);
  if (route === options.publicPath + "/404") {
    if (!title.includes("404"))
      console.log('⚠  404 page title does not contain "404" string');
    mkdirp.sync(path.dirname(filePath));
    fs.writeFileSync(`${filePath}.html`, minifiedContent);
  } else {
    if (title.includes("404")) console.log(`⚠  page not found ${route}`);
    mkdirp.sync(filePath);
    fs.writeFileSync(path.join(filePath, "index.html"), minifiedContent);
  }
};

const saveAsPng = ({ page, filePath, options, route }) => {
  mkdirp.sync(path.dirname(filePath));
  let screenshotPath;
  if (route === "/") {
    screenshotPath = `${filePath}/index.png`;
  } else {
    screenshotPath = `${filePath.replace(/\/$/, "")}.png`;
  }
  return page.screenshot({ path: screenshotPath });
};

const run = async userOptions => {
  const options = defaults(userOptions);

  const sourceDir = path.normalize(`${process.cwd()}/${options.source}`);
  const destinationDir = path.normalize(
    `${process.cwd()}/${options.destination}`
  );
  const startServer = options => {
    const app = express()
      .use(options.publicPath, serveStatic(sourceDir))
      .use(fallback("200.html", { root: sourceDir }));
    const server = http.createServer(app);
    server.listen(options.port);
    return server;
  };

  if (
    destinationDir === sourceDir &&
    options.saveAs === "html" &&
    fs.existsSync(path.join(sourceDir, "200.html"))
  ) {
    console.log(
      `200.html is present in the sourceDir (${sourceDir}). You can not run react-snap twice - this will brake the build`
    );
    process.exit(1);
  }

  fs
    .createReadStream(path.join(sourceDir, "index.html"))
    .pipe(fs.createWriteStream(path.join(sourceDir, "200.html")));

  if (destinationDir !== sourceDir && options.saveAs === "html") {
    mkdirp.sync(destinationDir);
    fs
      .createReadStream(path.join(sourceDir, "index.html"))
      .pipe(fs.createWriteStream(path.join(destinationDir, "200.html")));
  }

  const server = options.externalServer ? null : startServer(options);

  const basePath = `http://localhost:${options.port}`;
  const publicPath = options.publicPath;

  await crawl({
    options,
    basePath,
    publicPath,
    beforeFetch: async ({ page }) => {
      const {
        preloadImages,
        cacheAjaxRequests,
        preconnectThirdParty
      } = options;
      if (preloadImages || cacheAjaxRequests || preconnectThirdParty)
        preloadResources({
          page,
          basePath,
          preloadImages,
          cacheAjaxRequests,
          preconnectThirdParty
        });
    },
    afterFetch: async ({ page, route, browser }) => {
      const pageUrl = `${basePath}${route}`;
      if (options.removeStyleTags) await removeStyleTags({ page });
      if (options.removeBlobs) await removeBlobs({ page });
      if (options.inlineCss)
        await inlineCss({
          page,
          pageUrl,
          options,
          basePath,
          browser
        });
      if (options.fixWebpackChunksIssue) {
        await fixWebpackChunksIssue({
          page,
          basePath
        });
      }
      if (options.asyncJs) await asyncJs({ page });
      const routePath = route.replace(publicPath, "");
      const filePath = path.join(destinationDir, routePath);
      if (options.saveAs === "html") {
        await saveAsHtml({ page, filePath, options, route });
      } else if (options.saveAs === "png") {
        await saveAsPng({ page, filePath, options, route });
      } else {
        throw new Error(`Unexpected value for saveAs: ${options.saveAs}`);
      }
    },
    onEnd: () => {
      if (server) server.close();
    }
  });
};

exports.defaultOptions = defaultOptions;
exports.run = run;
<commit_msg>removeScriptTags<commit_after>const crawl = require("./src/puppeteer_utils.js").crawl;
const http = require("http");
const express = require("express");
const serveStatic = require("serve-static");
// @ts-ignore
const fallback = require("express-history-api-fallback");
const path = require("path");
const fs = require("fs");
const mkdirp = require("mkdirp");
const minify = require("html-minifier").minify;
const url = require("url");
// @ts-ignore https://github.com/peterbe/minimalcss/pull/30
const minimalcss = require("minimalcss");
const CleanCSS = require("clean-css");
const twentyKb = 20 * 1024;

const defaultOptions = {
  //# stable configurations
  port: 45678,
  source: "build",
  destination: null,
  concurrency: 4,
  include: ["/"],
  userAgent: "ReactSnap",
  headless: true,
  puppeteerArgs: [],
  publicPath: "/",
  minifyCSS: {},
  minifyHTML: {
    collapseBooleanAttributes: true,
    collapseWhitespace: true,
    decodeEntities: true,
    keepClosingSlash: true,
    sortAttributes: true,
    sortClassName: true
  },
  viewport: false, // TODO: set to mobile viewport by default?
  //# feature creeps to generate screenshots
  saveAs: "html",
  crawl: true,
  waitFor: false,
  externalServer: false,
  //# workarounds
  fixWebpackChunksIssue: true,
  removeBlobs: true,
  skipThirdPartyRequests: false,
  //# unstable configurations
  preconnectThirdParty: true,
  // Experimental. This config stands for two strategies inline and critical.
  // TODO: inline strategy can contain errors, like, confuse relative urls
  // TODO: critical strategy miss noscript fallback
  inlineCss: false,
  // Experimental. TODO: need to fix issues with sourcemaps
  sourceMaps: false,
  cacheAjaxRequests: false,
  //# even more workarounds
  removeStyleTags: false,
  preloadImages: false,
  // add async true to scripts and move them to the header, to start download earlier
  // can use <link rel="preload"> instead
  asyncJs: false,
  //# another feature creep
  // tribute to Netflix Server Side Only React https://twitter.com/NetflixUIE/status/923374215041912833
  removeScriptTags: false
};

/**
 *
 * @param {{source: ?string, destination: ?string, include: ?Array<string>, sourceMaps: ?boolean, skipThirdPartyRequests: ?boolean }} userOptions
 * @return {*}
 */
const defaults = userOptions => {
  const options = {
    ...defaultOptions,
    ...userOptions
  };
  options.destination = options.destination || options.source;
  if (!options.include || !options.include.length)
    throw new Error("include should be an array");

  if (options.preloadResources) {
    console.log(
      "preloadResources option deprecated. Use preloadImages or cacheAjaxRequests"
    );
    process.exit(1);
  }
  if (options.minifyOptions) {
    console.log("minifyOptions option renamed to minifyHTML");
    process.exit(1);
  }

  if (options.minifyHTML && !options.minifyHTML.minifyCSS) {
    options.minifyHTML.minifyCSS = options.minifyCSS;
  }

  if (!options.publicPath.startsWith("/")) {
    options.publicPath = `/${options.publicPath}`;
  }
  options.publicPath = options.publicPath.replace(/\/$/, "");

  options.include = options.include.map(
    include => options.publicPath + include
  );
  return options;
};

/**
 *
 * @param {{page: Page, basePath: string}} opt
 */
const preloadResources = opt => {
  const {
    page,
    basePath,
    preloadImages,
    cacheAjaxRequests,
    preconnectThirdParty
  } = opt;
  const uniqueResources = new Set();
  page.on("response", async response => {
    const responseUrl = response.url;
    if (/^data:/i.test(responseUrl)) return;
    const ct = response.headers["content-type"] || "";
    const route = responseUrl.replace(basePath, "");
    if (/^http:\/\/localhost/i.test(responseUrl)) {
      if (uniqueResources.has(responseUrl)) return;
      if (preloadImages && /\.(png|jpg|jpeg|webp|gif)$/.test(responseUrl)) {
        await page.evaluate(route => {
          var linkTag = document.createElement("link");
          linkTag.setAttribute("rel", "preload");
          linkTag.setAttribute("as", "image");
          linkTag.setAttribute("href", route);
          document.body.appendChild(linkTag);
        }, route);
      } else if (cacheAjaxRequests && ct.indexOf("json") > -1) {
        const json = await response.json();
        await page.evaluate(
          (route, json) => {
            var scriptTag = document.createElement("script");
            scriptTag.type = "text/javascript";
            scriptTag.text = [
              'window.snapStore = window.snapStore || {}; window.snapStore["',
              route,
              '"] = ',
              JSON.stringify(json),
              ";"
            ].join("");
            document.body.appendChild(scriptTag);
          },
          route,
          json
        );
      }
      uniqueResources.add(responseUrl);
    } else if (preconnectThirdParty) {
      const urlObj = url.parse(responseUrl);
      const domain = `${urlObj.protocol}//${urlObj.host}`;
      if (uniqueResources.has(domain)) return;
      await page.evaluate(route => {
        var linkTag = document.createElement("link");
        linkTag.setAttribute("rel", "preconnect");
        linkTag.setAttribute("href", route);
        document.head.appendChild(linkTag);
      }, domain);
      uniqueResources.add(domain);
    }
  });
};

const removeStyleTags = ({ page }) =>
  page.evaluate(() => {
    var x = Array.from(document.querySelectorAll("style"));
    for (var i = x.length - 1; i >= 0; i--) {
      const ell = x[i];
      ell.parentElement && ell.parentElement.removeChild(ell);
    }
  });

const removeScriptTags = ({ page }) =>
  page.evaluate(() => {
    var x = Array.from(document.querySelectorAll("script"));
    for (var i = x.length - 1; i >= 0; i--) {
      const ell = x[i];
      ell.parentElement && ell.parentElement.removeChild(ell);
    }
  });

const preloadPolyfill = fs.readFileSync(
  `${__dirname}/vendor/preload_polyfill.min.js`,
  "utf8"
);

/**
 * TODO: do we need to remove blobs for js?
 * @param {{page: Page}} opt
 * @return Promise
 */
const removeBlobs = async opt => {
  const { page } = opt;
  return page.evaluate(() => {
    var stylesheets = Array.from(
      document.querySelectorAll("link[rel=stylesheet]")
    );
    stylesheets.forEach(link => {
      if (link.href && link.href.startsWith("blob:")) {
        link.parentNode && link.parentNode.removeChild(link);
      }
    });
  });
};

/**
 * @param {{page: Page, pageUrl: string, options: {skipThirdPartyRequests: boolean, userAgent: string}, basePath: string, browser: Browser}} opt
 * @return {Promise}
 */
const inlineCss = async opt => {
  const { page, pageUrl, options, basePath, browser } = opt;

  const minimalcssResult = await minimalcss.minimize({
    urls: [pageUrl],
    skippable: request =>
      options.skipThirdPartyRequests && !request.url.startsWith(basePath),
    browser: browser,
    userAgent: options.userAgent
  });
  const criticalCss = minimalcssResult.finalCss;
  const criticalCssSize = Buffer.byteLength(criticalCss, "utf8");

  const result = await page.evaluate(async () => {
    var stylesheets = Array.from(
      document.querySelectorAll("link[rel=stylesheet]")
    );
    const cssArray = await Promise.all(
      stylesheets.map(async link => {
        const response = await fetch(link.href);
        return response.text();
      })
    );
    return cssArray.join("");
  });
  const allCss = new CleanCSS(options.minifyCSS).minify(result).styles;
  const allCssSize = Buffer.byteLength(allCss, "utf8");

  let cssStrategy, cssSize;
  if (criticalCssSize * 2 >= allCssSize) {
    cssStrategy = "inline";
    cssSize = criticalCssSize;
  } else {
    cssStrategy = "critical";
    cssSize = allCssSize;
  }

  if (cssSize > twentyKb) console.log("⚠  inlining CSS more than 20kb");

  if (cssStrategy === "critical") {
    return page.evaluate(
      (criticalCss, preloadPolyfill) => {
        var head = document.head || document.getElementsByTagName("head")[0],
          style = document.createElement("style");
        style.type = "text/css";
        style.appendChild(document.createTextNode(criticalCss));
        head.appendChild(style);

        var stylesheets = Array.from(
          document.querySelectorAll("link[rel=stylesheet]")
        );
        stylesheets.forEach(link => {
          // noscript fallback which doesn't work
          // var wrap = document.createElement('div');
          // wrap.appendChild(link.cloneNode(false));
          // var noscriptTag = document.createElement('noscript');
          // noscriptTag.innerHTML = wrap.innerHTML;
          // document.head.appendChild(noscriptTag);
          link.parentNode && link.parentNode.removeChild(link);
          link.setAttribute("rel", "preload");
          link.setAttribute("as", "style");
          link.setAttribute("onload", "this.rel='stylesheet'");
          document.head.appendChild(link);
        });

        var scriptTag = document.createElement("script");
        scriptTag.type = "text/javascript";
        scriptTag.text = preloadPolyfill;
        // scriptTag.id = "preloadPolyfill";
        document.body.appendChild(scriptTag);
      },
      criticalCss,
      preloadPolyfill
    );
  } else {
    return page.evaluate(allCss => {
      var head = document.head || document.getElementsByTagName("head")[0],
        style = document.createElement("style");
      style.type = "text/css";
      style.appendChild(document.createTextNode(allCss));
      head.appendChild(style);

      var stylesheets = Array.from(
        document.querySelectorAll("link[rel=stylesheet]")
      );
      stylesheets.forEach(link => {
        link.parentNode && link.parentNode.removeChild(link);
      });
    }, allCss);
  }
};

const asyncJs = ({ page }) => {
  return page.evaluate(() => {
    Array.from(document.querySelectorAll("script[src]")).forEach(x => {
      x.parentNode && x.parentNode.removeChild(x);
      x.setAttribute("async", "true");
      document.head.appendChild(x);
    });
  });
};

const fixWebpackChunksIssue = ({ page, basePath }) => {
  return page.evaluate(basePath => {
    const localScripts = Array.from(document.scripts).filter(
      x => x.src && x.src.startsWith(basePath)
    );
    const mainRegexp = /main\.[\w]{8}.js/;
    const mainScript = localScripts.filter(x => mainRegexp.test(x.src))[0];
    const chunkRegexp = /\.[\w]{8}\.chunk\.js/;
    const chunkSripts = localScripts.filter(x => chunkRegexp.test(x.src));
    chunkSripts.forEach(x => {
      if (x.parentElement && mainScript.parentNode) {
        x.parentElement.removeChild(x);
        // this will produce links like http://localhost:45678/static...
        // and will confuse order of load
        // const linkTag = document.createElement("link");
        // linkTag.setAttribute("rel", "preload");
        // linkTag.setAttribute("as", "script");
        // linkTag.setAttribute("href", x.src);
        // mainScript.parentNode.insertBefore(linkTag, mainScript.nextSibling);
      }
    });
  }, basePath);
};

const saveAsHtml = async ({ page, filePath, options, route }) => {
  const content = await page.content();
  const title = await page.title();
  const minifiedContent = options.minifyHTML
    ? minify(content, options.minifyHTML)
    : content;
  filePath = filePath.replace(/\//g, path.sep);
  if (route === options.publicPath + "/404") {
    if (!title.includes("404"))
      console.log('⚠  404 page title does not contain "404" string');
    mkdirp.sync(path.dirname(filePath));
    fs.writeFileSync(`${filePath}.html`, minifiedContent);
  } else {
    if (title.includes("404")) console.log(`⚠  page not found ${route}`);
    mkdirp.sync(filePath);
    fs.writeFileSync(path.join(filePath, "index.html"), minifiedContent);
  }
};

const saveAsPng = ({ page, filePath, options, route }) => {
  mkdirp.sync(path.dirname(filePath));
  let screenshotPath;
  if (route === "/") {
    screenshotPath = `${filePath}/index.png`;
  } else {
    screenshotPath = `${filePath.replace(/\/$/, "")}.png`;
  }
  return page.screenshot({ path: screenshotPath });
};

const run = async userOptions => {
  const options = defaults(userOptions);

  const sourceDir = path.normalize(`${process.cwd()}/${options.source}`);
  const destinationDir = path.normalize(
    `${process.cwd()}/${options.destination}`
  );
  const startServer = options => {
    const app = express()
      .use(options.publicPath, serveStatic(sourceDir))
      .use(fallback("200.html", { root: sourceDir }));
    const server = http.createServer(app);
    server.listen(options.port);
    return server;
  };

  if (
    destinationDir === sourceDir &&
    options.saveAs === "html" &&
    fs.existsSync(path.join(sourceDir, "200.html"))
  ) {
    console.log(
      `200.html is present in the sourceDir (${sourceDir}). You can not run react-snap twice - this will brake the build`
    );
    process.exit(1);
  }

  fs
    .createReadStream(path.join(sourceDir, "index.html"))
    .pipe(fs.createWriteStream(path.join(sourceDir, "200.html")));

  if (destinationDir !== sourceDir && options.saveAs === "html") {
    mkdirp.sync(destinationDir);
    fs
      .createReadStream(path.join(sourceDir, "index.html"))
      .pipe(fs.createWriteStream(path.join(destinationDir, "200.html")));
  }

  const server = options.externalServer ? null : startServer(options);

  const basePath = `http://localhost:${options.port}`;
  const publicPath = options.publicPath;

  await crawl({
    options,
    basePath,
    publicPath,
    beforeFetch: async ({ page }) => {
      const {
        preloadImages,
        cacheAjaxRequests,
        preconnectThirdParty
      } = options;
      if (preloadImages || cacheAjaxRequests || preconnectThirdParty)
        preloadResources({
          page,
          basePath,
          preloadImages,
          cacheAjaxRequests,
          preconnectThirdParty
        });
    },
    afterFetch: async ({ page, route, browser }) => {
      const pageUrl = `${basePath}${route}`;
      if (options.removeStyleTags) await removeStyleTags({ page });
      if (options.removeScriptTags) await removeScriptTags({ page });
      if (options.removeBlobs) await removeBlobs({ page });
      if (options.inlineCss)
        await inlineCss({
          page,
          pageUrl,
          options,
          basePath,
          browser
        });
      if (options.fixWebpackChunksIssue) {
        await fixWebpackChunksIssue({
          page,
          basePath
        });
      }
      if (options.asyncJs) await asyncJs({ page });
      const routePath = route.replace(publicPath, "");
      const filePath = path.join(destinationDir, routePath);
      if (options.saveAs === "html") {
        await saveAsHtml({ page, filePath, options, route });
      } else if (options.saveAs === "png") {
        await saveAsPng({ page, filePath, options, route });
      } else {
        throw new Error(`Unexpected value for saveAs: ${options.saveAs}`);
      }
    },
    onEnd: () => {
      if (server) server.close();
    }
  });
};

exports.defaultOptions = defaultOptions;
exports.run = run;
<|endoftext|>


<commit_before>const Eris = require('eris');
const config = require('./config.js');

var redis = require('redis'),
    client = redis.createClient(null, null);

var Trello = require("node-trello");
var t = new Trello(config.trelloKey, config.trelloToken);

var bot = new Eris(config.botToken, {
  maxShards: 2
});

bot.on("error", err => {
    console.log("@" + bot.user.username + " - " + "ERROR:\n" + err.stack);
});

bot.on("ready", () => {
    console.log('Ready!');
});

var canarySKey;
var iOSSKey;
var androidSKey;

bot.on('messageCreate', (msg) => {
  var messageSplit = msg.content.split(' ');
  var command = messageSplit.shift();

  if(command === "!submit"){

    var dev = msg.member.roles.indexOf(config.dev);
    var hunter = msg.member.roles.indexOf(config.hunter);
    var admin = msg.member.roles.indexOf(config.admin);

    var user = msg.author.username + "#" + msg.author.discriminator;

    if(dev > -1 || hunter > -1 || admin > -1){
      if(msg.channel.id === config.ios){
        sendToRedis("i", messageSplit.join(' '), msg.channel.id, user);

      }else if(msg.channel.id === config.android){
        sendToRedis("a", messageSplit.join(' '), msg.channel.id, user);

      }else if(msg.channel.id === config.canary){
        sendToRedis("c", messageSplit.join(' '), msg.channel.id, user);

      }
    }
  }

  if(command === "!header"){

    var dev = msg.member.roles.indexOf(config.dev);
    var hunter = msg.member.roles.indexOf(config.hunter);
    var admin = msg.member.roles.indexOf(config.admin);

    if(dev > -1 || hunter > -1 || admin > -1){
      var getKey;
      var header;
      switch (msg.channel.id) {
        case config.ios:
          if(!iOSSKey){
            getKey = messageSplit.shift();
          }else if(!!iOSSKey && messageSplit[0] === iOSSKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = iOSSKey;
            header = messageSplit.join(' ');
          }
          break;
        case config.android:
          if(!androidSKey){
            getKey = messageSplit.shift();
          }else if(!!androidSKey && messageSplit[0] === androidSKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = androidSKey;
            header = messageSplit.join(' ');
          }
          break;
        case config.canary:
          if(!canarySKey){
            getKey = messageSplit.shift();
          }else if(!!canarySKey && messageSplit[0] === canarySKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = canarySKey;
            header = messageSplit.join(' ');
          }
          break;

      }
      client.hexists(getKey, "os", function(err, reply){
        if(reply === 1){
          header = messageSplit.join(' ');
          setupTrello(getKey, header, msg.channel.id);
        }else if(reply === 0){
          bot.createMessage(msg.channel.id, "Report key is incorrect or missing.");
        }
      });
    }
  }
});

function sendToRedis(system, bugReport, channelID, user){
  var key = system + Math.floor((Math.random() * 8999) + 1000);

  client.hexists(key, "os", function(err, reply){
    if(reply === 1){
      sendToRedis(system, bugReport);
    }else{
      client.hset(key, "os", system);
      client.hset(key, "report", user + "\n" + bugReport);

      if(system === "c"){
        if(!canarySKey){
          canarySKey = key;
        }else if(!!canarySKey){
          canarySKey = undefined;
        }
      }else if(system === "a"){
        if(!androidSKey){
          androidSKey = key;
        }else if(!!androidSKey){
          androidSKey = undefined;
        }
      }else if(system === "i"){
        if(!iOSSKey){
          iOSSKey = key;
        }else if(!!iOSSKey){
          iOSSKey = undefined;
        }
      }
      bot.createMessage(channelID, "Type `!header <key> <name of bug>` to finish the bug report. Your key is: `" + key + "`");
    }
  });
}

function setupTrello(key, header, channelID){
  client.hget(key, "os", function(err, os){
    client.hget(key, "report", function(err, report){
      if(!!err){
        console.log(err);
      }
      if(os.toString() === "c"){
        var listID = config.canaryTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }else if(os.toString() === "a"){
        var listID = config.androidTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }else if(os.toString() === "i"){
        var listID = config.iosTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }
    });
  });
}

function sendToTrello(listID, header, report, channelID, key){
  var creationSuccess = function(data) {
    bot.createMessage(channelID, "Report `" + key + "` added to Trello");
    canarySKey = undefined;
    iOSSKey = undefined;
    androidSKey = undefined;
    client.del(key);
  };
  var newCard = {
    name: header,
    desc: report,
    idList: listID,
    pos: 'top'
  };
  t.post('/1/cards/', newCard, creationSuccess);
}
bot.connect();
<commit_msg>large (giant)<commit_after>const Eris = require('eris');
const config = require('./config.js');

var redis = require('redis'),
    client = redis.createClient(null, null);

var Trello = require("node-trello");
var t = new Trello(config.trelloKey, config.trelloToken);

var bot = new Eris(config.token, {
  maxShards: 2
});

bot.on("error", err => {
    console.log("@" + bot.user.username + " - " + "ERROR:\n" + err.stack);
});

bot.on("ready", () => {
    console.log('Ready!');
});
var canarySKey;
var iOSSKey;
var androidSKey;

bot.on('messageCreate', (msg) => {
  var messageSplit = msg.content.split(' ');
  var command = messageSplit.shift();

  if(command === "!submit"){

    var dev = msg.member.roles.indexOf(config.dev);
    var hunter = msg.member.roles.indexOf(config.hunter);
    var admin = msg.member.roles.indexOf(config.admin);

    var user = msg.author.username + "#" + msg.author.discriminator;

    if(dev > -1 || hunter > -1 || admin > -1){
      if(msg.channel.id === config.ios){
        sendToRedis("i", messageSplit.join(' '), msg.channel.id, user, msg.author.id);

      }else if(msg.channel.id === config.android){
        sendToRedis("a", messageSplit.join(' '), msg.channel.id, user, msg.author.id);

      }else if(msg.channel.id === config.canary){
        sendToRedis("c", messageSplit.join(' '), msg.channel.id, user, msg.author.id);

      }
    }
  }

  if(command === "!header"){

    var dev = msg.member.roles.indexOf(config.dev);
    var hunter = msg.member.roles.indexOf(config.hunter);
    var admin = msg.member.roles.indexOf(config.admin);

    if(dev > -1 || hunter > -1 || admin > -1){
      var getKey;
      var header;
      switch (msg.channel.id) {
        case config.ios:
          if(!iOSSKey){
            getKey = messageSplit.shift();
          }else if(!!iOSSKey && messageSplit[0] === iOSSKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = iOSSKey;
            header = messageSplit.join(' ');
          }
          break;
        case config.android:
          if(!androidSKey){
            getKey = messageSplit.shift();
          }else if(!!androidSKey && messageSplit[0] === androidSKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = androidSKey;
            header = messageSplit.join(' ');
          }
          break;
        case config.canary:
          if(!canarySKey){
            getKey = messageSplit.shift();
          }else if(!!canarySKey && messageSplit[0] === canarySKey){
            getKey = messageSplit.shift();
            header = messageSplit.join(' ');
          }else{
            getKey = canarySKey;
            header = messageSplit.join(' ');
          }
          break;

      }
      client.hexists(getKey, "os", function(err, reply){
        if(reply === 1){
          header = messageSplit.join(' ');
          setupTrello(getKey, header, msg.channel.id);
        }else if(reply === 0){
          bot.createMessage(msg.channel.id, "Report key is incorrect or missing.");
        }
      });
    }
  }
});

function sendToRedis(system, bugReport, channelID, user, userID){
  var key = system + Math.floor((Math.random() * 8999) + 1000);

  client.hexists(key, "os", function(err, reply){
    if(reply === 1){
      sendToRedis(system, bugReport);
    }else{
      client.hset(key, "os", system);
      client.hset(key, "report", user + "\n" + bugReport);

      if(system === "c"){
        if(!canarySKey){
          canarySKey = key;
        }else if(!!canarySKey){
          canarySKey = undefined;
        }
      }else if(system === "a"){
        if(!androidSKey){
          androidSKey = key;
        }else if(!!androidSKey){
          androidSKey = undefined;
        }
      }else if(system === "i"){
        if(!iOSSKey){
          iOSSKey = key;
        }else if(!!iOSSKey){
          iOSSKey = undefined;
        }
      }
      bot.createMessage(channelID, "<@" + userID +"> Type `!header <key> <name of bug>` to finish the bug report. Your key is: `" + key + "`");
    }
  });
}

function setupTrello(key, header, channelID){
  client.hget(key, "os", function(err, os){
    client.hget(key, "report", function(err, report){
      if(!!err){
        console.log(err);
      }
      if(os.toString() === "c"){
        var listID = config.canaryTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }else if(os.toString() === "a"){
        var listID = config.androidTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }else if(os.toString() === "i"){
        var listID = config.iosTrello;
        sendToTrello(listID, header, report.toString(), channelID, key);
      }
    });
  });
}

function sendToTrello(listID, header, report, channelID, key){
  var creationSuccess = function(data) {
    bot.createMessage(channelID, "Report `" + key + "` added to Trello");
    canarySKey = undefined;
    iOSSKey = undefined;
    androidSKey = undefined;
    client.del(key);
  };
  var newCard = {
    name: header,
    desc: report,
    idList: listID,
    pos: 'top'
  };
  t.post('/1/cards/', newCard, creationSuccess);
}
bot.connect();
<|endoftext|>


<commit_before>import React, { PropTypes, PureComponent } from 'react';
import { View, Platform, WebView, ActivityIndicator } from 'react-native';

export default class SVGImage extends PureComponent {
  static propTypes = {
    style: PropTypes.any,
    source: PropTypes.shape({
      uri: PropTypes.string,
    }).isRequired,
    showWebviewLoader: PropTypes.bool,
    height: PropTypes.number,
  };

  static defaultProps = {
    style: {},
    source: { uri: '' },
    showWebviewLoader: Platform.OS === 'android',
    height: null,
  };

  renderLoader = () => (
    <View style={[this.props.style, { flex: 1, alignItems: 'center', justifyContent: 'center' }]}>
      <ActivityIndicator />
    </View>
  );

  render() {
    const { showWebviewLoader, source: { uri }, height, ...props } = this.props;

    const html = `
      <!DOCTYPE html>\n

    `;

    return (
      <WebView
        source={{ html }}
        startInLoadingState={showWebviewLoader}
        renderLoading={showWebviewLoader ? this.renderLoader : null}
        {...props}
      />
    );
  }
}
<commit_msg>Remove extra margin around SVG images<commit_after>import React, { PropTypes, PureComponent } from 'react';
import { View, Platform, WebView, ActivityIndicator } from 'react-native';

export default class SVGImage extends PureComponent {
  static propTypes = {
    style: PropTypes.any,
    source: PropTypes.shape({
      uri: PropTypes.string,
    }).isRequired,
    showWebviewLoader: PropTypes.bool,
    height: PropTypes.number,
  };

  static defaultProps = {
    style: {},
    source: { uri: '' },
    showWebviewLoader: Platform.OS === 'android',
    height: null,
  };

  renderLoader = () => (
    <View style={[this.props.style, { flex: 1, alignItems: 'center', justifyContent: 'center' }]}>
      <ActivityIndicator />
    </View>
  );

  render() {
    const { showWebviewLoader, source: { uri }, height, ...restOfProps } = this.props;

    const html = `
      <!DOCTYPE html>\n

    `;

    return (
      <WebView
        startInLoadingState={showWebviewLoader}
        renderLoading={showWebviewLoader ? this.renderLoader : null}
        {...restOfProps}
        source={{ html }}
      />
    );
  }
}
<|endoftext|>