<commit_before>var exec = require('exec-queue'),
    _ = require('lodash'),
    log4js = require('log4js'),
    logger = log4js.getLogger('GooglePlay'),
    config = require('config'),
    download = require('./download.js'),
    re = new RegExp("[^;]+;([^;]+);");

log4js.configure({
  appenders: [
    { type: 'console' },
    { type: 'file', filename: 'play.log' }
  ]
});

function init(keyword, outputDir, appsem, callback) {
  var crawlerPath = config.get('plugin.GooglePlayCrawlerPath');
  COMMAND = "java -jar " + crawlerPath + "/googleplaycrawler-0.3.jar -f " + crawlerPath + "/crawler.conf search " + keyword;
  exec(COMMAND, function(error, stdout, stderr) {
    if (stderr || error) {
      return logger.error("init::STDerr:" + stderr + ". Error:" + error);
    }
    var output = stdout.split(/\n/);
    var packages = [];
    output = _.rest(output);
    output = _.compact(output);
    logger.info("init::" + output.length + " APKs detected for keyword [" + keyword + "]");

    output.forEach(function(outputLine) {
      var matchArr = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/);
      if (matchArr.length == 3) {
        packageName = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/)[1];
        price = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/)[2];
        if (price.toUpperCase() == "FREE") {
          packages.push(packageName);
        }
      }
      else {
        logger.error("init::Nonconformed app query information: " + outputLine);
      }
    })

    logger.info("init::" + output.length + " free apps for keyword [" + keyword + "]");
    packages.forEach(function(packageName) {
      appsem.take(function() {
        var last = false;
        if (packageName === packages[packages.length - 1]) {
          last = true;
        }
        else {
          last = false;
        }
        logger.info("init::Begin download " + packageName)
        download(packageName, outputDir, crawlerPath, function(tempAPKPath, htmlPath, tempCommentPath) {
          callback(tempAPKPath, htmlPath, tempCommentPath, last);
        })
      });
    });
  });
}

module.exports.init = init
<commit_msg>Error handling<commit_after>var exec = require('exec-queue'),
    _ = require('lodash'),
    log4js = require('log4js'),
    logger = log4js.getLogger('GooglePlay'),
    config = require('config'),
    download = require('./download.js'),
    re = new RegExp("[^;]+;([^;]+);");

log4js.configure({
  appenders: [
    { type: 'console' },
    { type: 'file', filename: 'play.log' }
  ]
});

function init(keyword, outputDir, appsem, callback) {
  var crawlerPath = config.get('plugin.GooglePlayCrawlerPath');
  COMMAND = "java -jar " + crawlerPath + "/googleplaycrawler-0.3.jar -f " + crawlerPath + "/crawler.conf search " + keyword;
  exec(COMMAND, function(error, stdout, stderr) {
    if (stderr || error) {
      logger.error("init::STDerr:" + stderr + ". Error:" + error);
      callback(true, null, null, null, null);
    }
    var output = stdout.split(/\n/);
    var packages = [];
    output = _.rest(output);
    output = _.compact(output);
    logger.info("init::" + output.length + " APKs detected for keyword [" + keyword + "]");

    output.forEach(function(outputLine) {
      var matchArr = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/);
      if (matchArr.length == 3) {
        packageName = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/)[1];
        price = outputLine.match(/[^;]+;([^;]+);[^;]+;([^;]+);/)[2];
        if (price.toUpperCase() == "FREE") {
          packages.push(packageName);
        }
      }
      else {
        logger.error("init::Nonconformed app query information: " + outputLine);
        callback(true, null, null, null, null);
      }
    })

    logger.info("init::" + output.length + " free apps for keyword [" + keyword + "]");
    packages.forEach(function(packageName) {
      appsem.take(function() {
        var last = false;
        if (packageName === packages[packages.length - 1]) {
          last = true;
        }
        else {
          last = false;
        }
        logger.info("init::Begin download " + packageName)
        download(packageName, outputDir, crawlerPath, function(tempAPKPath, htmlPath, tempCommentPath) {
          callback(null, tempAPKPath, htmlPath, tempCommentPath, last);
        })
      });
    });
  });
}

module.exports.init = init
<|endoftext|>


<commit_before>var
	cheerio = require('cheerio'),
	request = require('./request'),

	baseHost = '.craigslist.org',
	defaultRequestOptions = {
		hostname : '',
		path : '',
	    secure : true
	},
	searchMaxAsk = '&maxAsk=',
	searchMinAsk = '&minAsk=';
	// searchPath = '/search/sss?sort=rel&query=';

    var initialize = (function (self) {
	'use strict';

	self = self || {};
	self.options = {};
	/*
		Accepts string of HTML and parses that string to find all pertinent listings.
	*/
	function getRssListings (options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var category = $('channel').find('title').text()

		$('item').each(function (i, element) {
			var children = $(element).children()

            var url = $(element).attr('rdf:about')
			var idx = url.lastIndexOf('/')
			var idx1 = url.indexOf('.', idx)
			var listing = { category : category, url: url, pid: url.substring(idx + 1, idx1) }

			for (var i=0; i<children.length; i++) {
				var t, child = children[i]
				var name = child.name

				if (name == 'title') {
					t = child.children[0].data.trim()
					if (t.indexOf('[CDATA[') != -1)
						t = t.substring(7, t.length - 2).trim().replace(/^\&\#x0024\;/g, '').replace(/&\#x0024\;/g, '$')

					var priceIdx = t.indexOf('$')
					if (priceIdx != -1)  {
						var sidx = t.indexOf(' ', priceIdx)
						listing.price = sidx == -1 ? t.substring(priceIdx) : t.substring(priceIdx, sidx)
					}
					var idx = t.lastIndexOf('(')
				    if (idx == -1)
				    	listing.title = t
				    else {
				    	var idx1 = t.indexOf(')', idx)
				    	listing.location = t.substring(idx + 1, idx1).trim()
				    	listing.title = t.substring(0, idx).trim()
				    }

				}
				else if (name == 'description') {
					t = child.children[0].data.trim()
					if (t.indexOf('[CDATA[') != -1)
						t = t.substring(7, t.length - 2)
					listing.description = t.replace(/&\#x0024\;/g, '$').trim()
				}
				else if (name == 'dc:date')
					listing.date = $(child).text().trim()
				else if (name == 'enc:enclosure') {
					listing.hasPic = true
					listing.img = $(child).attr('resource')
				}
			}

					// coordinates : {
					// 	lat : $(element).attr('data-latitude'),
					// 	lon : $(element).attr('data-longitude')
					// },

				// make sure lat / lon is valid
				// if (typeof listing.coordinates.lat === 'undefined' ||
				// 	typeof listing.coordinates.lon === 'undefined') {
				// 	delete listing.coordinates;
				// }
			listings.push(listing);
		});

		return listings;
	}

	/*
		Accepts string of HTML and parses that string to find all pertinent listings.
	*/
	function getListings (options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

		$('div.content').find('p.row').each(function (i, element) {
			listing = {
				category : $(element)
					.find('span.l2 a.gc')
					.text(),
				coordinates : {
					lat : $(element).attr('data-latitude'),
					lon : $(element).attr('data-longitude')
				},
				date : $(element)
					.find('span.date')
					.text(),
				hasPic : ($(element)
					.find('span.l2 span.p')
					.text()
					.trim()) !== '',
				location : $(element)
					.find('span.pnr small')
					.text()
					.replace(/[\(,\)]/g, '').trim(), // santize
				pid : $(element)
					.attr('data-pid'),
				price : $(element)
					.find('span.l2 span.price')
					.text()
					.replace(/^\&\#x0024\;/g, ''), // sanitize
				title : $(element)
					.find('span.pl a')
					.text().trim()
				// url :
				// 	(options.secure ? 'https://' : 'http://') +
				// 	options.hostname + $(element).find('span.pl a').attr('href')
			}
			var rurl = $(element).find('span.pl a').attr('href');
			if (rurl) {
			  if (rurl.indexOf('http') == -1)
			    rurl = (options.secure ? 'https://' : 'http://') + options.hostname + rurl
			  listing.url = rurl;
		    }
		    if (options.cityName)
		      listing.cityName = options.cityName

			if (listing.hasPic) {
				var dId = $(element).find('a.i').attr('data-id')
				if (dId)
				  listing.pic = 'http://images.craigslist.org/' + dId.substring(2) + '_300x300.jpg'
			}
			var d = $(element).find('span.date')
			if (!d  ||  d.text().length == 0) {
				d = $(element).find('time').attr('datetime')
				listing.date = d;
			}

			// make sure lat / lon is valid
			if (typeof listing.coordinates.lat === 'undefined' ||
				typeof listing.coordinates.lon === 'undefined') {
				delete listing.coordinates;
			}

			listings.push(listing);
		});

		return listings;
	}

	function getCities(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

		$('div.colmask').find('a').each(function (i, element) {
           var href = $(element).attr('href')
           var idx = href.indexOf('//'), idx1 = href.indexOf('.');

           listing = { city: href.substring(idx + 2, idx1),
           			   href: href,
                       cityName: $(element).text().replace(/\w\S*/g, function(txt){return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase()})
                     }

           listings.push(listing)
		})
		return listings
	}

	function getCategories(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var city = $('h2.no-mobile').html();
        var idx = city.indexOf('<');
        if (idx != -1)
          city = city.substring(0, idx)

		$('div#center').find('a').each(function (i, element) {
           var href = $(element).attr('href')
           if (href.indexOf('/search/') != 0)
           	return;
           var idx = href.indexOf('//'), idx1 = href.indexOf('.');

           listing = { categoryName: $(element).text(),
           			   categoryHref: href,
                       city: city
                     }

           listings.push(listing)
		})
		return listings
	}
	function getSubCities(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var city = $('h2.no-mobile').html();
        var idx = city.indexOf('<');
        if (idx != -1)
          city = city.substring(0, idx)

		$('ul.sublinks').find('a').each(function (i, element) {
       var href = $(element).attr('href')

       listing = { subCityHref: options.hostname + href,
                   subCity: $(element).attr('title')
                 }

       listings.push(listing)
		})
		// nearby cities
		var rightPanelElms = $('div#rightbar').find('.menu .collapsible').find('li');
		for (var i=0; i<rightPanelElms; i++) {
			var li = rightPaneElms.get(i);
			if ($(li).find('h5').get(0).text() != 'nearby cl')
				continue;
			$(li).find('a').each(function(i, element) {
				listing = { city: $(element).text(),
					        href: $(element).attr('href')
				          }
			})

		}
		return listings
	}

    // Extracting full description for the listing
	function getDescription(options, html) {
		var
			$ = cheerio.load(html);

		var description = $('section#postingbody').html()
		var listing = {}
		if (description)
	  	  listing.description = description.trim()
		var attr = $('p.attrgroup')
        if (!attr)
          return listing
    	attr.find('span').each(function(i, p) {
    	  if ($(p).attr('class'))
    	  	listing.date = $(p).find('a').text()
    	  else {
    	  	var pVal, pName = $(p).text().trim()

    	  	var idx = pName.indexOf(':')
    	  	if (idx == -1) {
    	  	  pVal = pName
    	  	  pName = 'p' + i
    	  	}
    	  	else {
    	  	  pName = pName.substring(0, idx).trim()
    	  	  if (pName.charAt(0) == '#')
    	  	    pName = ('Number of' + pName.substring(1))
    	  	  pVal = $(p).find('b');
    	  	}

    	  	listing[pName] = pVal
    	  }
        })
        return listing
	}
	/*
		Accepts options, iterates through the known acceptable keys from defaultOptions
		and if found in input options, uses that. If not found in input options to method,
		falls back to the options specified when the module was initialized. If not found
		in initialization options, uses the default options setting. All keys provided in
		the input options variable are retained.
	*/
	function getRequestOptions (options, query) {
		var requestOptions = JSON.parse(JSON.stringify(defaultRequestOptions));
		if (options.fullListing  &&  options.path) {
			requestOptions.path = options.path
			requestOptions.hostname = options.hostname
			return requestOptions
		}

		// ensure default options are set, even if omitted from input options
		requestOptions.hostname = options.allCities? 'www' + baseHost : (options.city || self.options.city || '') + baseHost;

		// preserve any extraneous input option keys (may have addition instructions for underlying request object)
		Object.keys(options).forEach(function (key) {
			if (
				// key !== 'maxAsk'   &&
				// key !== 'minAsk'   &&
				// key !== 'category' &&
				// key !== 'hasPic'   &&
				// key !== 's'        &&
				typeof requestOptions[key] === 'undefined' &&
				typeof defaultRequestOptions[key] === 'undefined') {
				requestOptions[key] = options[key];
			}
		});

		// set path
		if (options.allCities)
		  requestOptions.path = '/about/sites'
		else if (options.categoriesOnly)
		  requestOptions.path = ''
		else
		  requestOptions.path = '/search/' + (options.category ? options.category : 'sss') + '?sort=rel' +
  		                                     (options.s ? '&s=' + options.s : '') +
		                                     (options.hasPic && (options.hasPic == '1' || options.hasPic == 'true') ? '&hasPic=1' : '') +
		                                     (options.rss ? '&format=rss' : '') +
		                                     (options.query ? '&query=' + encodeURIComponent(options.query) : '');

		// add min and max asking price
		if (typeof options.minAsk !== 'undefined') {
			requestOptions.path += searchMinAsk + options.minAsk;
		}

		if (typeof options.maxAsk !== 'undefined') {
			requestOptions.path += searchMaxAsk + options.maxAsk;
		}

		return requestOptions;
	}

	/*
		options = {
			city : '',
			maxAsk : '',
			minAsk : '',
		}
	*/
	self.search = function (options, query, callback) {
		if (typeof query === 'function' && typeof callback === 'undefined') {
			callback = query;
			query = options;
			options = {};
		}

        // The original set of options will have something like
        //
        //  options = {
		//   city: 'newyork',
		//   category: 'tia',
		//   hasPic: 1,
		//   fullListing: true,
		//   offset: 100,
		//   query: 'concert'
		// }
		//
		// only if particular listing's url is known then we can request the listing's page and description
		//
        var isFullDescriptionRequest = options.fullListing  &&  options.path
		options = getRequestOptions(options, query);

		self.request.get(options, function (err, data) {
			if (err)
			  return callback(err)

   //          if (options.allCities)
   //            return callback(null, getCities(options, data))
   //          if (options.rss)
   //            return callback(null, getRssListings(options, data))
   //          if (isFullDescriptionRequest)
   //            return callback(null, getDescription(options, data))
   //          if (options.categoriesOnly)
   //            return callback(null, getCategories(options, data))

			// return callback(null, getListings(options, data))
            if (options.allCities)
              return callback(null, data))
            if (options.rss)
              return callback(null, data))
            if (isFullDescriptionRequest)
              return callback(null, data))
            if (options.categoriesOnly)
              return callback(null, data))

			return callback(null, getListings(options, data))
		});
	};

	/*
		the module
	*/
	return function (options) {
		options = options || {};

		self.options = options;
		self.request = request.initialize();
		return self;
	};
}({}));

exports = module.exports = initialize;
exports.initialize = initialize;
<commit_msg>getting all craigslist categories<commit_after>var
	cheerio = require('cheerio'),
	request = require('./request'),

	baseHost = '.craigslist.org',
	defaultRequestOptions = {
		hostname : '',
		path : '',
	    secure : true
	},
	searchMaxAsk = '&maxAsk=',
	searchMinAsk = '&minAsk=';
	// searchPath = '/search/sss?sort=rel&query=';

    var initialize = (function (self) {
	'use strict';

	self = self || {};
	self.options = {};
	/*
		Accepts string of HTML and parses that string to find all pertinent listings.
	*/
	function getRssListings (options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var category = $('channel').find('title').text()

		$('item').each(function (i, element) {
			var children = $(element).children()

            var url = $(element).attr('rdf:about')
			var idx = url.lastIndexOf('/')
			var idx1 = url.indexOf('.', idx)
			var listing = { category : category, url: url, pid: url.substring(idx + 1, idx1) }

			for (var i=0; i<children.length; i++) {
				var t, child = children[i]
				var name = child.name

				if (name == 'title') {
					t = child.children[0].data.trim()
					if (t.indexOf('[CDATA[') != -1)
						t = t.substring(7, t.length - 2).trim().replace(/^\&\#x0024\;/g, '').replace(/&\#x0024\;/g, '$')

					var priceIdx = t.indexOf('$')
					if (priceIdx != -1)  {
						var sidx = t.indexOf(' ', priceIdx)
						listing.price = sidx == -1 ? t.substring(priceIdx) : t.substring(priceIdx, sidx)
					}
					var idx = t.lastIndexOf('(')
				    if (idx == -1)
				    	listing.title = t
				    else {
				    	var idx1 = t.indexOf(')', idx)
				    	listing.location = t.substring(idx + 1, idx1).trim()
				    	listing.title = t.substring(0, idx).trim()
				    }

				}
				else if (name == 'description') {
					t = child.children[0].data.trim()
					if (t.indexOf('[CDATA[') != -1)
						t = t.substring(7, t.length - 2)
					listing.description = t.replace(/&\#x0024\;/g, '$').trim()
				}
				else if (name == 'dc:date')
					listing.date = $(child).text().trim()
				else if (name == 'enc:enclosure') {
					listing.hasPic = true
					listing.img = $(child).attr('resource')
				}
			}

					// coordinates : {
					// 	lat : $(element).attr('data-latitude'),
					// 	lon : $(element).attr('data-longitude')
					// },

				// make sure lat / lon is valid
				// if (typeof listing.coordinates.lat === 'undefined' ||
				// 	typeof listing.coordinates.lon === 'undefined') {
				// 	delete listing.coordinates;
				// }
			listings.push(listing);
		});

		return listings;
	}

	/*
		Accepts string of HTML and parses that string to find all pertinent listings.
	*/
	function getListings (options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

		$('div.content').find('p.row').each(function (i, element) {
			listing = {
				category : $(element)
					.find('span.l2 a.gc')
					.text(),
				coordinates : {
					lat : $(element).attr('data-latitude'),
					lon : $(element).attr('data-longitude')
				},
				date : $(element)
					.find('span.date')
					.text(),
				hasPic : ($(element)
					.find('span.l2 span.p')
					.text()
					.trim()) !== '',
				location : $(element)
					.find('span.pnr small')
					.text()
					.replace(/[\(,\)]/g, '').trim(), // santize
				pid : $(element)
					.attr('data-pid'),
				price : $(element)
					.find('span.l2 span.price')
					.text()
					.replace(/^\&\#x0024\;/g, ''), // sanitize
				title : $(element)
					.find('span.pl a')
					.text().trim()
				// url :
				// 	(options.secure ? 'https://' : 'http://') +
				// 	options.hostname + $(element).find('span.pl a').attr('href')
			}
			var rurl = $(element).find('span.pl a').attr('href');
			if (rurl) {
			  if (rurl.indexOf('http') == -1)
			    rurl = (options.secure ? 'https://' : 'http://') + options.hostname + rurl
			  listing.url = rurl;
		    }
		    if (options.cityName)
		      listing.cityName = options.cityName

			if (listing.hasPic) {
				var dId = $(element).find('a.i').attr('data-id')
				if (dId)
				  listing.pic = 'http://images.craigslist.org/' + dId.substring(2) + '_300x300.jpg'
			}
			var d = $(element).find('span.date')
			if (!d  ||  d.text().length == 0) {
				d = $(element).find('time').attr('datetime')
				listing.date = d;
			}

			// make sure lat / lon is valid
			if (typeof listing.coordinates.lat === 'undefined' ||
				typeof listing.coordinates.lon === 'undefined') {
				delete listing.coordinates;
			}

			listings.push(listing);
		});

		return listings;
	}

	function getCities(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

		$('div.colmask').find('a').each(function (i, element) {
           var href = $(element).attr('href')
           var idx = href.indexOf('//'), idx1 = href.indexOf('.');

           listing = { city: href.substring(idx + 2, idx1),
           			   href: href,
                       cityName: $(element).text().replace(/\w\S*/g, function(txt){return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase()})
                     }

           listings.push(listing)
		})
		return listings
	}

	function getCategories(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var city = $('h2.no-mobile').html();
        var idx = city.indexOf('<');
        if (idx != -1)
          city = city.substring(0, idx)

		$('div#center').find('a').each(function (i, element) {
           var href = $(element).attr('href')
           if (href.indexOf('/search/') != 0)
           	return;
           var idx = href.indexOf('//'), idx1 = href.indexOf('.');

           listing = { categoryName: $(element).text(),
           			   categoryHref: href,
                       city: city
                     }

           listings.push(listing)
		})
		return listings
	}
	function getSubCities(options, html) {
		var
			$ = cheerio.load(html),
			listing = {},
			listings = [];

        var city = $('h2.no-mobile').html();
        var idx = city.indexOf('<');
        if (idx != -1)
          city = city.substring(0, idx)

		$('ul.sublinks').find('a').each(function (i, element) {
       var href = $(element).attr('href')

       listing = { subCityHref: options.hostname + href,
                   subCity: $(element).attr('title')
                 }

       listings.push(listing)
		})
		// nearby cities
		var rightPanelElms = $('div#rightbar').find('.menu .collapsible').find('li');
		for (var i=0; i<rightPanelElms; i++) {
			var li = rightPaneElms.get(i);
			if ($(li).find('h5').get(0).text() != 'nearby cl')
				continue;
			$(li).find('a').each(function(i, element) {
				listing = { city: $(element).text(),
					        href: $(element).attr('href')
				          }
			})

		}
		return listings
	}

    // Extracting full description for the listing
	function getDescription(options, html) {
		var
			$ = cheerio.load(html);

		var description = $('section#postingbody').html()
		var listing = {}
		if (description)
	  	  listing.description = description.trim()
		var attr = $('p.attrgroup')
        if (!attr)
          return listing
    	attr.find('span').each(function(i, p) {
    	  if ($(p).attr('class'))
    	  	listing.date = $(p).find('a').text()
    	  else {
    	  	var pVal, pName = $(p).text().trim()

    	  	var idx = pName.indexOf(':')
    	  	if (idx == -1) {
    	  	  pVal = pName
    	  	  pName = 'p' + i
    	  	}
    	  	else {
    	  	  pName = pName.substring(0, idx).trim()
    	  	  if (pName.charAt(0) == '#')
    	  	    pName = ('Number of' + pName.substring(1))
    	  	  pVal = $(p).find('b');
    	  	}

    	  	listing[pName] = pVal
    	  }
        })
        return listing
	}
	/*
		Accepts options, iterates through the known acceptable keys from defaultOptions
		and if found in input options, uses that. If not found in input options to method,
		falls back to the options specified when the module was initialized. If not found
		in initialization options, uses the default options setting. All keys provided in
		the input options variable are retained.
	*/
	function getRequestOptions (options, query) {
		var requestOptions = JSON.parse(JSON.stringify(defaultRequestOptions));
		if (options.fullListing  &&  options.path) {
			requestOptions.path = options.path
			requestOptions.hostname = options.hostname
			return requestOptions
		}

		// ensure default options are set, even if omitted from input options
		requestOptions.hostname = options.allCities? 'www' + baseHost : (options.city || self.options.city || '') + baseHost;

		// preserve any extraneous input option keys (may have addition instructions for underlying request object)
		Object.keys(options).forEach(function (key) {
			if (
				// key !== 'maxAsk'   &&
				// key !== 'minAsk'   &&
				// key !== 'category' &&
				// key !== 'hasPic'   &&
				// key !== 's'        &&
				typeof requestOptions[key] === 'undefined' &&
				typeof defaultRequestOptions[key] === 'undefined') {
				requestOptions[key] = options[key];
			}
		});

		// set path
		if (options.allCities)
		  requestOptions.path = '/about/sites'
		else if (options.categoriesOnly)
		  requestOptions.path = ''
		else
		  requestOptions.path = '/search/' + (options.category ? options.category : 'sss') + '?sort=rel' +
  		                                     (options.s ? '&s=' + options.s : '') +
		                                     (options.hasPic && (options.hasPic == '1' || options.hasPic == 'true') ? '&hasPic=1' : '') +
		                                     (options.rss ? '&format=rss' : '') +
		                                     (options.query ? '&query=' + encodeURIComponent(options.query) : '');

		// add min and max asking price
		if (typeof options.minAsk !== 'undefined') {
			requestOptions.path += searchMinAsk + options.minAsk;
		}

		if (typeof options.maxAsk !== 'undefined') {
			requestOptions.path += searchMaxAsk + options.maxAsk;
		}

		return requestOptions;
	}

	/*
		options = {
			city : '',
			maxAsk : '',
			minAsk : '',
		}
	*/
	self.search = function (options, query, callback) {
		if (typeof query === 'function' && typeof callback === 'undefined') {
			callback = query;
			query = options;
			options = {};
		}

        // The original set of options will have something like
        //
        //  options = {
		//   city: 'newyork',
		//   category: 'tia',
		//   hasPic: 1,
		//   fullListing: true,
		//   offset: 100,
		//   query: 'concert'
		// }
		//
		// only if particular listing's url is known then we can request the listing's page and description
		//
        var isFullDescriptionRequest = options.fullListing  &&  options.path
		options = getRequestOptions(options, query);

		self.request.get(options, function (err, data) {
			if (err)
			  return callback(err)

   //          if (options.allCities)
   //            return callback(null, getCities(options, data))
   //          if (options.rss)
   //            return callback(null, getRssListings(options, data))
   //          if (isFullDescriptionRequest)
   //            return callback(null, getDescription(options, data))
   //          if (options.categoriesOnly)
   //            return callback(null, getCategories(options, data))

			// return callback(null, getListings(options, data))
            if (options.allCities)
              return callback(null, data)
            if (options.rss)
              return callback(null, data)
            if (isFullDescriptionRequest)
              return callback(null, data)
            if (options.categoriesOnly)
              return callback(null, data)

			return callback(null, getListings(options, data))
		});
	};

	/*
		the module
	*/
	return function (options) {
		options = options || {};

		self.options = options;
		self.request = request.initialize();
		return self;
	};
}({}));

exports = module.exports = initialize;
exports.initialize = initialize;
<|endoftext|>


<commit_before>//noinspection BadExpressionStatementJS
'format es6';

import { nnNgConfigurations } from 'nn-ng-utils';

function configureUiStateSecurity ( $rootScope, $state, authService, cnst ) {
  //noinspection BadExpressionStatementJS
  'ngInject';

  $rootScope.$on( cnst.events.stateChangeStart, ( event, toState ) => {
    if ( unknownUserGoesToPrivateState() ) {
      goTo( 'root.user.login' );
    }

    if ( loggedInUserGoesToStateForUnknownUser() ) {
      goTo( '??????????????????' );
    }

    function unknownUserGoesToPrivateState () {
      return toState.data.requiresAuthentication &&
        Boolean( authService.userIsNotLoggedIn() );
    }

    function loggedInUserGoesToStateForUnknownUser () {
      return authService.userIsLoggedIn() && toStateIsOnlyForUnknownUser();

      function toStateIsOnlyForUnknownUser () {
        return toState.name === 'root.user.login' ||
               toState.name === 'root.user.signup' ||
               toState.name === 'root.user.signuptoken';
      }
    }

    function goTo ( targetState ) {
      $state.transitionTo( targetState );
      event.preventDefault();
    }
  });
}

nnNgConfigurations.registerForRunPhase( configureUiStateSecurity );
<commit_msg>add TODO to test if canceling spinner on state transition redirects (due to access permissions) is necessary<commit_after>//noinspection BadExpressionStatementJS
'format es6';

import { nnNgConfigurations } from 'nn-ng-utils';

function configureUiStateSecurity ( $rootScope, $state, authService, cnst ) {
  //noinspection BadExpressionStatementJS
  'ngInject';

  $rootScope.$on( cnst.events.stateChangeStart, ( event, toState ) => {
    if ( unknownUserGoesToPrivateState() ) {
      goTo( 'root.user.login' );
    }

    if ( loggedInUserGoesToStateForUnknownUser() ) {
      goTo( '??????????????????' );
    }

    function unknownUserGoesToPrivateState () {
      return toState.data.requiresAuthentication &&
        Boolean( authService.userIsNotLoggedIn() );
    }

    function loggedInUserGoesToStateForUnknownUser () {
      return authService.userIsLoggedIn() && toStateIsOnlyForUnknownUser();

      function toStateIsOnlyForUnknownUser () {
        return toState.name === 'root.user.login' ||
               toState.name === 'root.user.signup' ||
               toState.name === 'root.user.signuptoken';
      }
    }

    function goTo ( targetState ) {
      // TODO: cancel spinner
      $state.transitionTo( targetState );
      event.preventDefault();
    }
  });
}

nnNgConfigurations.registerForRunPhase( configureUiStateSecurity );
<|endoftext|>


<commit_before>'use strict';
/**
 * @file Easy access to AnyFetch API.
 * @TODO: wrap into an Anyfetch class (use prototype)
 * @TODO: split into smaller files and / or functions
 * @TODO: allow all parameters when starting with `_` (e.g. `_meta`)
 * @see http://anyFetch.com
 */

var request = require('supertest');

var isFunction = require('./helpers/is-function.js');
var isMongoId = require('./helpers/is-mongo-id.js');

var configuration = require('../config/configuration.js');

/**
 * Supported syntaxes:
 * - `new Anyfetch(accessToken)` (token authentication)
 * - `new Anyfetch(login, password)` (basic authentication)
 */
var Anyfetch = function(accessToken, password) {
  if(password) {
    // This is actually username:password
    var payload = new Buffer(accessToken + ':' + password);
    this.authHeader = 'Basic ' + payload.toString('base64');
  }
  else {
    this.authHeader = 'Bearer ' + accessToken;
  }

  this.apiHost = configuration.apiHost;
};

Anyfetch.prototype.setApiHost = function(host) {
  this.apiHost = host;
};

/**
 * Generate a simple API mapping function from an endpoint description
 * @param {String} name to give to the function
 * @param {Object} config Description of the endpoint
 * @see `config/json/api-descriptors.json` for examples
 */
var generateMapping = function(name, config) {
  /**
   * Example of accepted call syntaxes:
   * (depending on the requirements of the API endpoint)
   * apiMapping(cb)
   * apiMapping(id, cb)
   * apiMapping(body, cb)
   * apiMapping(id, body, cb)
   * apiMapping(id, params, cb)
   * apiMapping(params, cb)
   */
  return function apiMapping() {
    // Convert arguments to an array
    var args = Array.prototype.slice.call(arguments);

    // We must support quite a lot of different call syntaxes
    var endpoint = config.endpoint;
    var params = {};
    var body = {};
    var key;

    // cb is always the last argument
    var cb = args.pop();
    // cb must be a function
    if (!isFunction(cb)) {
      throw new Error('Argument error in ' + name + ':' + ' the last argument must be a function (callback)');
    }

    if (config.body) {
      body = args.pop() || {};
      // Make sure that every field of the body is accepted
      for(key in body) {
        if(config.body.indexOf(key) === -1) {
          return cb(new Error('Argument error in ' + name + ':' + ' the key ' + key + ' is not allowed in this request\'s body'));
        }
      }
    }

    if (config.params) {
      params = args.pop() || {};
      // Make sure that every GET parameter is accepted
      for(key in params) {
        if(config.params.indexOf(key) === -1) {
          return cb(new Error('Argument error in ' + name + ':' + ' the key ' + key + ' is not allowed in this request\'s GET parameters'));
        }
      }
    }

    // id is always the first parameter (when it's actually required)
    // An `identifier` is not necessarily numeric
    if(config.requireId) {
      var id = args.pop();
      if(!id || !isMongoId(id)) {
        return cb(new Error('Argument error in ' + name + ':' + ' the first parameter must be a numeric id'));
      }

      // Substitute id parameter in endpoint string
      endpoint = endpoint.replace('{id}', id);
    }

    if(config.requireIdentifier) {
      var identifier = encodeURI(args.pop());
      // Substitute id parameter in endpoint string
      endpoint = endpoint.replace('{identifier}', identifier);
    }

    // Build request (using supertest)
    var r = request(this.apiHost)[config.method](endpoint)
      .query(params)
      .set('Authorization', this.authHeader);

    if (config.body) {
      r = r.send(body);
    }

    r.expect(config.expectedStatus)
     .end(cb);
  };
};

/**
 * Add the nice semantic aliases described in `configuration.aliases`
 * E.g. postSubcompany --> postSubcompanies
 * (What it does is to post ONE company, so it makes more sense)
 * @param {Object} aliases The object which associates new name to old name
 */
var generateAliases = function(aliases) {
  for(var newName in aliases) {
    Anyfetch.prototype[newName] = Anyfetch.prototype[aliases[newName]];
  }
};

/**
 * Allow more complex syntax for API calls that are related to an element.
 * Examples:
 * - `getDocumentById(123).getRelated(cb)`
 * - `getDocumentByIdentifier(123).postFile(file, cb)`
 *
 * @param {String} extendedEndpoint The name of the function to extend
 * @param {Object} baseConfig Descriptor of the endpoint being extended
 * @param {Function} singleStepFunction Initial (single-step) version of the function
 * @warning getDocumentById(123, cb) remains valid as well!
 */
var generateSubFunctionMappings = function(extendedEndpoint, baseConfig, singleStepFunction) {
  // Descriptors of the second-step calls available
  var subFunctionsConfig = baseConfig.subFunctions;

  // Two-step call syntax
  var subFunctionsGenerator = {};
  // For each endpoint declared as a second step
  Object.keys(subFunctionsConfig).forEach(function(name) {
    // Local copy
    var thisConfig = {};
    for (var i in subFunctionsConfig[name]) {
      thisConfig[i] = subFunctionsConfig[name][i];
    }

    subFunctionsGenerator[name] = generateMapping(name, thisConfig);
  });

  /**
   * Support two call syntaxes:
   * - Single step: theEndpoint(123, cb)
   * - Two steps: theEndpoint(123).getRelated(cb)
   * Error handling is always done at the last step so as
   * to avoid throwing.
   */
  return function(id, cb) {
    // Single-step call syntax
    if (isFunction(cb)) {
      if (baseConfig.requireId && (!id || !isMongoId(id))) {
        return cb(new Error('Argument error in ' + extendedEndpoint + ': the first argument must be a numeric id'));
      }

      return singleStepFunction.call(this, id, arguments[1]);
    }

    /**
     * Currying FTW
     */
    var subFunctions = {};
    var self = this;
    Object.keys(subFunctionsConfig).forEach(function(name) {
      subFunctions[name] = function() {
        var args = Array.prototype.slice.call(arguments);
        args.unshift(id);
        subFunctionsGenerator[name].apply(self, args);
      };
    });

    return subFunctions;
  };
};

/**
 * Manually add the `postFile` function to `getDocumentById`
 * in Anyfetch's prototype
 */
var addPostFileFunctions = function() {

  var makePostFileFunction = function(extendedEndpoint, id) {
    var idMustBeValid = configuration.apiDescriptors[extendedEndpoint].requireId;
    var baseEndpoint = configuration.apiDescriptors[extendedEndpoint].endpoint;

    /**
     * Add a file to a previously uploaded document
     *
     * @param {Object} Must at least contain a `file` key, which can either be a stream (e.g. `fs.createReadStream`) or a Buffer object. Can also contains a `contentType` key (for MIME type), and a `filename`.
     * @param {Function} cb Callback with error if any.
     * @Warning: unfortunately, due to the variety of Stream, we can't type-check, so unexpected errors will occur if you specify weird file parameters.
     */
    return function postFile(config, cb) {
      if (idMustBeValid && (!id || !isMongoId(id))) {
        return cb(new Error('Argument error in ' + extendedEndpoint + '.postFile: the first argument must be a numeric id'));
      }

      if(isFunction(config)) {
        config = config();
      }
      if (!config.file) {
        return cb(new Error('The config parameter must contain a `file` key'));
      }

      var endpoint;
      if(configuration.apiDescriptors[extendedEndpoint].requireId) {
        endpoint = baseEndpoint.replace('{id}', id) + '/file';
      }
      if(configuration.apiDescriptors[extendedEndpoint].requireIdentifier) {
        endpoint = baseEndpoint.replace('{identifier}', id) + '/file';
      }

      var options = {
        filename: config.filename || '',
        contentType: config.contentType || ''
      };

      request(this.apiHost).post(endpoint)
        .set('Authorization', this.authHeader)
        .attach('file', config.file, options)
        .expect(204)
        .end(cb);
    };
  };

  var addPostFileFunction = function(endpointToExtend) {
    var previousVersion = Anyfetch.prototype[endpointToExtend];
    Anyfetch.prototype[endpointToExtend] = function(id, cb) {
      var ret = previousVersion.call(this, id, cb);
      // Two-step call syntax
      // In this case, `ret` contains the various "extended" functions
      if (!isFunction(cb)) {
        // Manually add the postFile function
        var f = makePostFileFunction(endpointToExtend, id);
        ret.postFile = f.bind(this);
      }
      return ret;
    };
  };

  addPostFileFunction('getDocumentsById');
  addPostFileFunction('getDocumentsByIdentifier');
};

/**
 * Generate the associated function for each endpoint
 * described in `configuration.apiDescriptors`.
 * If a key `subFunctions` is specified, advanced mappings (two-steps)
 * are added as well.
 */
var generateMappings = function(descriptors) {
  Object.keys(descriptors).forEach(function(name) {
    var config = descriptors[name];

    Anyfetch.prototype[name] = generateMapping(name, config);

    if(config.subFunctions) {
      Anyfetch.prototype[name] = generateSubFunctionMappings(name, config, Anyfetch.prototype[name]);
    }
  });
};

generateMappings(configuration.apiDescriptors);
addPostFileFunctions();
generateAliases(configuration.aliases);

module.exports = Anyfetch;<commit_msg>Allow `param` and `body` to be omitted when `requireId`<commit_after>'use strict';
/**
 * @file Easy access to AnyFetch API.
 * @TODO: allow all parameters when starting with `_` (e.g. `_meta`)
 * @see http://anyFetch.com
 */

var request = require('supertest');
var util = require('util');

var isFunction = require('./helpers/is-function.js');
var isMongoId = require('./helpers/is-mongo-id.js');

var configuration = require('../config/configuration.js');

/**
 * Supported syntaxes:
 * - `new Anyfetch(accessToken)` (token authentication)
 * - `new Anyfetch(login, password)` (basic authentication)
 */
var Anyfetch = function(accessToken, password) {
  if(password) {
    // This is actually username:password
    var payload = new Buffer(accessToken + ':' + password);
    this.authHeader = 'Basic ' + payload.toString('base64');
  }
  else {
    this.authHeader = 'Bearer ' + accessToken;
  }

  this.apiHost = configuration.apiHost;
};

Anyfetch.prototype.setApiHost = function(host) {
  this.apiHost = host;
};

/**
 * Generate a simple API mapping function from an endpoint description
 * @param {String} name to give to the function
 * @param {Object} config Description of the endpoint
 * @see `config/json/api-descriptors.json` for examples
 */
var generateMapping = function(name, config) {
  /**
   * Example of accepted call syntaxes:
   * (depending on the requirements of the API endpoint)
   * apiMapping(cb)
   * apiMapping(id, cb)
   * apiMapping(body, cb)
   * apiMapping(id, [body], cb)    (optionnal middle parameter)
   * apiMapping(id, [params], cb)  (optionnal middle parameter)
   * apiMapping(params, cb)
   */
  return function apiMapping() {
    // Convert arguments to an array
    var args = Array.prototype.slice.call(arguments);

    // We must support quite a lot of different call syntaxes
    var endpoint = config.endpoint;
    var params = {};
    var body = {};
    var key;

    // cb is always the last argument
    var cb = args.pop();
    // cb must be a function
    if (!isFunction(cb)) {
      throw new Error('Argument error in ' + name + ':' + ' the last argument must be a function (callback)');
    }

    if (config.body) {
      body = args.pop() || {};
      // Make sure that every field of the body is accepted
      if(util.isArray(body)) {
        for(key in body) {
          if(config.body.indexOf(key) === -1) {
            return cb(new Error('Argument error in ' + name + ':' + ' the key ' + key + ' is not allowed in this request\'s body'));
          }
        }
      }
    }

    if (config.params) {
      params = args.pop() || {};
      // Make sure that every GET parameter is accepted
      if(util.isArray(params)) {
        for(key in params) {
          if(config.params.indexOf(key) === -1) {
            return cb(new Error('Argument error in ' + name + ':' + ' the key ' + key + ' is not allowed in this request\'s GET parameters'));
          }
        }
      }
    }

    // id is always the first parameter (when it's actually required)
    // An `identifier` is not necessarily numeric
    if(config.requireId) {
      var id = args.pop();
      // We allow `params` and / or `body` to be omitted

      id = id || params || body;
      if(!id || !isMongoId(id)) {
        return cb(new Error('Argument error in ' + name + ':' + ' the first parameter must be a numeric id'));
      }

      // Substitute id parameter in endpoint string
      endpoint = endpoint.replace('{id}', id);
    }

    if(config.requireIdentifier) {
      var identifier = encodeURI(args.pop());
      // Substitute id parameter in endpoint string
      endpoint = endpoint.replace('{identifier}', identifier);
    }

    // Build request (using supertest)
    var r = request(this.apiHost)[config.method](endpoint)
      .query(params)
      .set('Authorization', this.authHeader);

    if (config.body) {
      r = r.send(body);
    }

    r.expect(config.expectedStatus)
     .end(cb);
  };
};

/**
 * Add the nice semantic aliases described in `configuration.aliases`
 * E.g. postSubcompany --> postSubcompanies
 * (What it does is to post ONE company, so it makes more sense)
 * @param {Object} aliases The object which associates new name to old name
 */
var generateAliases = function(aliases) {
  for(var newName in aliases) {
    Anyfetch.prototype[newName] = Anyfetch.prototype[aliases[newName]];
  }
};

/**
 * Allow more complex syntax for API calls that are related to an element.
 * Examples:
 * - `getDocumentById(123).getRelated(cb)`
 * - `getDocumentByIdentifier(123).postFile(file, cb)`
 *
 * @param {String} extendedEndpoint The name of the function to extend
 * @param {Object} baseConfig Descriptor of the endpoint being extended
 * @param {Function} singleStepFunction Initial (single-step) version of the function
 * @warning getDocumentById(123, cb) remains valid as well!
 */
var generateSubFunctionMappings = function(extendedEndpoint, baseConfig, singleStepFunction) {
  // Descriptors of the second-step calls available
  var subFunctionsConfig = baseConfig.subFunctions;

  // Two-step call syntax
  var subFunctionsGenerator = {};
  // For each endpoint declared as a second step
  Object.keys(subFunctionsConfig).forEach(function(name) {
    // Local copy
    var thisConfig = {};
    for (var i in subFunctionsConfig[name]) {
      thisConfig[i] = subFunctionsConfig[name][i];
    }

    subFunctionsGenerator[name] = generateMapping(name, thisConfig);
  });

  /**
   * Support two call syntaxes:
   * - Single step: theEndpoint(123, cb)
   * - Two steps: theEndpoint(123).getRelated(cb)
   * Error handling is always done at the last step so as
   * to avoid throwing.
   */
  return function(id, cb) {
    // Single-step call syntax
    if (isFunction(cb)) {
      if (baseConfig.requireId && (!id || !isMongoId(id))) {
        return cb(new Error('Argument error in ' + extendedEndpoint + ': the first argument must be a numeric id'));
      }

      return singleStepFunction.call(this, id, arguments[1]);
    }

    /**
     * Currying FTW
     */
    var subFunctions = {};
    var self = this;
    Object.keys(subFunctionsConfig).forEach(function(name) {
      subFunctions[name] = function() {
        var args = Array.prototype.slice.call(arguments);
        args.unshift(id);
        subFunctionsGenerator[name].apply(self, args);
      };
    });

    return subFunctions;
  };
};

/**
 * Manually add the `postFile` function to `getDocumentById`
 * in Anyfetch's prototype
 */
var addPostFileFunctions = function() {

  var makePostFileFunction = function(extendedEndpoint, id) {
    var idMustBeValid = configuration.apiDescriptors[extendedEndpoint].requireId;
    var baseEndpoint = configuration.apiDescriptors[extendedEndpoint].endpoint;

    /**
     * Add a file to a previously uploaded document
     *
     * @param {Object} Must at least contain a `file` key, which can either be a stream (e.g. `fs.createReadStream`) or a Buffer object. Can also contains a `contentType` key (for MIME type), and a `filename`.
     * @param {Function} cb Callback with error if any.
     * @Warning: unfortunately, due to the variety of Stream, we can't type-check, so unexpected errors will occur if you specify weird file parameters.
     */
    return function postFile(config, cb) {
      if (idMustBeValid && (!id || !isMongoId(id))) {
        return cb(new Error('Argument error in ' + extendedEndpoint + '.postFile: the first argument must be a numeric id'));
      }

      if(isFunction(config)) {
        config = config();
      }
      if (!config.file) {
        return cb(new Error('The config parameter must contain a `file` key'));
      }

      var endpoint;
      if(configuration.apiDescriptors[extendedEndpoint].requireId) {
        endpoint = baseEndpoint.replace('{id}', id) + '/file';
      }
      if(configuration.apiDescriptors[extendedEndpoint].requireIdentifier) {
        endpoint = baseEndpoint.replace('{identifier}', id) + '/file';
      }

      var options = {
        filename: config.filename || '',
        contentType: config.contentType || ''
      };

      request(this.apiHost).post(endpoint)
        .set('Authorization', this.authHeader)
        .attach('file', config.file, options)
        .expect(204)
        .end(cb);
    };
  };

  var addPostFileFunction = function(endpointToExtend) {
    var previousVersion = Anyfetch.prototype[endpointToExtend];
    Anyfetch.prototype[endpointToExtend] = function(id, cb) {
      var ret = previousVersion.call(this, id, cb);
      // Two-step call syntax
      // In this case, `ret` contains the various "extended" functions
      if (!isFunction(cb)) {
        // Manually add the postFile function
        var f = makePostFileFunction(endpointToExtend, id);
        ret.postFile = f.bind(this);
      }
      return ret;
    };
  };

  addPostFileFunction('getDocumentsById');
  addPostFileFunction('getDocumentsByIdentifier');
};

/**
 * Generate the associated function for each endpoint
 * described in `configuration.apiDescriptors`.
 * If a key `subFunctions` is specified, advanced mappings (two-steps)
 * are added as well.
 */
var generateMappings = function(descriptors) {
  Object.keys(descriptors).forEach(function(name) {
    var config = descriptors[name];

    Anyfetch.prototype[name] = generateMapping(name, config);

    if(config.subFunctions) {
      Anyfetch.prototype[name] = generateSubFunctionMappings(name, config, Anyfetch.prototype[name]);
    }
  });
};

generateMappings(configuration.apiDescriptors);
addPostFileFunctions();
generateAliases(configuration.aliases);

module.exports = Anyfetch;<|endoftext|>


<commit_before>'use strict';

var url = require('url');
var EventEmitter = require('events').EventEmitter;
var inherits = require('util').inherits;
var _ = require('lodash');
var propagate = require('propagate');

// skiff parts
var Skiff = require('skiff-redis');
var Gossip = require('sombrero-gossip');
var defaultOptions = require('./defaultOptions.js');
var Peer = require('./peer.js');
var callbackQueue = require('callback-queue');

/**
 * Creates task dispatcher node
 * @param {String} skiffURL
 * @param {Object} options
 */
function Node(skiffURL, options) {
    var connectionData;

    // instanciate ee
    EventEmitter.call(this);

    // bind methods and get default options
    _.bindAll(this);
    _.defaults(options, defaultOptions);

    if (!skiffURL) {
        throw new Error('need skiff URL');
    }

    this.id = skiffURL;
    this._options = options;

    this.callbackQueue = callbackQueue;

    // init metadata
    connectionData = url.parse(skiffURL);
    this.metadata = {
        gossipPort: options.gossip.port,
        hostname: connectionData.hostname,
        port: options.port
    };
    options.skiff.metadata = this.metadata;

    // init skiff node
    // determines cluster leader and performs fail over if leader dies
    this.skiff = new Skiff(skiffURL, options.skiff);
    this.skiff.on('joined', this._onPeerJoined);
    this.skiff.on('reconnected', this._onPeerJoined);
    this.skiff.on('leader', this._onLeader);
    this.skiff.on('left', this._removePeerGossip);

    propagate(this.skiff, this);

    // setup gossip cluster - basically in-memory shared storage
    // useful for holding shared cluster state
    this._remotes = {};

    // these proxy methods will grant access to other operations that are
    // defined in _allowedRemoteCalls variable
    this._remoteCallMethods = _.keys(this._allowedRemoteCalls);

    // instantiate transport server
    this._transportServer = options.transport.listen(options.port, options.hostname, _.pick(this, this._remoteCallMethods), this._transportListening);
    this._transportServer.on('error', this._proxyError);

    this.gossip = new Gossip(this._options.gossip);
    this.gossip.on('error', this._proxyError);
    this.gossip.on('cluster change', this._onGossipClusterChange);
}
inherits(Node, EventEmitter);

/**
 * Static function: attaches remote call
 * @param {String}   name     - resource name
 * @param {Function} function - function to perform, will be invoked with local scope
 */
Node.attachRemoteCall = function (name, method) {
    if (typeof Node.prototype[name] !== 'undefined') {
        throw new Error('Function ' + name + ' is already defined');
    }

    Node.prototype._allowedRemoteCalls[name] = true;
    Node.prototype[name] = method;
};

//
// Cluster rpc-allowed operations
//

/**
 * Method that can be performed by an RPC call
 * @type {Object}
 */
Node.prototype._allowedRemoteCalls = {
    getsetnx: true,
    update: true
};

/**
 * Returns operation key that is used for callback caching
 * @return {String}
 */
Node.prototype.ok = function () {
    var length = arguments.length;
    if (length === 0) {
        throw new Error('arguments must not be empty');
    }

    var arr = new Array(length);
    for (var i = 0; i < length; i++) {
        arr[i] = arguments[i];
    }

    return arr.join('~');
};

/**
 * Get resourceId of resourceType, if not exists - create it first.
 * The operation flow is the following: check local storage (gossip cluster),
 * if there is no result of this operation, queue this operation locally
 * and make a remote call
 *
 * @param  {String}   resourceId   - unique resource id, used for request queueing
 * @param  {String}   resourceType - resource type, determines what has to be done
 * @param  {Array}    args         - args to be applied to resource type
 * @param  {Function} next         - invoked when method completes
 *
 */
Node.prototype.getsetnx = function (resourceId, resourceType, args, next) {

    if (typeof resourceId !== 'string' || !resourceId) {
        return setImmediate(next, new Error('resourceId must be a truthy string'));
    }

    if (typeof resourceType !== 'string' || !this._allowedRemoteCalls[resourceType]) {
        return setImmediate(next, new Error('remote call ' + resourceType + ' is not allowed'));
    }

    var cluster = this.gossip.cluster;
    var operationKey = [ resourceType, resourceId ].join('~');
    var operationResult = cluster.get(operationKey);

    if (operationResult !== null && operationResult !== undefined) {
        return setImmediate(next, null, operationResult);
    }

    var callback = callbackQueue.add(operationKey, next);
    if (!callback) {
        // callback will be invoked later when ongoing operation will be completed
        return;
    }

    var currentLeader = cluster.get('leader');
    if (currentLeader === this.id) {
        args.unshift(operationKey, resourceId);
        args.push(callback);
        return this[resourceType].apply(this, args);
    }

    // push args there
    args.unshift(resourceId, resourceType);

    setImmediate(this._remoteCall, currentLeader, 'getsetnx', args, callback);
};

/**
 * Deletes specific resource type from shared storage, can be performed from either node
 * @param  {String}   resourceId
 * @param  {String}   resourceType
 * @param  {Function} next
 */
Node.prototype.del = function (resourceId, resourceType, next) {
    this.update(resourceId, resourceType, null, next);
};

/**
 * Updates specific resource type from shared storage, can be performed from either node
 *
 * @param  {String}   resourceId
 * @param  {String}   resourceType
 * @param  {Mixed}    value
 * @param  {Function} next
 *
 */
Node.prototype.update = function (resourceId, resourceType, value, next) {

    if (typeof resourceId !== 'string' || !resourceId) {
        return setImmediate(next, new Error('resourceId must be a truthy string'));
    }

    if (typeof resourceType !== 'string' || !this._allowedRemoteCalls[resourceType]) {
        return setImmediate(next, new Error('remote call ' + resourceType + ' is not allowed'));
    }

    if (value === undefined) {
        return setImmediate(next, new Error('value cant be set to undefined'));
    }

    var cluster = this.gossip.cluster;
    var operationKey = [ resourceType, resourceId ].join('~');
    var currentValue = cluster.get(operationKey);

    if (currentValue === value) {
        return setImmediate(next);
    }

    var currentLeader = cluster.get('leader');
    if (currentLeader === this.id) {
        cluster.set(operationKey, value);
        return setImmediate(next);
    }

    setImmediate(this._remoteCall, currentLeader, 'update', [ resourceId, resourceType, value ], next);
};

//
// Cluster topology section, public
//

Node.prototype.join = function join(url, options, next) {
    this.skiff.join(url, options || null, next);
};

Node.prototype.leave = function leave(url, next) {
    this.skiff.leave(url, next);
};

Node.prototype.open = function open(next) {
    this.skiff.open(next);
};

Node.prototype.close = function close(next) {
    var self = this;
    this._transportServer.close(function () {
        self.gossip.stop(function () {
            self.skiff.close(next);
        });
    });
};

//
// cluster setup section, private
//

/**
 * Performs RPC call
 * @param {String}   node   - node id to perform the operation on
 * @param {String}   method - method to invoke
 * @param {Array}    args   - arguments to pass
 * @param {Function} next   - callback that will be eventually called
 */
Node.prototype._remoteCall = function (node, method, args, next) {
    var remote = this._remote(node);
    var invoke = function (client) {
        var m = client[method];
        if (!m) {
            return next(new Error('Method not found: ' + method));
        }
        args.push(next);
        m.apply(client, args);
    };

    if (!remote) {
        return next(new Error('could not find remote metadata for URL ' + node));
    } else if (remote.connected) {
        invoke(remote.client);
    } else {
        remote.once('connect', invoke);
    }
};

/**
 * Establishes peer connection when it's first required to
 * @param  {String} node - node id
 * @return {TransportConnection}
 */
Node.prototype._remote = function (node) {
    var remote = this.peerOnline(node);
    if (!remote) {
        var meta = this.skiff.peerMeta(node);
        if (meta) {
            remote = this._remotes[node] = new Peer(this._remoteCallMethods, meta.hostname, meta.port);
        }
    }

    return remote;
};

/**
 * Returns peer reference or null/undefined if it's not online
 * @param {String} node id
 */
Node.prototype.peerOnline = function (node) {
    return this._remotes[node];
};

/**
 * Sets waitForNode option. It means that operation will only complete
 * after the command that was performed on the leader is propagated to the
 * local client
 *
 * @param {Object} options
 * @param {String} nodeId - optionally overwrite, which node we are waiting to commit
 */
Node.prototype._waitForNode = function (options, nodeId) {
    if (!options) {
        options = {};
    }
    options.waitForNode = nodeId || this.id;
    return options;
};

/**
 * When peer joins cluster, make it join gossip cluster as well
 * @param {Object} peer
 */
Node.prototype._onPeerJoined = function (peer) {
    if (peer && peer.id !== this.id && peer.metadata && peer.metadata.gossipPort) {
        this.gossip.addPeer({
            id: peer.id,
            hostname: peer.metadata.hostname,
            port: peer.metadata.gossipPort
        });
    }
};

/**
 * When node becomes a leader, set appropriate record in the shared doc
 */
Node.prototype._onLeader = function () {
    this._gossipLeader(this.id);
};

/**
 * When peer disconnected, remove it from shared store, too
 * @param {Object} peer
 */
Node.prototype._removePeerGossip = function (peer) {
    this._remotes[peer.id] = null;
    this.gossip.removePeer(peer.id);
};

/**
 * Promote this node to leader on the gossip cluster
 * @param {String} leaderId
 */
Node.prototype._gossipLeader = function (leaderId) {
    var cluster = this.gossip.cluster;
    var currentLeader = cluster.get('leader');
    if (currentLeader !== leaderId) {
        cluster.set('leader', leaderId);
    }
};

/**
 * Notifies when transport (gossip cluster) had connected to it
 * @param {Error} err
 */
Node.prototype._transportListening = function (err) {
    if (err) {
        this.emit('error', err);
        return;
    }

    this.emit('transport-connected');
};

/**
 * Proxies error emitted by some other EE
 * @param {Error} err
 */
Node.prototype._proxyError = function (err) {
    if (err) {
        this.emit('error', err);
    }
};

/**
 * Emitted when cluster change event was emitted on the gossip cluster
 */
Node.prototype._onGossipClusterChange = function () {
    this.emit('cluster change');
};

module.exports = Node;
<commit_msg>Add .randomPeerOrSelf - this way we would prefer other nodes to perform operations on, but if we dont have any of them fallback to ourselves<commit_after>'use strict';

var url = require('url');
var EventEmitter = require('events').EventEmitter;
var inherits = require('util').inherits;
var _ = require('lodash');
var propagate = require('propagate');

// skiff parts
var Skiff = require('skiff-redis');
var Gossip = require('sombrero-gossip');
var defaultOptions = require('./defaultOptions.js');
var Peer = require('./peer.js');
var callbackQueue = require('callback-queue');

/**
 * Creates task dispatcher node
 * @param {String} skiffURL
 * @param {Object} options
 */
function Node(skiffURL, options) {
    var connectionData;

    // instanciate ee
    EventEmitter.call(this);

    // bind methods and get default options
    _.bindAll(this);
    _.defaults(options, defaultOptions);

    if (!skiffURL) {
        throw new Error('need skiff URL');
    }

    this.id = skiffURL;
    this._options = options;

    this.callbackQueue = callbackQueue;

    // init metadata
    connectionData = url.parse(skiffURL);
    this.metadata = {
        gossipPort: options.gossip.port,
        hostname: connectionData.hostname,
        port: options.port
    };
    options.skiff.metadata = this.metadata;

    // init skiff node
    // determines cluster leader and performs fail over if leader dies
    this.skiff = new Skiff(skiffURL, options.skiff);
    this.skiff.on('joined', this._onPeerJoined);
    this.skiff.on('reconnected', this._onPeerJoined);
    this.skiff.on('leader', this._onLeader);
    this.skiff.on('left', this._removePeerGossip);

    propagate(this.skiff, this);

    // setup gossip cluster - basically in-memory shared storage
    // useful for holding shared cluster state
    this._remotes = {};
    this._remoteIds = [];

    // these proxy methods will grant access to other operations that are
    // defined in _allowedRemoteCalls variable
    this._remoteCallMethods = _.keys(this._allowedRemoteCalls);

    // instantiate transport server
    this._transportServer = options.transport.listen(options.port, options.hostname, _.pick(this, this._remoteCallMethods), this._transportListening);
    this._transportServer.on('error', this._proxyError);

    this.gossip = new Gossip(this._options.gossip);
    this.gossip.on('error', this._proxyError);
    this.gossip.on('cluster change', this._onGossipClusterChange);
}
inherits(Node, EventEmitter);

/**
 * Static function: attaches remote call
 * @param {String}   name     - resource name
 * @param {Function} function - function to perform, will be invoked with local scope
 */
Node.attachRemoteCall = function (name, method) {
    if (typeof Node.prototype[name] !== 'undefined') {
        throw new Error('Function ' + name + ' is already defined');
    }

    Node.prototype._allowedRemoteCalls[name] = true;
    Node.prototype[name] = method;
};

//
// Cluster rpc-allowed operations
//

/**
 * Method that can be performed by an RPC call
 * @type {Object}
 */
Node.prototype._allowedRemoteCalls = {
    getsetnx: true,
    update: true
};

/**
 * Returns operation key that is used for callback caching
 * @return {String}
 */
Node.prototype.ok = function () {
    var length = arguments.length;
    if (length === 0) {
        throw new Error('arguments must not be empty');
    }

    var arr = new Array(length);
    for (var i = 0; i < length; i++) {
        arr[i] = arguments[i];
    }

    return arr.join('~');
};

/**
 * Get resourceId of resourceType, if not exists - create it first.
 * The operation flow is the following: check local storage (gossip cluster),
 * if there is no result of this operation, queue this operation locally
 * and make a remote call
 *
 * @param  {String}   resourceId   - unique resource id, used for request queueing
 * @param  {String}   resourceType - resource type, determines what has to be done
 * @param  {Array}    args         - args to be applied to resource type
 * @param  {Function} next         - invoked when method completes
 *
 */
Node.prototype.getsetnx = function (resourceId, resourceType, args, next) {

    if (typeof resourceId !== 'string' || !resourceId) {
        return setImmediate(next, new Error('resourceId must be a truthy string'));
    }

    if (typeof resourceType !== 'string' || !this._allowedRemoteCalls[resourceType]) {
        return setImmediate(next, new Error('remote call ' + resourceType + ' is not allowed'));
    }

    var cluster = this.gossip.cluster;
    var operationKey = [ resourceType, resourceId ].join('~');
    var operationResult = cluster.get(operationKey);

    if (operationResult !== null && operationResult !== undefined) {
        return setImmediate(next, null, operationResult);
    }

    var callback = callbackQueue.add(operationKey, next);
    if (!callback) {
        // callback will be invoked later when ongoing operation will be completed
        return;
    }

    var currentLeader = cluster.get('leader');
    if (currentLeader === this.id) {
        args.unshift(operationKey, resourceId);
        args.push(callback);
        return this[resourceType].apply(this, args);
    }

    // push args there
    args.unshift(resourceId, resourceType);

    setImmediate(this._remoteCall, currentLeader, 'getsetnx', args, callback);
};

/**
 * Deletes specific resource type from shared storage, can be performed from either node
 * @param  {String}   resourceId
 * @param  {String}   resourceType
 * @param  {Function} next
 */
Node.prototype.del = function (resourceId, resourceType, next) {
    this.update(resourceId, resourceType, null, next);
};

/**
 * Updates specific resource type from shared storage, can be performed from either node
 *
 * @param  {String}   resourceId
 * @param  {String}   resourceType
 * @param  {Mixed}    value
 * @param  {Function} next
 *
 */
Node.prototype.update = function (resourceId, resourceType, value, next) {

    if (typeof resourceId !== 'string' || !resourceId) {
        return setImmediate(next, new Error('resourceId must be a truthy string'));
    }

    if (typeof resourceType !== 'string' || !this._allowedRemoteCalls[resourceType]) {
        return setImmediate(next, new Error('remote call ' + resourceType + ' is not allowed'));
    }

    if (value === undefined) {
        return setImmediate(next, new Error('value cant be set to undefined'));
    }

    var cluster = this.gossip.cluster;
    var operationKey = [ resourceType, resourceId ].join('~');
    var currentValue = cluster.get(operationKey);

    if (currentValue === value) {
        return setImmediate(next);
    }

    var currentLeader = cluster.get('leader');
    if (currentLeader === this.id) {
        cluster.set(operationKey, value);
        return setImmediate(next);
    }

    setImmediate(this._remoteCall, currentLeader, 'update', [ resourceId, resourceType, value ], next);
};

//
// Cluster topology section, public
//

Node.prototype.join = function join(url, options, next) {
    this.skiff.join(url, options || null, next);
};

Node.prototype.leave = function leave(url, next) {
    this.skiff.leave(url, next);
};

Node.prototype.open = function open(next) {
    this.skiff.open(next);
};

Node.prototype.close = function close(next) {
    var self = this;
    this._transportServer.close(function () {
        self.gossip.stop(function () {
            self.skiff.close(next);
        });
    });
};

//
// cluster setup section, private
//

/**
 * Performs RPC call
 * @param {String}   node   - node id to perform the operation on
 * @param {String}   method - method to invoke
 * @param {Array}    args   - arguments to pass
 * @param {Function} next   - callback that will be eventually called
 */
Node.prototype._remoteCall = function (node, method, args, next) {
    var remote = this._remote(node);
    var invoke = function (client) {
        var m = client[method];
        if (!m) {
            return next(new Error('Method not found: ' + method));
        }
        args.push(next);
        m.apply(client, args);
    };

    if (!remote) {
        return next(new Error('could not find remote metadata for URL ' + node));
    } else if (remote.connected) {
        invoke(remote.client);
    } else {
        remote.once('connect', invoke);
    }
};

/**
 * Establishes peer connection when it's first required to
 * @param  {String} node - node id
 * @return {TransportConnection}
 */
Node.prototype._remote = function (node) {
    var remote = this.peerOnline(node);
    if (!remote) {
        var meta = this.skiff.peerMeta(node);
        if (meta) {
            remote = this._remotes[node] = new Peer(this._remoteCallMethods, meta.hostname, meta.port);
        }
    }

    return remote;
};

/**
 * Returns peer reference or null/undefined if it's not online
 * @param {String} node id
 */
Node.prototype.peerOnline = function (node) {
    return this._remotes[node];
};

/**
 * Returns random peer or self
 */
Node.prototype.randomPeerOrSelf = function () {
    var id = _.sample(this._remoteIds);

    if (!id) {
        // if we do not have followers, perform operation locally
        return this;
    }

    return this._remotes[id];
};

/**
 * Sets waitForNode option. It means that operation will only complete
 * after the command that was performed on the leader is propagated to the
 * local client
 *
 * @param {Object} options
 * @param {String} nodeId - optionally overwrite, which node we are waiting to commit
 */
Node.prototype._waitForNode = function (options, nodeId) {
    if (!options) {
        options = {};
    }
    options.waitForNode = nodeId || this.id;
    return options;
};

/**
 * When peer joins cluster, make it join gossip cluster as well
 * @param {Object} peer
 */
Node.prototype._onPeerJoined = function (peer) {
    if (peer && peer.id !== this.id && peer.metadata && peer.metadata.gossipPort) {
        this.gossip.addPeer({
            id: peer.id,
            hostname: peer.metadata.hostname,
            port: peer.metadata.gossipPort
        });
    }
};

/**
 * When node becomes a leader, set appropriate record in the shared doc
 */
Node.prototype._onLeader = function () {
    this._gossipLeader(this.id);
};

/**
 * When peer disconnected, remove it from shared store, too
 * @param {Object} peer
 */
Node.prototype._removePeerGossip = function (peer) {
    var idx = this._remoteIds.indexOf(peer.id);
    if (idx >= 0) {
        this._remoteIds.splice(idx, 1);
    }
    this._remotes[peer.id] = null;
    this.gossip.removePeer(peer.id);
};

/**
 * Promote this node to leader on the gossip cluster
 * @param {String} leaderId
 */
Node.prototype._gossipLeader = function (leaderId) {
    var cluster = this.gossip.cluster;
    var currentLeader = cluster.get('leader');
    if (currentLeader !== leaderId) {
        cluster.set('leader', leaderId);
    }
};

/**
 * Notifies when transport (gossip cluster) had connected to it
 * @param {Error} err
 */
Node.prototype._transportListening = function (err) {
    if (err) {
        this.emit('error', err);
        return;
    }

    this.emit('transport-connected');
};

/**
 * Proxies error emitted by some other EE
 * @param {Error} err
 */
Node.prototype._proxyError = function (err) {
    if (err) {
        this.emit('error', err);
    }
};

/**
 * Emitted when cluster change event was emitted on the gossip cluster
 */
Node.prototype._onGossipClusterChange = function () {
    this.emit('cluster change');
};

module.exports = Node;
<|endoftext|>


<commit_before>const { join } = require('path');
const { readValues } = require('./readvalues');

const config = require('loke-config').create('envirophatmqtt', { appPath: join(__dirname, '/../') });

let mqttClient;
let mqttConnected = false;
let interval;

const logger = console;
const mqttUri = 'mqtt://' + config.get('mqtt.host');
const topic = config.get('mqtt.topic');

exports.start = () => {
  mqttClient  = mqtt.connect(mqttUri);

  mqttClient.on('connect', function () {
    logger.info('MQTT connected');
    mqttConnected = true;
  });

  mqttClient.on('close', console.log);
  mqttClient.on('offline', console.log);
  // mqttClient.on('error', console.error);
  // mqttClient.on('message', console.log);

  // mqttClient.on('message', function (topic, message) {
  //   // message is Buffer
  //   console.log(message.toString())
  // })

  interval = setInterval(() => {
    readValues.then(emit);
  }, 60000);
};

exports.stop = () => {
  clearInterval(interval);
};

function emit(valueMap) {
  if (!mqttConnected) return;

  const data = Object.assign({}, valueMap, { timestamp: new Date() });
  mqttClient.publish(topic, data);
  logger.info(`Publish: ${topic} ${JSON.stringify(data)}`);
}
<commit_msg>missing mqtt<commit_after>const { join } = require('path');
const mqtt = require('mqtt');

const { readValues } = require('./readvalues');

const config = require('loke-config').create('envirophatmqtt', { appPath: join(__dirname, '/../') });

let mqttClient;
let mqttConnected = false;
let interval;

const logger = console;
const mqttUri = 'mqtt://' + config.get('mqtt.host');
const topic = config.get('mqtt.topic');

exports.start = () => {
  mqttClient  = mqtt.connect(mqttUri);

  mqttClient.on('connect', function () {
    logger.info('MQTT connected');
    mqttConnected = true;
  });

  mqttClient.on('close', console.log);
  mqttClient.on('offline', console.log);
  // mqttClient.on('error', console.error);
  // mqttClient.on('message', console.log);

  // mqttClient.on('message', function (topic, message) {
  //   // message is Buffer
  //   console.log(message.toString())
  // })

  interval = setInterval(() => {
    readValues.then(emit);
  }, 60000);
};

exports.stop = () => {
  clearInterval(interval);
};

function emit(valueMap) {
  if (!mqttConnected) return;

  const data = Object.assign({}, valueMap, { timestamp: new Date() });
  mqttClient.publish(topic, data);
  logger.info(`Publish: ${topic} ${JSON.stringify(data)}`);
}
<|endoftext|>


<commit_before>'use strict';

var each        = require('async').each
var Prismic     = require('prismic.io').Prismic
var debug       = require('debug')('metalsmith-prismic')
var _           = require('underscore')
var clone       = require('clone')
/**
 * Expose `plugin`.
 */

module.exports = plugin;

/**
 * Metalsmith plugin to retrieve content from Prismic.io and place in the file's metadata.
 *
 * @param {Object} options
 *   @property {String} url
 *   @property {String} accessToken (optional)
 * @return {Function}
 */

function plugin(config) {

    checkProperty(config, 'url');

    return function(files, metalsmith, done){
        // initialize the API
        Prismic.Api(config['url'], retrieveContent, config['accessToken']);

        function retrieveContent(err, api) {
            if (err) {
                console.error("Could not initialize API: " + err);
            }
            else {

                // obtain the reference
                var release = config['release'];
                var ref;
                var ctx = {};
                for (var currentRef in api.data.refs) {
                    if ((!release || release.toLowerCase() == "master") && (api.data.refs[currentRef].isMaster)) {
                        ref = api.data.refs[currentRef].ref;
                    }
                    else if (release == api.data.refs[currentRef].label) {
                        ref = api.data.refs[currentRef].ref;
                    }
                }

                // setup the context object
                ctx.api = api;
                ctx.linkResolver = getLinkResolver(ctx);
                ctx.ref = ref;
                ctx.OAuth = config['accessToken'];

                each(Object.keys(files), function (fileName, callback) {
                    retrieveContentForFile(fileName, callback, ctx)
                }, function(err) {debug("metalsmith-prismic done!"); done(err)})
            }
        }

        function getLinkResolver(ctx) {
            return function(doc, isBroken) {
                if (config['linkResolver'] != undefined) {
                    return config['linkResolver'](ctx, doc);
                } else {
                    if (isBroken) return false;
                    return '/' + doc.type + '/' + doc.id + '/' + doc.slug + (ctx.maybeRef ? '?ref=' + ctx.maybeRef : '');
                }
            }
        }

        // retrieves and processes the content for the given filename
        function retrieveContentForFile(fileName, callback, ctx) {
            var file = files[fileName];
            if (file.prismic) {
                debug("Pulling in content for file: %s", fileName);
                var queryKeys = {};
                var collectionQuery;

                // create associative array of query keys and their query & formName
                for (var queryKey in file.prismic) {
                    debug("processing %s", queryKey);
                    file.prismic[queryKey].results = [];
                    var queryString = file.prismic[queryKey].query;
                    var orderings = file.prismic[queryKey].orderings;
                    var fetchLinks = file.prismic[queryKey].fetchLinks;
                    var pageSize = file.prismic[queryKey].pageSize;
                    var allPages = file.prismic[queryKey].allPages;
                    var formName = file.prismic[queryKey].formName;
                    if (formName == null) {
                        formName = "everything"
                    }
                    var collectionMetaData = file.prismic[queryKey].collection;
                    if (collectionMetaData != null) {
                        if (collectionQuery != null) {
                            throw new TypeError('Only one query can be designated the collection query. Please update the configuration settings appropriately.');
                        } else if (typeof collectionMetaData != "boolean" && file.prismic[queryKey].collection.fileExtension == null) {
                            throw new TypeError('The file extension to use when generating the files for the collection must be specified.');
                        } else if (collectionMetaData === true || typeof collectionMetaData != "boolean") {
                            collectionQuery = queryKey;
                        }
                    }
                    queryKeys[queryKey] = {"queryString": queryString,
                                           "pageSize": pageSize,
                                           "allPages": allPages,
                                           "orderings": orderings,
                                           "fetchLinks": fetchLinks,
                                           "formName": formName};
                }

                // asynchronously retrieve and process each query
                each(Object.keys(queryKeys), function (currentQueryKey, queriedAndProcessedCallback) {
                    debug("%s: api.form(\"%s\").query('%s').ref(ctx.ref).submit()", queryKey, formName, queryString);
                    var totalPages;
                    var page = 1;

                    queryNextPage();

                    function queryNextPage() {
                        ctx.api.form(queryKeys[currentQueryKey].formName)
                            .query(queryKeys[currentQueryKey].queryString)
                            .pageSize(queryKeys[currentQueryKey].pageSize)
                            .page(page)
                            .orderings(queryKeys[currentQueryKey].orderings)
                            .fetchLinks(queryKeys[currentQueryKey].fetchLinks)
                            .ref(ctx.ref).submit(prismicCallback);
                    }

                    function prismicCallback(err, d) {
                        if (err) {
                            console.error(err);
                        }
                        else {
                            processRetrievedContent(d, file.prismic, currentQueryKey, ctx);
                            totalPages = d.total_pages;
                            if (queryKeys[currentQueryKey].allPages &&
                                page < totalPages) {
                                page++;
                                queryNextPage();
                            } else {
                                queriedAndProcessedCallback();
                            }
                        }
                    }

                }, function(err){
                    if(err != null) {
                        console.log(err);
                    }
                    generateCollectionFiles(fileName, collectionQuery, callback, ctx);
                });
            } else {
                callback();
            }
        }

        // processes the retrieved content and adds it to the metadata
        function processRetrievedContent(content, filePrismicMetadata, queryKey, ctx) {
            if (content.results != null && content.results.length > 0) {
                var queryMetadata = filePrismicMetadata[queryKey];
                for (var i = 0; i < content.results.length; i++) {

                    // add the complete result except for the data fragments
                    var result = _.omit(content.results[i], 'fragments');
                    result.data = {};

                    // process the data fragments, invoking helpers to make the data more usable
                    if (content.results[i].fragments != null && Object.keys(content.results[i].fragments).length  > 0) {
                        for (var fragmentFullName in content.results[i].fragments) {

                            // strip the document type from the fragment name
                            var fragmentName = fragmentFullName.substr(fragmentFullName.lastIndexOf('.') + 1);
                            var fragment = content.results[i].fragments[fragmentFullName];

                            function processSingleFragment(fragment) {
                              var fragmentObject = {
                                json: fragment,
                                html: fragment.asHtml(ctx.linkResolver)
                              };

                              // Add child fragments from links
                              if (fragment instanceof Prismic.Fragments.DocumentLink && queryMetadata.fetchLinks && fragment.document.data) {
                                fragmentObject.children = _.mapObject(fragment.document.data[fragment.type], function(field) {
                                  // Structured text is not fetched nicely by prismic
                                  if (field.type) {
                                    fragment = Prismic.Fragments.initField(field);
                                    return {
                                      json: fragment,
                                      html: fragment.asHtml(ctx.linkResolver)
                                    }
                                  } else {
                                    return {
                                      json: field
                                    }
                                  }
                                });
                              }

                              return fragmentObject;
                            }

                            if (queryMetadata.arrayFragments && Array.isArray(fragment)) {
                              var fragmentArray = content.results[i].getAll(fragmentFullName);
                              result.data[fragmentName] = fragmentArray.map(processSingleFragment);
                            } else {
                              result.data[fragmentName] = processSingleFragment(content.results[i].get(fragmentFullName));
                            }
                        }
                    }
                    queryMetadata.results.push(result);
                }
            }
        }

        // if any of the queries are designated the collection query, then generate a file for each of the results
        function generateCollectionFiles(fileName, collectionQuery, callback, ctx) {
            if (collectionQuery != null) {
                var file = files[fileName];
                var newFiles = {};
                var fileExtension = file.prismic[collectionQuery].collection.fileExtension;
                var fileSuffix = (fileExtension != undefined && fileExtension !== "")? "." + fileExtension:"";
                // for every result in the collection query
                for (var i = 0; i < file.prismic[collectionQuery].results.length; i++) {

                    // clone the file and replace the original collectionQuery results with the current result
                    var newFile = clone(file);
                    newFile.prismic[collectionQuery].results = [file.prismic[collectionQuery].results[i]];

                    // add the filename to the ctx object to make it available for use in the linkResolver function
                    ctx.path = fileName;

                    // use the linkResolver to generate the filename removing the leading slash if present
                    newFiles[(ctx.linkResolver(file.prismic[collectionQuery].results[i]) + fileSuffix).replace(/^\//g, '')] = newFile;
                }

                delete files[fileName];
                _.extend(files, newFiles);
            }

            callback();
        }

    }

    function checkProperty(object, property) {
        if ((object[property]) == null) {
            throw new TypeError('Missing property \'' + property + '\'. Please update the configuration settings appropriately.');
        }
    }
}
<commit_msg>Fix node 0.x compatibility<commit_after>'use strict';

var each        = require('async').each
var Prismic     = require('prismic.io').Prismic
var debug       = require('debug')('metalsmith-prismic')
var _           = require('underscore')
var clone       = require('clone')
/**
 * Expose `plugin`.
 */

module.exports = plugin;

/**
 * Metalsmith plugin to retrieve content from Prismic.io and place in the file's metadata.
 *
 * @param {Object} options
 *   @property {String} url
 *   @property {String} accessToken (optional)
 * @return {Function}
 */

function plugin(config) {

    checkProperty(config, 'url');

    return function(files, metalsmith, done){
        // initialize the API
        Prismic.Api(config['url'], retrieveContent, config['accessToken']);

        function retrieveContent(err, api) {
            if (err) {
                console.error("Could not initialize API: " + err);
            }
            else {

                // obtain the reference
                var release = config['release'];
                var ref;
                var ctx = {};
                for (var currentRef in api.data.refs) {
                    if ((!release || release.toLowerCase() == "master") && (api.data.refs[currentRef].isMaster)) {
                        ref = api.data.refs[currentRef].ref;
                    }
                    else if (release == api.data.refs[currentRef].label) {
                        ref = api.data.refs[currentRef].ref;
                    }
                }

                // setup the context object
                ctx.api = api;
                ctx.linkResolver = getLinkResolver(ctx);
                ctx.ref = ref;
                ctx.OAuth = config['accessToken'];

                each(Object.keys(files), function (fileName, callback) {
                    retrieveContentForFile(fileName, callback, ctx)
                }, function(err) {debug("metalsmith-prismic done!"); done(err)})
            }
        }

        function getLinkResolver(ctx) {
            return function(doc, isBroken) {
                if (config['linkResolver'] != undefined) {
                    return config['linkResolver'](ctx, doc);
                } else {
                    if (isBroken) return false;
                    return '/' + doc.type + '/' + doc.id + '/' + doc.slug + (ctx.maybeRef ? '?ref=' + ctx.maybeRef : '');
                }
            }
        }

        // retrieves and processes the content for the given filename
        function retrieveContentForFile(fileName, callback, ctx) {
            var file = files[fileName];
            if (file.prismic) {
                debug("Pulling in content for file: %s", fileName);
                var queryKeys = {};
                var collectionQuery;

                // create associative array of query keys and their query & formName
                for (var queryKey in file.prismic) {
                    debug("processing %s", queryKey);
                    file.prismic[queryKey].results = [];
                    var queryString = file.prismic[queryKey].query;
                    var orderings = file.prismic[queryKey].orderings;
                    var fetchLinks = file.prismic[queryKey].fetchLinks;
                    var pageSize = file.prismic[queryKey].pageSize;
                    var allPages = file.prismic[queryKey].allPages;
                    var formName = file.prismic[queryKey].formName;
                    if (formName == null) {
                        formName = "everything"
                    }
                    var collectionMetaData = file.prismic[queryKey].collection;
                    if (collectionMetaData != null) {
                        if (collectionQuery != null) {
                            throw new TypeError('Only one query can be designated the collection query. Please update the configuration settings appropriately.');
                        } else if (typeof collectionMetaData != "boolean" && file.prismic[queryKey].collection.fileExtension == null) {
                            throw new TypeError('The file extension to use when generating the files for the collection must be specified.');
                        } else if (collectionMetaData === true || typeof collectionMetaData != "boolean") {
                            collectionQuery = queryKey;
                        }
                    }
                    queryKeys[queryKey] = {"queryString": queryString,
                                           "pageSize": pageSize,
                                           "allPages": allPages,
                                           "orderings": orderings,
                                           "fetchLinks": fetchLinks,
                                           "formName": formName};
                }

                // asynchronously retrieve and process each query
                each(Object.keys(queryKeys), function (currentQueryKey, queriedAndProcessedCallback) {
                    debug("%s: api.form(\"%s\").query('%s').ref(ctx.ref).submit()", queryKey, formName, queryString);
                    var totalPages;
                    var page = 1;

                    queryNextPage();

                    function queryNextPage() {
                        ctx.api.form(queryKeys[currentQueryKey].formName)
                            .query(queryKeys[currentQueryKey].queryString)
                            .pageSize(queryKeys[currentQueryKey].pageSize)
                            .page(page)
                            .orderings(queryKeys[currentQueryKey].orderings)
                            .fetchLinks(queryKeys[currentQueryKey].fetchLinks)
                            .ref(ctx.ref).submit(prismicCallback);
                    }

                    function prismicCallback(err, d) {
                        if (err) {
                            console.error(err);
                        }
                        else {
                            processRetrievedContent(d, file.prismic, currentQueryKey, ctx);
                            totalPages = d.total_pages;
                            if (queryKeys[currentQueryKey].allPages &&
                                page < totalPages) {
                                page++;
                                queryNextPage();
                            } else {
                                queriedAndProcessedCallback();
                            }
                        }
                    }

                }, function(err){
                    if(err != null) {
                        console.log(err);
                    }
                    generateCollectionFiles(fileName, collectionQuery, callback, ctx);
                });
            } else {
                callback();
            }
        }

        // processes the retrieved content and adds it to the metadata
        function processRetrievedContent(content, filePrismicMetadata, queryKey, ctx) {
            if (content.results != null && content.results.length > 0) {
                var queryMetadata = filePrismicMetadata[queryKey];
                for (var i = 0; i < content.results.length; i++) {

                    // add the complete result except for the data fragments
                    var result = _.omit(content.results[i], 'fragments');
                    result.data = {};

                    // process the data fragments, invoking helpers to make the data more usable
                    if (content.results[i].fragments != null && Object.keys(content.results[i].fragments).length  > 0) {
                        for (var fragmentFullName in content.results[i].fragments) {

                            // strip the document type from the fragment name
                            var fragmentName = fragmentFullName.substr(fragmentFullName.lastIndexOf('.') + 1);
                            var fragment = content.results[i].fragments[fragmentFullName];

                            if (queryMetadata.arrayFragments && Array.isArray(fragment)) {
                              var fragmentArray = content.results[i].getAll(fragmentFullName);
                              result.data[fragmentName] = fragmentArray.map(function(subfragment) {
                                return processSingleFragment(subfragment, ctx);
                              });
                            } else {
                              result.data[fragmentName] = processSingleFragment(content.results[i].get(fragmentFullName), ctx);
                            }
                        }
                    }
                    queryMetadata.results.push(result);
                }
            }
        }

        // process a single fragment, and return the fragment object with json and html data
        function processSingleFragment(fragment, ctx) {
          var fragmentObject = {
            json: fragment,
            html: fragment.asHtml(ctx.linkResolver)
          };

          // Add child fragments from links
          if (fragment instanceof Prismic.Fragments.DocumentLink && fragment.document.data) {
            fragmentObject.children = _.mapObject(fragment.document.data[fragment.type], function(field) {
              // Structured text is not fetched nicely by prismic
              if (field.type) {
                fragment = Prismic.Fragments.initField(field);
                return {
                  json: fragment,
                  html: fragment.asHtml(ctx.linkResolver)
                }
              } else {
                return {
                  json: field
                }
              }
            });
          }

          return fragmentObject;
        }

        // if any of the queries are designated the collection query, then generate a file for each of the results
        function generateCollectionFiles(fileName, collectionQuery, callback, ctx) {
            if (collectionQuery != null) {
                var file = files[fileName];
                var newFiles = {};
                var fileExtension = file.prismic[collectionQuery].collection.fileExtension;
                var fileSuffix = (fileExtension != undefined && fileExtension !== "")? "." + fileExtension:"";
                // for every result in the collection query
                for (var i = 0; i < file.prismic[collectionQuery].results.length; i++) {

                    // clone the file and replace the original collectionQuery results with the current result
                    var newFile = clone(file);
                    newFile.prismic[collectionQuery].results = [file.prismic[collectionQuery].results[i]];

                    // add the filename to the ctx object to make it available for use in the linkResolver function
                    ctx.path = fileName;

                    // use the linkResolver to generate the filename removing the leading slash if present
                    newFiles[(ctx.linkResolver(file.prismic[collectionQuery].results[i]) + fileSuffix).replace(/^\//g, '')] = newFile;
                }

                delete files[fileName];
                _.extend(files, newFiles);
            }

            callback();
        }

    }

    function checkProperty(object, property) {
        if ((object[property]) == null) {
            throw new TypeError('Missing property \'' + property + '\'. Please update the configuration settings appropriately.');
        }
    }
}
<|endoftext|>