<commit_before>// @flow
import { BufferedProcess, Directory, GitRepository } from 'atom'
import chooseRepo from './models/choose-repo'

const reposByDirectory: Map<string, GitRepository> = new Map()

export async function getRepo(): Promise<GitRepository> {
  const activeEditor = atom.workspace.getCenter().getActiveTextEditor()
  if (activeEditor) {
    const path = activeEditor.getPath()
    const directory = new Directory(path)
    let repo = reposByDirectory.get(directory.getPath())
    if (repo) return repo

    repo = await atom.project.repositoryForDirectory(directory)
    if (repo) {
      reposByDirectory.set(directory.getPath(), repo)
      return repo
    }
  }

  const getRepoForDirectory: Directory => Promise<
    GitRepository
  > = atom.project.repositoryForDirectory.bind(atom.project)
  const repos: GitRepository[] = await Promise.all(
    atom.project.getDirectories().map(getRepoForDirectory)
  ).then(results => results.filter(Boolean))

  if (repos.length === 1) return repos[0]
  if (repos.length === 0) return null
  if (repos.length > 0) return chooseRepo(repos)
}

const defaultCmdOptions = { color: false, env: process.env }

type GitCliOptions = {
  cwd?: string,
  color?: boolean,
  env?: {}
}
export type GitCliResponse = {
  output: string,
  failed: boolean
}

export default async function cmd(
  args: string[],
  options: GitCliOptions = defaultCmdOptions
): Promise<GitCliResponse> {
  args = args.slice()
  if (options.color) {
    args.push('-c', 'color.ui=always')
    delete options.color
  }

  return new Promise((resolve, reject) => {
    let output = ''
    const gitProcess = new BufferedProcess({
      //$FlowFixMe
      command: atom.config.get('git-plus.general.gitPath') || 'git',
      args,
      options,
      stdout: data => (output += data.toString()),
      stderr: data => (output += data.toString()),
      exit: code => {
        resolve({
          output: output.trim(),
          failed: code !== 0
        })
      }
    })
    // TODO: clean up the disposable from this subscription
    gitProcess.onWillThrowError(_error => {
      atom.notifications.addError(
        'Git Plus is unable to locate the git command. Please ensure process.env.PATH can access git.'
      )
      reject(Error("Couldn't find git"))
    })
  })
}
<commit_msg>fix color options when calling git cli<commit_after>// @flow
import { BufferedProcess, Directory, GitRepository } from 'atom'
import chooseRepo from './models/choose-repo'

const reposByDirectory: Map<string, GitRepository> = new Map()

export async function getRepo(): Promise<GitRepository> {
  const activeEditor = atom.workspace.getCenter().getActiveTextEditor()
  if (activeEditor) {
    const path = activeEditor.getPath()
    const directory = new Directory(path)
    let repo = reposByDirectory.get(directory.getPath())
    if (repo) return repo

    repo = await atom.project.repositoryForDirectory(directory)
    if (repo) {
      reposByDirectory.set(directory.getPath(), repo)
      return repo
    }
  }

  const getRepoForDirectory: Directory => Promise<
    GitRepository
  > = atom.project.repositoryForDirectory.bind(atom.project)
  const repos: GitRepository[] = await Promise.all(
    atom.project.getDirectories().map(getRepoForDirectory)
  ).then(results => results.filter(Boolean))

  if (repos.length === 1) return repos[0]
  if (repos.length === 0) return null
  if (repos.length > 0) return chooseRepo(repos)
}

const defaultCmdOptions = { color: false, env: process.env }

type GitCliOptions = {
  cwd?: string,
  color?: boolean,
  env?: {}
}
export type GitCliResponse = {
  output: string,
  failed: boolean
}

export default async function cmd(
  args: string[],
  options: GitCliOptions = defaultCmdOptions
): Promise<GitCliResponse> {
  if (options.color) {
    args = ['-c', 'color.ui=always'].concat(args)
    delete options.color
  }

  return new Promise((resolve, reject) => {
    let output = ''
    const gitProcess = new BufferedProcess({
      //$FlowFixMe
      command: atom.config.get('git-plus.general.gitPath') || 'git',
      args,
      options,
      stdout: data => (output += data.toString()),
      stderr: data => (output += data.toString()),
      exit: code => {
        resolve({
          output: output.trim(),
          failed: code !== 0
        })
      }
    })
    // TODO: clean up the disposable from this subscription
    gitProcess.onWillThrowError(_error => {
      atom.notifications.addError(
        'Git Plus is unable to locate the git command. Please ensure process.env.PATH can access git.'
      )
      reject(Error("Couldn't find git"))
    })
  })
}
<|endoftext|>


<commit_before>var request = require('request')
var cheerio = require('cheerio')
var querystring = require('querystring')
var util = require('util')

var linkSel = 'h3.r a'
var descSel = 'div.s'
var itemSel = 'li.g'
var nextSel = 'td.b a span'

var URL = 'http://www.google.%s/search?hl=%s&q=%s&start=%s&sa=N&num=%s&ie=UTF-8&oe=UTF-8'

var nextTextErrorMsg = 'Translate `google.nextText` option to selected language to detect next results link.'

function google (query, callback) {
  igoogle(query, 0, callback)
}

google.resultsPerPage = 10
google.tld = 'com'
google.lang = 'en'
google.requestOptions = {}
google.nextText = 'Next'

var igoogle = function (query, start, callback) {
  if (google.resultsPerPage > 100) google.resultsPerPage = 100 // Google won't allow greater than 100 anyway
  if (google.lang !== 'en' && google.nextText === 'Next') console.warn(nextTextErrorMsg)

  // timeframe is optional. splice in if set
  if (google.timeSpan) {
    URL = URL.indexOf('tbs=qdr:') >= 0 ? URL.replace(/tbs=qdr:[snhdwmy]\d*/, 'tbs=qdr:' + google.timeSpan) : URL.concat('&tbs=qdr:', google.timeSpan)
  }

  var newUrl = util.format(URL, google.tld, google.lang, querystring.escape(query), start, google.resultsPerPage)
  var requestOptions = {
    url: newUrl,
    method: 'GET'
  }

  for (var k in google.requestOptions) {
    requestOptions[k] = google.requestOptions[k]
  }

  request(requestOptions, function (err, resp, body) {
    if ((err == null) && resp.statusCode === 200) {
      var $ = cheerio.load(body)
      var links = []

      $(itemSel).each(function (i, elem) {
        var linkElem = $(elem).find(linkSel)
        var descElem = $(elem).find(descSel)
        var item = {
          title: $(linkElem).first().text(),
          link: null,
          description: null,
          href: null
        }
        var qsObj = querystring.parse($(linkElem).attr('href'))

        if (qsObj['/url?q']) {
          item.link = qsObj['/url?q']
          item.href = item.link
        }

        $(descElem).find('div').remove()
        item.description = $(descElem).text()

        links.push(item)
      })

      var nextFunc = null
      if ($(nextSel).last().text() === google.nextText) {
        nextFunc = function () {
          igoogle(query, start + google.resultsPerPage, callback)
        }
      }

      callback(null, nextFunc, links)
    } else {
      callback(new Error('Error on response' + (resp ? ' (' + resp.statusCode + ')' : '') + ':' + err + ' : ' + body), null, null)
    }
  })
}

module.exports = google
<commit_msg>added optional start parameter, enabling programmatic paging<commit_after>var request = require('request')
var cheerio = require('cheerio')
var querystring = require('querystring')
var util = require('util')

var linkSel = 'h3.r a'
var descSel = 'div.s'
var itemSel = 'li.g'
var nextSel = 'td.b a span'

var URL = 'http://www.google.%s/search?hl=%s&q=%s&start=%s&sa=N&num=%s&ie=UTF-8&oe=UTF-8'

var nextTextErrorMsg = 'Translate `google.nextText` option to selected language to detect next results link.'

// start parameter is optional
function google (query, start, callback) {
  var startIndex = 0;
  if (typeof callback === 'undefined') {
    callback = start;
  } else {
    startIndex = start;
  }
  igoogle(query, startIndex, callback)
}

google.resultsPerPage = 10
google.tld = 'com'
google.lang = 'en'
google.requestOptions = {}
google.nextText = 'Next'

var igoogle = function (query, start, callback) {
  if (google.resultsPerPage > 100) google.resultsPerPage = 100 // Google won't allow greater than 100 anyway
  if (google.lang !== 'en' && google.nextText === 'Next') console.warn(nextTextErrorMsg)

  // timeframe is optional. splice in if set
  if (google.timeSpan) {
    URL = URL.indexOf('tbs=qdr:') >= 0 ? URL.replace(/tbs=qdr:[snhdwmy]\d*/, 'tbs=qdr:' + google.timeSpan) : URL.concat('&tbs=qdr:', google.timeSpan)
  }

  var newUrl = util.format(URL, google.tld, google.lang, querystring.escape(query), start, google.resultsPerPage)
  var requestOptions = {
    url: newUrl,
    method: 'GET'
  }

  for (var k in google.requestOptions) {
    requestOptions[k] = google.requestOptions[k]
  }

  request(requestOptions, function (err, resp, body) {
    if ((err == null) && resp.statusCode === 200) {
      var $ = cheerio.load(body)
      var links = []

      $(itemSel).each(function (i, elem) {
        var linkElem = $(elem).find(linkSel)
        var descElem = $(elem).find(descSel)
        var item = {
          title: $(linkElem).first().text(),
          link: null,
          description: null,
          href: null
        }
        var qsObj = querystring.parse($(linkElem).attr('href'))

        if (qsObj['/url?q']) {
          item.link = qsObj['/url?q']
          item.href = item.link
        }

        $(descElem).find('div').remove()
        item.description = $(descElem).text()

        links.push(item)
      })

      var nextFunc = null
      if ($(nextSel).last().text() === google.nextText) {
        nextFunc = function () {
          igoogle(query, start + google.resultsPerPage, callback)
        }
      }

      callback(null, nextFunc, links)
    } else {
      callback(new Error('Error on response' + (resp ? ' (' + resp.statusCode + ')' : '') + ':' + err + ' : ' + body), null, null)
    }
  })
}

module.exports = google
<|endoftext|>


<commit_before>'use strict';

function showHelp() {
  return [
    '',
    '$ google --help',
    '',
    'CLI tool to launch Google Search in the default browser',
    '',
    'Usage: google <search-query>',
    '',
    'Example Usage:',
    '  google',
    '  google how do I google on google',
    '  google --help',
    '',
    'Flags:',
    '  -h, --help',
    '    Show this help.',
    ''
  ].join('\n');
}

function getCommand(os) {
  return os === 'darwin' ? 'open'
       : os === 'linux'  ? 'xdg-open'
       : os === 'win32'  ? 'explorer'
       : null;
}

function getQuery(raw) {
  return raw.map((s) => s.replace(/ /g, '+')).join('+');
}

function search(os, exec, query) {
  let cmd = getCommand(os);
  if (cmd === null)
    throw('Operating system not supported');

  let fullQuery = [];
  if (query.length) {
    if (query[0].match(/^-(h|-help)$/)) {
      console.info(showHelp());
      return;
    }

    fullQuery = [
      '/search?q',
      '=',
      getQuery(query)
    ];
  }

  exec([
    cmd,
    ' ',
    'https://www.google.com',
    ...fullQuery
  ].join(''));
}

module.exports = exports = query => {
  const os   = require('os').platform();
  const exec = require('child_process').exec;

  search(os, exec, query);
};
exports.getCommand  = getCommand;
exports.getQuery    = getQuery;
exports.showHelp    = showHelp;
exports.search      = search;
<commit_msg>prettier help<commit_after>'use strict';

function showHelp() {
  return [
    '',
    '  CLI tool to launch Google Search in the default browser',
    '',
    '  Usage: google <search-query>',
    '',
    '  Example Usage:',
    '    google',
    '    google how do I google on google',
    '    google --help',
    '',
    '  Flags:',
    '    -h, --help',
    '      Show this help.',
    ''
  ].join('\n');
}

function getCommand(os) {
  return os === 'darwin' ? 'open'
       : os === 'linux'  ? 'xdg-open'
       : os === 'win32'  ? 'explorer'
       : null;
}

function getQuery(raw) {
  return raw.map((s) => s.replace(/ /g, '+')).join('+');
}

function search(os, exec, query) {
  let cmd = getCommand(os);
  if (cmd === null)
    throw('Operating system not supported');

  let fullQuery = [];
  if (query.length) {
    if (query[0].match(/^-(h|-help)$/)) {
      console.info(showHelp());
      return;
    }

    fullQuery = [
      '/search?q',
      '=',
      getQuery(query)
    ];
  }

  exec([
    cmd,
    ' ',
    'https://www.google.com',
    ...fullQuery
  ].join(''));
}

module.exports = exports = query => {
  const os   = require('os').platform();
  const exec = require('child_process').exec;

  search(os, exec, query);
};
exports.getCommand  = getCommand;
exports.getQuery    = getQuery;
exports.showHelp    = showHelp;
exports.search      = search;
<|endoftext|>


<commit_before>var EventEmitter = require('events').EventEmitter,
	urlBuilder = require('url'),
	util = require('util'),
	request = require('superagent'),
	schema = require(__dirname + '/schema.json'),
	pkg = require(__dirname + '/../package.json')

/**
 * [utility function, ignore]
 *
 * Array Remove - By John Resig (MIT Licensed)
 */
Array.prototype.remove = function(from, to) {
	var rest = this.slice((to || from) + 1 || this.length)
	this.length = from < 0 ? this.length + from : from
	return this.push.apply(this, rest)
}

const ETOKENEXPIRED = 'expired_token'
function heelloApi(options) {
	var self = this
	if(options === undefined) options = {}

	// init, build up api handlers via __defineGetter__...
	Object.keys(schema).forEach(function(controller) {
		var actions = schema[controller]
		self.__defineGetter__(controller, function() {
			return new heelloController(self, controller, actions)
		})
	})

	this.conf = {
		appId: options.appId || false,
		appSecret: options.appSecret || false,
		protocol: options.protocol || 'https',
		domain: options.domain || 'api.heello.com',
		userAgent: options.userAgent || 'node-heello_v' + pkg.version,
		callbackURI: options.callbackURI || '',
		//authType: options.authType || 'web_server',
		apiVersion: 'v1', // locking ourselves to api v1, updates will likely be necessary in the event of api changes
	}
	Object.freeze(this.conf)

	this.rateLimitRemaining = -1, this.rateLimitMax = -1, this.rateLimitReset = null
	this.accessToken = null, this.refreshToken = null, this.code = null

	if(this.conf.appId === false || this.conf.appSecret === false)
		throw new Error('appId and appSecret must be provided')

	EventEmitter.call(self)
}
// heelloApi will inherit from EventEmitter, and will emit on severe errors (API limit, Bad Key, etc.)
util.inherits(heelloApi, EventEmitter)

heelloApi.prototype.getAuthURI = function(state) {
	var query = {
		client_id: this.conf.appId,
		redirect_uri: this.conf.callbackURI,
		response_type: 'code',
	}
	if(state !== undefined && state !== null)
		query.state = state

	return urlBuilder.format({
		protocol: this.conf.protocol,
		hostname: this.conf.domain,
		pathname: '/oauth/authorize',
		query: query
	})
}

heelloApi.prototype.refreshTokens = function(code, fn) {
	var self = this
	try {
		this.code = code
		var query = {
			client_id: this.conf.appId,
			client_secret: this.conf.appSecret,
			redirect_uri: this.conf.callbackURI,
			code: this.code,
			grant_type: 'authorization_code',
		}

		var req = request('POST', this.conf.protocol + '://' + this.conf.domain + '/oauth/token')

		req.set('Accept', 'application/vnd.heello.' + this.conf.apiVersion)
			.set('User-Agent', this.conf.userAgent)
			.timeout(15 * 1000) // 15sec timeout
			.type('form')
			.send(query)
			.end(function(err, res) {
				if(err) throw err
				self.loadTokens(res.body.refresh_token, res.body.access_token)
				fn(null)
			})
	} catch(err) {
		fn(err)
	}
}

heelloApi.prototype.loadTokens = function(refreshToken, accessToken) {
	this.refreshToken = refreshToken, this.accessToken = accessToken

	return this
}

heelloApi.prototype.request = function(options, fn) {
	var self = this
	try {
		var req = request(options.method || 'GET', this.conf.protocol + '://' + this.conf.domain + options.url)
		req.set('Accept', 'application/vnd.heello.' + this.conf.apiVersion)
			.set('User-Agent', this.conf.userAgent)
			.timeout(15 * 1000) // 15sec timeout

		if(options.method === 'POST' || options.method === 'PUT') {
			req.type('form')
				.send(options.queryData || {})

			if(options.media) {
				// @todo file upload support
				// maybe when I'm not so aggravated by oauth2 docs
			}
		} else {
			req.query(options.queryData || {})
		}

		// workaround for superagent bug: https://github.com/visionmedia/superagent/issues/206
		if(!req.req._headers.cookie) {
			delete req.req._headers.cookie;
			delete req.req._headerNames.cookie;
		}

		req.end(function(err, res) {
			if(err) throw err
			// grab rate-limit info.
			if(res.header['x-ratelimit-limit']) {
				self.rateLimitMax = res.header['x-ratelimit-limit']
				self.rateLimitRemaining = res.header['x-ratelimit-remaining']
				self.rateLimitReset = new Date(parseInt(res.header['x-ratelimit-reset']) * 1000)
			}
			if(!res.ok) {
				if(res.status === 429 || res.status === 503) {
					// hoping heello will move to use HTTP 429 only for api-limited response, as per RFC 6585.
					// <https://heello.com/katana/6803710> and <https://heello.com/noah/6805308>
					self.emit('apiLimit', res.error)
					throw new Error('API limit reached')
				} else if(res.status === 401 && res.body.error === ETOKENEXPIRED) {
					throw new Error(ETOKENEXPIRED)
				} else if(res.status === 401) {
					// bad key basically, auth details rejected for some reason. emit an error.
					self.emit('apiAuthError', res.error)
					throw new Error('Authorization failure')
				} else {
					//throw res.error
					fn(res.error, null, res)
				}
			} else {
				fn(null, (res.body) ? res.body : null, res)
			}
		})
	} catch(err) {
		fn(err)
	}
}

function heelloController(api, controller, actions) {
	var self = this
	this.api = api, this.controllerName = controller

	Object.keys(actions).forEach(function(action) {
		var details = actions[action]
		self.__defineGetter__(action, function() {
			return new heelloAction(api, controller, action, details)
		})
	})
}

function heelloAction(api, controller, action, details) {
	var self = this
	this.api = api, this.controllerName = controller, this.actionName = action

	// let's break it down...
	this.method = details.method || 'GET'
	this.url = (details.url) ? util.format('/%s/%s.json', controller, details.url) : '/' + controller + '.json'
	this.needsAuth = details.needsAuth || false // assume no auth needed unless stated otherwise

	// extract required URL params from this.url
	this.urlParams = []
	var re = /:([a-z0-9]+)/ig, res
	while((res = re.exec(this.url)) !== null)
		this.urlParams.push(res[1])

	this.queryParams = details.reqQueryParams || []
	this.mediaParams = details.mediaParams || []

	// automagical adding of access_token, if we need auth, this is how we get it.
	if(this.needsAuth)
		this.queryParams.push('access_token')

	this.reqParams = this.urlParams.slice().concat(this.queryParams)

	var ret = function(params, fn) {
		var self = this, reqParams = this.reqParams.slice(), options
		if(params === null) params = {}
		options = {
			method: this.method,
			url: this.url,
			params: {},
			media: {},
		}
		try {
			// sanity checks - all params we need present? do we have credentials required?
			if(this.needsAuth && self.api.creds === null)
				throw new Error(util.format('api call %s/%s requires auth, none provided', self.controllerName, self.actionName))

			// @todo include access_token anyways when we have one...? talking to heello devs about this,
			//   would help alleviate limiting when encountering users behind heavy NAT
			if(this.needsAuth) {
				params['access_token'] = this.api.accessToken
			} else {
				params['key'] = this.api.conf.appId
			}

			// check to make sure all needed params are present
			Object.keys(params).forEach(function(param) {
				if(reqParams.indexOf(param) !== -1)
					reqParams.remove(reqParams.indexOf(param))
			})
			if(reqParams.length > 0)
				throw new Error(util.format('api call %s/%s missing required parameter(s) [%s], none provided', self.controllerName, self.actionName, reqParams.join(', ')))

			// start query preparations
			this.urlParams.forEach(function(p) {
				// we should expect that ALL url params are required...
				options.url = options.url.replace(':' + p, params[p])
				delete params[p]
			})
			this.mediaParams.forEach(function(p) {
				if(params[p] !== undefined) {
					media[p] = params[p]
					delete params[p]
				}
			})
			if(Object.keys(options.media).length === 0)
				delete options.media

			// load params into options, fire.
			options.queryData = params
			this.api.request(options, function(err, json, res) {
				if(err && err.message === ETOKENEXPIRED) {
					// swap out with new token, try again pls
					self.api.refreshTokens(function() {
						options.queryData['access_token'] = this.api.accessToken
						self.api.request(options, fn)
					})
				} else {
					fn(err, json, res)
				}
			})
		} catch(err) {
			fn(err)
		}
	}

	return ret.bind(self)
}

module.exports = heelloApi
<commit_msg>Refactor token obtainment<commit_after>var EventEmitter = require('events').EventEmitter,
	urlBuilder = require('url'),
	util = require('util'),
	request = require('superagent'),
	schema = require(__dirname + '/schema.json'),
	pkg = require(__dirname + '/../package.json')

/**
 * [utility function, ignore]
 *
 * Array Remove - By John Resig (MIT Licensed)
 */
Array.prototype.remove = function(from, to) {
	var rest = this.slice((to || from) + 1 || this.length)
	this.length = from < 0 ? this.length + from : from
	return this.push.apply(this, rest)
}

const ETOKENEXPIRED = 'expired_token'
function heelloApi(options) {
	var self = this
	if(options === undefined) options = {}

	// init, build up api handlers via __defineGetter__...
	Object.keys(schema).forEach(function(controller) {
		var actions = schema[controller]
		self.__defineGetter__(controller, function() {
			return new heelloController(self, controller, actions)
		})
	})

	this.conf = {
		appId: options.appId || false,
		appSecret: options.appSecret || false,
		protocol: options.protocol || 'https',
		domain: options.domain || 'api.heello.com',
		userAgent: options.userAgent || 'node-heello_v' + pkg.version,
		callbackURI: options.callbackURI || '',
		//authType: options.authType || 'web_server',
		apiVersion: 'v1', // locking ourselves to api v1, updates will likely be necessary in the event of api changes
	}
	Object.freeze(this.conf)

	this.rateLimitRemaining = -1, this.rateLimitMax = -1, this.rateLimitReset = null
	this.accessToken = null, this.refreshToken = null, this.code = null

	if(this.conf.appId === false || this.conf.appSecret === false)
		throw new Error('appId and appSecret must be provided')

	EventEmitter.call(self)
}
// heelloApi will inherit from EventEmitter, and will emit on severe errors (API limit, Bad Key, etc.)
util.inherits(heelloApi, EventEmitter)

heelloApi.prototype.getAuthURI = function(state) {
	var query = {
		client_id: this.conf.appId,
		redirect_uri: this.conf.callbackURI,
		response_type: 'code',
	}
	if(state !== undefined && state !== null)
		query.state = state

	return urlBuilder.format({
		protocol: this.conf.protocol,
		hostname: this.conf.domain,
		pathname: '/oauth/authorize',
		query: query
	})
}

heelloApi.prototype.getTokens = function(code, fn) {
	this._updateTokens(code, 'authorization_code', fn)
}

heelloApi.prototype.refreshTokens = function(refreshToken, fn) {
	this._updateTokens(code, 'refresh_token', fn)
}

heelloApi.prototype._updateTokens = function(code, type, fn) {
	var self = this
	try {
		this.code = code
		var query = {
			client_id: this.conf.appId,
			client_secret: this.conf.appSecret,
			redirect_uri: this.conf.callbackURI,
			code: this.code,
			grant_type: type,
		}

		if(type === 'authorization_code') {
			query['code'] = code
		} else if(type === 'refresh_token') {
			query['refresh_token'] = code
		} else {
			throw new Error('unknown token update request')
		}

		var req = request('POST', this.conf.protocol + '://' + this.conf.domain + '/oauth/token')

		req.set('Accept', 'application/vnd.heello.' + this.conf.apiVersion)
			.set('User-Agent', this.conf.userAgent)
			.timeout(15 * 1000) // 15sec timeout
			.type('form')
			.send(query)
			.end(function(err, res) {
				if(err) throw err
				self.refreshToken = res.body.refresh_token,
				self.accessToken = res.body.access_token
				fn(null)
			})
	} catch(err) {
		fn(err)
	}
}

heelloApi.prototype.loadTokens = function(refreshToken, accessToken) {
	this.refreshToken = refreshToken, this.accessToken = accessToken

	return this
}

heelloApi.prototype.request = function(options, fn) {
	var self = this
	try {
		var req = request(options.method || 'GET', this.conf.protocol + '://' + this.conf.domain + options.url)
		req.set('Accept', 'application/vnd.heello.' + this.conf.apiVersion)
			.set('User-Agent', this.conf.userAgent)
			.timeout(15 * 1000) // 15sec timeout

		if(options.method === 'POST' || options.method === 'PUT') {
			req.type('form')
				.send(options.queryData || {})

			if(options.media) {
				// @todo file upload support
				// maybe when I'm not so aggravated by oauth2 docs
			}
		} else {
			req.query(options.queryData || {})
		}

		// workaround for superagent bug: https://github.com/visionmedia/superagent/issues/206
		if(!req.req._headers.cookie) {
			delete req.req._headers.cookie;
			delete req.req._headerNames.cookie;
		}

		req.end(function(err, res) {
			if(err) throw err
			// grab rate-limit info.
			if(res.header['x-ratelimit-limit']) {
				self.rateLimitMax = res.header['x-ratelimit-limit']
				self.rateLimitRemaining = res.header['x-ratelimit-remaining']
				self.rateLimitReset = new Date(parseInt(res.header['x-ratelimit-reset']) * 1000)
			}
			if(!res.ok) {
				if(res.status === 429 || res.status === 503) {
					// hoping heello will move to use HTTP 429 only for api-limited response, as per RFC 6585.
					// <https://heello.com/katana/6803710> and <https://heello.com/noah/6805308>
					self.emit('apiLimit', res.error)
					throw new Error('API limit reached')
				} else if(res.status === 401 && res.body.error === ETOKENEXPIRED) {
					throw new Error(ETOKENEXPIRED)
				} else if(res.status === 401) {
					// bad key basically, auth details rejected for some reason. emit an error.
					self.emit('apiAuthError', res.error)
					throw new Error('Authorization failure')
				} else {
					//throw res.error
					fn(res.error, null, res)
				}
			} else {
				fn(null, (res.body) ? res.body : null, res)
			}
		})
	} catch(err) {
		fn(err)
	}
}

function heelloController(api, controller, actions) {
	var self = this
	this.api = api, this.controllerName = controller

	Object.keys(actions).forEach(function(action) {
		var details = actions[action]
		self.__defineGetter__(action, function() {
			return new heelloAction(api, controller, action, details)
		})
	})
}

function heelloAction(api, controller, action, details) {
	var self = this
	this.api = api, this.controllerName = controller, this.actionName = action

	// let's break it down...
	this.method = details.method || 'GET'
	this.url = (details.url) ? util.format('/%s/%s.json', controller, details.url) : '/' + controller + '.json'
	this.needsAuth = details.needsAuth || false // assume no auth needed unless stated otherwise

	// extract required URL params from this.url
	this.urlParams = []
	var re = /:([a-z0-9]+)/ig, res
	while((res = re.exec(this.url)) !== null)
		this.urlParams.push(res[1])

	this.queryParams = details.reqQueryParams || []
	this.mediaParams = details.mediaParams || []

	// automagical adding of access_token, if we need auth, this is how we get it.
	if(this.needsAuth)
		this.queryParams.push('access_token')

	this.reqParams = this.urlParams.slice().concat(this.queryParams)

	var ret = function(params, fn) {
		var self = this, reqParams = this.reqParams.slice(), options
		if(params === null) params = {}
		options = {
			method: this.method,
			url: this.url,
			params: {},
			media: {},
		}
		try {
			// sanity checks - all params we need present? do we have credentials required?
			if(this.needsAuth && self.api.creds === null)
				throw new Error(util.format('api call %s/%s requires auth, none provided', self.controllerName, self.actionName))

			// @todo include access_token anyways when we have one...? talking to heello devs about this,
			//   would help alleviate limiting when encountering users behind heavy NAT
			if(this.needsAuth) {
				params['access_token'] = this.api.accessToken
			} else {
				params['key'] = this.api.conf.appId
			}

			// check to make sure all needed params are present
			Object.keys(params).forEach(function(param) {
				if(reqParams.indexOf(param) !== -1)
					reqParams.remove(reqParams.indexOf(param))
			})
			if(reqParams.length > 0)
				throw new Error(util.format('api call %s/%s missing required parameter(s) [%s], none provided', self.controllerName, self.actionName, reqParams.join(', ')))

			// start query preparations
			this.urlParams.forEach(function(p) {
				// we should expect that ALL url params are required...
				options.url = options.url.replace(':' + p, params[p])
				delete params[p]
			})
			this.mediaParams.forEach(function(p) {
				if(params[p] !== undefined) {
					media[p] = params[p]
					delete params[p]
				}
			})
			if(Object.keys(options.media).length === 0)
				delete options.media

			// load params into options, fire.
			options.queryData = params
			this.api.request(options, function(err, json, res) {
				if(err && err.message === ETOKENEXPIRED) {
					// swap out with new token, try again pls
					self.api.refreshTokens(function() {
						options.queryData['access_token'] = this.api.accessToken
						self.api.request(options, fn)
					})
				} else {
					fn(err, json, res)
				}
			})
		} catch(err) {
			fn(err)
		}
	}

	return ret.bind(self)
}

module.exports = heelloApi
<|endoftext|>


<commit_before>'use strict';

var openiLogger  = require('openi-logger');
var openiUtils   = require('openi-cloudlet-utils');
var dbc         = require('dbc');

var type         = ['type','object'];
var level        = ['app','cloudlet'];
var crud         = ['create','read','update','delete'];
var config;

var init = function(conf){
   config = conf;
   this.logger  = openiLogger(config.logger_params);
};

var parsePermissions = function( client_perms, errCallback ){

   var server_perms = { '@objects' : {}, '@types' : {} };

   for ( var i = 0; i < client_perms.length; i++ ){
      var perm = client_perms[i];

      if( undefined === perm.type || undefined === perm.access_level || undefined === perm.ref || undefined === perm.access_type ){
         errCallback("Invalid permission: ", perm);
         return;
      }

      if( -1 === type.indexOf(perm.type.toLowerCase())){
         errCallback("Incorrect type, must be one of: type or object");
         return;
      }

      if( -1 === level.indexOf(perm.access_level.toLowerCase())){
         errCallback("Incorrect Access Level, must be one of: APP or CLOUDLET");
         return;
      }

      if( -1 === crud.indexOf(perm.access_type.toLowerCase())){
         errCallback("Incorrect Access Type, must be one of: CREATE, READ, UPDATE, or DELETE");
         return;
      }

      if ( 'object' === perm.type){
         if(!openiUtils.isObjectId(perm.ref)) {
            errCallback("Object Reference: Invalid Object id");
            return;
         }
         if ( undefined === server_perms['@objects'][perm.ref] ){
            server_perms['@objects'][perm.ref] = {}
         }
         server_perms['@objects'][perm.ref][perm.access_type.toLowerCase()] = true
      }
      else if ( 'type' === perm.type ){

         if(!openiUtils.isTypeId(perm.ref)) {
            errCallback("Type Reference: Invalid Type ID");
            return;
         }
         if ( undefined === server_perms['@types'][perm.ref] ){
            server_perms['@types'][perm.ref] = {'@app_level' : {}, '@cloudlet_level' : {}}
         }

         if ('app' === perm.access_level.toLowerCase()){
            server_perms['@types'][perm.ref]['@app_level'][perm.access_type.toLowerCase()] = true
         }
         else{
            server_perms['@types'][perm.ref]['@cloudlet_level'][perm.access_type.toLowerCase()] = true
         }
      }
      else if ( 'GraphObject' === perm.type ){
         console.log("TODO: Add type id mapping")
      }

   }

   return server_perms;
};

var getPermissions = function(msg){

   var cloudlet_id = msg.token.cloudlet;
   var client_id   = msg.token.client_id;
   var third_party = msg.token.context;

   //var key         = third_party + "+" + client_id + "+" + cloudlet_id
   var key         = third_party + "+" + cloudlet_id;

   return {
      'dao_actions'      : [
         {
            'action'      : 'GET',
            'database'    : key,
            'cloudlet_id' : cloudlet_id,
            'third_party' : third_party,
            'resp_type'   : 'permissions',
            'bucket'      : 'permissions'
         }
      ],
      'clients'      : [
         {
            'uuid' : msg.uuid,
            'connId' : msg.connId
         }
      ]
   };

};

var postPermissions = function(msg, errCallback){

   var cloudlet_id = msg.token.cloudlet;
   var third_party = msg.token.context;

   //var key         = third_party + "+" + client_id + "+" + cloudlet_id
   var key         = third_party + "+" + cloudlet_id;

   var permissions = {
      _date_created : new Date().toJSON(),
      status        : 'unpropagated',
      perms         : parsePermissions(msg.json, errCallback)
   };

   if(undefined !== permissions.perms) {
      return {
         'dao_actions': [
            {
               'action'     : 'UPDATE_PERMISSIONS',
               'database'   : key,
               'object_data': permissions,
               'cloudlet_id': cloudlet_id,
               'third_party': third_party,
               'resp_type'  : 'permissions_create',
               'bucket'     : 'permissions'
            }
         ],
         'clients'    : [
            {
               'uuid'  : msg.uuid,
               'connId': msg.connId
            }
         ]
      };
   }

};

var authorizePermissionAccess = function( msg, senderToDao, errcallback ){

   if(msg['json']['app_api_key'] === null || msg['json']['app_api_key'] === undefined) {
      errcallback("'app_api_key' required");
   }

   console.log("<><><><><><><><><><><><><><><><><><><><><><><><><><>");

   var meta = {
      "limit"       : 1000,
      "offset"      : 0,
      "total_count" : 0,
      "prev"        : null,
      "next"        : null
   };

   var action = {
      'dao_actions' : [
         {
            'action'     : 'GENERIC_VIEW',
            'start_key'  : msg.token.cloudlet,
            'end_key'    : msg.token.cloudlet + '\uefff',
            'design_doc' : 'clients_views',
            'view_name'  : "clients_by_cloudlet_id",
            'meta'       : meta,
            'resp_type'  : 'clients',
            'cloudlet'   : msg.token.cloudlet,
            'bucket'     : 'clients',
            'data'       : msg.json
         }
      ],
      'mongrel_sink' : config.internal_handler.sink,
      'clients'     : [
         {
            'uuid'  : msg.uuid,
            'connId': msg.connId
         }
      ]
   };

   //Send Data to dao to retrieve list of client. Client-session authentication performed by internalWorker.js
   senderToDao.send(action);
};

var getAppPermissions = function(msg) {

   var key = msg.path.replace('/api/v1/app_permissions/', '');

   //TODO: make more robust
   //extract key
   //return error if not there

   var startKey = [key];
   var endKey   = [key + '\uefff' ];

   return {
      'dao_actions': [
         {
            'action'     : 'VIEW',
            'database'   : key,
            'design_doc' : "permission_views",
            'view_name'  : "app_permissions",
            'start_key'   : startKey ,
            'end_key'     : endKey,
            'resp_type'  : 'permissions',
            'bucket'     : 'app_permissions'
         }
      ],
      'clients'    : [
         {
            'uuid'  : msg.uuid,
            'connId': msg.connId
         }
      ]
   };
};

var processMongrel2Message = function (msg, senderToDao,errCallback) {

   this.logger.log('debug', msg);

   if ( msg.headers.METHOD === 'GET' ){
      if(msg.headers.PATH.indexOf("app_") >= 0){
         return getAppPermissions(msg);
      }
      else {
         return getPermissions(msg);
      }
   }
   else if ( msg.headers.METHOD === 'POST' ){
      return postPermissions(msg,errCallback);
   }
   else if ( msg.headers.METHOD === 'PUT' ){
      return authorizePermissionAccess(msg, senderToDao, errCallback);
   }
};

module.exports.init                   = init;
module.exports.processMongrel2Message = processMongrel2Message;<commit_msg>Fixed issue with users submitting empty body in permissions api and crashes the system.<commit_after>'use strict';

var openiLogger  = require('openi-logger');
var openiUtils   = require('openi-cloudlet-utils');
var dbc         = require('dbc');

var type         = ['type','object'];
var level        = ['app','cloudlet'];
var crud         = ['create','read','update','delete'];
var config;

var init = function(conf){
   config = conf;
   this.logger  = openiLogger(config.logger_params);
};

var parsePermissions = function( client_perms, errCallback ){

   if (null === client_perms || undefined === client_perms) {
      errCallback("Invalid format: permissions body not found.");
      return;
   }

   var server_perms = { '@objects' : {}, '@types' : {} };

   for ( var i = 0; i < client_perms.length; i++ ){
      var perm = client_perms[i];

      if( undefined === perm.type || undefined === perm.access_level || undefined === perm.ref || undefined === perm.access_type ){
         errCallback("Invalid permission: ", perm);
         return;
      }

      if( -1 === type.indexOf(perm.type.toLowerCase())){
         errCallback("Incorrect type, must be one of: type or object");
         return;
      }

      if( -1 === level.indexOf(perm.access_level.toLowerCase())){
         errCallback("Incorrect Access Level, must be one of: APP or CLOUDLET");
         return;
      }

      if( -1 === crud.indexOf(perm.access_type.toLowerCase())){
         errCallback("Incorrect Access Type, must be one of: CREATE, READ, UPDATE, or DELETE");
         return;
      }

      if ( 'object' === perm.type){
         if(!openiUtils.isObjectId(perm.ref)) {
            errCallback("Object Reference: Invalid Object id");
            return;
         }
         if ( undefined === server_perms['@objects'][perm.ref] ){
            server_perms['@objects'][perm.ref] = {}
         }
         server_perms['@objects'][perm.ref][perm.access_type.toLowerCase()] = true
      }
      else if ( 'type' === perm.type ){

         if(!openiUtils.isTypeId(perm.ref)) {
            errCallback("Type Reference: Invalid Type ID");
            return;
         }
         if ( undefined === server_perms['@types'][perm.ref] ){
            server_perms['@types'][perm.ref] = {'@app_level' : {}, '@cloudlet_level' : {}}
         }

         if ('app' === perm.access_level.toLowerCase()){
            server_perms['@types'][perm.ref]['@app_level'][perm.access_type.toLowerCase()] = true
         }
         else{
            server_perms['@types'][perm.ref]['@cloudlet_level'][perm.access_type.toLowerCase()] = true
         }
      }
      else if ( 'GraphObject' === perm.type ){
         console.log("TODO: Add type id mapping")
      }

   }

   return server_perms;
};

var getPermissions = function(msg){

   var cloudlet_id = msg.token.cloudlet;
   var client_id   = msg.token.client_id;
   var third_party = msg.token.context;

   //var key         = third_party + "+" + client_id + "+" + cloudlet_id
   var key         = third_party + "+" + cloudlet_id;

   return {
      'dao_actions'      : [
         {
            'action'      : 'GET',
            'database'    : key,
            'cloudlet_id' : cloudlet_id,
            'third_party' : third_party,
            'resp_type'   : 'permissions',
            'bucket'      : 'permissions'
         }
      ],
      'clients'      : [
         {
            'uuid' : msg.uuid,
            'connId' : msg.connId
         }
      ]
   };

};

var postPermissions = function(msg, errCallback){

   var cloudlet_id = msg.token.cloudlet;
   var third_party = msg.token.context;

   //var key         = third_party + "+" + client_id + "+" + cloudlet_id
   var key         = third_party + "+" + cloudlet_id;

   var permissions = {
      _date_created : new Date().toJSON(),
      status        : 'unpropagated',
      perms         : parsePermissions(msg.json, errCallback)
   };

   if(undefined !== permissions.perms) {
      return {
         'dao_actions': [
            {
               'action'     : 'UPDATE_PERMISSIONS',
               'database'   : key,
               'object_data': permissions,
               'cloudlet_id': cloudlet_id,
               'third_party': third_party,
               'resp_type'  : 'permissions_create',
               'bucket'     : 'permissions'
            }
         ],
         'clients'    : [
            {
               'uuid'  : msg.uuid,
               'connId': msg.connId
            }
         ]
      };
   }

};

var authorizePermissionAccess = function( msg, senderToDao, errcallback ){

   if(msg['json']['app_api_key'] === null || msg['json']['app_api_key'] === undefined) {
      errcallback("'app_api_key' required");
   }

   var meta = {
      "limit"       : 1000,
      "offset"      : 0,
      "total_count" : 0,
      "prev"        : null,
      "next"        : null
   };

   var action = {
      'dao_actions' : [
         {
            'action'     : 'GENERIC_VIEW',
            'start_key'  : msg.token.cloudlet,
            'end_key'    : msg.token.cloudlet + '\uefff',
            'design_doc' : 'clients_views',
            'view_name'  : "clients_by_cloudlet_id",
            'meta'       : meta,
            'resp_type'  : 'clients',
            'cloudlet'   : msg.token.cloudlet,
            'bucket'     : 'clients',
            'data'       : msg.json
         }
      ],
      'mongrel_sink' : config.internal_handler.sink,
      'clients'     : [
         {
            'uuid'  : msg.uuid,
            'connId': msg.connId
         }
      ]
   };

   //Send Data to dao to retrieve list of client. Client-session authentication performed by internalWorker.js
   senderToDao.send(action);
};

var getAppPermissions = function(msg) {

   var key = msg.path.replace('/api/v1/app_permissions/', '');

   //TODO: make more robust
   //extract key
   //return error if not there

   var startKey = [key];
   var endKey   = [key + '\uefff' ];

   return {
      'dao_actions': [
         {
            'action'     : 'VIEW',
            'database'   : key,
            'design_doc' : "permission_views",
            'view_name'  : "app_permissions",
            'start_key'   : startKey ,
            'end_key'     : endKey,
            //'order'      : 'descending',
            'resp_type'  : 'permissions',
            'bucket'     : 'app_permissions'
         }
      ],
      'clients'    : [
         {
            'uuid'  : msg.uuid,
            'connId': msg.connId
         }
      ]
   };
};

var processMongrel2Message = function (msg, senderToDao,errCallback) {

   this.logger.log('debug', msg);

   if ( msg.headers.METHOD === 'GET' ){
      if(msg.headers.PATH.indexOf("app_") >= 0){
         return getAppPermissions(msg);
      }
      else {
         return getPermissions(msg);
      }
   }
   else if ( msg.headers.METHOD === 'POST' ){
      return postPermissions(msg,errCallback);
   }
   else if ( msg.headers.METHOD === 'PUT' ){
      return authorizePermissionAccess(msg, senderToDao, errCallback);
   }
};

module.exports.init                   = init;
module.exports.processMongrel2Message = processMongrel2Message;<|endoftext|>


<commit_before>/* eslint-disable no-use-extend-native/no-use-extend-native */

const path = require('path');

const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const stripBomStream = require('strip-bom-stream');
const tmp = require('tmp-promise');
const untildify = require('untildify');
const Promise = require('bluebird');

const models = require('../models/models');
const { openDb } = require('./db');
const { unzip } = require('./file-utils');
const logUtils = require('./log-utils');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const response = await fetch(task.agency_url, { method: 'GET', headers: task.agency_headers || {} });

  if (response.status !== 200) {
    throw new Error('Couldn’t download files');
  }

  const buffer = await response.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const getTextFiles = async folderPath => {
  const files = await fs.readdir(folderPath);
  return files.filter(filename => filename.slice(-3) === 'txt');
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await unzip(gtfsPath, task.downloadDir);
      const textFiles = await getTextFiles(task.downloadDir);

      // If no .txt files in this directory, check for subdirectories and copy them here
      if (textFiles.length === 0) {
        const files = await fs.readdir(task.downloadDir);
        const folders = files.map(filename => path.join(task.downloadDir, filename)).filter(source => fs.lstatSync(source).isDirectory());

        if (folders.length > 1) {
          throw new Error(`More than one subfolder found in zip file at ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        } else if (folders.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }

        const subfolderName = folders[0];
        const directoryTextFiles = await getTextFiles(subfolderName);

        if (directoryTextFiles.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }

        await Promise.all(directoryTextFiles.map(async fileName => fs.rename(path.join(subfolderName, fileName), path.join(task.downloadDir, fileName))));
      }
    } catch (error) {
      task.error(error);
      console.error(error);
      throw new Error(`Unable to unzip file ${task.path}`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);
  }
};

const createEmptyTables = async db => {
  return Promise.all(models.map(async model => {
    if (!model.schema) {
      return;
    }

    await db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`);

    const columns = model.schema.map(column => {
      let check = '';
      if (column.min !== undefined && column.max) {
        check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
      } else if (column.min) {
        check = `CHECK( ${column.name} >= ${column.min} )`;
      } else if (column.max) {
        check = `CHECK( ${column.name} <= ${column.max} )`;
      }

      const primary = column.primary ? 'PRIMARY KEY' : '';
      const required = column.required ? 'NOT NULL' : '';
      const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
      return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault}`;
    });
    await db.run(`CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`);

    await Promise.all(model.schema.map(async column => {
      if (column.index) {
        const unique = column.index === 'unique' ? 'UNIQUE' : '';
        await db.run(`CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`);
      }
    }));
  }));
};

const formatLine = (line, model, totalLineCount) => {
  const lineNumber = totalLineCount + 1;
  for (const fieldName of Object.keys(line)) {
    const columnSchema = model.schema.find(schema => schema.name === fieldName);

    // Remove columns not part of model
    if (!columnSchema) {
      delete line[fieldName];
      continue;
    }

    // Remove null values
    if (line[fieldName] === null || line[fieldName] === '') {
      delete line[fieldName];
    }

    // Convert fields that should be integer
    if (columnSchema.type === 'integer') {
      const value = Number.parseInt(line[fieldName], 10);

      if (Number.isNaN(value)) {
        delete line[fieldName];
      } else {
        line[fieldName] = value;
      }
    }

    // Convert fields that should be float
    if (columnSchema.type === 'real') {
      const value = Number.parseFloat(line[fieldName]);

      if (Number.isNaN(value)) {
        delete line[fieldName];
      } else {
        line[fieldName] = value;
      }
    }

    // Validate required
    if (columnSchema.required === true) {
      if (line[fieldName] === undefined || line[fieldName] === '') {
        throw new Error(`Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`);
      }
    }

    // Validate minimum
    if (columnSchema.min !== undefined) {
      if (line[fieldName] < columnSchema.min) {
        throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`);
      }
    }

    // Validate maximum
    if (columnSchema.max !== undefined) {
      if (line[fieldName] > columnSchema.max) {
        throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`);
      }
    }
  }

  // Convert to midnight timestamp
  const timestampFormat = [
    'start_time',
    'end_time',
    'arrival_time',
    'departure_time'
  ];

  for (const fieldName of timestampFormat) {
    if (line[fieldName]) {
      line[`${fieldName}stamp`] = utils.calculateHourTimestamp(line[fieldName]);
    }
  }

  return line;
};

const importLines = async (task, lines, model, totalLineCount) => {
  if (lines.length === 0) {
    return;
  }

  const linesToImportCount = lines.length;
  const fieldNames = model.schema.map(column => column.name);
  const placeholders = [];
  const values = [];

  while (lines.length) {
    const line = lines.pop();
    placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
    for (const fieldName of fieldNames) {
      values.push(line[fieldName]);
    }
  }

  try {
    await task.db.run(`INSERT INTO ${model.filenameBase}(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`, values);
  } catch (error) {
    task.warn(`Check ${model.filenameBase}.txt for invalid data between lines ${totalLineCount - linesToImportCount} and ${totalLineCount}.`);
    throw error;
  }

  task.log(`Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`, true);
};

const importFiles = task => {
  // Loop through each GTFS file
  return Promise.mapSeries(models, model => {
    return new Promise((resolve, reject) => {
      // Filter out excluded files from config
      if (task.exclude && task.exclude.includes(model.filenameBase)) {
        task.log(`Skipping - ${model.filenameBase}.txt\r`);
        return resolve();
      }

      const filepath = path.join(task.downloadDir, `${model.filenameBase}.txt`);

      if (!fs.existsSync(filepath)) {
        if (!model.nonstandard) {
          task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
        }

        return resolve();
      }

      task.log(`Importing - ${model.filenameBase}.txt\r`);

      const lines = [];
      let totalLineCount = 0;
      const maxInsertVariables = 800;
      const parser = parse({
        columns: true,
        relax: true,
        trim: true,
        skip_empty_lines: true,
        ...task.csvOptions
      });

      parser.on('readable', async () => {
        let record;
        /* eslint-disable-next-line no-cond-assign */
        while (record = parser.read()) {
          try {
            totalLineCount += 1;
            lines.push(formatLine(record, model, totalLineCount));

            // If we have a bunch of lines ready to insert, then do it
            if (lines.length >= maxInsertVariables / model.schema.length) {
              /* eslint-disable-next-line no-await-in-loop */
              await importLines(task, lines, model, totalLineCount);
            }
          } catch (error) {
            reject(error);
          }
        }
      });

      parser.on('end', async () => {
        // Insert all remaining lines
        await importLines(task, lines, model, totalLineCount).catch(reject);
        resolve();
      });

      parser.on('error', reject);

      fs.createReadStream(filepath)
        .pipe(stripBomStream())
        .pipe(parser);
    })
      .catch(error => {
        throw error;
      });
  });
};

module.exports = async config => {
  const log = logUtils.log(config);
  const logError = logUtils.logError(config);
  const logWarning = logUtils.logWarning(config);
  const db = await openDb(config);

  const agencyCount = config.agencies.length;
  log(`Starting GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}`);

  await createEmptyTables(db);

  await Promise.mapSeries(config.agencies, async agency => {
    if (!agency.agency_key) {
      throw new Error('No Agency Key provided.');
    }

    if (!agency.url && !agency.path) {
      throw new Error('No Agency URL or path provided.');
    }

    const { path, cleanup } = await tmp.dir({ unsafeCleanup: true });

    const task = {
      exclude: agency.exclude,
      agency_key: agency.agency_key,
      agency_url: agency.url,
      agency_headers: agency.headers || false,
      downloadDir: path,
      path: agency.path,
      csvOptions: config.csvOptions || {},
      db,
      log: (message, overwrite) => {
        log(`${task.agency_key}: ${message}`, overwrite);
      },
      warn: message => {
        logWarning(message);
      },
      error: message => {
        logError(message);
      }
    };

    if (task.agency_url) {
      await downloadFiles(task);
    }

    await readFiles(task);
    await importFiles(task);

    cleanup();
    task.log('Completed GTFS import');
  });

  log(`Completed GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}\n`);
};
<commit_msg>Only create tables & associated indexes for filenames not in agencies.exclude (#119)<commit_after>/* eslint-disable no-use-extend-native/no-use-extend-native */

const path = require('path');

const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const stripBomStream = require('strip-bom-stream');
const tmp = require('tmp-promise');
const untildify = require('untildify');
const Promise = require('bluebird');

const models = require('../models/models');
const { openDb } = require('./db');
const { unzip } = require('./file-utils');
const logUtils = require('./log-utils');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const response = await fetch(task.agency_url, { method: 'GET', headers: task.agency_headers || {} });

  if (response.status !== 200) {
    throw new Error('Couldn’t download files');
  }

  const buffer = await response.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const getTextFiles = async folderPath => {
  const files = await fs.readdir(folderPath);
  return files.filter(filename => filename.slice(-3) === 'txt');
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await unzip(gtfsPath, task.downloadDir);
      const textFiles = await getTextFiles(task.downloadDir);

      // If no .txt files in this directory, check for subdirectories and copy them here
      if (textFiles.length === 0) {
        const files = await fs.readdir(task.downloadDir);
        const folders = files.map(filename => path.join(task.downloadDir, filename)).filter(source => fs.lstatSync(source).isDirectory());

        if (folders.length > 1) {
          throw new Error(`More than one subfolder found in zip file at ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        } else if (folders.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }

        const subfolderName = folders[0];
        const directoryTextFiles = await getTextFiles(subfolderName);

        if (directoryTextFiles.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }

        await Promise.all(directoryTextFiles.map(async fileName => fs.rename(path.join(subfolderName, fileName), path.join(task.downloadDir, fileName))));
      }
    } catch (error) {
      task.error(error);
      console.error(error);
      throw new Error(`Unable to unzip file ${task.path}`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);
  }
};

const createEmptyTables = async task => {

  return Promise.all(models.map(async model => {
    if (!model.schema) {
      return;
    }

    await task.db.run(`DROP TABLE IF EXISTS ${model.filenameBase};`);

    const columns = model.schema.map(column => {
      let check = '';
      if (column.min !== undefined && column.max) {
        check = `CHECK( ${column.name} >= ${column.min} AND ${column.name} <= ${column.max} )`;
      } else if (column.min) {
        check = `CHECK( ${column.name} >= ${column.min} )`;
      } else if (column.max) {
        check = `CHECK( ${column.name} <= ${column.max} )`;
      }

      const primary = column.primary ? 'PRIMARY KEY' : '';
      const required = column.required ? 'NOT NULL' : '';
      const columnDefault = column.default ? 'DEFAULT ' + column.default : '';
      return `${column.name} ${column.type} ${check} ${primary} ${required} ${columnDefault}`;
    });

    if (!task.exclude.includes(model.filenameBase)) {
      await task.db.run(`CREATE TABLE ${model.filenameBase} (${columns.join(', ')});`);

      await Promise.all(model.schema.map(async column => {
        if (column.index) {
          const unique = column.index === 'unique' ? 'UNIQUE' : '';
          await task.db.run(`CREATE ${unique} INDEX idx_${model.filenameBase}_${column.name} ON ${model.filenameBase} (${column.name});`);
        }
      }));
    }
  }));
};

const formatLine = (line, model, totalLineCount) => {
  const lineNumber = totalLineCount + 1;
  for (const fieldName of Object.keys(line)) {
    const columnSchema = model.schema.find(schema => schema.name === fieldName);

    // Remove columns not part of model
    if (!columnSchema) {
      delete line[fieldName];
      continue;
    }

    // Remove null values
    if (line[fieldName] === null || line[fieldName] === '') {
      delete line[fieldName];
    }

    // Convert fields that should be integer
    if (columnSchema.type === 'integer') {
      const value = Number.parseInt(line[fieldName], 10);

      if (Number.isNaN(value)) {
        delete line[fieldName];
      } else {
        line[fieldName] = value;
      }
    }

    // Convert fields that should be float
    if (columnSchema.type === 'real') {
      const value = Number.parseFloat(line[fieldName]);

      if (Number.isNaN(value)) {
        delete line[fieldName];
      } else {
        line[fieldName] = value;
      }
    }

    // Validate required
    if (columnSchema.required === true) {
      if (line[fieldName] === undefined || line[fieldName] === '') {
        throw new Error(`Missing required value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}.`);
      }
    }

    // Validate minimum
    if (columnSchema.min !== undefined) {
      if (line[fieldName] < columnSchema.min) {
        throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: below minimum value of ${columnSchema.min}.`);
      }
    }

    // Validate maximum
    if (columnSchema.max !== undefined) {
      if (line[fieldName] > columnSchema.max) {
        throw new Error(`Invalid value in ${model.filenameBase}.txt for ${fieldName} on line ${lineNumber}: above maximum value of ${columnSchema.max}.`);
      }
    }
  }

  // Convert to midnight timestamp
  const timestampFormat = [
    'start_time',
    'end_time',
    'arrival_time',
    'departure_time'
  ];

  for (const fieldName of timestampFormat) {
    if (line[fieldName]) {
      line[`${fieldName}stamp`] = utils.calculateHourTimestamp(line[fieldName]);
    }
  }

  return line;
};

const importLines = async (task, lines, model, totalLineCount) => {
  if (lines.length === 0) {
    return;
  }

  const linesToImportCount = lines.length;
  const fieldNames = model.schema.map(column => column.name);
  const placeholders = [];
  const values = [];

  while (lines.length) {
    const line = lines.pop();
    placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
    for (const fieldName of fieldNames) {
      values.push(line[fieldName]);
    }
  }

  try {
    await task.db.run(`INSERT INTO ${model.filenameBase}(${fieldNames.join(', ')}) VALUES${placeholders.join(',')}`, values);
  } catch (error) {
    task.warn(`Check ${model.filenameBase}.txt for invalid data between lines ${totalLineCount - linesToImportCount} and ${totalLineCount}.`);
    throw error;
  }

  task.log(`Importing - ${model.filenameBase}.txt - ${totalLineCount} lines imported\r`, true);
};

const importFiles = task => {
  // Loop through each GTFS file
  return Promise.mapSeries(models, model => {
    return new Promise((resolve, reject) => {
      // Filter out excluded files from config
      if (task.exclude && task.exclude.includes(model.filenameBase)) {
        task.log(`Skipping - ${model.filenameBase}.txt\r`);
        return resolve();
      }

      const filepath = path.join(task.downloadDir, `${model.filenameBase}.txt`);

      if (!fs.existsSync(filepath)) {
        if (!model.nonstandard) {
          task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
        }

        return resolve();
      }

      task.log(`Importing - ${model.filenameBase}.txt\r`);

      const lines = [];
      let totalLineCount = 0;
      const maxInsertVariables = 800;
      const parser = parse({
        columns: true,
        relax: true,
        trim: true,
        skip_empty_lines: true,
        ...task.csvOptions
      });

      parser.on('readable', async () => {
        let record;
        /* eslint-disable-next-line no-cond-assign */
        while (record = parser.read()) {
          try {
            totalLineCount += 1;
            lines.push(formatLine(record, model, totalLineCount));

            // If we have a bunch of lines ready to insert, then do it
            if (lines.length >= maxInsertVariables / model.schema.length) {
              /* eslint-disable-next-line no-await-in-loop */
              await importLines(task, lines, model, totalLineCount);
            }
          } catch (error) {
            reject(error);
          }
        }
      });

      parser.on('end', async () => {
        // Insert all remaining lines
        await importLines(task, lines, model, totalLineCount).catch(reject);
        resolve();
      });

      parser.on('error', reject);

      fs.createReadStream(filepath)
        .pipe(stripBomStream())
        .pipe(parser);
    })
      .catch(error => {
        throw error;
      });
  });
};

module.exports = async config => {
  const log = logUtils.log(config);
  const logError = logUtils.logError(config);
  const logWarning = logUtils.logWarning(config);
  const db = await openDb(config);

  const agencyCount = config.agencies.length;
  log(`Starting GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}`);

  await Promise.mapSeries(config.agencies, async agency => {
    if (!agency.agency_key) {
      throw new Error('No Agency Key provided.');
    }

    if (!agency.url && !agency.path) {
      throw new Error('No Agency URL or path provided.');
    }

    const { path, cleanup } = await tmp.dir({ unsafeCleanup: true });

    const task = {
      exclude: agency.exclude,
      agency_key: agency.agency_key,
      agency_url: agency.url,
      agency_headers: agency.headers || false,
      downloadDir: path,
      path: agency.path,
      csvOptions: config.csvOptions || {},
      db,
      log: (message, overwrite) => {
        log(`${task.agency_key}: ${message}`, overwrite);
      },
      warn: message => {
        logWarning(message);
      },
      error: message => {
        logError(message);
      }
    };

    if (task.agency_url) {
      await downloadFiles(task);
    }

    await createEmptyTables(task);
    await readFiles(task);
    await importFiles(task);

    cleanup();
    task.log('Completed GTFS import');
  });

  log(`Completed GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}\n`);
};
<|endoftext|>


<commit_before>(function(root, factory) {
  if (typeof define === 'function') {
    define(factory);
  } else if (typeof exports === 'object') {
    module.exports = factory();
  } else {
    root.jqlite = root.$ = factory();
  }
}(this, function() {

  var readyCalls = [];
  window.onload = function(x) {
    readyCalls.forEach(function(call) {
      call(x);
    });
  };

  function $(selector) {
    var itemType = typeof selector;

    if (itemType == "function") {
      readyCalls.push(selector);
    } else {
      function JQLite(elem, length) {
        this.element = elem;
        this.length = length || elem.length;
      }

      JQLite.prototype.html = function(newHTML) {
        if (newHTML === undefined) {
          return this.element[0].innerHTML;
        } else {
          return this.each(function() {
            this.innerHTML = newHTML;
          });
        }
      };
      JQLite.prototype.text = function(newText) {
        if (newText === undefined) {
          return String(this.element[0].innerHTML);
        } else {
          newHTML = String(newText);
          return this.each(function() {
            this.innerHTML = newText;
          });
        }
      };

      JQLite.prototype.click = function(fn) {
        return this.each(function() {
          this.onclick = fn;
        });
      };
      JQLite.prototype.blur = function(fn) {
        return this.each(function() {
          this.onblur = fn;
        });
      };
      JQLite.prototype.focus = function(fn) {
        if (fn) {
          this.each(function() {
            this.onfocus = fn;
          });
        } else {
          this.element[0].focus();
        }
        return this;
      };

      JQLite.prototype.append = function(newElem) {
        if (typeof newElem == "string") {
          newElem = document.createElement(newElem);
        }
        this.element[0].appendChild(newElem);
        return new JQLite([newElem]);
      };

      JQLite.prototype.remove = function() {
        return this.each(function(element) {
          element.parentElement.removeChild(element);
        });
      };

      JQLite.prototype.each = function(fn) {
        for (var x of Array.from(this.element)) {
          fn.call(x, x);
        }
        return this;
      };

      JQLite.prototype.children = function() {
        return new JQLite(this.element[0].children);
      };
      JQLite.prototype.parent = function() {
        return new JQLite([this.element[0].parentElement]);
      };
      JQLite.prototype.first = function() {
        return this.get(0);
      };
      JQLite.prototype.last = function() {
        return this.get(-1);
      };
      JQLite.prototype.get = function(num) {
        if (Math.abs(num) > this.length) {
          throw "bad error";
        }
        if (num < 0) {
          return this.element[this.length + num];
        } else {
          return this.element[num];
        }
      };
      JQLite.prototype.slice = function(start, end) {
        return new JQLite(this.toArray()
          .slice(start, end));
      };

      JQLite.prototype.toArray = function() {
        if (Array.isArray(this.element)) {
          return this.element;
        } else {
          return Array.prototype.slice.call(this.element);
        }
      };

      JQLite.prototype.addClass = function(classname) {
        return this.each(function() {
          if (!this.className.match(classname)) {
            this.className += " " + classname;
          }
        });
      };
      JQLite.prototype.removeClass = function(classname) {
        return this.each(function() {
          this.className = this.className.replace(classname, "");
        });
      };

      JQLite.prototype.toggleClass = function(classname) {
        return this.each(function() {
          if (!this.className.match(classname)) {
            this.className += " " + classname;
          } else {
            this.className = this.className.replace(classname, "");
          }
        });
      };
      JQLite.prototype.css = function(style, rule) {
        return this.each(function() {
          if (typeof style == 'string') {
            this.style[style] = rule;
          } else {
            for (var styleKey in style) {
              if (style.hasOwnProperty(styleKey)) {
                this.style[style[styleKey]] = style[styleKey];
              }
            }
          }
        });
      }
      JQLite.prototype.attr = function(attribute, newValue) {
        return this.each(function() {
          if (typeof attribute == 'string') {
            this[attribute] = newValue;
          }
        });
      }

      JQLite.prototype.wrap = function(wrappingElem) {
        return this.each(function() {
          var wrapper = document.createElement(wrappingElem),
            parent = this.parentElement;
          parent.replaceChild(wrapper, this);
          wrapper.appendChild(this);
        });
      };
      JQLite.prototype.wrapAll = function(wrappingElem) {
        var wrapper = document.createElement(wrappingElem);
        this.each(function() {
          this.parentElement.removeChild(this);
          wrapper.appendChild(this);
        });
        return new JQLite([wrapper]);
      };

      JQLite.prototype.load = function(urlToLoad, ajaxer) {
        var that = this;
        ajaxer.get(urlToLoad, function(responseText) {
          that.html(responseText);
        });
      };

      JQLite.prototype.val = function() {
        return this.element[0].value;
      };
      JQLite.prototype.value = function() {
        return this.val();
      };

      /*
      addAttr
      removeAttr
      */

      /*
      change
      dlbclick
      hover
      submit
      */

      /*
      unwrap
      appendTo
      prepend
      before
      after

      replaceWith
      replaceAll
      data
      filter
      not
      hide
      */

      /*
      now()
      trim()
      isArray
      isFunction
      isNumeric
      each(object, fn)

      parseHTML
      stringify

      table
      */
      if (itemType == "string") {
        var selections = document.querySelectorAll(selector);
        return new JQLite(selections);
      } else if (itemType == "object") {
        return new JQLite([selector]);
      }
    }
  }
  return $;
}));
<commit_msg>Add simple animation functions for sliding: slide, _doSlide. Add css getter functionality in css function. Add hasClass function.<commit_after>(function(root, factory) {
  if (typeof define === 'function') {
    define(factory);
  } else if (typeof exports === 'object') {
    module.exports = factory();
  } else {
    root.jqlite = root.$ = factory();
  }
}(this, function() {

  var readyCalls = [];
  window.onload = function(x) {
    readyCalls.forEach(function(call) {
      call(x);
    });
  };

  function $(selector) {
    var itemType = typeof selector;

    if (itemType == "function") {
      readyCalls.push(selector);
    } else {
      function JQLite(elem, length) {
        this.element = elem;
        this.length = length || elem.length;
      }

      JQLite.prototype.html = function(newHTML) {
        if (newHTML === undefined) {
          return this.element[0].innerHTML;
        } else {
          return this.each(function() {
            this.innerHTML = newHTML;
          });
        }
      };
      JQLite.prototype.text = function(newText) {
        if (newText === undefined) {
          return String(this.element[0].innerHTML);
        } else {
          newHTML = String(newText);
          return this.each(function() {
            this.innerHTML = newText;
          });
        }
      };

      JQLite.prototype.click = function(fn) {
        return this.each(function() {
          this.onclick = fn;
        });
      };
      JQLite.prototype.blur = function(fn) {
        return this.each(function() {
          this.onblur = fn;
        });
      };
      JQLite.prototype.focus = function(fn) {
        if (fn) {
          this.each(function() {
            this.onfocus = fn;
          });
        } else {
          this.element[0].focus();
        }
        return this;
      };

      JQLite.prototype.append = function(newElem) {
        if (typeof newElem == "string") {
          newElem = document.createElement(newElem);
        }
        this.element[0].appendChild(newElem);
        return new JQLite([newElem]);
      };

      JQLite.prototype.remove = function() {
        return this.each(function(element) {
          element.parentElement.removeChild(element);
        });
      };

      JQLite.prototype.each = function(fn) {
        for (var x of Array.from(this.element)) {
          fn.call(x, x);
        }
        return this;
      };

      JQLite.prototype.children = function() {
        return new JQLite(this.element[0].children);
      };
      JQLite.prototype.parent = function() {
        return new JQLite([this.element[0].parentElement]);
      };
      JQLite.prototype.first = function() {
        return this.get(0);
      };
      JQLite.prototype.last = function() {
        return this.get(-1);
      };
      JQLite.prototype.get = function(num) {
        if (Math.abs(num) > this.length) {
          throw "bad error";
        }
        if (num < 0) {
          return this.element[this.length + num];
        } else {
          return this.element[num];
        }
      };
      JQLite.prototype.slice = function(start, end) {
        return new JQLite(this.toArray()
          .slice(start, end));
      };

      JQLite.prototype.toArray = function() {
        if (Array.isArray(this.element)) {
          return this.element;
        } else {
          return Array.prototype.slice.call(this.element);
        }
      };

      JQLite.prototype.addClass = function(classname) {
        return this.each(function() {
          if (!this.className.match(classname)) {
            this.className += " " + classname;
          }
        });
      };
      JQLite.prototype.removeClass = function(classname) {
        return this.each(function() {
          this.className = this.className.replace(classname, "");
        });
      };

      JQLite.prototype.toggleClass = function(classname) {
        return this.each(function() {
          if (!this.className.match(classname)) {
            this.className += " " + classname;
          } else {
            this.className = this.className.replace(classname, "");
          }
        });
      };
      JQLite.prototype.css = function(style, rule) {
        if (rule === undefined) {
          var cssProp = window.getComputedStyle(this.element[0], null);
          return cssProp.getPropertyValue(style);
        }

        return this.each(function() {
          if (typeof style == 'string') {
            this.style[style] = rule;
          } else {
            for (var styleKey in style) {
              if (style.hasOwnProperty(styleKey)) {
                this.style[style[styleKey]] = style[styleKey];
              }
            }
          }
        });
      }
      JQLite.prototype.attr = function(attribute, newValue) {
        return this.each(function() {
          if (typeof attribute == 'string') {
            this[attribute] = newValue;
          }
        });
      }

      JQLite.prototype.wrap = function(wrappingElem) {
        return this.each(function() {
          var wrapper = document.createElement(wrappingElem),
            parent = this.parentElement;
          parent.replaceChild(wrapper, this);
          wrapper.appendChild(this);
        });
      };
      JQLite.prototype.wrapAll = function(wrappingElem) {
        var wrapper = document.createElement(wrappingElem);
        this.each(function() {
          this.parentElement.removeChild(this);
          wrapper.appendChild(this);
        });
        return new JQLite([wrapper]);
      };

      JQLite.prototype.load = function(urlToLoad, ajaxer) {
        var that = this;
        ajaxer.get(urlToLoad, function(responseText) {
          that.html(responseText);
        });
      };

      JQLite.prototype.val = function() {
        return this.element[0].value;
      };
      JQLite.prototype.value = function() {
        return this.val();
      };

      JQLite.prototype.slide = function(destination, duration) {
        this.toggleClass(":animating");
        var left = this.css('left');
        var start = left === "auto" ? 0 : parseInt(left.slice(0, left.length-2));
        this._startTime = new Date().getTime();
        this._doSlide(start, destination, duration);
      };

      JQLite.prototype._doSlide = function(start, destination, duration) {
        var curTime = new Date().getTime();
        var delta = curTime - this._startTime;
        var progress = delta/duration;

        var difference = start - destination;
        progress = progress > 1.0 ? 1.0 : progress;
        var currentPos = destination + (difference - progress*difference);

        this.css("left", "" + currentPos + "px");
        if (progress === 1.0) {
          this.toggleClass(":animating");
          return;
        }
        var that = this;
        window.requestAnimationFrame( function() {
          that._doSlide(start, destination, duration);
        });
      };

      JQLite.prototype.hasClass = function(className) {
        if (this.element[0].className.indexOf(className) >= 0) {
          return true;
        }
        return false;
      };

      /*
      addAttr
      removeAttr
      */

      /*
      change
      dlbclick
      hover
      submit
      */

      /*
      unwrap
      appendTo
      prepend
      before
      after
      replaceWith
      replaceAll
      data
      filter
      not
      hide
      */

      /*
      now()
      trim()
      isArray
      isFunction
      isNumeric
      each(object, fn)
      parseHTML
      stringify
      table
      */
      if (itemType == "string") {
        var selections = document.querySelectorAll(selector);
        return new JQLite(selections);
      } else if (itemType == "object") {
        return new JQLite([selector]);
      }
    }
  }
  return $;
}));
<|endoftext|>


<commit_before>var dfm={'hooks':{}};

dfm.object=function(){
	this.actions = [];
}

dfm.object.ifnull=function(nullVal){
	this.actions.push(['ifnull',nullVal]);
}

dfm.object.ifblank=function(blankVal){
	this.actions.push(['ifblank',blankVal]);
}

dfm.object.int=function(){
	this.actions.push(['int']);
}

dfm.object.float=function(){
	this.actions.push(['float']);
}

dfm.construct=function(){
	var obj = new dfm.object;
	return obj;
}

dfm.object.prototype.apply=function(inData){
	var out = inData;
	for(var i=0;i<this.actions.length;i++){
		switch(this.actions[0]){
			case 'ifnull':
				out = (out+'' == 'undefined')?this.actions[1]:out;
				break;
			case 'ifblank':
				out = (out+'' == '')?this.actions[1]:out;
				break;
			case 'int':
				out = parseInt(out);
				break;
			case 'float':
				out = parseFloat(out);
				break;
			case 'add':
				out += this.actions[1];
				break;
			case 'subtract':
				out -= this.actions[1];
				break;
			default:
				if(typeof(dfm.hooks[this.actions[0]]) == 'function'){
					out = dfm.hooks[this.actions[0]](out);
				}else{
					alert('Could not find formatting function: '+this.actions[0]);
				}
				break;
		}
	}
	return out;
}<commit_msg>updating libs<commit_after>var dfm={'hooks':{}};

dfm.object=function(){
	this.actions = [];
}

dfm.object.ifnull=function(nullVal){
	this.actions.push(['ifnull',nullVal]);
}

dfm.object.ifblank=function(blankVal){
	this.actions.push(['ifblank',blankVal]);
}

dfm.object.int=function(){
	this.actions.push(['int']);
}

dfm.object.float=function(){
	this.actions.push(['float']);
}

dfm.construct=function(){
	var obj = new dfm.object;
	return obj;
}

dfm.object.prototype.apply=function(inData){
	var out = inData;
	for(var i=0;i<this.actions.length;i++){
		switch(this.actions[0]){
			case 'ifnull':
				out = (out+'' == 'undefined')?this.actions[1]:out;
				break;
			case 'ifblank':
				out = (out+'' == '')?this.actions[1]:out;
				break;
			case 'int':
				out = parseInt(out);
				break;
			case 'float':
				out = parseFloat(out);
				break;
			case 'add':
				out += this.actions[1];
				break;
			case 'subtract':
				out -= this.actions[1];
				break;
			default:
				if(typeof(dfm.hooks[this.actions[0]]) == 'function'){
					out = dfm.hooks[this.actions[0]](out);
				}else{
					alert('Could not find formatting function: '+this.actions[0]);
				}
				break;
		}
	}
	return out;
};
<|endoftext|>


<commit_before>'use strict';

var postcssLess = require('postcss-less');
var sortBy = require('lodash.sortby');
var merge = require('lodash.merge');
var postcss = require('postcss');
var path = require('path');

var defaultLinters = [
    require('./linters/attribute_quotes'),
    require('./linters/border_zero'),
    require('./linters/comment'),
    require('./linters/decimal_zero'),
    require('./linters/depth_level'),
    require('./linters/duplicate_property'),
    require('./linters/empty_rule'),
    require('./linters/final_newline'),
    require('./linters/hex_length'),
    require('./linters/hex_notation'),
    require('./linters/hex_validation'),
    require('./linters/id_selector'),
    require('./linters/import_path'),
    require('./linters/important_rule'),
    require('./linters/property_ordering'),
    require('./linters/property_units'),
    require('./linters/qualifying_element'),
    require('./linters/selector_naming'),
    require('./linters/single_line_per_property'),
    require('./linters/single_line_per_selector'),
    require('./linters/space_after_property_colon'),
    require('./linters/space_after_property_name'),
    require('./linters/space_after_property_value'),
    require('./linters/space_around_comma'),
    require('./linters/space_around_operator'),
    require('./linters/space_before_brace'),
    require('./linters/space_between_parens'),
    require('./linters/string_quotes'),
    require('./linters/trailing_semicolon'),
    require('./linters/trailing_whitespace'),
    require('./linters/url_format'),
    require('./linters/url_quotes'),
    require('./linters/zero_unit')
];

var parseRules = function (node) {
    var rules = node.text.match(/^\s*lesshint\s+(.*)/);

    if (!rules) {
        return {};
    }

    rules = rules[1].trim();
    rules = rules.replace(/([a-z0-9]+)\s*:/ig, '"$1":');

    try {
        rules = JSON.parse('{' + rules + '}');
    } catch (e) {
        throw new Error('Invalid inline option on line ' + node.start.line);
    }

    // If it's only shorthand enabled or disabled, account for that
    Object.keys(rules).forEach(function (rule) {
        if (typeof rules[rule] === 'boolean') {
            rules[rule] = {
                enabled: rules[rule]
            };
        }
    });

    return rules;
};

var getInlineOptions = function (node) {
    var start;

    node = node || {};

    if (node.type !== 'comment') {
        return {};
    }

    start = node.source.start;

    // Not the first thing in the file, bail
    if (start.line !== 1 && start.column !== 1) {
        return {};
    }

    return parseRules(node);
};

var runChecks = function (ast, fullPath, lines, config) {
    var filename = path.basename(fullPath);
    var lineOptions = [];
    var results = [];
    var inlineOptions;
    var linters;

    // Freeze the AST so linters won't accidentally change it
    Object.freeze(ast);

    // Fetch inline options and merge it with whatever passed
    inlineOptions = getInlineOptions(ast.root.first);
    config = merge(config, inlineOptions);

    // Load any additional linters
    linters = mergeLintersFromConfig(defaultLinters, config);

    // Fetch all single line options before we start the actual linting
    ast.root.walkComments(function (node) {
        var option = parseRules(node);
        var key = Object.keys(option)[0];

        option[key] = merge(option[key], {
            line: node.source.start.line
        });

        lineOptions.push(option);
    });

    ast.root.walk(function (node) {
        linters.forEach(function (linter) {
            var options = config[linter.name];
            var lint;

            // Bail if the linter isn't wanted
            if (!options || (options && !options.enabled)) {
                return;
            }

            // Node type(s) not applicable, bail
            if (Array.isArray(linter.nodeTypes) && linter.nodeTypes.indexOf(node.type) === -1) {
                return;
            }

            lint = linter.lint.call(linter, options, node);
            if (lint) {
                if (!Array.isArray(lint)) {
                    throw new Error('Linter ' + linter.name + ' must return an array.');
                }

                lint.forEach(function (piece) {
                    var line = piece.line || node.source.start.line;
                    var shouldIgnore = lineOptions.some(function (lineOption) {
                        lineOption = lineOption[linter.name];

                        if (!lineOption) {
                            return false;
                        }

                        return lineOption.line === line && !lineOption.enabled;
                    });

                    if (shouldIgnore) {
                        return;
                    }

                    /**
                     * Each piece of lint (linter result) can override
                     * the default properties of the result we're adding
                     * to the collection
                     */
                    results.push(merge({
                        column: node.source.start.column,
                        file: filename,
                        fullPath: fullPath,
                        line: line,
                        linter: linter.name,
                        message: 'Unknown linting error. Source: ' + linter.name,
                        severity: options.severity || exports.resultSeverity.warning,
                        source: line ? lines[line - 1] : ''
                    }, piece));
                });
            }
        });
    });

    return results;
};

var mergeLintersFromConfig = function (defaultLinters, config) {
    if (!config.linters) {
        return defaultLinters;
    }

    if (!(config.linters instanceof Array)) {
        throw new Error('linters should be an array of file paths and/or linters');
    }

    return defaultLinters.concat(config.linters.map(requireConfigLinter));
};

var requireConfigLinter = function (linter) {
    if (typeof linter === 'string') {
        return require(linter);
    }

    return linter;
};

exports.resultSeverity = {
    error: 'error',
    warning: 'warning'
};

exports.lint = function (input, fullPath, config) {
    var results;
    var parser;
    var lines;
    var ast;

    input = input.replace(/\r\n|\r|\n/g, '\n'); // Normalize line endings. See #40
    lines = input.split('\n');

    try {
        parser = this.getParser(input);
        ast = parser.root.toResult();

        results = runChecks(ast, fullPath, lines, config);
        results = sortBy(results, 'column');
        results = sortBy(results, 'line');

        return results;
    } catch (e) {
        // Just rethrow the exception and let someone else handle it
        throw e;
    }
};

exports.getParser = function (input) {
    return postcss([]).process(input, {
        syntax: postcssLess
    });
};
<commit_msg>checking rules for pseudo class/selectors. postcss-less erroneously adds params property to nodes with them.<commit_after>'use strict';

var postcssLess = require('postcss-less');
var sortBy = require('lodash.sortby');
var merge = require('lodash.merge');
var postcss = require('postcss');
var path = require('path');

var defaultLinters = [
    require('./linters/attribute_quotes'),
    require('./linters/border_zero'),
    require('./linters/comment'),
    require('./linters/decimal_zero'),
    require('./linters/depth_level'),
    require('./linters/duplicate_property'),
    require('./linters/empty_rule'),
    require('./linters/final_newline'),
    require('./linters/hex_length'),
    require('./linters/hex_notation'),
    require('./linters/hex_validation'),
    require('./linters/id_selector'),
    require('./linters/import_path'),
    require('./linters/important_rule'),
    require('./linters/property_ordering'),
    require('./linters/property_units'),
    require('./linters/qualifying_element'),
    require('./linters/selector_naming'),
    require('./linters/single_line_per_property'),
    require('./linters/single_line_per_selector'),
    require('./linters/space_after_property_colon'),
    require('./linters/space_after_property_name'),
    require('./linters/space_after_property_value'),
    require('./linters/space_around_comma'),
    require('./linters/space_around_operator'),
    require('./linters/space_before_brace'),
    require('./linters/space_between_parens'),
    require('./linters/string_quotes'),
    require('./linters/trailing_semicolon'),
    require('./linters/trailing_whitespace'),
    require('./linters/url_format'),
    require('./linters/url_quotes'),
    require('./linters/zero_unit')
];

var parseRules = function (node) {
    var rules = node.text.match(/^\s*lesshint\s+(.*)/);

    if (!rules) {
        return {};
    }

    rules = rules[1].trim();
    rules = rules.replace(/([a-z0-9]+)\s*:/ig, '"$1":');

    try {
        rules = JSON.parse('{' + rules + '}');
    } catch (e) {
        throw new Error('Invalid inline option on line ' + node.start.line);
    }

    // If it's only shorthand enabled or disabled, account for that
    Object.keys(rules).forEach(function (rule) {
        if (typeof rules[rule] === 'boolean') {
            rules[rule] = {
                enabled: rules[rule]
            };
        }
    });

    return rules;
};

var getInlineOptions = function (node) {
    var start;

    node = node || {};

    if (node.type !== 'comment') {
        return {};
    }

    start = node.source.start;

    // Not the first thing in the file, bail
    if (start.line !== 1 && start.column !== 1) {
        return {};
    }

    return parseRules(node);
};

var runChecks = function (ast, fullPath, lines, config) {
    var filename = path.basename(fullPath);
    var lineOptions = [];
    var results = [];
    var inlineOptions;
    var linters;

    // tests selectors for pseudo classes/selectors. eg. :not(), :active
    var rPseudo = /::?[^ ,:.]+/g;

    // Freeze the AST so linters won't accidentally change it
    Object.freeze(ast);

    // Fetch inline options and merge it with whatever passed
    inlineOptions = getInlineOptions(ast.root.first);
    config = merge(config, inlineOptions);

    // Load any additional linters
    linters = mergeLintersFromConfig(defaultLinters, config);

    // Fetch all single line options before we start the actual linting
    ast.root.walkComments(function (node) {
        var option = parseRules(node);
        var key = Object.keys(option)[0];

        option[key] = merge(option[key], {
            line: node.source.start.line
        });

        lineOptions.push(option);
    });

    ast.root.walk(function (node) {
        linters.forEach(function (linter) {
            var options = config[linter.name];
            var lint;
            var matches;
            var pseudoParams;

            // Bail if the linter isn't wanted
            if (!options || (options && !options.enabled)) {
                return;
            }

            // Node type(s) not applicable, bail
            if (Array.isArray(linter.nodeTypes) && linter.nodeTypes.indexOf(node.type) === -1) {
                return;
            }

            // if we're dealing with a regular Rule (or other) node, which isn't
            // an actual Mixin or AtRule, and its selector contains a pseudo
            // class or selector, then clean up the raws and params properties.
            // tracking: https://github.com/webschik/postcss-less/issues/56
            if (node.params && rPseudo.test(node.selector)) {
              delete node.params;

              // this just started showing up in postcss-less@0.14.0. not sure
              // if it's sticking around, but making sure we're thorough.
              if (node.raws.params) {
                delete node.raws.params;
              }
            }

            lint = linter.lint.call(linter, options, node);

            if (lint) {
                if (!Array.isArray(lint)) {
                    throw new Error('Linter ' + linter.name + ' must return an array.');
                }

                lint.forEach(function (piece) {
                    var line = piece.line || node.source.start.line;
                    var shouldIgnore = lineOptions.some(function (lineOption) {
                        lineOption = lineOption[linter.name];

                        if (!lineOption) {
                            return false;
                        }

                        return lineOption.line === line && !lineOption.enabled;
                    });

                    if (shouldIgnore) {
                        return;
                    }

                    /**
                     * Each piece of lint (linter result) can override
                     * the default properties of the result we're adding
                     * to the collection
                     */
                    results.push(merge({
                        column: node.source.start.column,
                        file: filename,
                        fullPath: fullPath,
                        line: line,
                        linter: linter.name,
                        message: 'Unknown linting error. Source: ' + linter.name,
                        severity: options.severity || exports.resultSeverity.warning,
                        source: line ? lines[line - 1] : ''
                    }, piece));
                });
            }
        });
    });

    return results;
};

var mergeLintersFromConfig = function (defaultLinters, config) {
    if (!config.linters) {
        return defaultLinters;
    }

    if (!(config.linters instanceof Array)) {
        throw new Error('linters should be an array of file paths and/or linters');
    }

    return defaultLinters.concat(config.linters.map(requireConfigLinter));
};

var requireConfigLinter = function (linter) {
    if (typeof linter === 'string') {
        return require(linter);
    }

    return linter;
};

exports.resultSeverity = {
    error: 'error',
    warning: 'warning'
};

exports.lint = function (input, fullPath, config) {
    var results;
    var parser;
    var lines;
    var ast;

    input = input.replace(/\r\n|\r|\n/g, '\n'); // Normalize line endings. See #40
    lines = input.split('\n');

    try {
        parser = this.getParser(input);
        ast = parser.root.toResult();

        results = runChecks(ast, fullPath, lines, config);
        results = sortBy(results, 'column');
        results = sortBy(results, 'line');

        return results;
    } catch (e) {
        // Just rethrow the exception and let someone else handle it
        throw e;
    }
};

exports.getParser = function (input) {
    return postcss([]).process(input, {
        syntax: postcssLess
    });
};
<|endoftext|>