<commit_before>'use strict';
var QEmitter = require('qemitter'),
    util = require('util'),
    path = require('path'),

    _ = require('lodash'),
    chalk = require('chalk'),
    ScreenShooter = require('./screen-shooter'),
    Tester = require('./tester'),
    Config = require('./config'),
    Suite = require('./suite'),
    suiteUtils = require('./suite-util'),

    pathUtils = require('./path-utils'),
    publicApi = require('./public-api'),
    RunnerEvents = require('./constants/runner-events'),

    exposeTestsApi = require('./tests-api'),

    GeminiError = require('./errors/gemini-error'),
    Image = require('./image'),

    DEFAULT_CFG_NAME = '.gemini.yml',
    DEFAULT_SPECS_DIR = 'gemini';

function requireWithNoCache(moduleName) {
    var result = require(moduleName);
    delete require.cache[moduleName];
    return result;
}

function Gemini(config, overrides) {
    QEmitter.call(this);
    config = config || DEFAULT_CFG_NAME;
    this.config = new Config(config, overrides);
    var _this = this;

    require('./plugins').load(_this, this.config);

    function getDefaultSpecsDir() {
        return path.join(_this.config.projectRoot, DEFAULT_SPECS_DIR);
    }

    function executeRunner(runnerInstance, paths, options) {
        if (!options) {
            //if there are only two arguments, they are
            //(runnerInstance, options) and paths are
            //the default.
            options = paths;
            paths = [getDefaultSpecsDir()];
        }
        options = options || {};
        options.reporters = options.reporters || [];

        var envBrowsers = process.env.GEMINI_BROWSERS? process.env.GEMINI_BROWSERS.split(',') : null;
        options.browsers = options.browsers || envBrowsers;

        if (options.browsers) {
            validateBrowsers(options.browsers);
            runnerInstance.setTestBrowsers(options.browsers);
        }

        return _this.readTests(paths)
            .then(function(rootSuite) {
                return _this.emitAndWait(RunnerEvents.START_RUNNER, runnerInstance)
                    .thenResolve(rootSuite);
            })
            .then(function(rootSuite) {
                var suites = suiteUtils.flattenSuites(rootSuite);

                if (options.grep) {
                    suites = suites.filter(function(suite) {
                        return options.grep.test(suite.fullName);
                    });
                }

                options.reporters.forEach(_.partial(applyReporter, runnerInstance));
                return runnerInstance.run(suites);
            })
            .then(function(data) {
                return _this.emitAndWait(RunnerEvents.END_RUNNER, runnerInstance, data)
                    .thenResolve(data);
            });
    }

    function validateBrowsers(browsers) {
        var browserIds = _this.browserIds;
        browsers.forEach(function(id) {
            if (browserIds.indexOf(id) === -1) {
                throw new GeminiError('Unknown browser id: ' + id,
                    'Use one of the browser ids specified in config file: ' +
                    browserIds.join(', '));
            }
        });
    }

    this.readTests = function(paths) {
        paths = paths || [getDefaultSpecsDir()];
        return pathUtils.expandPaths(paths)
            .then(function(expanded) {
                var rootSuite = Suite.create('');
                exposeTestsApi(publicApi, rootSuite);

                expanded.forEach(requireWithNoCache);
                return rootSuite;
            });
    };

    this.gather = function(paths, options) {
        return executeRunner(
            new ScreenShooter(this.config),
            paths,
            options);
    };

    this.test = function(paths, options) {
        return executeRunner(
            new Tester(this.config, {tempDir: options.tempDir}),
            paths,
            options
        );
    };

    this.getScreenshotPath = function(suite, stateName, browserId) {
        return this.config.getScreenshotPath(suite, stateName, browserId);
    };

    this.buildDiff = util.deprecate(function(referencePath, currentPath, diffPath) {
            return Image.buildDiff({
                reference: referencePath,
                current: currentPath,
                diff: diffPath,
                diffColor: this.config.diffColor,
                strictComparison: this.config.strictComparison,
                tolerance: this.config.tolerance
            });
        }, 'gemini.buildDiff' + chalk.red(' is deprecated.\n') +
        'Use ' + chalk.green('testResult.saveDiffTo(path)') +
        ' in ' + chalk.green('endTest') + ' event handler for more accurate diff'
    );

    this.getBrowserCapabilites = function(browserId) {
        return this.config.browsers[browserId];
    };

    Object.defineProperty(this, 'browserIds', {
        enumerable: true,
        get: function() {
            return Object.keys(this.config.browsers);
        }
    });
}

// Gemini needs to inherit from QEmitter rather then
// EventEmmiter because it needs to allow `startRunner`
// event handler delay the execution of tests
util.inherits(Gemini, QEmitter);

function applyReporter(runner, reporter) {
    if (typeof reporter === 'string') {
        try {
            reporter = require('./reporters/' + reporter);
        } catch (e) {
            if (e.code === 'MODULE_NOT_FOUND') {
                throw new Error('No such reporter: ' + reporter);
            }
            throw e;
        }
    }
    if (typeof reporter !== 'function') {
        throw new TypeError('Reporter must be a string or a function');
    }

    reporter(runner);
}

module.exports = Gemini;
<commit_msg>Renamed validateBrowsers to areAllBrowsersFromConfig<commit_after>'use strict';
var QEmitter = require('qemitter'),
    util = require('util'),
    path = require('path'),

    _ = require('lodash'),
    chalk = require('chalk'),
    ScreenShooter = require('./screen-shooter'),
    Tester = require('./tester'),
    Config = require('./config'),
    Suite = require('./suite'),
    suiteUtils = require('./suite-util'),

    pathUtils = require('./path-utils'),
    publicApi = require('./public-api'),
    RunnerEvents = require('./constants/runner-events'),

    exposeTestsApi = require('./tests-api'),

    GeminiError = require('./errors/gemini-error'),
    Image = require('./image'),

    DEFAULT_CFG_NAME = '.gemini.yml',
    DEFAULT_SPECS_DIR = 'gemini';

function requireWithNoCache(moduleName) {
    var result = require(moduleName);
    delete require.cache[moduleName];
    return result;
}

function Gemini(config, overrides) {
    QEmitter.call(this);
    config = config || DEFAULT_CFG_NAME;
    this.config = new Config(config, overrides);
    var _this = this;

    require('./plugins').load(_this, this.config);

    function getDefaultSpecsDir() {
        return path.join(_this.config.projectRoot, DEFAULT_SPECS_DIR);
    }

    function executeRunner(runnerInstance, paths, options) {
        if (!options) {
            //if there are only two arguments, they are
            //(runnerInstance, options) and paths are
            //the default.
            options = paths;
            paths = [getDefaultSpecsDir()];
        }
        options = options || {};
        options.reporters = options.reporters || [];

        var envBrowsers = process.env.GEMINI_BROWSERS? process.env.GEMINI_BROWSERS.split(',') : null;
        options.browsers = options.browsers || envBrowsers;

        if (options.browsers) {
            areAllBrowsersFromConfig(options.browsers, _this.browserIds);
            runnerInstance.setTestBrowsers(options.browsers);
        }

        return _this.readTests(paths)
            .then(function(rootSuite) {
                return _this.emitAndWait(RunnerEvents.START_RUNNER, runnerInstance)
                    .thenResolve(rootSuite);
            })
            .then(function(rootSuite) {
                var suites = suiteUtils.flattenSuites(rootSuite);

                if (options.grep) {
                    suites = suites.filter(function(suite) {
                        return options.grep.test(suite.fullName);
                    });
                }

                options.reporters.forEach(_.partial(applyReporter, runnerInstance));
                return runnerInstance.run(suites);
            })
            .then(function(data) {
                return _this.emitAndWait(RunnerEvents.END_RUNNER, runnerInstance, data)
                    .thenResolve(data);
            });
    }

    function areAllBrowsersFromConfig(browsers, browsersFromConfig) {
        browsers.forEach(function(name) {
            if (browsersFromConfig.indexOf(name) === -1) {
                throw new GeminiError('Unknown browser id: ' + name,
                    'Use one of the browser ids specified in config file: ' +
                    browsersFromConfig.join(', '));
            }
        });
    }

    this.readTests = function(paths) {
        paths = paths || [getDefaultSpecsDir()];
        return pathUtils.expandPaths(paths)
            .then(function(expanded) {
                var rootSuite = Suite.create('');
                exposeTestsApi(publicApi, rootSuite);

                expanded.forEach(requireWithNoCache);
                return rootSuite;
            });
    };

    this.gather = function(paths, options) {
        return executeRunner(
            new ScreenShooter(this.config),
            paths,
            options);
    };

    this.test = function(paths, options) {
        return executeRunner(
            new Tester(this.config, {tempDir: options.tempDir}),
            paths,
            options
        );
    };

    this.getScreenshotPath = function(suite, stateName, browserId) {
        return this.config.getScreenshotPath(suite, stateName, browserId);
    };

    this.buildDiff = util.deprecate(function(referencePath, currentPath, diffPath) {
            return Image.buildDiff({
                reference: referencePath,
                current: currentPath,
                diff: diffPath,
                diffColor: this.config.diffColor,
                strictComparison: this.config.strictComparison,
                tolerance: this.config.tolerance
            });
        }, 'gemini.buildDiff' + chalk.red(' is deprecated.\n') +
        'Use ' + chalk.green('testResult.saveDiffTo(path)') +
        ' in ' + chalk.green('endTest') + ' event handler for more accurate diff'
    );

    this.getBrowserCapabilites = function(browserId) {
        return this.config.browsers[browserId];
    };

    Object.defineProperty(this, 'browserIds', {
        enumerable: true,
        get: function() {
            return Object.keys(this.config.browsers);
        }
    });
}

// Gemini needs to inherit from QEmitter rather then
// EventEmmiter because it needs to allow `startRunner`
// event handler delay the execution of tests
util.inherits(Gemini, QEmitter);

function applyReporter(runner, reporter) {
    if (typeof reporter === 'string') {
        try {
            reporter = require('./reporters/' + reporter);
        } catch (e) {
            if (e.code === 'MODULE_NOT_FOUND') {
                throw new Error('No such reporter: ' + reporter);
            }
            throw e;
        }
    }
    if (typeof reporter !== 'function') {
        throw new TypeError('Reporter must be a string or a function');
    }

    reporter(runner);
}

module.exports = Gemini;
<|endoftext|>


<commit_before>// Native
import path from 'path'

// Packages
import fs from 'fs-promise'
import fetch from 'node-fetch'
import download from 'download'
import tmp from 'tmp-promise'

// Ours
import {error} from './error'

const exists = async repoPath => {
  const apiURL = `https://api.github.com/repos/${repoPath}`

  let request
  let response

  try {
    request = await fetch(apiURL)
    response = await request.json()
  } catch (err) {
    error(`Not able to check if repo exists - ${err.message}`)
    return false
  }

  if (!response.name) {
    return false
  }

  return response
}

const downloadRepo = async repoPath => {
  const url = `https://api.github.com/repos/${repoPath}/tarball`

  const tmpDir = await tmp.dir({
    // We'll remove it manually once deployment is done
    keep: true,
    // Recursively remove directory when calling respective method
    unsafeCleanup: true
  })

  await download(url, tmpDir.path, {
    extract: true
  })

  const tmpContents = await fs.readdir(tmpDir.path)
  tmpDir.path = path.join(tmpDir.path, tmpContents[0])

  return tmpDir
}

export const isRepoPath = path => {
  if (!path) {
    return false
  }

  const slashCount = path.split('/').length - 1

  if (!slashCount || slashCount > 1) {
    return false
  }

  return true
}

export const onGitHub = async (path, debug) => {
  let repo = await exists(path)

  try {
    repo = await exists(path)
  } catch (err) {
    if (debug) {
      console.log(`Repository "${path}" does not exist on GitHub`)
    }

    return false
  }

  if (!repo) {
    return false
  }

  let tmpDir

  try {
    tmpDir = await downloadRepo(path)
  } catch (err) {
    error(`Not able to download repo: ${err.stack}`)
    process.exit(1)
  }

  return tmpDir
}
<commit_msg>Check if path is a repo path via regex<commit_after>// Native
import path from 'path'

// Packages
import fs from 'fs-promise'
import fetch from 'node-fetch'
import download from 'download'
import tmp from 'tmp-promise'

// Ours
import {error} from './error'

const exists = async repoPath => {
  const apiURL = `https://api.github.com/repos/${repoPath}`

  let request
  let response

  try {
    request = await fetch(apiURL)
    response = await request.json()
  } catch (err) {
    error(`Not able to check if repo exists - ${err.message}`)
    return false
  }

  if (!response.name) {
    return false
  }

  return response
}

const downloadRepo = async repoPath => {
  const url = `https://api.github.com/repos/${repoPath}/tarball`

  const tmpDir = await tmp.dir({
    // We'll remove it manually once deployment is done
    keep: true,
    // Recursively remove directory when calling respective method
    unsafeCleanup: true
  })

  await download(url, tmpDir.path, {
    extract: true
  })

  const tmpContents = await fs.readdir(tmpDir.path)
  tmpDir.path = path.join(tmpDir.path, tmpContents[0])

  return tmpDir
}

export const isRepoPath = path => {
  if (!path) {
    return false
  }

  return /[^\s\\]\/[^\s\\]/g.test(path)
}

export const onGitHub = async (path, debug) => {
  let repo = await exists(path)

  try {
    repo = await exists(path)
  } catch (err) {
    if (debug) {
      console.log(`Repository "${path}" does not exist on GitHub`)
    }

    return false
  }

  if (!repo) {
    return false
  }

  let tmpDir

  try {
    tmpDir = await downloadRepo(path)
  } catch (err) {
    error(`Not able to download repo: ${err.stack}`)
    process.exit(1)
  }

  return tmpDir
}
<|endoftext|>


<commit_before>const log = require('debug')('git3po:github')
const request = require('superagent-bluebird-promise')

const GITHUB_URL = 'https://api.github.com'

class GitHubApi {
  constructor(token, repo, dryRun) {
    this._token = token
    this._repo = repo
    this._dryRun = dryRun
  }

  get url() {
    return `${GITHUB_URL}/repos/${this._repo}`
  }

  _authenticatedRequest(url, method = 'get') {
    return request[method](url).set('Authorization', `token ${this._token}`)
  }

  async *_fetchUntilFinished(request) {
    let merged = []
    let pages = 0
    while (request) {
      pages++

      log('fetching page %d - %d results so far', pages, merged.length)
      const response = await request
      const body = response.body
      const next = response.links && response.links.next

      await (yield body)
      merged = merged.concat(body)
      request = next ? this._authenticatedRequest(next).promise() : null
    }

    log('fetched %d pages - %d total results', pages, merged.length)
    return merged
  }

  fetchIssuesSince(date, state = 'all') {
    const url = `${this.url}/issues`
    const since = date.toISOString()
    log(`fetching issues since=${since} state=${state}`)

    const query = this._authenticatedRequest(url)
      .query({since, state})
      .promise()
    return this._fetchUntilFinished(query)
  }

  async dryRunGuardedRequest(url, method, action) {
    if (this._dryRun) {
      log(`dry run, skipping ${method.toUpperCase()} request to ${url}`)
      return Promise.resolve()
    }

    let promiseLike = action(
      this._authenticatedRequest(`${this.url}${url}`, method)
    )
    if (typeof promiseLike.promise === 'function') {
      promiseLike = promiseLike.promise()
    }

    const result = await promiseLike
    log(`received successful response from ${url}`)
    return result
  }

  addComment(issue, comment) {
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}/comments`,
      'post',
      request => request.type('json').send({body: comment})
    )
  }

  addLabel(issue, label) {
    const labels = issue.labels.map(l => l.name).concat(label)
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}`,
      'patch',
      request => request.type('json').send({labels})
    )
  }

  closeIssue(issue) {
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}`,
      'patch',
      request => request.type('json').send({state: 'closed'})
    )
  }
}

module.exports = GitHubApi
<commit_msg>test: fix lint errors<commit_after>const log = require('debug')('git3po:github')
const request = require('superagent-bluebird-promise')

const GITHUB_URL = 'https://api.github.com'

class GitHubApi {
  constructor(token, repo, dryRun) {
    this._token = token
    this._repo = repo
    this._dryRun = dryRun
  }

  get url() {
    return `${GITHUB_URL}/repos/${this._repo}`
  }

  _authenticatedRequest(url, method = 'get') {
    return request[method](url).set('Authorization', `token ${this._token}`)
  }

  async * _fetchUntilFinished(request) {
    let merged = []
    let pages = 0
    while (request) {
      pages++

      log('fetching page %d - %d results so far', pages, merged.length)
      const response = await request // eslint-disable-line no-await-in-loop
      const body = response.body
      const next = response.links && response.links.next

      await (yield body) // eslint-disable-line no-await-in-loop
      merged = merged.concat(body)
      request = next ? this._authenticatedRequest(next).promise() : null
    }

    log('fetched %d pages - %d total results', pages, merged.length)
    return merged
  }

  fetchIssuesSince(date, state = 'all') {
    const url = `${this.url}/issues`
    const since = date.toISOString()
    log(`fetching issues since=${since} state=${state}`)

    const query = this._authenticatedRequest(url)
      .query({since, state})
      .promise()
    return this._fetchUntilFinished(query)
  }

  async dryRunGuardedRequest(url, method, action) {
    if (this._dryRun) {
      log(`dry run, skipping ${method.toUpperCase()} request to ${url}`)
      return Promise.resolve()
    }

    let promiseLike = action(
      this._authenticatedRequest(`${this.url}${url}`, method)
    )
    if (typeof promiseLike.promise === 'function') {
      promiseLike = promiseLike.promise()
    }

    const result = await promiseLike
    log(`received successful response from ${url}`)
    return result
  }

  addComment(issue, comment) {
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}/comments`,
      'post',
      request => request.type('json').send({body: comment})
    )
  }

  addLabel(issue, label) {
    const labels = issue.labels.map(l => l.name).concat(label)
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}`,
      'patch',
      request => request.type('json').send({labels})
    )
  }

  closeIssue(issue) {
    return this.dryRunGuardedRequest(
      `/issues/${issue.number}`,
      'patch',
      request => request.type('json').send({state: 'closed'})
    )
  }
}

module.exports = GitHubApi
<|endoftext|>


<commit_before>
/**
 * This seemingly useless file actually gets populated by NodObjC throughout
 * use. When symbols are imported from Objective-C, they get placed here.
 *
 * This allows the same object to be re-used for "module-mode" NodObjC as well
 * as "global-mode" NodObjC.
 */

module.exports = {}
<commit_msg>docs: better global docs<commit_after>
/**
 * NodObjC actually has 2 "modes" of operation:
 *
 * ### "module-mode"
 *
 * "module-mode" is the default mode that happens by simply requiring the
 * `NodObjC` module and working with the returned exports. The name of the
 * variable throughout the examples is `$`.
 *
 * ``` js
 * var $ = require('NodObjC');
 *
 * $.import('Foundation');
 *
 * var obj = $.NSObject('alloc')('init');
 * …
 * ```
 *
 * ### "global-mode"
 *
 * "global-mode" is where the Objective-C symbols also get added to the global
 * scope of the current program, so that the `$` all the time is not
 * necessary. Global mode is usually a more pleasant environment to work in,
 * but it should only be used by end programs, and not by modules that depend
 * on NodObjC. To enable global mode you simply require `NodObjC/global`, and
 * after that all NodObjC symbols will be available globally
 *
 * ``` js
 * require('NodObjC/global');
 *
 * importFramework('Foundation');
 *
 * var obj = NSObject('alloc')('init');
 * …
 * ```
 *
 */

module.exports = {}
<|endoftext|>


<commit_before>
/**
 * Module dependencies.
 */

var thunkify = require('thunkify-wrap');
var request = require('request');
var cheerio = require('cheerio');
var get = thunkify(request.get);

/**
 * Static variables.
 */

var google = 'https://www.google.com/search?q=';

/**
 * Get first page text results.
 *
 * @param {String} query
 *
 * @return {Array}
 */

module.exports.search = function *(query) {
  var res = yield get(google + encodeURI(query));
  var $ = cheerio.load(res[0].body);
  var results = [];
  // var url = /url\?q=(.*)?(?=&sa)/;
  $('li.g').each(function(i, el) {
    var link = 'https://www.google.com/' + $(el).find('h3 a').attr('href');
    // if (link.match(url)) link = link.match(url)[1];
    results.push({
      title: $(el).find('h3 a').text(),
      link: link,
      blurb: $(el).find('span.st').text()
    });
  });
  return results;
};
<commit_msg>removed a slash<commit_after>
/**
 * Module dependencies.
 */

var thunkify = require('thunkify-wrap');
var request = require('request');
var cheerio = require('cheerio');
var get = thunkify(request.get);

/**
 * Static variables.
 */

var google = 'https://www.google.com/search?q=';

/**
 * Get first page text results.
 *
 * @param {String} query
 *
 * @return {Array}
 */

module.exports.search = function *(query) {
  var res = yield get(google + encodeURI(query));
  var $ = cheerio.load(res[0].body);
  var results = [];
  // var url = /url\?q=(.*)?(?=&sa)/;
  $('li.g').each(function(i, el) {
    var link = 'https://www.google.com' + $(el).find('h3 a').attr('href');
    // if (link.match(url)) link = link.match(url)[1];
    results.push({
      title: $(el).find('h3 a').text(),
      link: link,
      blurb: $(el).find('span.st').text()
    });
  });
  return results;
};
<|endoftext|>


<commit_before>var util = require('util');
var Q = require('q');

module.exports.splitKey = function(key){
    return key.split('.');
};

/**
 * Traverses the given translationObject to find the given Key.
 * @param {string} key
 * @param {object} translationObject
 * @returns {promise} Resolves to an object with the following properties:
 *  - translation: The actual value that is associated to the overhanded key.
 *  - history: Array of properties that were traversed to find the key.
 *  - propName Short name of the property. For Example: If key is named 'TEST.KEY', the short name
 *  would be KEY.
 *  Resolves to null if the given key could not be found.
 */
module.exports.search = function(key, translationObject){
    var splitted = module.exports.splitKey(key);
    var obj = translationObject;
    var length = splitted.length;
    var prop;
    var history = [translationObject];

    for(var i = 0; i < length -1; i++){
        prop = obj[splitted[i]];

        if(util.isNullOrUndefined(prop)){
            return Q();
        } else if(util.isString(prop)){
            return Q();
        } else if(util.isObject(prop)){
            /* ignore */
        } else {
            return Q.reject(new TypeError('unexpected type'));
        }

        obj = prop;
        history.push(prop);
    }

    prop = obj[splitted[length-1]];

    if(util.isNullOrUndefined(prop)){
        return Q(null);
    } else if(util.isString(prop)){
        return Q({translation: obj[splitted[length-1]], history: history, propName: splitted[length-1]});
    } else if(util.isObject(prop)){
        return Q(null);
    } else {
        return Q.reject(new TypeError('unexpected type'));
    }
};<commit_msg>Removed isNullOrUndefined function usage<commit_after>var util = require('util');
var Q = require('q');

module.exports.splitKey = function(key){
    return key.split('.');
};

/**
 * Traverses the given translationObject to find the given Key.
 * @param {string} key
 * @param {object} translationObject
 * @returns {promise} Resolves to an object with the following properties:
 *  - translation: The actual value that is associated to the overhanded key.
 *  - history: Array of properties that were traversed to find the key.
 *  - propName Short name of the property. For Example: If key is named 'TEST.KEY', the short name
 *  would be KEY.
 *  Resolves to null if the given key could not be found.
 */
module.exports.search = function(key, translationObject){
    var splitted = module.exports.splitKey(key);
    var obj = translationObject;
    var length = splitted.length;
    var prop;
    var history = [translationObject];

    for(var i = 0; i < length -1; i++){
        prop = obj[splitted[i]];

        if(!prop){
            return Q();
        } else if(util.isString(prop)){
            return Q();
        } else if(util.isObject(prop)){
            /* ignore */
        } else {
            return Q.reject(new TypeError('unexpected type'));
        }

        obj = prop;
        history.push(prop);
    }

    prop = obj[splitted[length-1]];

    if(!prop){
        return Q(null);
    } else if(util.isString(prop)){
        return Q({translation: obj[splitted[length-1]], history: history, propName: splitted[length-1]});
    } else if(util.isObject(prop)){
        return Q(null);
    } else {
        return Q.reject(new TypeError('unexpected type'));
    }
};<|endoftext|>


<commit_before>/*
 * openi_data_api
 * openi-ict.eu
 *
 * Copyright (c) 2013 dmccarthy
 * Licensed under the MIT license.
 */

'use strict';

var openiLogger  = require('openi-logger')
var openiUtils   = require('openi-cloudlet-utils')

var init = function(logger_params){
   this.logger = openiLogger(logger_params);
}

var getType = function (path) {

   var parts   = path.split('/')
   var namePos = 4;

   return parts[namePos]

}

var genPostMessage = function(msg){

   var typeId = 't_' + openiUtils.hash(msg.json)

   var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + typeId;

   return {
      'dao_actions'      : [
         { 'action'       : 'POST',
            'database'    : typeId,
            'object_name' : 'meta',
            'object_data' : msg.json,
            'rest_uuid'   : typeRestUrl }
      ],
      'mongrel_resp' : {'value':true, data : {type_id : typeId} },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var genGetMessage = function(msg){

   var type =  getType(msg.path).replace('/api/v1/types/','')

   var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + type;

   return {
      'dao_actions'      : [
         { 'action' : 'GET', 'database': type, 'object_name'  : {}, 'rest_uuid'   : typeRestUrl }
      ],
      'mongrel_resp' : { 'value':true },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var genPatchMessage = function(msg){

   var actions = []

   for ( var i =0; i < msg.json.length; i++){
      var entry       = msg.json[i]
      var typeId      = 't_' + openiUtils.hash(entry)
      var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + typeId;

      actions.push({ 'action' : 'POST',
         'database'    : typeId,
         'object_name' : 'meta',
         'object_data' : entry,
         'rest_uuid'   : typeRestUrl });
   }

   return {
      'dao_actions'  : actions,
      'mongrel_resp' : {'value':true },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var processMongrel2Message = function (msg) {

   this.logger.logMongrel2Message(msg)

   var dao_msg = null;

   switch(msg.headers['METHOD']){
   case 'POST':
      dao_msg = genPostMessage(msg)
      break;
   case 'GET':
      dao_msg = genGetMessage(msg)
      break;
   case 'PATCH':
      dao_msg = genPatchMessage(msg)
      break;
   default:
      break;
   }

   this.logger.log('debug', dao_msg)

   return dao_msg
}

module.exports.init                   = init
module.exports.getObject              = getType
module.exports.processMongrel2Message = processMongrel2Message<commit_msg>Porting master to Couchbase<commit_after>/*
 * openi_data_api
 * openi-ict.eu
 *
 * Copyright (c) 2013 dmccarthy
 * Licensed under the MIT license.
 */

'use strict';

var openiLogger  = require('openi-logger')
var openiUtils   = require('openi-cloudlet-utils')

var init = function(logger_params){
   this.logger = openiLogger(logger_params);
}

var getType = function (path) {

   var parts   = path.split('/')
   var namePos = 4;

   return parts[namePos]

}

var validateType = function(type){
   //validate that it:
   // has a valid _reference
   // has a _context
   // Check that type is valid
   // --- needs to be of type {integer, long, float, double, string, boolean, byte, date, OPENi}
   // pull down reference and check that context is apt
}

var genPostMessage = function(msg){

   var typeId = 't_' + openiUtils.hash(msg.json)

   var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + typeId;

   msg.json._id = typeRestUrl;

   //validateType(msg.json)

   return {
      'dao_actions'      : [
         { 'action'       : 'POST',
            'database'    : typeId,
            'object_name' : 'meta',
            'object_data' : msg.json,
            'rest_uuid'   : typeRestUrl }
      ],
      'mongrel_resp' : {'value':true, data : {type_id : typeId} },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var genGetMessage = function(msg){

   var type =  getType(msg.path).replace('/api/v1/types/','')

   var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + type;

   return {
      'dao_actions'      : [
         { 'action' : 'GET', 'database': type, 'object_name'  : {}, 'rest_uuid'   : typeRestUrl }
      ],
      'mongrel_resp' : { 'value':true },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var genPatchMessage = function(msg){

   var actions = []

   for ( var i =0; i < msg.json.length; i++){
      var entry       = msg.json[i]
      var typeId      = 't_' + openiUtils.hash(entry)
      var typeRestUrl = "http://" + msg.headers.host + "/api/v1/types/" + typeId;

      actions.push({ 'action' : 'POST',
         'database'    : typeId,
         'object_name' : 'meta',
         'object_data' : entry,
         'rest_uuid'   : typeRestUrl });
   }

   return {
      'dao_actions'  : actions,
      'mongrel_resp' : {'value':true },
      'clients'      : [
         {'uuid' : msg.uuid, 'connId' : msg.connId }
      ]
   }
}

var processMongrel2Message = function (msg) {

   this.logger.logMongrel2Message(msg)

   var dao_msg = null;

   switch(msg.headers['METHOD']){
   case 'POST':
      dao_msg = genPostMessage(msg)
      break;
   case 'GET':
      dao_msg = genGetMessage(msg)
      break;
   case 'PATCH':
      dao_msg = genPatchMessage(msg)
      break;
   default:
      break;
   }

   this.logger.log('debug', dao_msg)

   return dao_msg
}

module.exports.init                   = init
module.exports.getObject              = getType
module.exports.processMongrel2Message = processMongrel2Message<|endoftext|>


<commit_before>var async = require( 'async' )
var msgpack5 = require( 'msgpack5' )
var sublevel = require( 'level-sublevel' )

/**
 * HexaDB Constructor
 * @return {HexaDB}
 */
function HexaDB( db, options ) {

  if( !(this instanceof HexaDB) )
    return new HexaDB( db, options )

  this.format = msgpack5()
  this.encoding = {
    encode: function( value ) {
      return this.format.encode( value )
        .slice() // bl != instanceof Buffer
    }.bind( this ),
    decode: this.format.decode,
    buffer: true,
    type: 'msgpack',
  }

  this.db = db
  this.level = sublevel( this.db, {
    keyEncoding: 'utf8',
    valueEncoding: this.encoding,
  })

  // Maps (s,p,o) hashes to each other
  this.graph = this.level.sublevel( 'graph' )
  // Maps values to hashes
  this.data = this.level.sublevel( 'data' )

}

/**
 * Query Variable Constructor
 * @type {Function}
 */
HexaDB.Variable = require( './variable' )

/**
 * Triple Constructor
 * @type {Function}
 */
HexaDB.Triple = require( './triple' )

/**
 * TripleStream that converts decoded values
 * to instances of HexaDB.Triple
 * @type {Function}
 */
HexaDB.TripleStream = require( './triple-stream' )

/**
 * Query Constructor
 * @type {Function}
 */
HexaDB.Query = require( './query' )

/**
 * Solution Constructor
 * @type {Function}
 */
HexaDB.Solution = require( './solution' )

/**
 * HexaDB Prototype
 * @type {Object}
 */
HexaDB.prototype = {

  constructor: HexaDB,

  /**
   * Register a type with the msgpack encoder
   * @param  {Number}   type
   * @param  {Function} ctor
   * @param  {Function} encode( object )
   * @param  {Function} decode( buffer )
   * @return {HexaDB}
   */
  registerType: function( type, ctor, encode, decode ) {
    this.format.register( type, ctor, encode, decode )
    return this
  },

  approximateSize: function( start, end, callback ) {

    var self = this
    var done = callback.bind( this )

    // TODO: Generate triple indices, and key range
    // to use as `start` & `end` to determine the approximate
    // size in bytes of the given triple range

    setImmediate( function() {
      done( new Error( 'Not implemented' ) )
    })

    return this

    // if( this.db.db && this.db.db.approximateSize ) {
    //   this.db.db.approximateSize( start, end )
    // }

  },

  /**
   * Retrieves triples matching the given triple pattern
   * @param  {Triple}   pattern
   * @param  {Object}   options
   * @param  {Function} callback
   * @return {HexaDB}
   */
  get: function( pattern, options, callback ) {

    if( typeof options === 'function' ) {
      callback = options
      options = null
    }

    var done = callback.bind( this )
    var values = []

    var tripleStream = this.getStream( pattern, options )

    tripleStream.on( 'error', done )
    tripleStream.on( 'readable', function() {
      var value, data = null
      while( value = this.read() ) {
        values.push( value )
      }
    })

    tripleStream.on( 'end', function() {
      done( null, values )
    })

    return this

  },

  /**
   * Stores a given triple in the database
   * @param  {Triple}   triple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  put: function( triple, callback ) {

    var self = this
    var done = callback.bind( this )
    var triple = HexaDB.Triple.create( triple )

    if( triple.isSparse ) {
      return done( new Error( 'Triple must not be sparse' ) )
    }

    // Generate all six permutation indices
    var tripleKeys = triple.generateKeys()
    // The SPO index = identity index,
    // used for triple data retrieval in .get()
    // NOTE: By using a single identity for all six
    // indices, we allow for good compression of values
    var identity = tripleKeys[0]

    this.data.get( identity, function( error, value ) {

      if( error != null && !error.notFound )
        return done( error )

      async.series([
        function( next ) {
          if( error == null || !error.notFound )
            return next()
          self.data.put( identity, triple.toArray(), next )
        },
        function( next ) {
          self.graph.batch( tripleKeys.map( function( key ) {
            return { type: 'put', key: key, value: identity }
          }), next )
        }
      ], done )

    })

    return this

  },

  /**
   * Deletes a given triple from the database
   * @param  {Triple}   triple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  delete: function( triple, callback ) {

    var self = this
    var done = callback.bind( this )
    var triple = HexaDB.Triple.create( triple )

    if( triple.isSparse ) {
      return done( new Error( 'Triple must not be sparse' ) )
    }

    var tripleKeys = triple.generateKeys()
    var identity = tripleKeys[0]
    var batch = tripleKeys.map( function( key ) {
      return { type: 'del', key: key }
    })

    async.series([
      this.data.del.bind( this.data, identity ),
      this.graph.batch.bind( this.graph, batch )
    ], done )

    return this

  },

  /**
   * Replaces a given triple with another triple
   * @param  {Triple}   oldTriple
   * @param  {Triple}   newTriple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  update: function( oldTriple, newTriple, callback ) {

    var self = this
    var done = callback.bind( this )

    this.delete( oldTriple, function( error ) {
      if( error != null )
        return done( error )
      self.put( newTriple, done )
    })

    return this

  },

  /**
   * Searches witha given query
   * @param  {Array}    query
   * @param  {Object}   options
   * @param  {Function} callback
   * @return {HexaDB}
   */
  search: function( query, options, callback ) {

    if( typeof options === 'function' ) {
      callback = options
      options = null
    }

    var self = this
    var done = callback.bind( this )
    var result = []

    this.searchStream( query, options )
      .on( 'error', done )
      .on( 'readable', function() {
        var value = null
        while( value = this.read() ) {
          result.push( value )
        }
      })
      .on( 'end', function() {
        done( null, result )
      })

    return this

  },

  navigate: function( start ) {
    // return new Navigator( this, start )
  },

  getStream: function( pattern, options ) {

    options = options != null ?
      options : {}

    var query = !( pattern instanceof HexaDB.Query ) ?
      new HexaDB.Query( pattern, options ) :
      pattern

    var tripleStream = new HexaDB.TripleStream( this )
    var stream = this.graph.createValueStream( query )

    tripleStream.query = query

    return stream.pipe( tripleStream )

  },

  putStream: function( options ) {
    var done = callback.bind( this )
    done( new Error( 'Not implemented' ) )
  },

  deleteStream: function( options ) {
    var done = callback.bind( this )
    done( new Error( 'Not implemented' ) )
  },

  /**
   * Creates a search stream
   * @param  {Array}  query
   * @param  {Object} options
   * @return {Stream}
   */
  searchStream: function( query, options ) {

    var self = this
    var solution = new HexaDB.Solution()

    // Map triple patterns (query elements) to queries,
    // and immediately bind variables to solution
    var queries = query.map( function( pattern ) {
      return new HexaDB.Query( pattern, options )
    })

    // Order queries by approximate size (smallest first)
    // TODO: Maybe utilize db.approximateSize() (slower?)
    queries.sort( function( a, b ) {
      var x = Object.keys( a.variables ).length
      var y = Object.keys( b.variables ).length
      if( x < y ) return -1
      if( x > y ) return  1
      return 0
    })

    queries.forEach( function( query ) {
      query.bindTo( solution )
    })

    var iteration = 0

    // Run each query in series, piping each one into the solution
    async.eachSeries( queries, function iterator( query, next ) {

      var stream = self.getStream( query, options )

      // Automatically end() the solution with the
      // end of the last query
      stream.pipe( solution, {
        end: iteration++ === queries.length - 1
      })

      stream.on( 'error', next )
      stream.on( 'end', function() {
        this.unpipe( solution )
        next()
      })

    })

    return solution

  },

}

// Exports
module.exports = HexaDB

<commit_msg>Updated lib/hexadb; Fixed typo in .search() doc<commit_after>var async = require( 'async' )
var msgpack5 = require( 'msgpack5' )
var sublevel = require( 'level-sublevel' )

/**
 * HexaDB Constructor
 * @return {HexaDB}
 */
function HexaDB( db, options ) {

  if( !(this instanceof HexaDB) )
    return new HexaDB( db, options )

  this.format = msgpack5()
  this.encoding = {
    encode: function( value ) {
      return this.format.encode( value )
        .slice() // bl != instanceof Buffer
    }.bind( this ),
    decode: this.format.decode,
    buffer: true,
    type: 'msgpack',
  }

  this.db = db
  this.level = sublevel( this.db, {
    keyEncoding: 'utf8',
    valueEncoding: this.encoding,
  })

  // Maps (s,p,o) hashes to each other
  this.graph = this.level.sublevel( 'graph' )
  // Maps values to hashes
  this.data = this.level.sublevel( 'data' )

}

/**
 * Query Variable Constructor
 * @type {Function}
 */
HexaDB.Variable = require( './variable' )

/**
 * Triple Constructor
 * @type {Function}
 */
HexaDB.Triple = require( './triple' )

/**
 * TripleStream that converts decoded values
 * to instances of HexaDB.Triple
 * @type {Function}
 */
HexaDB.TripleStream = require( './triple-stream' )

/**
 * Query Constructor
 * @type {Function}
 */
HexaDB.Query = require( './query' )

/**
 * Solution Constructor
 * @type {Function}
 */
HexaDB.Solution = require( './solution' )

/**
 * HexaDB Prototype
 * @type {Object}
 */
HexaDB.prototype = {

  constructor: HexaDB,

  /**
   * Register a type with the msgpack encoder
   * @param  {Number}   type
   * @param  {Function} ctor
   * @param  {Function} encode( object )
   * @param  {Function} decode( buffer )
   * @return {HexaDB}
   */
  registerType: function( type, ctor, encode, decode ) {
    this.format.register( type, ctor, encode, decode )
    return this
  },

  approximateSize: function( start, end, callback ) {

    var self = this
    var done = callback.bind( this )

    // TODO: Generate triple indices, and key range
    // to use as `start` & `end` to determine the approximate
    // size in bytes of the given triple range

    setImmediate( function() {
      done( new Error( 'Not implemented' ) )
    })

    return this

    // if( this.db.db && this.db.db.approximateSize ) {
    //   this.db.db.approximateSize( start, end )
    // }

  },

  /**
   * Retrieves triples matching the given triple pattern
   * @param  {Triple}   pattern
   * @param  {Object}   options
   * @param  {Function} callback
   * @return {HexaDB}
   */
  get: function( pattern, options, callback ) {

    if( typeof options === 'function' ) {
      callback = options
      options = null
    }

    var done = callback.bind( this )
    var values = []

    var tripleStream = this.getStream( pattern, options )

    tripleStream.on( 'error', done )
    tripleStream.on( 'readable', function() {
      var value, data = null
      while( value = this.read() ) {
        values.push( value )
      }
    })

    tripleStream.on( 'end', function() {
      done( null, values )
    })

    return this

  },

  /**
   * Stores a given triple in the database
   * @param  {Triple}   triple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  put: function( triple, callback ) {

    var self = this
    var done = callback.bind( this )
    var triple = HexaDB.Triple.create( triple )

    if( triple.isSparse ) {
      return done( new Error( 'Triple must not be sparse' ) )
    }

    // Generate all six permutation indices
    var tripleKeys = triple.generateKeys()
    // The SPO index = identity index,
    // used for triple data retrieval in .get()
    // NOTE: By using a single identity for all six
    // indices, we allow for good compression of values
    var identity = tripleKeys[0]

    this.data.get( identity, function( error, value ) {

      if( error != null && !error.notFound )
        return done( error )

      async.series([
        function( next ) {
          if( error == null || !error.notFound )
            return next()
          self.data.put( identity, triple.toArray(), next )
        },
        function( next ) {
          self.graph.batch( tripleKeys.map( function( key ) {
            return { type: 'put', key: key, value: identity }
          }), next )
        }
      ], done )

    })

    return this

  },

  /**
   * Deletes a given triple from the database
   * @param  {Triple}   triple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  delete: function( triple, callback ) {

    var self = this
    var done = callback.bind( this )
    var triple = HexaDB.Triple.create( triple )

    if( triple.isSparse ) {
      return done( new Error( 'Triple must not be sparse' ) )
    }

    var tripleKeys = triple.generateKeys()
    var identity = tripleKeys[0]
    var batch = tripleKeys.map( function( key ) {
      return { type: 'del', key: key }
    })

    async.series([
      this.data.del.bind( this.data, identity ),
      this.graph.batch.bind( this.graph, batch )
    ], done )

    return this

  },

  /**
   * Replaces a given triple with another triple
   * @param  {Triple}   oldTriple
   * @param  {Triple}   newTriple
   * @param  {Function} callback
   * @return {HexaDB}
   */
  update: function( oldTriple, newTriple, callback ) {

    var self = this
    var done = callback.bind( this )

    this.delete( oldTriple, function( error ) {
      if( error != null )
        return done( error )
      self.put( newTriple, done )
    })

    return this

  },

  /**
   * Searches with a given query
   * @param  {Array}    query
   * @param  {Object}   options
   * @param  {Function} callback
   * @return {HexaDB}
   */
  search: function( query, options, callback ) {

    if( typeof options === 'function' ) {
      callback = options
      options = null
    }

    var self = this
    var done = callback.bind( this )
    var result = []

    this.searchStream( query, options )
      .on( 'error', done )
      .on( 'readable', function() {
        var value = null
        while( value = this.read() ) {
          result.push( value )
        }
      })
      .on( 'end', function() {
        done( null, result )
      })

    return this

  },

  navigate: function( start ) {
    // return new Navigator( this, start )
  },

  getStream: function( pattern, options ) {

    options = options != null ?
      options : {}

    var query = !( pattern instanceof HexaDB.Query ) ?
      new HexaDB.Query( pattern, options ) :
      pattern

    var tripleStream = new HexaDB.TripleStream( this )
    var stream = this.graph.createValueStream( query )

    tripleStream.query = query

    return stream.pipe( tripleStream )

  },

  putStream: function( options ) {
    var done = callback.bind( this )
    done( new Error( 'Not implemented' ) )
  },

  deleteStream: function( options ) {
    var done = callback.bind( this )
    done( new Error( 'Not implemented' ) )
  },

  /**
   * Creates a search stream
   * @param  {Array}  query
   * @param  {Object} options
   * @return {Stream}
   */
  searchStream: function( query, options ) {

    var self = this
    var solution = new HexaDB.Solution()

    // Map triple patterns (query elements) to queries,
    // and immediately bind variables to solution
    var queries = query.map( function( pattern ) {
      return new HexaDB.Query( pattern, options )
    })

    // Order queries by approximate size (smallest first)
    // TODO: Maybe utilize db.approximateSize() (slower?)
    queries.sort( function( a, b ) {
      var x = Object.keys( a.variables ).length
      var y = Object.keys( b.variables ).length
      if( x < y ) return -1
      if( x > y ) return  1
      return 0
    })

    queries.forEach( function( query ) {
      query.bindTo( solution )
    })

    var iteration = 0

    // Run each query in series, piping each one into the solution
    async.eachSeries( queries, function iterator( query, next ) {

      var stream = self.getStream( query, options )

      // Automatically end() the solution with the
      // end of the last query
      stream.pipe( solution, {
        end: iteration++ === queries.length - 1
      })

      stream.on( 'error', next )
      stream.on( 'end', function() {
        this.unpipe( solution )
        next()
      })

    })

    return solution

  },

}

// Exports
module.exports = HexaDB

<|endoftext|>


<commit_before>

module.exports = function(dolphin){
  var opts = dolphin.opts;

  /**
   *
   * filters – a JSON encoded value of the filters (a map[string][]string) to process on the images list. Available filters:
      dangling=true
      label=key or label="key=value" of an image label
      before=(<image-name>[:<tag>], <image id> or <image@digest>)
      since=(<image-name>[:<tag>], <image id> or <image@digest>)
   **/
  var images = function(imageName){
    //return dolphin._list('images', filters, opts);
    return dolphin._get('images/' + imageName + '/json', null, opts);
  }

/**
 *
 *  We use docker command line since building images is quite a complex
 *  operation.
 *  */
  images.build = function (url, name, versions, params) {
    var args = [].concat.apply([], versions.map(function(tag){
      return ['-t', name + ':' + tag];
    }));

    if(params.Labels){
      Object.keys(params.Labels).forEach(function(label){
        args = args.concat('--label', label + '=' + params.Labels[label]);
      });
    }

    args.unshift('build');
    args.push(url);

    return dolphin.docker(args).then(function(imageId){
      return {
        Id: imageId
      }
    })
  }

  images.push = function(image, tag){
    var url = 'images/'+image+'/push';
    if(tag){
      url += '?tag=' + tag;
    }
    return dolphin._post(url, null, opts);
  }

  return images;
}
<commit_msg>corrected image build to return imageid<commit_after>

module.exports = function(dolphin){
  var opts = dolphin.opts;

  /**
   *
   * filters – a JSON encoded value of the filters (a map[string][]string) to process on the images list. Available filters:
      dangling=true
      label=key or label="key=value" of an image label
      before=(<image-name>[:<tag>], <image id> or <image@digest>)
      since=(<image-name>[:<tag>], <image id> or <image@digest>)
   **/
  var images = function(imageName){
    //return dolphin._list('images', filters, opts);
    return dolphin._get('images/' + imageName + '/json', null, opts);
  }

/**
 *
 *  We use docker command line since building images is quite a complex
 *  operation.
 *  */
  images.build = function (url, name, versions, params) {
    var args = [].concat.apply([], versions.map(function(tag){
      return ['-t', name + ':' + tag];
    }));

    if(params.Labels){
      Object.keys(params.Labels).forEach(function(label){
        args = args.concat('--label', label + '=' + params.Labels[label]);
      });
    }

    args.unshift('build');
    args.push('-q')
    args.push(url);

    return dolphin.docker(args).then(function(imageId){
      return {
        Id: imageId.split(':')[1]
      }
    })
  }

  images.push = function(image, tag){
    var url = 'images/'+image+'/push';
    if(tag){
      url += '?tag=' + tag;
    }
    return dolphin._post(url, null, opts);
  }

  return images;
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
 *
 * Client library for the SDC Image API (IMGAPI)
 */

var util = require('util'),
    format = util.format;
var qs = require('querystring');
var fs = require('fs');
var crypto = require('crypto');

var WError = require('verror').WError;
var assert = require('assert-plus');
var restify = require('restify');

// ---- client errors

function ChecksumError(cause, actual, expected) {
    if (expected === undefined) {
        actual = cause;
        expected = actual;
        cause = undefined;
    }
    assert.optionalObject(cause);
    assert.string(actual);
    assert.string(expected);

    var args = [];
    if (cause) args.push(cause);
    args = args.concat('content-md5 expected to be %s, but was %s',
        expected, actual);
    WError.apply(this, args);
}
util.inherits(ChecksumError, WError);

// ---- client API

/**
 * Create an IMGAPI client.
 *
 * @param options {Object}
 *      - `url` {String} IMGAPI url
 */
function IMGAPI(options) {
    assert.object(options, 'options');
    assert.string(options.url, 'options.url');
    this.client = restify.createJsonClient(options);
    this.rawClient = restify.createClient(options);
}

/**
 * Ping. <https://mo.joyent.com/docs/imgapi/master/#Ping>
 *
 * TODO: support 'error' query param
 * @param error {String} Optional error code. If given, the ping is expected
 *      to respond with a sample error with that code (if supported).
 * @param callback {Function} `function (err, pong, res)`
 */
IMGAPI.prototype.ping = function ping(error, callback) {
    if (typeof (error) === 'function') {
        callback = error;
        error = undefined;
    }
    assert.optionalString(error, 'error');
    assert.func(callback, 'callback');

    var path = '/ping';
    if (error) {
        path += '?' + qs.stringify({error: error});
    }
    this.client.get(path, function (err, req, res, pong) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, pong, res);
        }
    });
};

/**
 * Lists all Images
 *
 * @param filters {Object} Optional filter params. NYI
 *      XXX Previous filters allowed? 'name', 'version', 'type', 'os',
 *          'restricted_to_uuid' & 'creator_uuid' params.
 * @param callback {Function} `function (err, images, res)`
 */
IMGAPI.prototype.listImages = function listImages(filters, callback) {
    if (typeof (filters) === 'function') {
        callback = filters;
        filters = {};
    }
    assert.func(callback, 'callback');
    assert.object(filters);

    var path = '/images';
    var query = qs.stringify(filters);
    if (query.length > 0) {
        path += '?' + query;
    }
    this.client.get(path, function (err, req, res, images) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, images, res);
        }
    });
};

/**
 * Gets an image by UUID.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {UUID} user : Optional. The UUID of the user who is querying. If
 *      given this will only return images accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.getImage = function getImage(uuid, user, callback) {
    assert.string(uuid, 'uuid');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = format('/images/%s', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    this.client.get(path, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

/**
 * Create an image.
 *
 * @param {String} data : the image data.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.createImage = function createImage(data, user, callback) {
    assert.object(data, 'data');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = '/images';
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    this.client.post(path, data, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

/**
 * Add an image file.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {String} filePath : Path to the image file.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.addImageFile = function addImageFile(uuid, filePath, user,
                                                      callback) {
    assert.string(uuid, 'uuid');
    assert.string(filePath, 'filePath');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var self = this;
    var path = format('/images/%s/file', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    fs.stat(filePath, function (statErr, stats) {
        if (statErr) {
            callback(statErr);
            return;
        }

        var opts = {
            path: path,
            headers: {
                'Content-Type': 'application/octet-stream',
                'Content-Length': stats.size,
                'Accept': 'application/json'
            }
        };
        self.rawClient.put(opts, function (connectErr, req) {
            if (connectErr) {
                callback(connectErr);
                return;
            }

            fs.createReadStream(filePath).pipe(req);

            req.on('result', function (resultErr, res) {
                if (resultErr) {
                    callback(resultErr, null, res);
                    return;
                }

                var chunks = [];
                res.setEncoding('utf8');
                res.on('data', function (chunk) {
                    chunks.push(chunk);
                });
                res.on('end', function () {
                    var body = chunks.join('');
                    var data;
                    try {
                        data = JSON.parse(body);
                    } catch (syntaxErr) {
                        callback(new WError(syntaxErr,
                            'invalid image data in response: \'%s\'', body));
                        return;
                    }
                    callback(null, data, res);
                });
            });
        });
    });
};

/**
 * Get an image file.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {String} filePath : Path to which to save the image file.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, res)`
 *
 * TODO: May want a version of this that returns a stream, `getImageFileStream`.
 */
IMGAPI.prototype.getImageFile = function getImageFile(uuid, filePath, user,
                                                      callback) {
    assert.string(uuid, 'uuid');
    assert.string(filePath, 'filePath');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var self = this;
    var path = format('/images/%s/file', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    // TODO: checksumming a la node-manta/lib/client.js#get
    self.rawClient.get(path, function (connectErr, req) {
        if (connectErr) {
            callback(connectErr);
            return;
        }
        req.on('result', function (resultErr, res) {
            if (resultErr) {
                callback(resultErr, res);
                return;
            }

            res.pause();
            res.pipe(fs.createWriteStream(filePath));

            var hash = crypto.createHash('md5');
            res.on('data', function (chunk) { hash.update(chunk); });

            var finished = false;
            function finish(err) {
                if (!finished) {
                    if (!err) {
                        var md5_expected = res.headers['content-md5'];
                        var md5_actual = hash.digest('base64');
                        if (md5_actual !== md5_expected) {
                            err = new ChecksumError(md5_actual, md5_expected);
                        }
                    }
                    callback(err, res);
                    finished = true;
                }
            }
            res.on('error', finish);
            res.on('end', finish);

            process.nextTick(function () {
                res.resume();
            });
        });
    });
};

/**
 * Activate an image.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.activateImage = function activateImage(uuid, user, callback) {
    assert.string(uuid, 'uuid');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = format('/images/%s', uuid);
    var query = {action: 'activate'};
    if (user) {
        query.user = user;
    }
    path += '?' + qs.stringify(query);
    this.client.post(path, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

module.exports = IMGAPI;
<commit_msg>IMGAPI-25: imgapi sdc-client<commit_after>/*
 * Copyright (c) 2012, Joyent, Inc. All rights reserved.
 *
 * Client library for the SDC Image API (IMGAPI)
 */

var util = require('util'),
    format = util.format;
var qs = require('querystring');
var fs = require('fs');
var crypto = require('crypto');

var WError = require('verror').WError;
var assert = require('assert-plus');
var restify = require('restify');

// ---- client errors

function ChecksumError(cause, actual, expected) {
    if (expected === undefined) {
        actual = cause;
        expected = actual;
        cause = undefined;
    }
    assert.optionalObject(cause);
    assert.string(actual);
    assert.string(expected);

    var args = [];
    if (cause) args.push(cause);
    args = args.concat('content-md5 expected to be %s, but was %s',
        expected, actual);
    WError.apply(this, args);
}
util.inherits(ChecksumError, WError);

// ---- client API

/**
 * Create an IMGAPI client.
 *
 * @param options {Object}
 *      - `url` {String} IMGAPI url
 */
function IMGAPI(options) {
    assert.object(options, 'options');
    assert.string(options.url, 'options.url');
    this.client = restify.createJsonClient(options);
    this.rawClient = restify.createClient(options);
}

/**
 * Ping. <https://mo.joyent.com/docs/imgapi/master/#Ping>
 *
 * TODO: support 'error' query param
 * @param error {String} Optional error code. If given, the ping is expected
 *      to respond with a sample error with that code (if supported).
 * @param callback {Function} `function (err, pong, res)`
 */
IMGAPI.prototype.ping = function ping(error, callback) {
    if (typeof (error) === 'function') {
        callback = error;
        error = undefined;
    }
    assert.optionalString(error, 'error');
    assert.func(callback, 'callback');

    var path = '/ping';
    if (error) {
        path += '?' + qs.stringify({error: error});
    }
    this.client.get(path, function (err, req, res, pong) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, pong, res);
        }
    });
};

/**
 * Lists all Images
 *
 * @param filters {Object} Optional filter params. NYI
 *      XXX Previous filters allowed? 'name', 'version', 'type', 'os',
 *          'restricted_to_uuid' & 'creator_uuid' params.
 * @param callback {Function} `function (err, images, res)`
 */
IMGAPI.prototype.listImages = function listImages(filters, callback) {
    if (typeof (filters) === 'function') {
        callback = filters;
        filters = {};
    }
    assert.func(callback, 'callback');
    assert.object(filters);

    var path = '/images';
    var query = qs.stringify(filters);
    if (query.length > 0) {
        path += '?' + query;
    }
    this.client.get(path, function (err, req, res, images) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, images, res);
        }
    });
};

/**
 * Gets an image by UUID.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {UUID} user : Optional. The UUID of the user who is querying. If
 *      given this will only return images accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.getImage = function getImage(uuid, user, callback) {
    assert.string(uuid, 'uuid');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = format('/images/%s', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    this.client.get(path, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

/**
 * Create an image.
 *
 * @param {String} data : the image data.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.createImage = function createImage(data, user, callback) {
    assert.object(data, 'data');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = '/images';
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    this.client.post(path, data, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

/**
 * Add an image file.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {String} filePath : Path to the image file.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.addImageFile = function addImageFile(uuid, filePath, user,
                                                      callback) {
    assert.string(uuid, 'uuid');
    assert.string(filePath, 'filePath');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var self = this;
    var path = format('/images/%s/file', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    fs.stat(filePath, function (statErr, stats) {
        if (statErr) {
            callback(statErr);
            return;
        }

        var opts = {
            path: path,
            headers: {
                'Content-Type': 'application/octet-stream',
                'Content-Length': stats.size,
                'Accept': 'application/json'
            }
        };
        self.rawClient.put(opts, function (connectErr, req) {
            if (connectErr) {
                callback(connectErr);
                return;
            }

            fs.createReadStream(filePath).pipe(req);

            req.on('result', function (resultErr, res) {
                if (resultErr) {
                    callback(resultErr, null, res);
                    return;
                }

                var chunks = [];
                res.setEncoding('utf8');
                res.on('data', function (chunk) {
                    chunks.push(chunk);
                });
                res.on('end', function () {
                    var body = chunks.join('');
                    var data;
                    try {
                        data = JSON.parse(body);
                    } catch (syntaxErr) {
                        callback(new WError(syntaxErr,
                            'invalid image data in response: \'%s\'', body));
                        return;
                    }
                    callback(null, data, res);
                });
            });
        });
    });
};

/**
 * Get an image file.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {String} filePath : Path to which to save the image file.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, res)`
 *
 * TODO: May want a version of this that returns a stream, `getImageFileStream`.
 */
IMGAPI.prototype.getImageFile = function getImageFile(uuid, filePath, user,
                                                      callback) {
    assert.string(uuid, 'uuid');
    assert.string(filePath, 'filePath');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var self = this;
    var path = format('/images/%s/file', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    // TODO: checksumming a la node-manta/lib/client.js#get
    self.rawClient.get(path, function (connectErr, req) {
        if (connectErr) {
            callback(connectErr);
            return;
        }
        req.on('result', function (resultErr, res) {
            if (resultErr) {
                callback(resultErr, res);
                return;
            }

            res.pause();
            res.pipe(fs.createWriteStream(filePath));

            var hash = crypto.createHash('md5');
            res.on('data', function (chunk) { hash.update(chunk); });

            var finished = false;
            function finish(err) {
                if (!finished) {
                    if (!err) {
                        var md5_expected = res.headers['content-md5'];
                        var md5_actual = hash.digest('base64');
                        if (md5_actual !== md5_expected) {
                            err = new ChecksumError(md5_actual, md5_expected);
                        }
                    }
                    callback(err, res);
                    finished = true;
                }
            }
            res.on('error', finish);
            res.on('end', finish);

            process.nextTick(function () {
                res.resume();
            });
        });
    });
};

/**
 * Activate an image.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will only return images
 *      accessible to that user.
 * @param {Function} callback : `function (err, image, res)`
 */
IMGAPI.prototype.activateImage = function activateImage(uuid, user, callback) {
    assert.string(uuid, 'uuid');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = format('/images/%s', uuid);
    var query = {action: 'activate'};
    if (user) {
        query.user = user;
    }
    path += '?' + qs.stringify(query);
    this.client.post(path, function (err, req, res, image) {
        if (err) {
            callback(err, null, res);
        } else {
            callback(null, image, res);
        }
    });
};

/**
 * Create an image.
 *
 * @param {String} uuid : the UUID of the image.
 * @param {UUID} user : Optional. The UUID of the user on behalf of whom this
 *      request is being made. If given this will restrict to images
 *      accessible to that user.
 * @param {Function} callback : `function (err, res)`
 */
IMGAPI.prototype.deleteImage = function deleteImage(uuid, user, callback) {
    assert.string(uuid, 'uuid');
    if (callback === undefined) {
        callback = user;
        user = undefined;
    }
    assert.optionalString(user, 'user');
    assert.func(callback, 'callback');

    var path = format('/images/%s', uuid);
    if (user) {
        path += '?' + qs.stringify({user: user});
    }
    this.client.del(path, function (err, req, res) {
        if (err) {
            callback(err, res);
        } else {
            callback(null, res);
        }
    });
};

module.exports = IMGAPI;
<|endoftext|>


<commit_before>/* eslint-disable no-use-extend-native/no-use-extend-native */

const path = require('path');

const _ = require('lodash');
const extract = require('extract-zip');
const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const proj4 = require('proj4');
const stripBomStream = require('strip-bom-stream');
const tmp = require('tmp-promise');
const untildify = require('untildify');
const Promise = require('bluebird');

const models = require('../models/models');
const logUtils = require('./log-utils');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const response = await fetch(task.agency_url, {method: 'GET', headers: task.agency_headers || {}});

  if (response.status !== 200) {
    throw new Error('Couldn’t download files');
  }

  const buffer = await response.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await extract(gtfsPath, {dir: task.downloadDir});
    } catch (error) {
      task.error(error);
      throw new Error(`Unable to unzip file ${task.path}`);
    }

    const files = await fs.readdir(task.downloadDir);
    const textFiles = files.filter(file => file.slice(-3) === 'txt');

    if (textFiles.length === 0) {
      throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, not in a subdirectory.`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);
  }
};

const removeData = task => {
  // Remove old db records based on agency_key
  return Promise.all(models.map(model => {
    return model.model.collection.deleteMany({agency_key: task.agency_key});
  }));
};

const importLines = (lines, model, cb) => {
  const bulk = model.collection.initializeUnorderedBulkOp();
  const count = lines.length;

  if (!bulk) {
    return cb(new Error('Unable to initialize Mongo bulk insert.'));
  }

  while (lines.length) {
    bulk.insert(lines.pop());
  }

  bulk.execute(err => {
    cb(err, count);
  });
};

const formatLine = (line, task, lineNumber) => {
  // Remove null values
  for (const key in line) {
    if (line[key] === null) {
      delete line[key];
    }
  }

  // Add agency_key
  line.agency_key = task.agency_key;

  // Add created_at
  line.created_at = task.created_at;

  // Convert fields that should be int
  const integerFields = [
    'monday',
    'tuesday',
    'wednesday',
    'thursday',
    'friday',
    'saturday',
    'sunday',
    'start_date',
    'end_date',
    'date',
    'exception_type',
    'shape_pt_sequence',
    'payment_method',
    'transfers',
    'transfer_duration',
    'feed_start_date',
    'feed_end_date',
    'headway_secs',
    'exact_times',
    'route_type',
    'direction_id',
    'location_type',
    'wheelchair_boarding',
    'stop_sequence',
    'pickup_type',
    'drop_off_type',
    'use_stop_sequence',
    'transfer_type',
    'min_transfer_time',
    'wheelchair_accessible',
    'bikes_allowed',
    'timepoint',
    'timetable_sequence',
    'include_exceptions',
    'show_trip_continuation'
  ];

  integerFields.forEach(fieldName => {
    if (line[fieldName]) {
      line[fieldName] = Number.parseInt(line[fieldName], 10);
    } else {
      delete line[fieldName];
    }
  });

  // Convert fields that should be float
  const floatFields = [
    'price',
    'shape_dist_traveled',
    'shape_pt_lat',
    'shape_pt_lon',
    'stop_lat',
    'stop_lon'
  ];

  floatFields.forEach(fieldName => {
    if (line[fieldName]) {
      line[fieldName] = Number.parseFloat(line[fieldName]);
    } else {
      delete line[fieldName];
    }
  });

  // Check for valid coordinates
  const latitudeFields = [
    'shape_pt_lat',
    'stop_lat'
  ];

  for (const fieldName of latitudeFields) {
    if (line[fieldName] !== undefined) {
      if (line[fieldName] < -90 || line[fieldName] > 90) {
        throw new Error(`Invalid latitude in ${fieldName} on ${lineNumber}`);
      }
    }
  }

  const longitudeFields = [
    'shape_pt_lon',
    'stop_lon'
  ];

  for (const fieldName of longitudeFields) {
    if (line[fieldName] !== undefined) {
      if (line[fieldName] < -180 || line[fieldName] > 180) {
        throw new Error(`Invalid longitude in ${fieldName} on ${lineNumber}`);
      }
    }
  }

  // Convert to midnight timestamp
  const timestampFormat = [
    'start_time',
    'end_time',
    'arrival_time',
    'departure_time'
  ];

  for (const fieldName of timestampFormat) {
    if (line[fieldName]) {
      line[`${fieldName}stamp`] = utils.calculateHourTimestamp(line[fieldName]);
    }
  }

  // Make lat/lon array for stops
  if (line.stop_lat && line.stop_lon) {
    line.loc = [
      line.stop_lon,
      line.stop_lat
    ];

    // If coordinates are not specified, use [0,0]
    if (Number.isNaN(line.loc[0])) {
      line.loc[0] = 0;
    }

    if (Number.isNaN(line.loc[1])) {
      line.loc[1] = 0;
    }

    // Convert to epsg4326 if needed
    if (task.agency_proj) {
      line.loc = proj4(task.agency_proj, 'WGS84', line.loc);
      line.stop_lon = line.loc[0];
      line.stop_lat = line.loc[1];
    }
  }

  // Make lat/long for shapes
  if (line.shape_pt_lat && line.shape_pt_lon) {
    line.loc = [line.shape_pt_lon, line.shape_pt_lat];
  }

  return line;
};

const importFiles = task => {
  // Loop through each GTFS file
  return Promise.mapSeries(models, model => {
    return new Promise((resolve, reject) => {
      // Filter out excluded files from config
      if (task.exclude && _.includes(task.exclude, model.filenameBase)) {
        task.log(`Skipping - ${model.filenameBase}.txt\r`);
        return resolve();
      }

      const filepath = path.join(task.downloadDir, `${model.filenameBase}.txt`);

      if (!fs.existsSync(filepath)) {
        if (!model.nonstandard) {
          task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
        }

        return resolve();
      }

      task.log(`Importing - ${model.filenameBase}.txt\r`);

      const lines = [];
      const chunkSize = 10000;
      let lineCount = 0;
      const parser = parse({
        columns: true,
        relax: true,
        trim: true,
        skip_empty_lines: true
      });

      parser.on('readable', async () => {
        let record;
        /* eslint-disable-next-line no-cond-assign */
        while (record = parser.read()) {
          const line = formatLine(record, task, parser.info.lines);

          // Calculate agency bounds
          if (line.loc) {
            task.agency_bounds = utils.extendBounds(task.agency_bounds, line.loc);
          }

          lines.push(line);

          // If we have a bunch of lines ready to insert, then do it
          if (lines.length >= chunkSize) {
            importLines(lines, model.model, (err, count) => {
              if (err) {
                return reject(err);
              }

              lineCount += count;
              task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
            });
          }
        }
      });

      parser.on('end', () => {
        // Insert all remaining lines
        if (lines.length > 0) {
          importLines(lines, model.model, (err, count) => {
            if (err) {
              return reject(err);
            }

            lineCount += count;
            task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
            resolve();
          });
        } else {
          task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
          resolve();
        }
      });

      parser.on('error', reject);

      fs.createReadStream(filepath)
        .pipe(stripBomStream())
        .pipe(parser);
    })
      .catch(error => {
        throw error;
      });
  });
};

const postProcess = async task => {
  task.log('Post Processing data');

  const agencyModel = _.find(models, {filenameBase: 'agency'});
  const agencyCenter = utils.boundsCenter(task.agency_bounds);

  if (agencyCenter) {
    await agencyModel.model.collection.updateMany({
      agency_key: task.agency_key
    }, {
      $set: {
        agency_bounds: task.agency_bounds,
        agency_center: agencyCenter
      }
    });
  }
};

const ensureIndexes = config => {
  return Promise.all(models.map(model => {
    if (config.dataExpireAfterSeconds === undefined) {
      // Remove any TTL expiration index and ignore errors
      model.model.collection.dropIndex({created_at: 1}).catch(() => null);
    } else {
      // Set TTL expiration index if specified in configs
      model.model.schema.index({created_at: 1}, {expireAfterSeconds: config.dataExpireAfterSeconds});
    }

    return model.model.ensureIndexes();
  }));
};

module.exports = async config => {
  const log = logUtils.log(config);
  const logError = logUtils.logError(config);

  const agencyCount = config.agencies.length;
  log(`Starting GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}`);

  await Promise.mapSeries(config.agencies, async agency => {
    if (!agency.agency_key) {
      throw new Error('No Agency Key provided.');
    }

    if (!agency.url && !agency.path) {
      throw new Error('No Agency URL or path provided.');
    }

    const {path, cleanup} = await tmp.dir({unsafeCleanup: true});

    const task = {
      exclude: agency.exclude,
      agency_key: agency.agency_key,
      agency_url: agency.url,
      agency_headers: agency.headers || false,
      downloadDir: path,
      path: agency.path,
      skipDelete: config.skipDelete,
      log: (message, overwrite) => {
        log(`${task.agency_key}: ${message}`, overwrite);
      },
      error: message => {
        logError(message);
      },
      created_at: new Date()
    };

    if (task.agency_url) {
      await downloadFiles(task);
    }

    await readFiles(task);

    // Override using --skipDelete command line argument or `skipDelete` in config.json
    if (task.skipDelete === true) {
      task.log('Skipping deletion of existing data');
    } else {
      await removeData(task);
    }

    await importFiles(task);
    await postProcess(task);
    await ensureIndexes(config);

    cleanup();
    task.log('Completed GTFS import');
  });

  log(`Completed GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}\n`);
};
<commit_msg>Handle subdirectories in GTFS zip files<commit_after>/* eslint-disable no-use-extend-native/no-use-extend-native */

const path = require('path');

const _ = require('lodash');
const extract = require('extract-zip');
const fetch = require('node-fetch');
const fs = require('fs-extra');
const parse = require('csv-parse');
const proj4 = require('proj4');
const stripBomStream = require('strip-bom-stream');
const tmp = require('tmp-promise');
const untildify = require('untildify');
const Promise = require('bluebird');

const models = require('../models/models');
const logUtils = require('./log-utils');
const utils = require('./utils');

const downloadFiles = async task => {
  task.log(`Downloading GTFS from ${task.agency_url}`);

  task.path = `${task.downloadDir}/${task.agency_key}-gtfs.zip`;

  const response = await fetch(task.agency_url, {method: 'GET', headers: task.agency_headers || {}});

  if (response.status !== 200) {
    throw new Error('Couldn’t download files');
  }

  const buffer = await response.buffer();

  await fs.writeFile(task.path, buffer);
  task.log('Download successful');
};

const getTextFiles = async folderPath => {
  const files = await fs.readdir(folderPath);
  return files.filter(filename => filename.slice(-3) === 'txt')
};

const readFiles = async task => {
  const gtfsPath = untildify(task.path);
  task.log(`Importing GTFS from ${task.path}\r`);
  if (path.extname(gtfsPath) === '.zip') {
    try {
      await extract(gtfsPath, {dir: task.downloadDir});
      const textFiles = await getTextFiles(task.downloadDir);

      // If no .txt files in this directory, check for subdirectories and copy them here
      if (textFiles.length === 0) {
        const files = await fs.readdir(task.downloadDir);
        const folders = files.map(filename => path.join(task.downloadDir, filename)).filter(source => fs.lstatSync(source).isDirectory());

        if (folders.length > 1) {
          throw new Error(`More than one subfolder found in zip file at ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        } else if (folders.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }
        const subfolderName = folders[0];
        const directoryTextFiles = await getTextFiles(subfolderName);

        if (directoryTextFiles.length === 0) {
          throw new Error(`No .txt files found in ${task.path}. Ensure that .txt files are in the top level of the zip file, or in a single subdirectory.`);
        }

        await Promise.all(directoryTextFiles.map(async fileName => fs.rename(path.join(subfolderName, fileName), path.join(task.downloadDir, fileName))));
      }
    } catch (error) {
      task.error(error);
      console.error(error)
      throw new Error(`Unable to unzip file ${task.path}`);
    }
  } else {
    // Local file is unzipped, just copy it from there.
    await fs.copy(gtfsPath, task.downloadDir);
  }
};

const removeData = task => {
  // Remove old db records based on agency_key
  return Promise.all(models.map(model => {
    return model.model.collection.deleteMany({agency_key: task.agency_key});
  }));
};

const importLines = (lines, model, cb) => {
  const bulk = model.collection.initializeUnorderedBulkOp();
  const count = lines.length;

  if (!bulk) {
    return cb(new Error('Unable to initialize Mongo bulk insert.'));
  }

  while (lines.length) {
    bulk.insert(lines.pop());
  }

  bulk.execute(err => {
    cb(err, count);
  });
};

const formatLine = (line, task, lineNumber) => {
  // Remove null values
  for (const key in line) {
    if (line[key] === null) {
      delete line[key];
    }
  }

  // Add agency_key
  line.agency_key = task.agency_key;

  // Add created_at
  line.created_at = task.created_at;

  // Convert fields that should be int
  const integerFields = [
    'monday',
    'tuesday',
    'wednesday',
    'thursday',
    'friday',
    'saturday',
    'sunday',
    'start_date',
    'end_date',
    'date',
    'exception_type',
    'shape_pt_sequence',
    'payment_method',
    'transfers',
    'transfer_duration',
    'feed_start_date',
    'feed_end_date',
    'headway_secs',
    'exact_times',
    'route_type',
    'direction_id',
    'location_type',
    'wheelchair_boarding',
    'stop_sequence',
    'pickup_type',
    'drop_off_type',
    'use_stop_sequence',
    'transfer_type',
    'min_transfer_time',
    'wheelchair_accessible',
    'bikes_allowed',
    'timepoint',
    'timetable_sequence',
    'include_exceptions',
    'show_trip_continuation'
  ];

  integerFields.forEach(fieldName => {
    if (line[fieldName]) {
      line[fieldName] = Number.parseInt(line[fieldName], 10);
    } else {
      delete line[fieldName];
    }
  });

  // Convert fields that should be float
  const floatFields = [
    'price',
    'shape_dist_traveled',
    'shape_pt_lat',
    'shape_pt_lon',
    'stop_lat',
    'stop_lon'
  ];

  floatFields.forEach(fieldName => {
    if (line[fieldName]) {
      line[fieldName] = Number.parseFloat(line[fieldName]);
    } else {
      delete line[fieldName];
    }
  });

  // Check for valid coordinates
  const latitudeFields = [
    'shape_pt_lat',
    'stop_lat'
  ];

  for (const fieldName of latitudeFields) {
    if (line[fieldName] !== undefined) {
      if (line[fieldName] < -90 || line[fieldName] > 90) {
        throw new Error(`Invalid latitude in ${fieldName} on ${lineNumber}`);
      }
    }
  }

  const longitudeFields = [
    'shape_pt_lon',
    'stop_lon'
  ];

  for (const fieldName of longitudeFields) {
    if (line[fieldName] !== undefined) {
      if (line[fieldName] < -180 || line[fieldName] > 180) {
        throw new Error(`Invalid longitude in ${fieldName} on ${lineNumber}`);
      }
    }
  }

  // Convert to midnight timestamp
  const timestampFormat = [
    'start_time',
    'end_time',
    'arrival_time',
    'departure_time'
  ];

  for (const fieldName of timestampFormat) {
    if (line[fieldName]) {
      line[`${fieldName}stamp`] = utils.calculateHourTimestamp(line[fieldName]);
    }
  }

  // Make lat/lon array for stops
  if (line.stop_lat && line.stop_lon) {
    line.loc = [
      line.stop_lon,
      line.stop_lat
    ];

    // If coordinates are not specified, use [0,0]
    if (Number.isNaN(line.loc[0])) {
      line.loc[0] = 0;
    }

    if (Number.isNaN(line.loc[1])) {
      line.loc[1] = 0;
    }

    // Convert to epsg4326 if needed
    if (task.agency_proj) {
      line.loc = proj4(task.agency_proj, 'WGS84', line.loc);
      line.stop_lon = line.loc[0];
      line.stop_lat = line.loc[1];
    }
  }

  // Make lat/long for shapes
  if (line.shape_pt_lat && line.shape_pt_lon) {
    line.loc = [line.shape_pt_lon, line.shape_pt_lat];
  }

  return line;
};

const importFiles = task => {
  // Loop through each GTFS file
  return Promise.mapSeries(models, model => {
    return new Promise((resolve, reject) => {
      // Filter out excluded files from config
      if (task.exclude && _.includes(task.exclude, model.filenameBase)) {
        task.log(`Skipping - ${model.filenameBase}.txt\r`);
        return resolve();
      }

      const filepath = path.join(task.downloadDir, `${model.filenameBase}.txt`);

      if (!fs.existsSync(filepath)) {
        if (!model.nonstandard) {
          task.log(`Importing - ${model.filenameBase}.txt - No file found\r`);
        }

        return resolve();
      }

      task.log(`Importing - ${model.filenameBase}.txt\r`);

      const lines = [];
      const chunkSize = 10000;
      let lineCount = 0;
      const parser = parse({
        columns: true,
        relax: true,
        trim: true,
        skip_empty_lines: true
      });

      parser.on('readable', async () => {
        let record;
        /* eslint-disable-next-line no-cond-assign */
        while (record = parser.read()) {
          const line = formatLine(record, task, parser.info.lines);

          // Calculate agency bounds
          if (line.loc) {
            task.agency_bounds = utils.extendBounds(task.agency_bounds, line.loc);
          }

          lines.push(line);

          // If we have a bunch of lines ready to insert, then do it
          if (lines.length >= chunkSize) {
            importLines(lines, model.model, (err, count) => {
              if (err) {
                return reject(err);
              }

              lineCount += count;
              task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
            });
          }
        }
      });

      parser.on('end', () => {
        // Insert all remaining lines
        if (lines.length > 0) {
          importLines(lines, model.model, (err, count) => {
            if (err) {
              return reject(err);
            }

            lineCount += count;
            task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
            resolve();
          });
        } else {
          task.log(`Importing - ${model.filenameBase}.txt - ${lineCount} lines imported\r`, true);
          resolve();
        }
      });

      parser.on('error', reject);

      fs.createReadStream(filepath)
        .pipe(stripBomStream())
        .pipe(parser);
    })
      .catch(error => {
        throw error;
      });
  });
};

const postProcess = async task => {
  task.log('Post Processing data');

  const agencyModel = _.find(models, {filenameBase: 'agency'});
  const agencyCenter = utils.boundsCenter(task.agency_bounds);

  if (agencyCenter) {
    await agencyModel.model.collection.updateMany({
      agency_key: task.agency_key
    }, {
      $set: {
        agency_bounds: task.agency_bounds,
        agency_center: agencyCenter
      }
    });
  }
};

const ensureIndexes = config => {
  return Promise.all(models.map(model => {
    if (config.dataExpireAfterSeconds === undefined) {
      // Remove any TTL expiration index and ignore errors
      model.model.collection.dropIndex({created_at: 1}).catch(() => null);
    } else {
      // Set TTL expiration index if specified in configs
      model.model.schema.index({created_at: 1}, {expireAfterSeconds: config.dataExpireAfterSeconds});
    }

    return model.model.ensureIndexes();
  }));
};

module.exports = async config => {
  const log = logUtils.log(config);
  const logError = logUtils.logError(config);

  const agencyCount = config.agencies.length;
  log(`Starting GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}`);

  await Promise.mapSeries(config.agencies, async agency => {
    if (!agency.agency_key) {
      throw new Error('No Agency Key provided.');
    }

    if (!agency.url && !agency.path) {
      throw new Error('No Agency URL or path provided.');
    }

    const {path, cleanup} = await tmp.dir({unsafeCleanup: true});

    const task = {
      exclude: agency.exclude,
      agency_key: agency.agency_key,
      agency_url: agency.url,
      agency_headers: agency.headers || false,
      downloadDir: path,
      path: agency.path,
      skipDelete: config.skipDelete,
      log: (message, overwrite) => {
        log(`${task.agency_key}: ${message}`, overwrite);
      },
      error: message => {
        logError(message);
      },
      created_at: new Date()
    };

    if (task.agency_url) {
      await downloadFiles(task);
    }

    await readFiles(task);

    // Override using --skipDelete command line argument or `skipDelete` in config.json
    if (task.skipDelete === true) {
      task.log('Skipping deletion of existing data');
    } else {
      await removeData(task);
    }

    await importFiles(task);
    await postProcess(task);
    await ensureIndexes(config);

    cleanup();
    task.log('Completed GTFS import');
  });

  log(`Completed GTFS import for ${agencyCount} ${utils.pluralize('file', agencyCount)}\n`);
};
<|endoftext|>