<commit_before>const { resolve, parse, join } = require('path');
const { existsSync } = require('fs');
const GroupHelper = require('./utils/group-helper');
const Compiler = require('./utils/compiler');

const webpackMerge = require('webpack-merge');

class WebpackCLI extends GroupHelper {
    constructor() {
        super();
        this.groupMap = new Map();
        this.groups = [];
        this.processingErrors = [];
    }
    setMappedGroups(args, yargsOptions) {
        const { _all } = args;
        Object.keys(_all).forEach(key => {
            this.setGroupMap(key, _all[key], yargsOptions);
        });
    }
    setGroupMap(key, val, yargsOptions) {
        const opt = yargsOptions.find(opt => opt.name === key);
        const groupName = opt.group;
        const namePrefix = groupName.slice(0, groupName.length - 9).toLowerCase();
        if (this.groupMap.has(namePrefix)) {
            const pushToMap = this.groupMap.get(namePrefix);
            pushToMap.push({ [opt.name]: val });
        } else {
            this.groupMap.set(namePrefix, [{ [opt.name]: val }]);
        }
    }

    checkDefaults(options, outputOptions) {
        if (Array.isArray(options)) {
            return options.map(opt => this.checkDefaults(opt, outputOptions));
        }
        const defaultEntry = 'index';
        const possibleFileNames = [
            `./${defaultEntry}`, `./${defaultEntry}.js`, `${defaultEntry}.js`, defaultEntry,
            `./src/${defaultEntry}`, `./src/${defaultEntry}.js`, `src/${defaultEntry}.js`
        ];
        if (options.entry && possibleFileNames.includes(options.entry)) {
            const absFilename = parse(options.entry);
            let tmpFileName = options.entry;
            if (absFilename.ext !== '.js') {
                tmpFileName += '.js';
            }
            const normalizedEntry = resolve(tmpFileName);
            if (!existsSync(normalizedEntry)) {
                const parsedPath = parse(normalizedEntry);
                const possibleEntries = possibleFileNames.map(f => {
                    return resolve(parsedPath.dir, f);
                }).filter(e => existsSync(e));

                if (possibleEntries.length) {
                    options.entry = possibleEntries[0];
                }

            }
        }
        if(outputOptions.devtool) {
            options.devtool = outputOptions.devtool;
        }
        return options;
    }

    resolveGroups() {
        let mode;
        for (const [key, value] of this.groupMap.entries()) {
            const fileName = join(__dirname, 'groups', key);
            const GroupClass = require(fileName);
            if (key === 'config') {
                value.push({ mode: mode });
            }
            const GroupInstance = new GroupClass(value);
            if (key === 'basic') {
                if (GroupInstance.opts.outputOptions.dev) {
                    mode = 'dev';
                } else {
                    mode = 'prod';
                }
                this.groups.push(GroupInstance);
            } else {
                this.groups.push(GroupInstance);
            }
        }
    }

    async runOptionGroups() {
        let groupResult = {
            options: {},
            outputOptions: {},
            processingErrors: [],
        };

        const tmpGroupResult = this.groups.map(Group => Group.run()).filter(e => e);
        const resolvedResults = await Promise.all(tmpGroupResult);
        resolvedResults.forEach((e, idx) => {
            let groupObject = resolvedResults[idx + 1];
            if (e.processingErrors) {
                groupResult.processingErrors = groupResult.processingErrors.concat(...e.processingErrors);
            }
            if (!groupObject) {
                groupObject = {
                    outputOptions: {},
                    options: {}
                }
            }
            if (!groupObject.outputOptions) {
                groupObject.outputOptions = {};
            }

            groupResult.outputOptions = Object.assign(groupObject.outputOptions, e.outputOptions);
            if (Array.isArray(e.options)) {
                const defaultArrayOptions = e.options.map(arrayConfigObject => {
                    return webpackMerge(groupResult.options, arrayConfigObject);
                })
                groupResult.options = defaultArrayOptions;
                return;
            }
            if (Array.isArray(groupResult.options)) {
                const defaultArrayOptions = groupResult.options.map(arrayConfigObject => {
                    return webpackMerge(e.options, arrayConfigObject);
                })
                groupResult.options = defaultArrayOptions;
                return;
            }
            groupResult.options = webpackMerge(groupResult.options, e.options);
        });
        const wrappedConfig = require('./utils/zero-config')(groupResult);
        wrappedConfig.options = this.checkDefaults(wrappedConfig.options, wrappedConfig.outputOptions);
        return wrappedConfig;
    }

    async processArgs(args, yargsOptions) {
        await this.setMappedGroups(args, yargsOptions);
        await this.resolveGroups();
        const groupResult = await this.runOptionGroups();
        return groupResult;
    }

    async getCompiler(args, yargsOptions) {
        const groupResult = await this.processArgs(args, yargsOptions);
        return await Compiler.getCompiler(groupResult);
    }

    async run(args, yargsOptions) {
        const groupResult = await this.processArgs(args, yargsOptions);
        const webpack = await Compiler.webpackInstance(groupResult);
        return webpack;
    }

    async runCommand(command, ...args) {
        // TODO: rename and depreciate init
        if (command.name === 'create') {
            command.name = 'init';
        } else if (command.name === 'loader') {
            command.name = 'generate-loader';
        } else if (command.name === 'plugin') {
            command.name = 'generate-plugin';
        }
        return await require('./commands/external').run(command.name, ...args);
    }

    runHelp(args) {
        const HelpGroup = require('./groups/help');
        const Commands = require('./utils/commands').names;
        const command = Object.keys(Commands).map( i => {
            if (args.includes(Commands[i])) {
                return Commands[i];
            }
        })[0];
        return new HelpGroup().outputHelp(command);
    }

    runVersion() {
        const HelpGroup = require('./groups/help');
        return new HelpGroup().outputVersion();
    }

}

module.exports = WebpackCLI;
<commit_msg>chore: use filter instead<commit_after>const { resolve, parse, join } = require('path');
const { existsSync } = require('fs');
const GroupHelper = require('./utils/group-helper');
const Compiler = require('./utils/compiler');

const webpackMerge = require('webpack-merge');

class WebpackCLI extends GroupHelper {
    constructor() {
        super();
        this.groupMap = new Map();
        this.groups = [];
        this.processingErrors = [];
    }
    setMappedGroups(args, yargsOptions) {
        const { _all } = args;
        Object.keys(_all).forEach(key => {
            this.setGroupMap(key, _all[key], yargsOptions);
        });
    }
    setGroupMap(key, val, yargsOptions) {
        const opt = yargsOptions.find(opt => opt.name === key);
        const groupName = opt.group;
        const namePrefix = groupName.slice(0, groupName.length - 9).toLowerCase();
        if (this.groupMap.has(namePrefix)) {
            const pushToMap = this.groupMap.get(namePrefix);
            pushToMap.push({ [opt.name]: val });
        } else {
            this.groupMap.set(namePrefix, [{ [opt.name]: val }]);
        }
    }

    checkDefaults(options, outputOptions) {
        if (Array.isArray(options)) {
            return options.map(opt => this.checkDefaults(opt, outputOptions));
        }
        const defaultEntry = 'index';
        const possibleFileNames = [
            `./${defaultEntry}`, `./${defaultEntry}.js`, `${defaultEntry}.js`, defaultEntry,
            `./src/${defaultEntry}`, `./src/${defaultEntry}.js`, `src/${defaultEntry}.js`
        ];
        if (options.entry && possibleFileNames.includes(options.entry)) {
            const absFilename = parse(options.entry);
            let tmpFileName = options.entry;
            if (absFilename.ext !== '.js') {
                tmpFileName += '.js';
            }
            const normalizedEntry = resolve(tmpFileName);
            if (!existsSync(normalizedEntry)) {
                const parsedPath = parse(normalizedEntry);
                const possibleEntries = possibleFileNames.map(f => {
                    return resolve(parsedPath.dir, f);
                }).filter(e => existsSync(e));

                if (possibleEntries.length) {
                    options.entry = possibleEntries[0];
                }

            }
        }
        if(outputOptions.devtool) {
            options.devtool = outputOptions.devtool;
        }
        return options;
    }

    resolveGroups() {
        let mode;
        for (const [key, value] of this.groupMap.entries()) {
            const fileName = join(__dirname, 'groups', key);
            const GroupClass = require(fileName);
            if (key === 'config') {
                value.push({ mode: mode });
            }
            const GroupInstance = new GroupClass(value);
            if (key === 'basic') {
                if (GroupInstance.opts.outputOptions.dev) {
                    mode = 'dev';
                } else {
                    mode = 'prod';
                }
                this.groups.push(GroupInstance);
            } else {
                this.groups.push(GroupInstance);
            }
        }
    }

    async runOptionGroups() {
        let groupResult = {
            options: {},
            outputOptions: {},
            processingErrors: [],
        };

        const tmpGroupResult = this.groups.map(Group => Group.run()).filter(e => e);
        const resolvedResults = await Promise.all(tmpGroupResult);
        resolvedResults.forEach((e, idx) => {
            let groupObject = resolvedResults[idx + 1];
            if (e.processingErrors) {
                groupResult.processingErrors = groupResult.processingErrors.concat(...e.processingErrors);
            }
            if (!groupObject) {
                groupObject = {
                    outputOptions: {},
                    options: {}
                }
            }
            if (!groupObject.outputOptions) {
                groupObject.outputOptions = {};
            }

            groupResult.outputOptions = Object.assign(groupObject.outputOptions, e.outputOptions);
            if (Array.isArray(e.options)) {
                const defaultArrayOptions = e.options.map(arrayConfigObject => {
                    return webpackMerge(groupResult.options, arrayConfigObject);
                })
                groupResult.options = defaultArrayOptions;
                return;
            }
            if (Array.isArray(groupResult.options)) {
                const defaultArrayOptions = groupResult.options.map(arrayConfigObject => {
                    return webpackMerge(e.options, arrayConfigObject);
                })
                groupResult.options = defaultArrayOptions;
                return;
            }
            groupResult.options = webpackMerge(groupResult.options, e.options);
        });
        const wrappedConfig = require('./utils/zero-config')(groupResult);
        wrappedConfig.options = this.checkDefaults(wrappedConfig.options, wrappedConfig.outputOptions);
        return wrappedConfig;
    }

    async processArgs(args, yargsOptions) {
        await this.setMappedGroups(args, yargsOptions);
        await this.resolveGroups();
        const groupResult = await this.runOptionGroups();
        return groupResult;
    }

    async getCompiler(args, yargsOptions) {
        const groupResult = await this.processArgs(args, yargsOptions);
        return await Compiler.getCompiler(groupResult);
    }

    async run(args, yargsOptions) {
        const groupResult = await this.processArgs(args, yargsOptions);
        const webpack = await Compiler.webpackInstance(groupResult);
        return webpack;
    }

    async runCommand(command, ...args) {
        // TODO: rename and depreciate init
        if (command.name === 'create') {
            command.name = 'init';
        } else if (command.name === 'loader') {
            command.name = 'generate-loader';
        } else if (command.name === 'plugin') {
            command.name = 'generate-plugin';
        }
        return await require('./commands/external').run(command.name, ...args);
    }

    runHelp(args) {
        const HelpGroup = require('./groups/help');
        const names = require('./utils/commands').names;
        const command = names.filter( name => {
            return args.includes(name);
        })[0];
        return new HelpGroup().outputHelp(command);
    }

    runVersion() {
        const HelpGroup = require('./groups/help');
        return new HelpGroup().outputVersion();
    }

}

module.exports = WebpackCLI;
<|endoftext|>


<commit_before>var
  gulp          = require('gulp-param')(require('gulp'), process.argv),
  gulpSequence  = require('gulp-sequence'),
  fs            = require('fs'),
  download      = require('gulp-download'),
  replace       = require('gulp-replace'),
  insert        = require('gulp-insert'),
  parse         = require('csv-parse'),
  jeditor       = require("gulp-json-editor"),
  jsonRefactor  = require('gulp-json-refactor'),
  rename        = require('gulp-rename'),
  slug          = require('slug'),
  concat        = require('gulp-concat'),
  ifElse        = require('gulp-if-else'),
  replace       = require('gulp-replace'),
  dir           = require('node-dir'),
  jsonlint      = require("gulp-jsonlint"),
  cartesian     = require('cartesian-product'),
  R             = require('ramda'),
  urlencode     = require('urlencode'),
  productHelper = require('../src/product_helper'),
  TEMPDIR       = './tmp/',
  RAWDIR        = './data/raw/',
  DATADIR       = './data/',
  OPTIONS       = ['class_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'unit_desc', 'domain_desc']
;

function getProductList() {
  var
    productList = RAWDIR + 'product-list.json',
    productsRaw = fs.readFileSync(productList),
    productJSON = JSON.parse(productsRaw),
    products    = productJSON["data"][0]["Values"]
  ;

  return products;
}

function getFilteredProductList() {
  var
    allProducts      = getProductList(),
    filteredProducts = productHelper.filterProducts(allProducts)
  ;

  return filteredProducts;
}

function getFilteredOptions(option, productOptions) {
  var
    filteredOptions = productHelper.filterOptions(option, productOptions)
  ;

  return filteredOptions;
}

function getProductProperties(product) {
  var
    productQSVar  = product,
    productSlug   = slug(productQSVar.toLowerCase(), '_'),
    productFolder = RAWDIR + productSlug
  ;

  return {
    qsVar:  productQSVar,
    slug:   productSlug,
    folder: productFolder
  }
}

function getOptionFilename(slug, option) {
  return slug + '_' + option + '.json';
}

gulp.task('product-list', function() {
  var
    url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=commodity_desc',
    filename = 'product-list.json'
  ;
  download(url)
    .pipe(rename(filename))
    .pipe(gulp.dest(RAWDIR))
});

gulp.task('product-metadata', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    OPTIONS.forEach(function(option) {
      var
        url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=' + option + '&commodity_desc=' + props.qsVar,
        filename = getOptionFilename(props.slug, option)
      ;

      download(url)
        .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
          if(!json || !json["data"]) { return []; }
          return json["data"][0]["Values"];
        }))
        .pipe(insert.prepend('"' + option + '":')) // add option name before value array
        .pipe(rename(filename)) // set filename
        .pipe(gulp.dest(props.folder)) // write to fs
    });
  });

});

gulp.task('product-concat', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*.json')
      .pipe(insert.append(',')) // add , to end of each of the files in the folder
      .pipe(concat(props.slug + '_options.json')) // combine all files into single options file
      .pipe(insert.prepend('{"' + props.slug + '":{ "commodity_desc": ["' + product + '"], "source_desc": ["CENSUS"], "year": ["2012"], ')) // wrap object with product name
      .pipe(insert.append('"agg_level_desc": ["NATIONAL", "STATE", "COUNTY"]}}')) // add item and end object
      .pipe(gulp.dest(props.folder)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-combinations', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props        = getProductProperties(product),
      optionKeys   = ['year', 'source_desc', 'agg_level_desc', 'commodity_desc', 'class_desc', 'unit_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'domain_desc'],
      optionFile   = props.folder + '/' + props.slug + '_options.json',
      outputFile   = props.folder + '/' + props.slug + '_queries.json',
      optionsRaw   = fs.readFileSync(optionFile),
      optionsJSON  = JSON.parse(optionsRaw),
      optionArr    = [],
      optionCombos = null,
      optionURLs   = [],
      result   = {}
    ;

    optionKeys.forEach(function(option) {
      optionArr.push(
        getFilteredOptions(option, optionsJSON[props.slug][option])
      );
    });

    optionCombos = cartesian(optionArr);

    optionCombos.forEach(function(combo) {
      var
        filenames = combo.map(v => slug(v.toLowerCase(), '_')).join('_'),
        zipped    = R.zip(optionKeys, combo),
        encoded   = zipped.map(kv => [kv[0], urlencode(kv[1])]),
        qspair    = encoded.map(kv => kv.join('='))
        qsvars    = qspair.join('&')
      ;

      optionURLs.push({
       filename:    filenames,
       querystring: qsvars
      });
    });

    result[props.slug] = {
      queries: optionURLs
    };

    fs.writeFileSync(outputFile, JSON.stringify(result));
  });

});

gulp.task('product-download', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props       = getProductProperties(product),
      baseURL     = 'http://nass-api.azurewebsites.net/api/api_get?',
      queriesFile = props.folder + '/' + props.slug + '_queries.json',
      queriesRaw  = fs.readFileSync(queriesFile),
      queriesJSON = JSON.parse(queriesRaw)
      queries     = queriesJSON[props.slug]["queries"]
    ;

    queries.forEach(function(query) {
      var
        url      = baseURL + query.querystring,
        filename = query.filename + '.json'
      ;
      download(url)
        .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
          if(!json) { throw "Error\t[" + url + "]"; }
          if(Object.keys(json).length == 0) { return false; }
          return json;
        }))
        .pipe(rename(filename)) // set filename
        .pipe(gulp.dest(props.folder)) // write to fs
    });

  });

});

gulp.task('products', gulpSequence('product-list', 'product-metadata', 'product-concat', 'product-combinations'));
<commit_msg>gulp -- output option keys/values on combination files<commit_after>var
  gulp          = require('gulp-param')(require('gulp'), process.argv),
  gulpSequence  = require('gulp-sequence'),
  fs            = require('fs'),
  download      = require('gulp-download'),
  replace       = require('gulp-replace'),
  insert        = require('gulp-insert'),
  parse         = require('csv-parse'),
  jeditor       = require("gulp-json-editor"),
  jsonRefactor  = require('gulp-json-refactor'),
  rename        = require('gulp-rename'),
  slug          = require('slug'),
  concat        = require('gulp-concat'),
  ifElse        = require('gulp-if-else'),
  replace       = require('gulp-replace'),
  dir           = require('node-dir'),
  jsonlint      = require("gulp-jsonlint"),
  cartesian     = require('cartesian-product'),
  R             = require('ramda'),
  urlencode     = require('urlencode'),
  productHelper = require('../src/product_helper'),
  TEMPDIR       = './tmp/',
  RAWDIR        = './data/raw/',
  DATADIR       = './data/',
  OPTIONS       = ['class_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'unit_desc', 'domain_desc']
;

function getProductList() {
  var
    productList = RAWDIR + 'product-list.json',
    productsRaw = fs.readFileSync(productList),
    productJSON = JSON.parse(productsRaw),
    products    = productJSON["data"][0]["Values"]
  ;

  return products;
}

function getFilteredProductList() {
  var
    allProducts      = getProductList(),
    filteredProducts = productHelper.filterProducts(allProducts)
  ;

  return filteredProducts;
}

function getFilteredOptions(option, productOptions) {
  var
    filteredOptions = productHelper.filterOptions(option, productOptions)
  ;

  return filteredOptions;
}

function getProductProperties(product) {
  var
    productQSVar  = product,
    productSlug   = slug(productQSVar.toLowerCase(), '_'),
    productFolder = RAWDIR + productSlug
  ;

  return {
    qsVar:  productQSVar,
    slug:   productSlug,
    folder: productFolder
  }
}

function getOptionFilename(slug, option) {
  return slug + '_' + option + '.json';
}

gulp.task('product-list', function() {
  var
    url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=commodity_desc',
    filename = 'product-list.json'
  ;
  download(url)
    .pipe(rename(filename))
    .pipe(gulp.dest(RAWDIR))
});

gulp.task('product-metadata', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    OPTIONS.forEach(function(option) {
      var
        url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=' + option + '&commodity_desc=' + props.qsVar,
        filename = getOptionFilename(props.slug, option)
      ;

      download(url)
        .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
          if(!json || !json["data"]) { return []; }
          return json["data"][0]["Values"];
        }))
        .pipe(insert.prepend('"' + option + '":')) // add option name before value array
        .pipe(rename(filename)) // set filename
        .pipe(gulp.dest(props.folder)) // write to fs
    });
  });

});

gulp.task('product-concat', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*.json')
      .pipe(insert.append(',')) // add , to end of each of the files in the folder
      .pipe(concat(props.slug + '_options.json')) // combine all files into single options file
      .pipe(insert.prepend('{"' + props.slug + '":{ "commodity_desc": ["' + product + '"], "source_desc": ["CENSUS"], "year": ["2012"], ')) // wrap object with product name
      .pipe(insert.append('"agg_level_desc": ["NATIONAL", "STATE", "COUNTY"]}}')) // add item and end object
      .pipe(gulp.dest(props.folder)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-combinations', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props        = getProductProperties(product),
      optionKeys   = ['year', 'source_desc', 'agg_level_desc', 'commodity_desc', 'class_desc', 'unit_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'domain_desc'],
      optionFile   = props.folder + '/' + props.slug + '_options.json',
      outputFile   = props.folder + '/' + props.slug + '_queries.json',
      optionsRaw   = fs.readFileSync(optionFile),
      optionsJSON  = JSON.parse(optionsRaw),
      optionArr    = [],
      optionCombos = null,
      optionURLs   = [],
      result   = {}
    ;

    optionKeys.forEach(function(option) {
      optionArr.push(
        getFilteredOptions(option, optionsJSON[props.slug][option])
      );
    });

    optionCombos = cartesian(optionArr);

    optionCombos.forEach(function(combo) {
      var
        filenames = combo.map(v => slug(v.toLowerCase(), '_')).join('_'),
        zipped    = R.zip(optionKeys, combo),
        encoded   = zipped.map(kv => [kv[0], urlencode(kv[1])]),
        qspair    = encoded.map(kv => kv.join('='))
        qsvars    = qspair.join('&')
      ;

      optionURLs.push({
       filename:    filenames,
       options:     zipped,
       querystring: qsvars
      });
    });

    result[props.slug] = {
      queries: optionURLs
    };

    fs.writeFileSync(outputFile, JSON.stringify(result));
  });

});

gulp.task('product-download', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props       = getProductProperties(product),
      baseURL     = 'http://nass-api.azurewebsites.net/api/api_get?',
      queriesFile = props.folder + '/' + props.slug + '_queries.json',
      queriesRaw  = fs.readFileSync(queriesFile),
      queriesJSON = JSON.parse(queriesRaw)
      queries     = queriesJSON[props.slug]["queries"]
    ;

    queries.forEach(function(query) {
      var
        url      = baseURL + query.querystring,
        filename = query.filename + '.json'
      ;
      download(url)
        .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
          if(!json) { throw "Error\t[" + url + "]"; }
          if(Object.keys(json).length == 0) { return false; }
          return json;
        }))
        .pipe(rename(filename)) // set filename
        .pipe(gulp.dest(props.folder)) // write to fs
    });

  });

});

gulp.task('products', gulpSequence('product-list', 'product-metadata', 'product-concat', 'product-combinations'));
<|endoftext|>


<commit_before>var
  gulp          = require('gulp-param')(require('gulp'), process.argv),
  gulpSequence  = require('gulp-sequence'),
  fs            = require('fs'),
  fileExists    = require('file-exists'),
  flatten       = require('flat'),
  download      = require('gulp-download'),
  request       = require('request'),
  replace       = require('gulp-replace'),
  source        = require('vinyl-source-stream'),
  streamify     = require('streamify'),
  insert        = require('gulp-insert'),
  parse         = require('csv-parse'),
  jeditor       = require("gulp-json-editor"),
  jsonRefactor  = require('gulp-json-refactor'),
  jsonTransform = require('gulp-json-transform'),
  rename        = require('gulp-rename'),
  slug          = require('slug'),
  concat        = require('gulp-concat'),
  ifElse        = require('gulp-if-else'),
  replace       = require('gulp-replace'),
  dir           = require('node-dir'),
  jsonlint      = require("gulp-jsonlint"),
  cartesian     = require('cartesian-product'),
  R             = require('ramda'),
  q             = require('q'),
  urlencode     = require('urlencode'),
  promise       = require("gulp-promise"),
  using         = require("gulp-using"),
  $size         = require("gulp-size"),
  productHelper = require('../src/product_helper'),
  TEMPDIR       = './tmp/',
  DATADIR       = './data/',
  RAWDIR        = './data/raw/',
  PRODDIR       = './data/products/',
  OPTIONS       = ['class_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'unit_desc', 'domain_desc']
;

function getProductList() {
  var
    productList = RAWDIR + 'product-list.json',
    productsRaw = fs.readFileSync(productList),
    productJSON = JSON.parse(productsRaw),
    products    = productJSON["data"][0]["Values"]
  ;

  return products;
}

function getFilteredProductList() {
  var
    allProducts      = getProductList(),
    filteredProducts = productHelper.filterProducts(allProducts)
  ;

  return filteredProducts;
}

function getFilteredOptions(option, productOptions) {
  var
    filteredOptions = productHelper.filterOptions(option, productOptions)
  ;

  return filteredOptions;
}

function getProductProperties(product) {
  var
    productQSVar  = product,
    productSlug   = slug(productQSVar.toLowerCase(), '_'),
    productFolder = RAWDIR + productSlug
  ;

  return {
    qsVar:  productQSVar,
    slug:   productSlug,
    folder: productFolder
  }
}

function getOptionFilename(slug, option) {
  return slug + '_' + option + '.json';
}

gulp.task('product-list', function() {
  var
    url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=commodity_desc',
    filename = 'product-list.json'
  ;
  download(url)
    .pipe(rename(filename))
    .pipe(gulp.dest(RAWDIR))
});

gulp.task('product-metadata', function(cb) {
  var
    products    = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    OPTIONS.forEach(function(option) {
      var
        url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=' + option + '&commodity_desc=' + props.qsVar,
        filename = getOptionFilename(props.slug, option),
        filepath = props.folder + '/' + filename
      ;

      if(fileExists(filepath)) {
        // skip
      } else {
        download(url)
          .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
            if(!json || !json["data"]) { return []; }
            return json["data"][0]["Values"];
          }))
          .pipe(insert.prepend('{ "' + props.slug + '": { "' + option + '":')) // add option name before value array
          .pipe(insert.append('}}')) // add option name before value array
          .pipe(rename(filename)) // set filename
          .pipe(gulp.dest(props.folder)) // write to fs
      }

    });
  });

});

gulp.task('product-concat', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*.json')
      .pipe(jsonTransform(function(json) {
        if(!json || !json[props.slug]) { return []; }
        var
          value           = json[props.slug],
          value_string    = JSON.stringify(json[props.slug]),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring
        ;
        return value_return;
      }))
      .pipe(insert.append(','))
      .pipe(concat(props.slug + '_options.json')) // combine all files into single options file
      .pipe(insert.prepend('{"' + props.slug + '":{ "commodity_desc": ["' + product + '"], "source_desc": ["CENSUS"], "year": ["2012"], ')) // wrap object with product name
      .pipe(insert.append('"agg_level_desc": ["NATIONAL", "STATE", "COUNTY"]}}')) // add item and end object
      .pipe(gulp.dest(props.folder)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-combinations', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props        = getProductProperties(product),
      optionKeys   = ['commodity_desc', 'class_desc', 'unit_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'domain_desc', 'year', 'source_desc', 'agg_level_desc'],
      optionFile   = props.folder + '/' + props.slug + '_options.json',
      outputFile   = props.folder + '/' + props.slug + '_queries.json',
      optionsRaw   = fs.readFileSync(optionFile),
      optionsJSON  = JSON.parse(optionsRaw),
      optionArr    = [],
      optionCombos = null,
      optionURLs   = [],
      result   = {}
    ;

    optionKeys.forEach(function(option) {
      optionArr.push(
        getFilteredOptions(option, optionsJSON[props.slug][option])
      );
    });

    optionCombos = cartesian(optionArr);

    optionCombos.forEach(function(combo) {
      var
        zipped    = R.zip(optionKeys, combo),
        zipHash   = zipped.map(kv => ({option: kv[0], value: kv[1]})),
        name      = productHelper.filenameFromOptions(zipHash),
        encoded   = zipped.map(kv => [kv[0], urlencode(kv[1])]),
        qspair    = encoded.map(kv => kv.join('=')),
        qsvars    = qspair.join('&')
      ;

      optionURLs.push({
       name:        name,
       options:     zipHash,
       querystring: qsvars
      });
    });

    result[props.slug] = {
      queries: optionURLs
    };

    fs.writeFileSync(outputFile, JSON.stringify(result));
  });

});

gulp.task('product-empty-queries', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*_queries.json')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }

        if(json[props.slug].queries.length == 0) {
          console.log("ZOMG THIS HAS ZERO QUERIES", props.slug);
        }

        return json;
      }))
      .pipe(jsonlint())
  });

});

gulp.task('product-download', function(cb) {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props       = getProductProperties(product),
      baseURL     = 'http://nass-api.azurewebsites.net/api/api_get?',
      queriesFile = props.folder + '/' + props.slug + '_queries.json',
      queriesRaw  = fs.readFileSync(queriesFile),
      queriesJSON = JSON.parse(queriesRaw)
      queries     = queriesJSON[props.slug]["queries"],
      promiselist = []
    ;

    queries.forEach(function(query) {
      promiselist.push(query);
    });

    promise.makePromises(promiselist, function () {
      if (cb) { cb(); }
    });

    queries.forEach(function(query) {
      var
        url      = baseURL + query.querystring,
        filename = query.name + '.json',
        filepath = props.folder + '/' + filename
      ;

      if(fileExists(filepath)) {
        // skip
      } else {
        request(url, {timeout: 120000})
          .on('response', function(response) {
            console.log(filename);
          })
          .on('error', function(err) {
            console.log("ERROR [" + err.code + "]", "Please run script again", query.name);
          })
          .pipe(source(filename))
          .pipe(insert.prepend('{ "' + query.name + '": '))
          .pipe(insert.append('}'))
          .pipe(gulp.dest(props.folder)) // write to fs
          .pipe(promise.deliverPromise(query))
        ;
      }

    });

  });

});

gulp.task('product-clean', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*{national,state,county}.json')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }
        var
          value           = json,
          primaryKey      = Object.keys(json)[0],
          innerValue      = json[primaryKey],
          emptyObject     = Object.keys(innerValue).length == 0,
          value_string    = JSON.stringify(value),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring,
          value_return    = value_return + ',' // not append outside of transform
        ;

        // don't return anything if the file was blank
        // situation arises when query is created from combinations, but actually has no data
        return(emptyObject ? '' : value_return);
      }))
      .pipe(concat(props.slug + '.json')) // combine all files into single options file
      .pipe(insert.prepend('{ "' + props.slug + '": {'))
      .pipe(insert.append('"ignore": {}}}'))
      .pipe(jsonTransform(function(dirtyJSON) {
        return productHelper.getCleanJSON(dirtyJSON);
      }))
      .pipe(gulp.dest(PRODDIR)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-jsonlint', function() {
  gulp.src(DATADIR + '/**/*.json')
    .pipe(jsonlint()) // ensure we created valid JSON object in file
    .pipe(jsonlint.reporter())
});

gulp.task('product-jsonlint-clean', function() {
  gulp.src(DATADIR + '/**/*.json')
    .pipe(jsonlint()) // ensure we created valid JSON object in file
    .pipe(jsonlint.reporter(function(file) {
      if(file.success !== true) {
        fs.unlinkSync(file.path);
        console.log('deleted', file.path);
      }
    }))
});

gulp.task('product-sanity-check', function() {
  gulp.src(PRODDIR + '*')
      .pipe(jsonTransform(function(json) {
        var
          product = Object.keys(json)[0],
          data    = json[product],
          keys    = Object.keys(data)
        ;

        keys.forEach(function(key) {
          var
            item     = data[key],
            stats    = item.stats
          ;

          if(stats) {
            var
              totals   = stats.totals,
              national = (totals.national || -1).toLocaleString(),
              state    = (totals.state || -1).toLocaleString(),
              county   = (totals.county || -1).toLocaleString()
            ;
            console.log(product + " => " + key + " => ( " + national + ' / ' + state + ' / ' + county + ' )');
          } else {
            console.log(product + " => " + key + " => ( no information available )");
          }
        });
        return json;
      }))
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  ;
});

gulp.task('product-combine-all', function() {
  gulp.src(PRODDIR + '*')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }
        var
          keys            = Object.keys(json),
          value           = json[keys[0]],
          value_string    = JSON.stringify(value),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring
        ;
        return value_return;
      }))
      .pipe(insert.append(','))
      .pipe(concat('products.json')) // combine all files into single options file
      .pipe(insert.prepend('{ "products": {'))
      .pipe(insert.append(' "ignore": {} }}'))
      .pipe(jsonTransform(function(json) {
        // was just a placeholder for concat
        delete json.products.ignore;
        return json;
      }))
      .pipe(gulp.dest(DATADIR))
      .pipe(jsonlint())
      .pipe(jsonlint.reporter())
  ;
});

gulp.task('products', gulpSequence('product-list', 'product-metadata', 'product-concat', 'product-combinations'));
<commit_msg>data -- product-combine-all -- skip blank files<commit_after>var
  gulp          = require('gulp-param')(require('gulp'), process.argv),
  gulpSequence  = require('gulp-sequence'),
  fs            = require('fs'),
  fileExists    = require('file-exists'),
  flatten       = require('flat'),
  download      = require('gulp-download'),
  request       = require('request'),
  replace       = require('gulp-replace'),
  source        = require('vinyl-source-stream'),
  streamify     = require('streamify'),
  insert        = require('gulp-insert'),
  parse         = require('csv-parse'),
  jeditor       = require("gulp-json-editor"),
  jsonRefactor  = require('gulp-json-refactor'),
  jsonTransform = require('gulp-json-transform'),
  rename        = require('gulp-rename'),
  slug          = require('slug'),
  concat        = require('gulp-concat'),
  ifElse        = require('gulp-if-else'),
  replace       = require('gulp-replace'),
  dir           = require('node-dir'),
  jsonlint      = require("gulp-jsonlint"),
  cartesian     = require('cartesian-product'),
  R             = require('ramda'),
  q             = require('q'),
  urlencode     = require('urlencode'),
  promise       = require("gulp-promise"),
  using         = require("gulp-using"),
  $size         = require("gulp-size"),
  productHelper = require('../src/product_helper'),
  TEMPDIR       = './tmp/',
  DATADIR       = './data/',
  RAWDIR        = './data/raw/',
  PRODDIR       = './data/products/',
  OPTIONS       = ['class_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'unit_desc', 'domain_desc']
;

function getProductList() {
  var
    productList = RAWDIR + 'product-list.json',
    productsRaw = fs.readFileSync(productList),
    productJSON = JSON.parse(productsRaw),
    products    = productJSON["data"][0]["Values"]
  ;

  return products;
}

function getFilteredProductList() {
  var
    allProducts      = getProductList(),
    filteredProducts = productHelper.filterProducts(allProducts)
  ;

  return filteredProducts;
}

function getFilteredOptions(option, productOptions) {
  var
    filteredOptions = productHelper.filterOptions(option, productOptions)
  ;

  return filteredOptions;
}

function getProductProperties(product) {
  var
    productQSVar  = product,
    productSlug   = slug(productQSVar.toLowerCase(), '_'),
    productFolder = RAWDIR + productSlug
  ;

  return {
    qsVar:  productQSVar,
    slug:   productSlug,
    folder: productFolder
  }
}

function getOptionFilename(slug, option) {
  return slug + '_' + option + '.json';
}

gulp.task('product-list', function() {
  var
    url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=commodity_desc',
    filename = 'product-list.json'
  ;
  download(url)
    .pipe(rename(filename))
    .pipe(gulp.dest(RAWDIR))
});

gulp.task('product-metadata', function(cb) {
  var
    products    = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    OPTIONS.forEach(function(option) {
      var
        url      = 'http://nass-api.azurewebsites.net/api/get_dependent_param_values?source_desc=CENSUS&year=2012&freq_desc=ANNUAL&agg_level_desc=COUNTY&distinctParams=' + option + '&commodity_desc=' + props.qsVar,
        filename = getOptionFilename(props.slug, option),
        filepath = props.folder + '/' + filename
      ;

      if(fileExists(filepath)) {
        // skip
      } else {
        download(url)
          .pipe(jeditor(function(json) { // get meat and potatoes of JSON object
            if(!json || !json["data"]) { return []; }
            return json["data"][0]["Values"];
          }))
          .pipe(insert.prepend('{ "' + props.slug + '": { "' + option + '":')) // add option name before value array
          .pipe(insert.append('}}')) // add option name before value array
          .pipe(rename(filename)) // set filename
          .pipe(gulp.dest(props.folder)) // write to fs
      }

    });
  });

});

gulp.task('product-concat', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*.json')
      .pipe(jsonTransform(function(json) {
        if(!json || !json[props.slug]) { return []; }
        var
          value           = json[props.slug],
          value_string    = JSON.stringify(json[props.slug]),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring
        ;
        return value_return;
      }))
      .pipe(insert.append(','))
      .pipe(concat(props.slug + '_options.json')) // combine all files into single options file
      .pipe(insert.prepend('{"' + props.slug + '":{ "commodity_desc": ["' + product + '"], "source_desc": ["CENSUS"], "year": ["2012"], ')) // wrap object with product name
      .pipe(insert.append('"agg_level_desc": ["NATIONAL", "STATE", "COUNTY"]}}')) // add item and end object
      .pipe(gulp.dest(props.folder)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-combinations', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props        = getProductProperties(product),
      optionKeys   = ['commodity_desc', 'class_desc', 'unit_desc', 'statisticcat_desc', 'util_practice_desc', 'prodn_practice_desc', 'domain_desc', 'year', 'source_desc', 'agg_level_desc'],
      optionFile   = props.folder + '/' + props.slug + '_options.json',
      outputFile   = props.folder + '/' + props.slug + '_queries.json',
      optionsRaw   = fs.readFileSync(optionFile),
      optionsJSON  = JSON.parse(optionsRaw),
      optionArr    = [],
      optionCombos = null,
      optionURLs   = [],
      result   = {}
    ;

    optionKeys.forEach(function(option) {
      optionArr.push(
        getFilteredOptions(option, optionsJSON[props.slug][option])
      );
    });

    optionCombos = cartesian(optionArr);

    optionCombos.forEach(function(combo) {
      var
        zipped    = R.zip(optionKeys, combo),
        zipHash   = zipped.map(kv => ({option: kv[0], value: kv[1]})),
        name      = productHelper.filenameFromOptions(zipHash),
        encoded   = zipped.map(kv => [kv[0], urlencode(kv[1])]),
        qspair    = encoded.map(kv => kv.join('=')),
        qsvars    = qspair.join('&')
      ;

      optionURLs.push({
       name:        name,
       options:     zipHash,
       querystring: qsvars
      });
    });

    result[props.slug] = {
      queries: optionURLs
    };

    fs.writeFileSync(outputFile, JSON.stringify(result));
  });

});

gulp.task('product-empty-queries', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*_queries.json')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }

        if(json[props.slug].queries.length == 0) {
          console.log("ZOMG THIS HAS ZERO QUERIES", props.slug);
        }

        return json;
      }))
      .pipe(jsonlint())
  });

});

gulp.task('product-download', function(cb) {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props       = getProductProperties(product),
      baseURL     = 'http://nass-api.azurewebsites.net/api/api_get?',
      queriesFile = props.folder + '/' + props.slug + '_queries.json',
      queriesRaw  = fs.readFileSync(queriesFile),
      queriesJSON = JSON.parse(queriesRaw)
      queries     = queriesJSON[props.slug]["queries"],
      promiselist = []
    ;

    queries.forEach(function(query) {
      promiselist.push(query);
    });

    promise.makePromises(promiselist, function () {
      if (cb) { cb(); }
    });

    queries.forEach(function(query) {
      var
        url      = baseURL + query.querystring,
        filename = query.name + '.json',
        filepath = props.folder + '/' + filename
      ;

      if(fileExists(filepath)) {
        // skip
      } else {
        request(url, {timeout: 120000})
          .on('response', function(response) {
            console.log(filename);
          })
          .on('error', function(err) {
            console.log("ERROR [" + err.code + "]", "Please run script again", query.name);
          })
          .pipe(source(filename))
          .pipe(insert.prepend('{ "' + query.name + '": '))
          .pipe(insert.append('}'))
          .pipe(gulp.dest(props.folder)) // write to fs
          .pipe(promise.deliverPromise(query))
        ;
      }

    });

  });

});

gulp.task('product-clean', function() {
  var
    products = getFilteredProductList()
  ;

  products.forEach(function(product) {
    var
      props = getProductProperties(product)
    ;

    gulp.src(props.folder + '/*{national,state,county}.json')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }
        var
          value           = json,
          primaryKey      = Object.keys(json)[0],
          innerValue      = json[primaryKey],
          emptyObject     = Object.keys(innerValue).length == 0,
          value_string    = JSON.stringify(value),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring,
          value_return    = value_return + ',' // not append outside of transform
        ;

        // don't return anything if the file was blank
        // situation arises when query is created from combinations, but actually has no data
        return(emptyObject ? '' : value_return);
      }))
      .pipe(concat(props.slug + '.json')) // combine all files into single options file
      .pipe(insert.prepend('{ "' + props.slug + '": {'))
      .pipe(insert.append('"ignore": {}}}'))
      .pipe(jsonTransform(function(dirtyJSON) {
        return productHelper.getCleanJSON(dirtyJSON);
      }))
      .pipe(gulp.dest(PRODDIR)) // write to fs
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  });

});

gulp.task('product-jsonlint', function() {
  gulp.src(DATADIR + '/**/*.json')
    .pipe(jsonlint()) // ensure we created valid JSON object in file
    .pipe(jsonlint.reporter())
});

gulp.task('product-jsonlint-clean', function() {
  gulp.src(DATADIR + '/**/*.json')
    .pipe(jsonlint()) // ensure we created valid JSON object in file
    .pipe(jsonlint.reporter(function(file) {
      if(file.success !== true) {
        fs.unlinkSync(file.path);
        console.log('deleted', file.path);
      }
    }))
});

gulp.task('product-sanity-check', function() {
  gulp.src(PRODDIR + '*')
      .pipe(jsonTransform(function(json) {
        var
          product = Object.keys(json)[0],
          data    = json[product],
          keys    = Object.keys(data)
        ;

        keys.forEach(function(key) {
          var
            item     = data[key],
            stats    = item.stats
          ;

          if(stats) {
            var
              totals   = stats.totals,
              national = (totals.national || -1).toLocaleString(),
              state    = (totals.state || -1).toLocaleString(),
              county   = (totals.county || -1).toLocaleString()
            ;
            console.log(product + " => " + key + " => ( " + national + ' / ' + state + ' / ' + county + ' )');
          } else {
            console.log(product + " => " + key + " => ( no information available )");
          }
        });
        return json;
      }))
      .pipe(jsonlint()) // ensure we created valid JSON object in file
      .pipe(jsonlint.reporter())
  ;
});

gulp.task('product-combine-all', function() {
  gulp.src(PRODDIR + '*')
      .pipe(jsonTransform(function(json) {
        if(!json) { return '!error!'; }
        var
          keys            = Object.keys(json),
          value           = json[keys[0]],
          emptyObject     = Object.keys(value).length == 0,
          value_string    = JSON.stringify(value),
          value_substring = value_string.substring(1, value_string.length - 1),
          value_return    = value_substring,
          value_return    = value_return + ','
        ;
        return (emptyObject ? '' : value_return);
      }))
      .pipe(concat('products.json')) // combine all files into single options file
      .pipe(insert.prepend('{ "products": {'))
      .pipe(insert.append(' "ignore": {} }}'))
      .pipe(jsonlint())
      .pipe(jsonlint.reporter())
      .pipe(gulp.dest(DATADIR))
  ;
});

gulp.task('products', gulpSequence('product-list', 'product-metadata', 'product-concat', 'product-combinations'));
<|endoftext|>


<commit_before>'use strict'

import settle from 'axios/lib/core/settle'
import buildURL from 'axios/lib/helpers/buildURL'
import createError from 'axios/lib/core/createError'

const supportedMethods = ['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT']

/**
 * 创建用于微信小程序的 axios adapter
 *
 * @param {AxiosInstance} axios 原始 axios 对象
 * @returns {AxiosAdapter} 用于微信小程序的 axios adapter
 */
export default function wxaAdapterFactory (axios) {
  /**
   * 用于微信小程序的 axios adapter
   *
   * @type {AxiosAdapter}
   */
  return function wxaAdapter (config) {
    return new Promise((resolve, reject) => {
      // 必须使用 https
      if (config.url.slice(0, 5).toLowerCase() !== 'https') {
        reject(createError('Requesting an http URL is not allowed', config))
      }

      if (config.responseType && config.responseType !== 'json' && config.responseType !== 'text') {
        // 只接受 json 或 text 的返回类型
        reject(createError('responseType only supports "json" or "text"', config))
      } else if (config.responseType === 'text' && config.transformResponse[0] === axios.defaults.transformResponse[0]) {
        // axios 默认会尝试将文本转为 json
        // 这里保证返回类型为 text 时返回的一定是文本
        config.transformResponse = []
      }

      // 检查小程序是否支持所使用的 method
      config.method = config.method.toUpperCase()
      if (supportedMethods.indexOf(config.method) === -1) {
        reject(createError('Unsupported request method', config))
      }

      // 生成用于 wx.request 的参数
      const request = {
        method: config.method.toUpperCase(),
        url: buildURL(config.url, config.params, config.paramsSerializer),
        data: config.data,
        header: config.headers,
        dataType: config.responseType
      }

      wx.request({
        ...request,
        success: response => {
          // 按照 axios 的返回格式构造返回值
          settle(resolve, reject, {
            data: response.data,
            status: response.statusCode,
            headers: response.header,
            config,
            request
          })
        },
        fail: response => {
          let err = response.errMsg

          // 增加错误消息可读性
          if (err === 'request:fail') {
            err = 'Request failed'
          }

          reject(createError(err, config))
        }
      })
    })
  }
}
<commit_msg>feat: 支持调用上传与下载接口<commit_after>'use strict'

import settle from 'axios/lib/core/settle'
import buildURL from 'axios/lib/helpers/buildURL'
import createError from 'axios/lib/core/createError'

const supportedMethods = ['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT']
const supportedResponseTypes = ['json', 'text', 'file']

/**
 * 创建用于微信小程序的 axios adapter
 *
 * @param {AxiosInstance} axios 原始 axios 对象
 * @returns {AxiosAdapter} 用于微信小程序的 axios adapter
 */
export default function wxaAdapterFactory (axios) {
  // 往 axios 的默认设置中加入 upload 项
  // 否则得到的 config 中将不会有 upload 项
  axios.defaults['upload'] = undefined

  /**
   * 用于微信小程序的 axios adapter
   *
   * @type {AxiosAdapter}
   */
  return function wxaAdapter (config) {
    return new Promise((resolve, reject) => {
      /** 发送普通请求或是上传、下载请求 */
      let requestType = 'request'

      /** 请求所使用的参数 */
      let request = {
        url: buildURL(config.url, config.params, config.paramsSerializer),
        header: config.headers
      }

      // 处理调用上传文件接口的情况
      if (config.upload) {
        const { filePath, name } = config.upload
        if (typeof filePath === 'string' && typeof name === 'string') {
          requestType = 'uploadFile'
          config.responseType = 'file'
          Object.assign(request, config.upload)
        } else {
          return reject(createError('Invalid upload paramaters', config))
        }
      }

      // 处理响应类型和下载接口类型
      if (config.responseType && supportedResponseTypes.indexOf(config.responseType) === -1) {
        // 只接受 json 或 text 的返回类型
        return reject(createError('Unsupported responseType', config))
      } else if (config.responseType === 'text' && config.transformResponse[0] === axios.defaults.transformResponse[0]) {
        // axios 默认会尝试将文本转为 json
        // 这里保证返回类型为 text 时返回的一定是文本
        config.transformResponse = []
      } else if (requestType === 'request') {
        // 确认是否调用下载文件接口
        if (config.responseType === 'file') {
          requestType = 'downloadFile'
        } else {
          // 必须使用 https
          if (config.url.slice(0, 5).toLowerCase() !== 'https') {
            return reject(createError('Requesting an http URL is not allowed', config))
          }

          // 检查小程序是否支持所使用的 method
          config.method = config.method.toUpperCase()
          if (supportedMethods.indexOf(config.method) === -1) {
            return reject(createError('Unsupported request method', config))
          }
        }
      }

      wx[requestType]({
        ...request,
        success: response => {
          // 按照 axios 的返回格式构造返回值
          settle(resolve, reject, {
            data: response.data || response.tempFilePath,
            status: response.statusCode,
            headers: response.header,
            config,
            request
          })
        },
        fail: response => {
          let err = response.errMsg

          // 增加错误消息可读性
          if (err === 'request:fail') {
            err = 'Request failed'
          }

          reject(createError(err, config))
        }
      })
    })
  }
}
<|endoftext|>


<commit_before>'use strict'

import settle from 'axios/lib/core/settle'
import buildURL from 'axios/lib/helpers/buildURL'
import createError from 'axios/lib/core/createError'

const supportedMethods = ['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT']

export default function wxaAdapterFactory (axios) {
  return function wxaAdapter (config) {
    return new Promise((resolve, reject) => {
      // 必须使用 https
      if (config.url.slice(0, 5).toLowerCase() !== 'https') {
        reject(createError('Requesting an http URL is not allowed', config))
      }

      if (config.responseType && config.responseType !== 'json' && config.responseType !== 'text') {
        // 只接受 json 或 text 的返回类型
        reject(createError('responseType only supports "json" or "text"', config))
      } else if (config.responseType === 'text' && config.transformResponse[0] === axios.defaults.transformResponse[0]) {
        // axios 默认会尝试将文本转为 json
        // 这里保证返回类型为 text 时返回的一定是文本
        config.transformResponse = []
      }

      // 检查小程序是否支持所使用的 method
      config.method = config.method.toUpperCase()
      if (supportedMethods.indexOf(config.method) === -1) {
        reject(createError('Unsupported request method', config))
      }

      // 生成用于 wx.request 的参数
      const request = {
        method: config.method.toUpperCase(),
        url: buildURL(config.url, config.params, config.paramsSerializer),
        data: config.data,
        header: config.headers,
        dataType: config.responseType
      }

      wx.request({
        ...request,
        success: response => {
          // 按照 axios 的返回格式构造返回值
          settle(resolve, reject, {
            data: response.data,
            status: response.statusCode,
            headers: response.header,
            config,
            request
          })
        },
        fail: response => {
          let err = response.errMsg

          // 增加错误消息可读性
          if (err === 'request:fail') {
            err = 'Request failed'
          }

          reject(createError(err, config))
        }
      })
    })
  }
}
<commit_msg>feat: 添加 jsdoc 注释<commit_after>'use strict'

import settle from 'axios/lib/core/settle'
import buildURL from 'axios/lib/helpers/buildURL'
import createError from 'axios/lib/core/createError'

const supportedMethods = ['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT']

/**
 * 创建用于微信小程序的 axios adapter
 *
 * @param {AxiosInstance} axios 原始 axios 对象
 * @returns {AxiosAdapter} 用于微信小程序的 axios adapter
 */
export default function wxaAdapterFactory (axios) {
  /**
   * 用于微信小程序的 axios adapter
   *
   * @type {AxiosAdapter}
   */
  return function wxaAdapter (config) {
    return new Promise((resolve, reject) => {
      // 必须使用 https
      if (config.url.slice(0, 5).toLowerCase() !== 'https') {
        reject(createError('Requesting an http URL is not allowed', config))
      }

      if (config.responseType && config.responseType !== 'json' && config.responseType !== 'text') {
        // 只接受 json 或 text 的返回类型
        reject(createError('responseType only supports "json" or "text"', config))
      } else if (config.responseType === 'text' && config.transformResponse[0] === axios.defaults.transformResponse[0]) {
        // axios 默认会尝试将文本转为 json
        // 这里保证返回类型为 text 时返回的一定是文本
        config.transformResponse = []
      }

      // 检查小程序是否支持所使用的 method
      config.method = config.method.toUpperCase()
      if (supportedMethods.indexOf(config.method) === -1) {
        reject(createError('Unsupported request method', config))
      }

      // 生成用于 wx.request 的参数
      const request = {
        method: config.method.toUpperCase(),
        url: buildURL(config.url, config.params, config.paramsSerializer),
        data: config.data,
        header: config.headers,
        dataType: config.responseType
      }

      wx.request({
        ...request,
        success: response => {
          // 按照 axios 的返回格式构造返回值
          settle(resolve, reject, {
            data: response.data,
            status: response.statusCode,
            headers: response.header,
            config,
            request
          })
        },
        fail: response => {
          let err = response.errMsg

          // 增加错误消息可读性
          if (err === 'request:fail') {
            err = 'Request failed'
          }

          reject(createError(err, config))
        }
      })
    })
  }
}
<|endoftext|>


<commit_before>var net = require('net');
var Server = require('./server');
var JID = require('./jid');
var StreamShaper = require('./../stream_shaper');
var StringPrep = require('node-stringprep').StringPrep;
var nameprep = new StringPrep('nameprep');

dbgStream = function(tag, stream) {
    stream.on('data', function(data) {
	console.log(tag + ' in: ' + data);
    });
    stream.on('error', function(e) {
	console.log(tag + ' error: ' + e.stack);
    });
    stream.on('close', function() {
	console.log(tag + ' close');
    });
    var oldSend = stream.send;
    stream.send = function(data) {
	console.log(tag + ' out: ' + data);
	oldSend.call(stream, data);
    };
};

/**
 * Represents a domain we host with connections to federated servers
 */
function DomainContext(router, domain) {
    this.router = router;
    this.domain = domain;
    this.s2sIn = {};
    this.s2sOut = {};
}

/**
 * Buffers until stream has been verified via Dialback
 */
DomainContext.prototype.send = function(stanza) {
    if (stanza.root)
	stanza = stanza.root();

    // TODO: return on empty to
    destDomain = new JID.JID(stanza.attrs.to).domain;
    var outStream = this.getOutStream(destDomain);

    if (outStream.isVerified)
	outStream.send(stanza);
    else {
	outStream.queue = outStream.queue || [];
	outStream.queue.push(stanza);
    }
};

/**
 * Does only buffer until stream is established, used for Dialback
 * communication itself.
 *
 * returns the stream
 */
DomainContext.prototype.sendRaw = function(stanza, destDomain) {
    if (stanza.root)
	stanza = stanza.root();

    var outStream = this.getOutStream(destDomain);
    var send = function() {
	outStream.send(stanza);
    };

    if (outStream.isOnline)
	send();
    else
	outStream.addListener('online', send);

    return outStream;
};

DomainContext.prototype.getOutStream = function(domain) {
    var self = this;

    // unfortunately we cannot use the incoming streams

    if (this.s2sOut.hasOwnProperty(domain)) {
	// There's one already
	return this.s2sOut[domain];
    } else {
	console.log("OUTGOING to " + domain);
	// Setup a new outgoing connection
	var outStream = Server.makeOutgoingServer(domain);
	this.s2sOut[domain] = outStream;
	dbgStream('outgoing', outStream);

	this.router.setupStream(outStream);
	this.setupStream(domain, outStream);

	outStream.addListener('close', function() {
	    // purge queue
	    if (outStream.queue) {
		outStream.queue.forEach(function(stanza) {
					    // do not provoke ping-pong effects
					    if (stanza.attrs.type === 'error')
						return;

					    var dest = stanza.attrs.to;
					    stanza.attrs.to = stanza.attrs.from;
					    stanza.attrs.from = dest;
					    stanza.attrs.type = 'error';
					    // TODO: add <error/> child if any programs wants to use that
					    self.receive(stanza);
					});
	    }
	    delete outStream.queue;

	    // remove from DomainContext
	    delete self.s2sOut[domain];
	});

	// Prepare dialback
	outStream.addListener('online', function() {
	    outStream.isOnline = true;
	    outStream.dbKey = generateKey();
	    outStream.send(Server.dialbackKey(self.domain, domain, outStream.dbKey));
	});
	outStream.addListener('dialbackResult', function(from, to, isValid) {
	    if (isValid) {
		outStream.isVerified = true;
		if (outStream.queue) {
		    outStream.queue.forEach(function(stanza) {
			outStream.send(stanza);
		    });
		    delete outStream.queue;
		}
	    } else {
		// we cannot do anything else with this stream that
		// failed dialback
		outStream.end();
	    }
	});

	return outStream;
    }
};

/**
 * Called by router when verification is done
 */
DomainContext.prototype.addInStream = function(domain, stream) {
    var self = this;

    if (this.s2sIn.hasOwnProperty(domain)) {
	// Replace old
	var oldStream = this.s2sIn[domain];
	oldStream.error('conflict', 'Connection replaced');
    }

    this.setupStream(domain, stream);
    stream.isOnline = true;
    stream.isVerified = true;
    stream.addListener('close', function() {
	if (self.s2sIn[domain] == stream)
	    delete self.s2sIn[domain];
    });
    this.s2sIn[domain] = stream;
};

DomainContext.prototype.setupStream = function(domain, stream) {
    var self = this;

    stream.addListener('stanza', function(stanza) {
	// Before verified they can send whatever they want
	if (!stream.isVerified)
	    return;

	if (!(typeof stanza.attrs.from === 'string' &&
	      typeof stanza.attrs.to === 'string')) {
	    stream.error('improper-addressing');
	    return;
	}

	// Only accept 'from' attribute JIDs that have the same domain
	// that we validated the stream for
	var fromDomain = (new JID.JID(stanza.attrs.from)).domain;
	if (fromDomain !== domain) {
	    stream.error('invalid-from');
	    return;
	}

	// Only accept 'to' attribute JIDs to this DomainContext
	var toDomain = (new JID.JID(stanza.attrs.to)).domain;
	if (toDomain !== self.domain) {
	    stream.error('improper-addressing');
	    return;
	}

	self.receive(stanza);
    });
};

DomainContext.prototype.verifyDialback = function(domain, id, key) {
    var outStream;
    if (this.s2sOut.hasOwnProperty(domain) &&
	(outStream = this.s2sOut[domain])) {

	var isValid = outStream.streamAttrs.id === id &&
	    outStream.dbKey === key;

	return isValid;
    } else
	return false;
};

DomainContext.prototype.receive = function(stanza) {
    if (this.stanzaListener)
	this.stanzaListener(stanza);
};

/**
 * TODO:
 * * connect timeouts, stream idle timeouts
 * * TLS
 */
function Router(s2sPort) {
    var self = this;
    this.ctxs = {};

    net.createServer(function(inStream) {
	console.log('INCOMING from ' + inStream.remoteAddress);
	self.acceptConnection(inStream);
    }).listen(s2sPort || 5269);
}
exports.Router = Router;

Router.prototype.rateLimit = 1E6;  // 100 KB/s, it's S2S after all
Router.prototype.maxStanzaSize = 65536;  // 64 KB, by convention
Router.prototype.keepAlive = 30 * 1000;  // 30s

Router.prototype.acceptConnection = function(inStream) {
    var self = this;

    dbgStream('incoming', inStream);
    this.setupStream(inStream);
    Server.makeIncomingServer(inStream);

    // incoming server wants to verify an outgoing connection of ours
    inStream.addListener('dialbackVerify', function(from, to, id, key) {
	from = nameprep(from);
	to = nameprep(to);
	var isValid = self.verifyDialback(from, to, id, key);
	inStream.send(Server.dialbackVerified(to, from, id, isValid));
    });
    // incoming connection wants to get verified
    inStream.addListener('dialbackKey', function(from, to, key) {
	from = nameprep(from);
	to = nameprep(to);
	var destDomain = to;
	if (self.hasContext(to)) {
	    var ctx = self.getContext(to);
	    var outStream = ctx.sendRaw(Server.dialbackVerify(to, from, inStream.streamId, key),
					from);

	    // TODO: hook inStream close
	    // these are needed before for removeListener()
	    var onVerified, onClose;
	    onVerified = function(from, to, id, isValid) {
		from = nameprep(from);
		to = nameprep(to);
		if (to !== destDomain ||
		    id != inStream.streamId)  // not for us
		    return;

		inStream.send(Server.dialbackResult(to, from, isValid));

		if (isValid && self.hasContext(to)) {
		    self.getContext(to).addInStream(from, inStream);
		} else {
		    // the connection isn't used for another domain, so
		    // closing is safe
		    inStream.send('</stream:stream>');
		    inStream.end();
		}

		outStream.removeListener('dialbackVerified', onVerified);
		outStream.removeListener('close', onClose);
	    };
	    onClose = function() {
		// outgoing connection didn't work out, tell the incoming
		// connection
		inStream.send(Server.dialbackResult(to, from, false));
	    };
	    outStream.addListener('dialbackVerified', onVerified);
	    outStream.addListener('close', onClose);
	} else {
	    inStream.error('host-unknown', to + ' is not served here');
	}
    });
};

Router.prototype.setupStream = function(stream) {
    stream.maxStanzaSize = this.maxStanzaSize;
    StreamShaper.attach(stream, this.rateLimit);
    stream.setKeepAlive(true, this.keepAlive);
};

/**
 * Create domain context & register a stanza listener callback
 */
Router.prototype.register = function(domain, listener) {
    domain = nameprep(domain);
    this.getContext(domain).stanzaListener = listener;
};

// TODO: unregister w/ connections teardown

Router.prototype.send = function(stanza) {
    if (stanza.root)
	stanza = stanza.root();

    if (stanza.attrs && stanza.attrs.from) {
	var domain = (new JID.JID(stanza.attrs.from)).domain;
	this.getContext(domain).send(stanza);
    } else
	throw 'Sending stanza without destination';
};

Router.prototype.hasContext = function(domain) {
    return this.ctxs.hasOwnProperty(domain);
};

Router.prototype.getContext = function(domain) {
    if (this.ctxs.hasOwnProperty(domain))
	return this.ctxs[domain];
    else
	return (this.ctxs[domain] = new DomainContext(this, domain));
};

Router.prototype.verifyDialback = function(from, to, id, key) {
    return this.hasContext(to) &&
	this.getContext(to).verifyDialback(from, id, key);
};

function generateKey() {
    var r = new Buffer(16);
    for(var i = 0; i < r.length; i++) {
	r[i] = 48 + Math.floor(Math.random() * 10);  // '0'..'9'
    }
    return r.toString();
}
<commit_msg>router: fix nameprepping<commit_after>var net = require('net');
var Server = require('./server');
var JID = require('./jid');
var StreamShaper = require('./../stream_shaper');
var StringPrep = require('node-stringprep').StringPrep;
var nameprep = new StringPrep('nameprep');

dbgStream = function(tag, stream) {
    stream.on('data', function(data) {
	console.log(tag + ' in: ' + data);
    });
    stream.on('error', function(e) {
	console.log(tag + ' error: ' + e.stack);
    });
    stream.on('close', function() {
	console.log(tag + ' close');
    });
    var oldSend = stream.send;
    stream.send = function(data) {
	console.log(tag + ' out: ' + data);
	oldSend.call(stream, data);
    };
};

/**
 * Represents a domain we host with connections to federated servers
 */
function DomainContext(router, domain) {
    this.router = router;
    this.domain = domain;
    this.s2sIn = {};
    this.s2sOut = {};
}

/**
 * Buffers until stream has been verified via Dialback
 */
DomainContext.prototype.send = function(stanza) {
    if (stanza.root)
	stanza = stanza.root();

    // TODO: return on empty to
    destDomain = new JID.JID(stanza.attrs.to).domain;
    var outStream = this.getOutStream(destDomain);

    if (outStream.isVerified)
	outStream.send(stanza);
    else {
	outStream.queue = outStream.queue || [];
	outStream.queue.push(stanza);
    }
};

/**
 * Does only buffer until stream is established, used for Dialback
 * communication itself.
 *
 * returns the stream
 */
DomainContext.prototype.sendRaw = function(stanza, destDomain) {
    if (stanza.root)
	stanza = stanza.root();

    var outStream = this.getOutStream(destDomain);
    var send = function() {
	outStream.send(stanza);
    };

    if (outStream.isOnline)
	send();
    else
	outStream.addListener('online', send);

    return outStream;
};

DomainContext.prototype.getOutStream = function(domain) {
    var self = this;

    // unfortunately we cannot use the incoming streams

    if (this.s2sOut.hasOwnProperty(domain)) {
	// There's one already
	return this.s2sOut[domain];
    } else {
	console.log("OUTGOING to " + domain);
	// Setup a new outgoing connection
	var outStream = Server.makeOutgoingServer(domain);
	this.s2sOut[domain] = outStream;
	dbgStream('outgoing', outStream);

	this.router.setupStream(outStream);
	this.setupStream(domain, outStream);

	outStream.addListener('close', function() {
	    // purge queue
	    if (outStream.queue) {
		outStream.queue.forEach(function(stanza) {
					    // do not provoke ping-pong effects
					    if (stanza.attrs.type === 'error')
						return;

					    var dest = stanza.attrs.to;
					    stanza.attrs.to = stanza.attrs.from;
					    stanza.attrs.from = dest;
					    stanza.attrs.type = 'error';
					    // TODO: add <error/> child if any programs wants to use that
					    self.receive(stanza);
					});
	    }
	    delete outStream.queue;

	    // remove from DomainContext
	    delete self.s2sOut[domain];
	});

	// Prepare dialback
	outStream.addListener('online', function() {
	    outStream.isOnline = true;
	    outStream.dbKey = generateKey();
	    outStream.send(Server.dialbackKey(self.domain, domain, outStream.dbKey));
	});
	outStream.addListener('dialbackResult', function(from, to, isValid) {
	    if (isValid) {
		outStream.isVerified = true;
		if (outStream.queue) {
		    outStream.queue.forEach(function(stanza) {
			outStream.send(stanza);
		    });
		    delete outStream.queue;
		}
	    } else {
		// we cannot do anything else with this stream that
		// failed dialback
		outStream.end();
	    }
	});

	return outStream;
    }
};

/**
 * Called by router when verification is done
 */
DomainContext.prototype.addInStream = function(domain, stream) {
    var self = this;

    if (this.s2sIn.hasOwnProperty(domain)) {
	// Replace old
	var oldStream = this.s2sIn[domain];
	oldStream.error('conflict', 'Connection replaced');
    }

    this.setupStream(domain, stream);
    stream.isOnline = true;
    stream.isVerified = true;
    stream.addListener('close', function() {
	if (self.s2sIn[domain] == stream)
	    delete self.s2sIn[domain];
    });
    this.s2sIn[domain] = stream;
};

DomainContext.prototype.setupStream = function(domain, stream) {
    var self = this;

    stream.addListener('stanza', function(stanza) {
	// Before verified they can send whatever they want
	if (!stream.isVerified)
	    return;

	if (!(typeof stanza.attrs.from === 'string' &&
	      typeof stanza.attrs.to === 'string')) {
	    stream.error('improper-addressing');
	    return;
	}

	// Only accept 'from' attribute JIDs that have the same domain
	// that we validated the stream for
	var fromDomain = (new JID.JID(stanza.attrs.from)).domain;
	if (fromDomain !== domain) {
	    stream.error('invalid-from');
	    return;
	}

	// Only accept 'to' attribute JIDs to this DomainContext
	var toDomain = (new JID.JID(stanza.attrs.to)).domain;
	if (toDomain !== self.domain) {
	    stream.error('improper-addressing');
	    return;
	}

	self.receive(stanza);
    });
};

DomainContext.prototype.verifyDialback = function(domain, id, key) {
    var outStream;
    if (this.s2sOut.hasOwnProperty(domain) &&
	(outStream = this.s2sOut[domain])) {

	var isValid = outStream.streamAttrs.id === id &&
	    outStream.dbKey === key;

	return isValid;
    } else
	return false;
};

DomainContext.prototype.receive = function(stanza) {
    if (this.stanzaListener)
	this.stanzaListener(stanza);
};

/**
 * TODO:
 * * connect timeouts, stream idle timeouts
 * * TLS
 */
function Router(s2sPort) {
    var self = this;
    this.ctxs = {};

    net.createServer(function(inStream) {
	console.log('INCOMING from ' + inStream.remoteAddress);
	self.acceptConnection(inStream);
    }).listen(s2sPort || 5269);
}
exports.Router = Router;

Router.prototype.rateLimit = 1E6;  // 100 KB/s, it's S2S after all
Router.prototype.maxStanzaSize = 65536;  // 64 KB, by convention
Router.prototype.keepAlive = 30 * 1000;  // 30s

Router.prototype.acceptConnection = function(inStream) {
    var self = this;

    dbgStream('incoming', inStream);
    this.setupStream(inStream);
    Server.makeIncomingServer(inStream);

    // incoming server wants to verify an outgoing connection of ours
    inStream.addListener('dialbackVerify', function(from, to, id, key) {
	from = nameprep.prepare(from);
	to = nameprep.prepare(to);
	var isValid = self.verifyDialback(from, to, id, key);
	inStream.send(Server.dialbackVerified(to, from, id, isValid));
    });
    // incoming connection wants to get verified
    inStream.addListener('dialbackKey', function(from, to, key) {
	from = nameprep.prepare(from);
	to = nameprep.prepare(to);
	var destDomain = to;
	if (self.hasContext(to)) {
	    var ctx = self.getContext(to);
	    var outStream = ctx.sendRaw(Server.dialbackVerify(to, from, inStream.streamId, key),
					from);

	    // TODO: hook inStream close
	    // these are needed before for removeListener()
	    var onVerified, onClose;
	    onVerified = function(from, to, id, isValid) {
		from = nameprep.prepare(from);
		to = nameprep.prepare(to);
		if (to !== destDomain ||
		    id != inStream.streamId)  // not for us
		    return;

		inStream.send(Server.dialbackResult(to, from, isValid));

		if (isValid && self.hasContext(to)) {
		    self.getContext(to).addInStream(from, inStream);
		} else {
		    // the connection isn't used for another domain, so
		    // closing is safe
		    inStream.send('</stream:stream>');
		    inStream.end();
		}

		outStream.removeListener('dialbackVerified', onVerified);
		outStream.removeListener('close', onClose);
	    };
	    onClose = function() {
		// outgoing connection didn't work out, tell the incoming
		// connection
		inStream.send(Server.dialbackResult(to, from, false));
	    };
	    outStream.addListener('dialbackVerified', onVerified);
	    outStream.addListener('close', onClose);
	} else {
	    inStream.error('host-unknown', to + ' is not served here');
	}
    });
};

Router.prototype.setupStream = function(stream) {
    stream.maxStanzaSize = this.maxStanzaSize;
    StreamShaper.attach(stream, this.rateLimit);
    stream.setKeepAlive(true, this.keepAlive);
};

/**
 * Create domain context & register a stanza listener callback
 */
Router.prototype.register = function(domain, listener) {
    domain = nameprep.prepare(domain);
    this.getContext(domain).stanzaListener = listener;
};

// TODO: unregister w/ connections teardown

Router.prototype.send = function(stanza) {
    if (stanza.root)
	stanza = stanza.root();

    if (stanza.attrs && stanza.attrs.from) {
	var domain = (new JID.JID(stanza.attrs.from)).domain;
	this.getContext(domain).send(stanza);
    } else
	throw 'Sending stanza without destination';
};

Router.prototype.hasContext = function(domain) {
    return this.ctxs.hasOwnProperty(domain);
};

Router.prototype.getContext = function(domain) {
    if (this.ctxs.hasOwnProperty(domain))
	return this.ctxs[domain];
    else
	return (this.ctxs[domain] = new DomainContext(this, domain));
};

Router.prototype.verifyDialback = function(from, to, id, key) {
    return this.hasContext(to) &&
	this.getContext(to).verifyDialback(from, id, key);
};

function generateKey() {
    var r = new Buffer(16);
    for(var i = 0; i < r.length; i++) {
	r[i] = 48 + Math.floor(Math.random() * 10);  // '0'..'9'
    }
    return r.toString();
}
<|endoftext|>


<commit_before>'use strict';

var dbg = 0;
var PATH = require('path'),
    BEM = require('bem'),
    Q = BEM.require('q'),
    ymPath = require.resolve('ym'),
    __assert = require('assert'),
    Deps = require('bem/lib/techs/v2/deps.js').Deps;

exports.baseTechName = 'vanilla.js';

exports.techMixin = {

    getWeakBuildSuffixesMap:function(){
        return { 'js' : ['vanilla.js', 'browser.js', 'js'] };
    },

    getBuildSuffixesMap:function(){
        return { 'js' : ['browser.js', 'js'] };
    },

    getCreateSuffixes : function() {
        return ['browser.js'];
    },

    transformBuildDecl: function(decl, adjacentTechs) {
        var _this = this;

        var decl_ = decl.then(function(decl){
            if(!decl.depsFull) {
                return decl;
            }

            var depsGraph = new Deps().parseDepsDecl(decl);

            var includeTechs = {
                'browser.js': true,
                'vanilla.js': true
            };

            adjacentTechs = adjacentTechs || {};

            var adjacentRoots = [];
            Object.keys(adjacentTechs).forEach(function(tech){
                adjacentRoots[tech] = [];
            });

            var                 uniq = {
                'browser.js':{},
                'vanilla.js':{},
                '':{}
            };
            var items = [];
            _walkN([depsGraph.rootItem.buildKey()], 'browser.js');

            return {deps:items,
              adjacentRoots:adjacentRoots};

            function _walkN(keys, buildTech){
                keys.forEach(function(key){
                    if(!uniq[buildTech][key]){
                        uniq[buildTech][key] = 1;
dbg && console.log('(walk',key);
                        var item = depsGraph.items[key];
                        var itemTech = item.item.tech;
                        if(!itemTech){
                            _walk1$(key, buildTech);
                        } else {
                            if (itemTech === buildTech) {
                                _walk1t(key, buildTech);
                            } else if (itemTech in includeTechs){
                                _walk1t(key, itemTech);
                            } else {
                                // skip item

dbg && console.log('(skip)');
                                if(itemTech in adjacentRoots){
dbg && console.log('(roots-push',key,')')
                                    adjacentRoots[itemTech].push(item)
                                }
                            }
                        }
dbg && console.log(')');
                    }
                })
            }

            function _walk1$(key, buildTech){
                var item = depsGraph.items[key];
                __assert(!item.item.tech);

                var item1 = item.clone();
                delete item1.key;
                item1.item.tech = buildTech;
                var key1 = item1.buildKey();
dbg && console.log('(walk-wild',key)
                if(depsGraph.items[key1] && !uniq[buildTech][key1]){
                    uniq[buildTech][key1] = 2;
                    __walk1t(key1, buildTech);
                }
                if(uniq[buildTech][key] === 1){
                    uniq[buildTech][key] = 2;
                    __walk1$(key, buildTech);
                }
dbg && console.log(')');
            }

            function _walk1t(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item.item.tech);

                var item1 = item.clone();
                delete item1.key;
                delete item1.item.tech;
                var key1 = item1.buildKey();
dbg && console.log('(walk-tech',key);
                if(!uniq[buildTech][key] === 1){
                    uniq[buildTech][key] = 2;
                    __walk1t(key, buildTech);
                }
                if(depsGraph.items[key1] && !uniq[buildTech][key1]){
                    uniq[buildTech][key1] = 2;
                    __walk1$(key1, buildTech);
                }
dbg && console.log(')');
            }

            function __walk1$(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item && !item.item.tech);
                _walkN(item.mustDeps, buildTech);
                _include(item, buildTech);
                _walkN(item.shouldDeps, buildTech);
            }

            function __walk1t(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item && item.item.tech === buildTech);
                _walkN(item.mustDeps, buildTech);
                _include(item, buildTech);
                _walkN(item.shouldDeps, buildTech);
            }

            function _include(item, buildTech){
                var key = item.buildKey();
                if(uniq[buildTech][key] === 2){
                    uniq[buildTech][key] = 3;

                    if(!item.item.tech){
                        item = item.clone();
                        delete item.key;
                        item.item.tech = buildTech;
                        key = item.buildKey();
                    }
                    if(!uniq[''][key]){
                        uniq[''][key] = 1;
                        items.push(item.item);
dbg && console.log('(include',key,')');
                    }
                }
            }
        })

        decl_.done();
        return decl_;

    },

    getYmChunk : function(output) {
        var outputDir = PATH.resolve(output, '..');
        var ymRelPath = PATH.relative(outputDir, ymPath);
        return this.getBuildResultChunk(ymRelPath, ymPath);
    },

    getBuildResult : function(files, suffix, output) {
        return Q.all([
                this.getYmChunk(output),
                this.__base.apply(this, arguments)
            ])
            .spread(function(ym, res) {
                return [ym].concat(res);
            });
    }

};

<commit_msg>undebug<commit_after>'use strict';

var PATH = require('path'),
    BEM = require('bem'),
    Q = BEM.require('q'),
    ymPath = require.resolve('ym'),
    __assert = require('assert'),
    Deps = require('bem/lib/techs/v2/deps.js').Deps;

exports.baseTechName = 'vanilla.js';

exports.techMixin = {

    getWeakBuildSuffixesMap:function(){
        return { 'js' : ['vanilla.js', 'browser.js', 'js'] };
    },

    getBuildSuffixesMap:function(){
        return { 'js' : ['browser.js', 'js'] };
    },

    getCreateSuffixes : function() {
        return ['browser.js'];
    },

    transformBuildDecl: function(decl, adjacentTechs) {
        var _this = this;

        var decl_ = decl.then(function(decl){
            if(!decl.depsFull) {
                return decl;
            }

            var depsGraph = new Deps().parseDepsDecl(decl);

            var includeTechs = {
                'browser.js': true,
                'vanilla.js': true
            };

            adjacentTechs = adjacentTechs || {};

            var adjacentRoots = [];
            Object.keys(adjacentTechs).forEach(function(tech){
                adjacentRoots[tech] = [];
            });

            var                 uniq = {
                'browser.js':{},
                'vanilla.js':{},
                '':{}
            };
            var items = [];
            _walkN([depsGraph.rootItem.buildKey()], 'browser.js');

            return {deps:items,
              adjacentRoots:adjacentRoots};

            function _walkN(keys, buildTech){
                keys.forEach(function(key){
                    if(!uniq[buildTech][key]){
                        uniq[buildTech][key] = 1;
                        var item = depsGraph.items[key];
                        var itemTech = item.item.tech;
                        if(!itemTech){
                            _walk1$(key, buildTech);
                        } else {
                            if (itemTech === buildTech) {
                                _walk1t(key, buildTech);
                            } else if (itemTech in includeTechs){
                                _walk1t(key, itemTech);
                            } else {
                                // skip item

                                if(itemTech in adjacentRoots){
                                    adjacentRoots[itemTech].push(item)
                                }
                            }
                        }
                    }
                })
            }

            function _walk1$(key, buildTech){
                var item = depsGraph.items[key];
                __assert(!item.item.tech);

                var item1 = item.clone();
                delete item1.key;
                item1.item.tech = buildTech;
                var key1 = item1.buildKey();
                if(depsGraph.items[key1] && !uniq[buildTech][key1]){
                    uniq[buildTech][key1] = 2;
                    __walk1t(key1, buildTech);
                }
                if(uniq[buildTech][key] === 1){
                    uniq[buildTech][key] = 2;
                    __walk1$(key, buildTech);
                }
            }

            function _walk1t(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item.item.tech);

                var item1 = item.clone();
                delete item1.key;
                delete item1.item.tech;
                var key1 = item1.buildKey();
                if(!uniq[buildTech][key] === 1){
                    uniq[buildTech][key] = 2;
                    __walk1t(key, buildTech);
                }
                if(depsGraph.items[key1] && !uniq[buildTech][key1]){
                    uniq[buildTech][key1] = 2;
                    __walk1$(key1, buildTech);
                }
            }

            function __walk1$(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item && !item.item.tech);
                _walkN(item.mustDeps, buildTech);
                _include(item, buildTech);
                _walkN(item.shouldDeps, buildTech);
            }

            function __walk1t(key, buildTech){
                var item = depsGraph.items[key];
                __assert(item && item.item.tech === buildTech);
                _walkN(item.mustDeps, buildTech);
                _include(item, buildTech);
                _walkN(item.shouldDeps, buildTech);
            }

            function _include(item, buildTech){
                var key = item.buildKey();
                if(uniq[buildTech][key] === 2){
                    uniq[buildTech][key] = 3;

                    if(!item.item.tech){
                        item = item.clone();
                        delete item.key;
                        item.item.tech = buildTech;
                        key = item.buildKey();
                    }
                    if(!uniq[''][key]){
                        uniq[''][key] = 1;
                        items.push(item.item);
                    }
                }
            }
        })

        decl_.done();
        return decl_;

    },

    getYmChunk : function(output) {
        var outputDir = PATH.resolve(output, '..');
        var ymRelPath = PATH.relative(outputDir, ymPath);
        return this.getBuildResultChunk(ymRelPath, ymPath);
    },

    getBuildResult : function(files, suffix, output) {
        return Q.all([
                this.getYmChunk(output),
                this.__base.apply(this, arguments)
            ])
            .spread(function(ym, res) {
                return [ym].concat(res);
            });
    }

};

<|endoftext|>