<commit_before>var util = require('util'),
    errors = require('../errors'),
    extendWithGettersAndSetters = require('../util/extendWithGettersAndSetters'),
    Text = require('./Text'),
    AssetGraph = require('../');

function SrcSet(config) {
    Text.call(this, config);
    if (!this._parseTree && !('_text' in this)) {
        throw new Error('SrcSet: Either parseTree or text must be specified');
    }
}

util.inherits(SrcSet, Text);

extendWithGettersAndSetters(SrcSet.prototype, {
    type: 'SrcSet',

    contentType: null, // Avoid reregistering application/octet-stream

    supportedExtensions: [],

    get parseTree() {
        if (!this._parseTree) {
            this._parseTree = [];
            this.text.split(/,/).forEach(function (entryStr) {
                entryStr = entryStr.replace(/^\s+|\s+$/g, '');
                var matchEntryStr = entryStr.match(/^([^\s]+)(.*)$/);
                if (matchEntryStr) {
                    var extraTokens = matchEntryStr[2].split(/\s+/).filter(function (extraToken) {
                        return !/^\s*$/.test(extraToken);
                    });
                    this._parseTree.push({
                        href: matchEntryStr[1],
                        extraTokens: extraTokens
                    });
                } else {
                    var warning = new errors.SyntaxError({message: 'SrcSet: Could not parse entry: ' + entry, asset: this});
                    if (this.assetGraph) {
                        this.assetGraph.emit('warn', warning);
                    } else {
                        console.warn(this.toString() + ': ' + warning.message);
                    }
                }
            }, this);
        }
        return this._parseTree;
    },

    set parseTree(parseTree) {
        this.unload();
        this._parseTree = parseTree;
        this.markDirty();
    },

    get text() {
        if (!('_text' in this)) {
            // We're in trouble if neither this._text, nor this._parseTree is found.
            this._text = this.parseTree.map(function (node) {
                return node.href + (node.extraTokens.length > 0 ? ' ' + node.extraTokens.join(' ') : '');
            }).join(', ');
        }
        return this._text;
    },

    findOutgoingRelationsInParseTree: function () {
        return this.parseTree.map(function (node) {
            return new AssetGraph.SrcSetEntry({
                from: this,
                node: node,
                to: {
                    url: node.href
                }
            });
        }, this);
    }
});

module.exports = SrcSet;
<commit_msg>Fix: SrcSet was referencing an undefined variable<commit_after>var util = require('util'),
    errors = require('../errors'),
    extendWithGettersAndSetters = require('../util/extendWithGettersAndSetters'),
    Text = require('./Text'),
    AssetGraph = require('../');

function SrcSet(config) {
    Text.call(this, config);
    if (!this._parseTree && !('_text' in this)) {
        throw new Error('SrcSet: Either parseTree or text must be specified');
    }
}

util.inherits(SrcSet, Text);

extendWithGettersAndSetters(SrcSet.prototype, {
    type: 'SrcSet',

    contentType: null, // Avoid reregistering application/octet-stream

    supportedExtensions: [],

    get parseTree() {
        if (!this._parseTree) {
            this._parseTree = [];
            this.text.split(/,/).forEach(function (entryStr) {
                entryStr = entryStr.replace(/^\s+|\s+$/g, '');
                var matchEntryStr = entryStr.match(/^([^\s]+)(.*)$/);
                if (matchEntryStr) {
                    var extraTokens = matchEntryStr[2].split(/\s+/).filter(function (extraToken) {
                        return !/^\s*$/.test(extraToken);
                    });
                    this._parseTree.push({
                        href: matchEntryStr[1],
                        extraTokens: extraTokens
                    });
                } else {
                    var warning = new errors.SyntaxError({message: 'SrcSet: Could not parse entry: ' + entryStr, asset: this});
                    if (this.assetGraph) {
                        this.assetGraph.emit('warn', warning);
                    } else {
                        console.warn(this.toString() + ': ' + warning.message);
                    }
                }
            }, this);
        }
        return this._parseTree;
    },

    set parseTree(parseTree) {
        this.unload();
        this._parseTree = parseTree;
        this.markDirty();
    },

    get text() {
        if (!('_text' in this)) {
            // We're in trouble if neither this._text, nor this._parseTree is found.
            this._text = this.parseTree.map(function (node) {
                return node.href + (node.extraTokens.length > 0 ? ' ' + node.extraTokens.join(' ') : '');
            }).join(', ');
        }
        return this._text;
    },

    findOutgoingRelationsInParseTree: function () {
        return this.parseTree.map(function (node) {
            return new AssetGraph.SrcSetEntry({
                from: this,
                node: node,
                to: {
                    url: node.href
                }
            });
        }, this);
    }
});

module.exports = SrcSet;
<|endoftext|>


<commit_before>const resolvePreset = function (presetName) {
  return require.resolve(`babel-preset-${presetName}`);
};

const resolvePlugin = function (pluginName) {
  return require.resolve(`babel-plugin-${pluginName}`);
};

module.exports = function createOpts(env, react) {
  const production = !env.endsWith('-dev');

  if (!production) {
    env = env.slice(0, -4);
  }

  if (env === 'doc') {
    return {
      presets: [
        'jsdoc',
        'jsdoc/object-rest',
        react && 'react',
        resolvePreset('stage-1'),
      ].filter(Boolean),
      plugins: [
        'add-jsdoc-annotations',
        resolvePlugin('transform-flow-strip-types'),
      ]
    };
  }

  let presets;
  let browser;
  let modernBrowsers = false;

  switch (env) {
    case 'es5':
      throw new Error('use olderNode instead.');

    case 'test':
    case 'node6':
      presets = [
        'es2015-node6/object-rest',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = false;
      break;

    case 'older-node':
      presets = [
        'es2015',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = false;
      break;

    case 'webpack':
      presets = [
        ['es2015', { modules: false }],
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      break;

    case 'webpack-modern-browsers':
      presets = [
        ['modern-browsers', { modules: false, objectRest: true }],
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      modernBrowsers = true;
      break;

    case 'browsers':
      presets = [
        'es2015',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      break;

    default:
      throw new Error(`Unsupported env ${env}`);

  }

  return {
    presets: presets.filter(Boolean),
    plugins: [
      resolvePlugin('syntax-flow'),
      [resolvePlugin('import-rename'), { '^([a-z\\-]+)/src(.*)$': '$1$2' }],
      !production && resolvePlugin('tcomb-forked'),
      resolvePlugin('transform-flow-strip-types'),
      react && resolvePlugin('react-require'),
      // browser && 'react-hot-loader/babel',
      !production && react && resolvePlugin('transform-react-jsx-self'),
      !production && react && resolvePlugin('transform-react-jsx-source'),

      [resolvePlugin('minify-replace'), {
          replacements: [
              {
                  identifierName: 'PRODUCTION',
                  replacement: { type: 'booleanLiteral', value: production },
              },
              {
                  identifierName: 'BROWSER',
                  replacement: { type: 'booleanLiteral', value: browser },
              },
              {
                  identifierName: 'SERVER',
                  replacement: { type: 'booleanLiteral', value: !browser },
              },
              {
                  identifierName: 'NODEJS',
                  replacement: { type: 'booleanLiteral', value: !browser },
              },
          ],
      }],
      resolvePlugin('minify-constant-folding'),
      [resolvePlugin('minify-dead-code-elimination'), { keepFnName: true, keepFnames: true }],
      resolvePlugin('minify-guarded-expressions'),
      resolvePlugin('discard-module-references'),
    ].filter(Boolean).concat(createOpts.plugins || [])
  };
};
<commit_msg>import-rename allow internal src module, for tests<commit_after>const resolvePreset = function (presetName) {
  return require.resolve(`babel-preset-${presetName}`);
};

const resolvePlugin = function (pluginName) {
  return require.resolve(`babel-plugin-${pluginName}`);
};

module.exports = function createOpts(env, react) {
  const production = !env.endsWith('-dev');

  if (!production) {
    env = env.slice(0, -4);
  }

  if (env === 'doc') {
    return {
      presets: [
        'jsdoc',
        'jsdoc/object-rest',
        react && 'react',
        resolvePreset('stage-1'),
      ].filter(Boolean),
      plugins: [
        'add-jsdoc-annotations',
        resolvePlugin('transform-flow-strip-types'),
      ]
    };
  }

  let presets;
  let browser;
  let modernBrowsers = false;

  switch (env) {
    case 'es5':
      throw new Error('use olderNode instead.');

    case 'test':
    case 'node6':
      presets = [
        'es2015-node6/object-rest',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = false;
      break;

    case 'older-node':
      presets = [
        'es2015',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = false;
      break;

    case 'webpack':
      presets = [
        ['es2015', { modules: false }],
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      break;

    case 'webpack-modern-browsers':
      presets = [
        ['modern-browsers', { modules: false, objectRest: true }],
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      modernBrowsers = true;
      break;

    case 'browsers':
      presets = [
        'es2015',
        react && 'react',
        resolvePreset('stage-1'),
      ];
      browser = true;
      break;

    default:
      throw new Error(`Unsupported env ${env}`);

  }

  return {
    presets: presets.filter(Boolean),
    plugins: [
      resolvePlugin('syntax-flow'),
      [resolvePlugin('import-rename'), { '^([a-z\\-]+|[\./]+)/src(.*)$': '$1$2' }],
      !production && resolvePlugin('tcomb-forked'),
      resolvePlugin('transform-flow-strip-types'),
      react && resolvePlugin('react-require'),
      // browser && 'react-hot-loader/babel',
      !production && react && resolvePlugin('transform-react-jsx-self'),
      !production && react && resolvePlugin('transform-react-jsx-source'),

      [resolvePlugin('minify-replace'), {
          replacements: [
              {
                  identifierName: 'PRODUCTION',
                  replacement: { type: 'booleanLiteral', value: production },
              },
              {
                  identifierName: 'BROWSER',
                  replacement: { type: 'booleanLiteral', value: browser },
              },
              {
                  identifierName: 'SERVER',
                  replacement: { type: 'booleanLiteral', value: !browser },
              },
              {
                  identifierName: 'NODEJS',
                  replacement: { type: 'booleanLiteral', value: !browser },
              },
          ],
      }],
      resolvePlugin('minify-constant-folding'),
      [resolvePlugin('minify-dead-code-elimination'), { keepFnName: true, keepFnames: true }],
      resolvePlugin('minify-guarded-expressions'),
      resolvePlugin('discard-module-references'),
    ].filter(Boolean).concat(createOpts.plugins || [])
  };
};
<|endoftext|>


<commit_before>var babel = require("babel-core");
var assign = require('object.assign');

module.exports = function(source, compileOptions, options){
	var opts = assign({}, options.babelOptions, {
		sourceMap: compileOptions.sourceMaps || false
	});

	opts.presets = opts.presets || [
		["es2015", {"loose": false, "modules": compileOptions.modules}],
		"babel-preset-react",
		"babel-preset-stage-0"
	];
	opts.plugins = opts.plugins || [];

	// Remove Babel 5 options
	delete opts.optional;
	delete opts.whitelist;
	delete opts.blacklist;

	if(opts.sourceMap) {
		opts.sourceMapTarget = options.sourceMapFileName;
		opts.sourceFileName = options.sourceMapFileName;
	}

	var result = babel.transform(source, opts);
	return result;
};
<commit_msg>the need of require otherwise it will try to find the dependencies in the user's own node_modules, thanks to matthew<commit_after>var babel = require("babel-core");
var assign = require('object.assign');

module.exports = function(source, compileOptions, options){
	var opts = assign({}, options.babelOptions, {
		sourceMap: compileOptions.sourceMaps || false
	});

	opts.presets = opts.presets || [
		require("babel-preset-es2015").buildPreset({},{"loose": false, "modules": compileOptions.modules}),
		require("babel-preset-react"),
		require("babel-preset-stage-0")
	];
	opts.plugins = opts.plugins || [];

	// Remove Babel 5 options
	delete opts.optional;
	delete opts.whitelist;
	delete opts.blacklist;

	if(opts.sourceMap) {
		opts.sourceMapTarget = options.sourceMapFileName;
		opts.sourceFileName = options.sourceMapFileName;
	}

	var result = babel.transform(source, opts);
	return result;
};
<|endoftext|>


<commit_before>// Copyright (c) 2012-2016, Matt Godbolt
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

var child_process = require('child_process'),
    temp = require('temp'),
    path = require('path'),
    fs = require('fs-extra'),
    Promise = require('promise'), // jshint ignore:line
    asm = require('./asm'),
    utils = require('./utils'
),
quote = require('shell-quote'),
    _ = require('underscore-node'),
    logger = require('./logger').logger;

function Compile(compiler, env) {
    this.compiler = compiler;
    this.env = env;
    this.asm = new asm.AsmParser(env.compilerProps);
}

Compile.prototype.newTempDir = function () {
    return new Promise(function (resolve, reject) {
        temp.mkdir('gcc-explorer-compiler', function (err, dirPath) {
            if (err)
                reject("Unable to open temp file: " + err);
            else
                resolve(dirPath);
        });
    });
};

Compile.prototype.writeFile = Promise.denodeify(fs.writeFile);
Compile.prototype.readFile = Promise.denodeify(fs.readFile);
Compile.prototype.stat = Promise.denodeify(fs.stat);

Compile.prototype.getRemote = function () {
    if (this.compiler.exe === null && this.compiler.remote)
        return this.compiler.remote;
    return false;
};

Compile.prototype.runCompiler = function (compiler, options, inputFilename) {
    return this.exec(compiler, options, {
        timeoutMs: this.env.gccProps("compileTimeoutMs", 100),
        maxErrorOutput: this.env.gccProps("max-error-output", 5000),
        env: this.env.getEnv(this.compiler.needsMulti),
        wrapper: this.env.compilerProps("compiler-wrapper")
    }).then(function (result) {
        result.stdout = utils.parseOutput(result.stdout, inputFilename);
        result.stderr = utils.parseOutput(result.stderr, inputFilename);
        return result;
    });
};

Compile.prototype.supportsObjdump = function () {
    return true;
};

Compile.prototype.objdump = function (outputFilename, result, maxSize, intelAsm) {
    var args = ["-d", "-C", outputFilename, "-l", "--insn-width=16"];
    if (intelAsm) args = args.concat(["-M", "intel"]);
    return this.exec("objdump", args, {maxOutput: maxSize})
        .then(function (objResult) {
            result.asm = objResult.stdout;
            if (objResult.code !== 0) {
                result.asm = "<No output: objdump returned " + objResult.code + ">";
            }
            return result;
        });
};

Compile.prototype.filename = function (fn) {
    return fn;
};

Compile.prototype.optionsForFilter = function (filters, outputFilename) {
    var options = ['-g', '-o', this.filename(outputFilename)];
    if (this.compiler.intelAsm && filters.intel && !filters.binary) {
        options = options.concat(this.compiler.intelAsm.split(" "));
    }
    if (!filters.binary) options = options.concat('-S');
    return options;
};

Compile.prototype.prepareArguments = function (userOptions, filters, inputFilename, outputFilename) {
    var options = this.optionsForFilter(filters, outputFilename);
    if (this.compiler.options) {
        options = options.concat(this.compiler.options.split(" "));
    }
    return options.concat(userOptions || []).concat([this.filename(inputFilename)]);
};

Compile.prototype.compile = function (source, options, filters) {
    var self = this;
    var optionsError = self.checkOptions(options);
    if (optionsError) return Promise.reject(optionsError);
    var sourceError = self.checkSource(source);
    if (sourceError) return Promise.reject(sourceError);

    // Don't run binary for unsupported compilers, even if we're asked.
    if (filters.binary && !self.compiler.supportsBinary) {
        delete filters.binary;
    }

    var key = JSON.stringify({compiler: this.compiler, source: source, options: options, filters: filters});
    var cached = this.env.cacheGet(key);
    if (cached) {
        return Promise.resolve(cached);
    }

    if (filters.binary && !source.match(this.env.compilerProps("stubRe"))) {
        source += "\n" + this.env.compilerProps("stubText") + "\n";
    }

    var tempFileAndDirPromise = Promise.resolve().then(function () {
        return self.newTempDir().then(function (dirPath) {
            var inputFilename = path.join(dirPath, self.env.compilerProps("compileFilename"));
            return self.writeFile(inputFilename, source).then(function () {
                return {inputFilename: inputFilename, dirPath: dirPath};
            });
        });
    });

    var compileToAsmPromise = tempFileAndDirPromise.then(function (info) {
        var inputFilename = info.inputFilename;
        var dirPath = info.dirPath;
        var outputFilename = path.join(dirPath, 'output.s'); // NB keep lower case as ldc compiler `tolower`s the output name
        options = self.prepareArguments(options, filters, inputFilename, outputFilename);

        options = options.filter(_.identity);
        return self.runCompiler(self.compiler.exe, options, self.filename(inputFilename))
            .then(function (result) {
                result.dirPath = dirPath;
                if (result.code !== 0) {
                    result.asm = "<Compilation failed>";
                    return result;
                }
                return self.postProcess(result, outputFilename, filters);
            });
    });

    return self.env.enqueue(function () {
        return compileToAsmPromise
            .then(function (result) {
                if (result.dirPath) {
                    fs.remove(result.dirPath);
                    result.dirPath = undefined;
                }
                if (result.okToCache) {
                    result.asm = self.asm.process(result.asm, filters);
                } else {
                    result.asm = {text: result.asm};
                }
                return result;
            })
            .then(_.bind(self.postProcessAsm, self))
            .then(function (result) {
                if (result.okToCache) self.env.cachePut(key, result);
                return result;
            });
    });
};

Compile.prototype.postProcessAsm = function (result) {
    if (!result.okToCache) return result;
    var demangler = this.compiler.demangler;
    if (!demangler) return result;
    return this.exec(demangler, [], {input: _.pluck(result.asm, 'text').join("\n")})
        .then(function (demangleResult) {
            var lines = utils.splitLines(demangleResult.stdout);
            for (var i = 0; i < result.asm.length; ++i)
                result.asm[i].text = lines[i];
            return result;
        });
};

Compile.prototype.postProcess = function (result, outputFilename, filters) {
    var postProcess = this.compiler.postProcess.filter(_.identity);
    var maxSize = this.env.gccProps("max-asm-size", 8 * 1024 * 1024);
    if (filters.binary && this.supportsObjdump()) {
        return this.objdump(outputFilename, result, maxSize, filters.intel);
    }
    return this.stat(outputFilename).then(_.bind(function (stat) {
            if (stat.size >= maxSize) {
                result.asm = "<No output: generated assembly was too large (" + stat.size + " > " + maxSize + " bytes)>";
                return result;
            }
            if (postProcess.length) {
                var postCommand = 'cat "' + outputFilename + '" | ' + postProcess.join(" | ");
                return this.exec("bash", ["-c", postCommand], {maxOutput: maxSize})
                    .then(function (postResult) {
                        result.asm = postResult.stdout;
                        if (postResult.code !== 0) {
                            result.asm = "<Error during post processing: " + postResult.code + ">";
                        }
                        return result;
                    });
            } else {
                return this.readFile(outputFilename).then(function (contents) {
                    result.asm = contents.toString();
                    return Promise.resolve(result);
                });
            }
        }, this),
        function () {
            result.asm = "<No output file>";
            return result;
        }
    );
};

Compile.prototype.checkOptions = function (options) {
    var error = this.env.findBadOptions(options);
    if (error.length > 0) return "Bad options: " + error.join(", ");
    return null;
};

Compile.prototype.checkSource = function (source) {
    var re = /^\s*#\s*i(nclude|mport)(_next)?\s+["<"](\/|.*\.\.)/;
    var failed = [];
    utils.splitLines(source).forEach(function (line, index) {
        if (line.match(re)) {
            failed.push("<stdin>:" + (index + 1) + ":1: no absolute or relative includes please");
        }
    });
    if (failed.length > 0) return failed.join("\n");
    return null;
};

Compile.prototype.exec = function (command, args, options) {
    options = options || {};
    var maxOutput = options.maxOutput || 1024 * 1024;
    var timeoutMs = options.timeoutMs || 0;
    var env = options.env;

    if (options.wrapper) {
        args.unshift(command);
        command = options.wrapper;
    }

    var okToCache = true;
    logger.debug({type: "executing", command: command, args: args});
    var child = child_process.spawn(command, args, {
        env: env,
        detached: process.platform == 'linux'
    });
    var stderr = "";
    var stdout = "";
    var timeout;
    if (timeoutMs) timeout = setTimeout(function () {
        okToCache = false;
        child.kill();
        stderr += "\nKilled - processing time exceeded";
    }, timeoutMs);
    var truncated = false;
    child.stdout.on('data', function (data) {
        if (truncated) return;
        if (stdout.length > maxOutput) {
            stdout += "\n[Truncated]";
            truncated = true;
            child.kill();
            return;
        }
        stdout += data;
    });
    child.stderr.on('data', function (data) {
        if (truncated) return;
        if (stderr.length > maxOutput) {
            stderr += "\n[Truncated]";
            truncated = true;
            child.kill();
            return;
        }
        stderr += data;
    });
    return new Promise(function (resolve, reject) {
        child.on('error', function (e) {
            logger.debug("Error with " + command + "args", args, ":", e);
            reject(e);
        });
        child.on('exit', function (code) {
            if (timeout !== undefined) clearTimeout(timeout);
            var result = {
                code: code,
                stdout: stdout,
                stderr: stderr,
                okToCache: okToCache
            };
            logger.debug({type: "executed", command: command, args: args, result: result});
            // Why is this apparently needed in some cases (e.g. when I used to use this to do getMultiarch)?
            // Without it, I apparently get stdout/stderr callbacks *after* the exit...
            setTimeout(function () {
                resolve(result);
            }, 0);
        });
        if (options.input) child.stdin.write(options.input);
        child.stdin.end();
    });
};

Compile.prototype.initialise = function () {
    if (this.getRemote()) return Promise.resolve(this);
    var compiler = this.compiler.exe;
    var versionFlag = this.compiler.versionFlag || '--version';
    var versionRe = new RegExp(this.compiler.versionRe || '.*');
    logger.info("Gathering version information on", compiler);
    return this.exec(compiler, [versionFlag])
        .then(_.bind(function (result) {
            if (result.code !== 0) {
                logger.error("Unable to get version for compiler '" + compiler + "' - non-zero result " + result.code);
                return null;
            }
            var version = "";
            _.each(utils.splitLines(result.stdout + result.stderr), function (line) {
                if (version) return;
                var match = line.match(versionRe);
                if (match) version = match[0];
            });
            if (!version) {
                logger.error("Unable to get compiler version for '" + compiler + "'");
                return null;
            }
            logger.info(compiler + " is version '" + version + "'");
            this.compiler.version = version;
            if (this.compiler.intelAsm) return this;
            return this.exec(compiler, ['--target-help'])
                .then(_.bind(function (result) {
                    var options = {};
                    if (result.code === 0) {
                        var splitness = /--?[-a-zA-Z]+( ?[-a-zA-Z]+)/;
                        utils.eachLine(result.stdout + result.stderr, function (line) {
                            var match = line.match(splitness);
                            if (!match) return;
                            options[match[0]] = true;
                        });
                    }
                    if (options['-masm']) {
                        this.compiler.intelAsm = "-masm=intel";
                    }

                    logger.debug("compiler options: ", options);

                    return this;
                }, this));
        }, this));
};

Compile.prototype.getInfo = function () {
    return this.compiler;
};

module.exports = Compile;<commit_msg>Handle missing compilers more gracefully<commit_after>// Copyright (c) 2012-2016, Matt Godbolt
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

var child_process = require('child_process'),
    temp = require('temp'),
    path = require('path'),
    fs = require('fs-extra'),
    Promise = require('promise'), // jshint ignore:line
    asm = require('./asm'),
    utils = require('./utils'
    ),
    quote = require('shell-quote'),
    _ = require('underscore-node'),
    logger = require('./logger').logger;

function Compile(compiler, env) {
    this.compiler = compiler;
    this.env = env;
    this.asm = new asm.AsmParser(env.compilerProps);
}

Compile.prototype.newTempDir = function () {
    return new Promise(function (resolve, reject) {
        temp.mkdir('gcc-explorer-compiler', function (err, dirPath) {
            if (err)
                reject("Unable to open temp file: " + err);
            else
                resolve(dirPath);
        });
    });
};

Compile.prototype.writeFile = Promise.denodeify(fs.writeFile);
Compile.prototype.readFile = Promise.denodeify(fs.readFile);
Compile.prototype.stat = Promise.denodeify(fs.stat);

Compile.prototype.getRemote = function () {
    if (this.compiler.exe === null && this.compiler.remote)
        return this.compiler.remote;
    return false;
};

Compile.prototype.runCompiler = function (compiler, options, inputFilename) {
    return this.exec(compiler, options, {
        timeoutMs: this.env.gccProps("compileTimeoutMs", 100),
        maxErrorOutput: this.env.gccProps("max-error-output", 5000),
        env: this.env.getEnv(this.compiler.needsMulti),
        wrapper: this.env.compilerProps("compiler-wrapper")
    }).then(function (result) {
        result.stdout = utils.parseOutput(result.stdout, inputFilename);
        result.stderr = utils.parseOutput(result.stderr, inputFilename);
        return result;
    });
};

Compile.prototype.supportsObjdump = function () {
    return true;
};

Compile.prototype.objdump = function (outputFilename, result, maxSize, intelAsm) {
    var args = ["-d", "-C", outputFilename, "-l", "--insn-width=16"];
    if (intelAsm) args = args.concat(["-M", "intel"]);
    return this.exec("objdump", args, {maxOutput: maxSize})
        .then(function (objResult) {
            result.asm = objResult.stdout;
            if (objResult.code !== 0) {
                result.asm = "<No output: objdump returned " + objResult.code + ">";
            }
            return result;
        });
};

Compile.prototype.filename = function (fn) {
    return fn;
};

Compile.prototype.optionsForFilter = function (filters, outputFilename) {
    var options = ['-g', '-o', this.filename(outputFilename)];
    if (this.compiler.intelAsm && filters.intel && !filters.binary) {
        options = options.concat(this.compiler.intelAsm.split(" "));
    }
    if (!filters.binary) options = options.concat('-S');
    return options;
};

Compile.prototype.prepareArguments = function (userOptions, filters, inputFilename, outputFilename) {
    var options = this.optionsForFilter(filters, outputFilename);
    if (this.compiler.options) {
        options = options.concat(this.compiler.options.split(" "));
    }
    return options.concat(userOptions || []).concat([this.filename(inputFilename)]);
};

Compile.prototype.compile = function (source, options, filters) {
    var self = this;
    var optionsError = self.checkOptions(options);
    if (optionsError) return Promise.reject(optionsError);
    var sourceError = self.checkSource(source);
    if (sourceError) return Promise.reject(sourceError);

    // Don't run binary for unsupported compilers, even if we're asked.
    if (filters.binary && !self.compiler.supportsBinary) {
        delete filters.binary;
    }

    var key = JSON.stringify({compiler: this.compiler, source: source, options: options, filters: filters});
    var cached = this.env.cacheGet(key);
    if (cached) {
        return Promise.resolve(cached);
    }

    if (filters.binary && !source.match(this.env.compilerProps("stubRe"))) {
        source += "\n" + this.env.compilerProps("stubText") + "\n";
    }

    var tempFileAndDirPromise = Promise.resolve().then(function () {
        return self.newTempDir().then(function (dirPath) {
            var inputFilename = path.join(dirPath, self.env.compilerProps("compileFilename"));
            return self.writeFile(inputFilename, source).then(function () {
                return {inputFilename: inputFilename, dirPath: dirPath};
            });
        });
    });

    var compileToAsmPromise = tempFileAndDirPromise.then(function (info) {
        var inputFilename = info.inputFilename;
        var dirPath = info.dirPath;
        var outputFilename = path.join(dirPath, 'output.s'); // NB keep lower case as ldc compiler `tolower`s the output name
        options = self.prepareArguments(options, filters, inputFilename, outputFilename);

        options = options.filter(_.identity);
        return self.runCompiler(self.compiler.exe, options, self.filename(inputFilename))
            .then(function (result) {
                result.dirPath = dirPath;
                if (result.code !== 0) {
                    result.asm = "<Compilation failed>";
                    return result;
                }
                return self.postProcess(result, outputFilename, filters);
            });
    });

    return self.env.enqueue(function () {
        return compileToAsmPromise
            .then(function (result) {
                if (result.dirPath) {
                    fs.remove(result.dirPath);
                    result.dirPath = undefined;
                }
                if (result.okToCache) {
                    result.asm = self.asm.process(result.asm, filters);
                } else {
                    result.asm = {text: result.asm};
                }
                return result;
            })
            .then(_.bind(self.postProcessAsm, self))
            .then(function (result) {
                if (result.okToCache) self.env.cachePut(key, result);
                return result;
            });
    });
};

Compile.prototype.postProcessAsm = function (result) {
    if (!result.okToCache) return result;
    var demangler = this.compiler.demangler;
    if (!demangler) return result;
    return this.exec(demangler, [], {input: _.pluck(result.asm, 'text').join("\n")})
        .then(function (demangleResult) {
            var lines = utils.splitLines(demangleResult.stdout);
            for (var i = 0; i < result.asm.length; ++i)
                result.asm[i].text = lines[i];
            return result;
        });
};

Compile.prototype.postProcess = function (result, outputFilename, filters) {
    var postProcess = this.compiler.postProcess.filter(_.identity);
    var maxSize = this.env.gccProps("max-asm-size", 8 * 1024 * 1024);
    if (filters.binary && this.supportsObjdump()) {
        return this.objdump(outputFilename, result, maxSize, filters.intel);
    }
    return this.stat(outputFilename).then(_.bind(function (stat) {
            if (stat.size >= maxSize) {
                result.asm = "<No output: generated assembly was too large (" + stat.size + " > " + maxSize + " bytes)>";
                return result;
            }
            if (postProcess.length) {
                var postCommand = 'cat "' + outputFilename + '" | ' + postProcess.join(" | ");
                return this.exec("bash", ["-c", postCommand], {maxOutput: maxSize})
                    .then(function (postResult) {
                        result.asm = postResult.stdout;
                        if (postResult.code !== 0) {
                            result.asm = "<Error during post processing: " + postResult.code + ">";
                        }
                        return result;
                    });
            } else {
                return this.readFile(outputFilename).then(function (contents) {
                    result.asm = contents.toString();
                    return Promise.resolve(result);
                });
            }
        }, this),
        function () {
            result.asm = "<No output file>";
            return result;
        }
    );
};

Compile.prototype.checkOptions = function (options) {
    var error = this.env.findBadOptions(options);
    if (error.length > 0) return "Bad options: " + error.join(", ");
    return null;
};

Compile.prototype.checkSource = function (source) {
    var re = /^\s*#\s*i(nclude|mport)(_next)?\s+["<"](\/|.*\.\.)/;
    var failed = [];
    utils.splitLines(source).forEach(function (line, index) {
        if (line.match(re)) {
            failed.push("<stdin>:" + (index + 1) + ":1: no absolute or relative includes please");
        }
    });
    if (failed.length > 0) return failed.join("\n");
    return null;
};

Compile.prototype.exec = function (command, args, options) {
    options = options || {};
    var maxOutput = options.maxOutput || 1024 * 1024;
    var timeoutMs = options.timeoutMs || 0;
    var env = options.env;

    if (options.wrapper) {
        args.unshift(command);
        command = options.wrapper;
    }

    var okToCache = true;
    logger.debug({type: "executing", command: command, args: args});
    var child = child_process.spawn(command, args, {
        env: env,
        detached: process.platform == 'linux'
    });
    var stderr = "";
    var stdout = "";
    var timeout;
    if (timeoutMs) timeout = setTimeout(function () {
        okToCache = false;
        child.kill();
        stderr += "\nKilled - processing time exceeded";
    }, timeoutMs);
    var truncated = false;
    child.stdout.on('data', function (data) {
        if (truncated) return;
        if (stdout.length > maxOutput) {
            stdout += "\n[Truncated]";
            truncated = true;
            child.kill();
            return;
        }
        stdout += data;
    });
    child.stderr.on('data', function (data) {
        if (truncated) return;
        if (stderr.length > maxOutput) {
            stderr += "\n[Truncated]";
            truncated = true;
            child.kill();
            return;
        }
        stderr += data;
    });
    return new Promise(function (resolve, reject) {
        child.on('error', function (e) {
            logger.debug("Error with " + command + "args", args, ":", e);
            reject(e);
        });
        child.on('exit', function (code) {
            if (timeout !== undefined) clearTimeout(timeout);
            var result = {
                code: code,
                stdout: stdout,
                stderr: stderr,
                okToCache: okToCache
            };
            logger.debug({type: "executed", command: command, args: args, result: result});
            // Why is this apparently needed in some cases (e.g. when I used to use this to do getMultiarch)?
            // Without it, I apparently get stdout/stderr callbacks *after* the exit...
            setTimeout(function () {
                resolve(result);
            }, 0);
        });
        if (options.input) child.stdin.write(options.input);
        child.stdin.end();
    });
};

Compile.prototype.initialise = function () {
    if (this.getRemote()) return Promise.resolve(this);
    var compiler = this.compiler.exe;
    var versionFlag = this.compiler.versionFlag || '--version';
    var versionRe = new RegExp(this.compiler.versionRe || '.*');
    logger.info("Gathering version information on", compiler);
    return this.exec(compiler, [versionFlag])
        .then(_.bind(function (result) {
                if (result.code !== 0) {
                    logger.error("Unable to get version for compiler '" + compiler + "' - non-zero result " + result.code);
                    return null;
                }
                var version = "";
                _.each(utils.splitLines(result.stdout + result.stderr), function (line) {
                    if (version) return;
                    var match = line.match(versionRe);
                    if (match) version = match[0];
                });
                if (!version) {
                    logger.error("Unable to get compiler version for '" + compiler + "'");
                    return null;
                }
                logger.info(compiler + " is version '" + version + "'");
                this.compiler.version = version;
                if (this.compiler.intelAsm) return this;
                return this.exec(compiler, ['--target-help'])
                    .then(_.bind(function (result) {
                        var options = {};
                        if (result.code === 0) {
                            var splitness = /--?[-a-zA-Z]+( ?[-a-zA-Z]+)/;
                            utils.eachLine(result.stdout + result.stderr, function (line) {
                                var match = line.match(splitness);
                                if (!match) return;
                                options[match[0]] = true;
                            });
                        }
                        if (options['-masm']) {
                            this.compiler.intelAsm = "-masm=intel";
                        }

                        logger.debug("compiler options: ", options);

                        return this;
                    }, this));
            }, this),
            _.bind(function (err) {
                logger.error("Unable to get version for compiler '" + compiler + "' - " + err);
                return null;
            }, this));
};

Compile.prototype.getInfo = function () {
    return this.compiler;
};

module.exports = Compile;<|endoftext|>


<commit_before>"use strict";

const P = require('bluebird');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('hyperswitch').HTTPError;
const kafka = require('node-rdkafka');

const utils = require('./utils');

/**
 * The default number of tasks that could be run concurrently
 *
 * @type {number}
 * @const
 */
const DEFAULT_CONCURRENCY = 30;

/**
 * The default maximum delay to commit the offsets
 *
 * @const
 * @type {number}
 */
const DEFAULT_COMMIT_INTERVAL = 500;

class BaseExecutor {

    /**
     * Creates a new instance of a rule executor
     *
     * @param {Rule} rule
     * @param {KafkaFactory} kafkaFactory
     * @param {Object} hyper
     * @param {function} log
     * @param {Object} options
     * @constructor
     */
    constructor(rule, kafkaFactory, hyper, log, options) {
        if (this.constructor.name === 'BaseExecutor') {
            throw new Error('BaseService is abstract. Create Master or Worker instance.');
        }

        this.rule = rule;
        this.kafkaFactory = kafkaFactory;
        this.hyper = hyper;
        this.log = log;
        this.concurrency = rule.spec.concurrency || options.concurrency || DEFAULT_CONCURRENCY;

        this._commitTimeout = null;
        this._pendingMsgs = [];
        this._toCommit = undefined;
        this._consuming = false;
    }

    subscribe() {
        return this.kafkaFactory.createConsumer(
            `change-prop-${this.rule.name}`,
            this.subscribeTopic,
            this.hyper.metrics
        )
        .then((consumer) => {
            this.consumer = consumer;
            P.delay(this.kafkaFactory.startup_delay).then(() => this._consume());
        });
    }

    _safeParse(payload) {
        try {
            return JSON.parse(payload);
        } catch (e) {
            this.log(`error/${this.rule.name}`, e);
            this.hyper.post({
                uri: '/sys/queue/events',
                body: [ this._constructErrorMessage(e, payload) ]
            });
        }
    }

    _consume() {
        this._consuming = true;
        this.consumer.consumeAsync()
        .then((msg) => {
            this.hyper.metrics.increment(`${this.statName}_dequeue`, 1, 0.1);

            const message = this._safeParse(msg.value.toString('utf8'));
            const handler = this.getHandler(message);
            if (handler) {
                // We're pushing it to pending messages only if it matched so that items
                // that don't match don't count against the concurrency limit.
                this._pendingMsgs.push(msg);
                // Note: we don't return the promise here since we wanna process messages
                // asynchronously from consuming them to be able to fill up the pendingMsg
                // queue and achieve the level of concurrency we want.
                this.processMessage(message, handler)
                .finally(() => {
                    this._notifyFinished(msg);
                    if (this._pendingMsgs.length < this.concurrency && !this._consuming) {
                        this._consume();
                    }
                });
            }
        })
        .catch((e) => {
            // This errors must come from the KafkaConsumer
            // since the actual handler must never throw errors
            /* eslint-disable indent */
            switch (e.code) {
                case kafka.CODES.ERRORS.ERR__PARTITION_EOF:
                case kafka.CODES.ERRORS.ERR__TIMED_OUT:
                    // We're reading to fast, nothing is there, slow down a little bit
                    return P.delay(100);

                default:
                    this.log(`error/consumer/${this.rule.name}`, e);
            }
            /* eslint-enable indent */
        })
        .finally(() => {
            if (this._pendingMsgs.length < this.concurrency) {
                this._consume();
            } else {
                this._consuming = false;
            }
        });
    }

    _notifyFinished(finishedMsg) {
        this._pendingMsgs = this._pendingMsgs.filter((o) => o.offset !== finishedMsg.offset);
        if (this._pendingMsgs.length) {
            this._toCommit = this._pendingMsgs.sort((msg1, msg2) => {
                return msg1.offset - msg2.offset;
            })[0];
        } else {
            this._toCommit = finishedMsg;
            // We need to commit the offset of the next message we want to consume
            this._toCommit.offset += 1;
        }

        if (!this._commitTimeout) {
            this._commitTimeout = setTimeout(() => {
                this._commitTimeout = null;
                if (this._toCommit) {
                    const committing = this._toCommit;
                    return this.consumer.commitAsync(committing)
                    .then(() => {
                        if (this._toCommit && this._toCommit.offset === committing.offset) {
                            // Don't commit what we've just committed
                            this._toCommit = undefined;
                        }
                    })
                    .catch((e) => {
                        this.log(`error/commit/${this.rule.name}`, {
                            msg: 'Commit failed',
                            offset: committing.offset,
                            raw_event: committing.value.toString(),
                            description: e.toString()
                        });
                    });
                }
            }, DEFAULT_COMMIT_INTERVAL);
        }
    }

    /** Private methods */

    _test(event) {
        try {
            const optionIndex = this.rule.test(event);
            if (optionIndex === -1) {
                // no match, drop the message
                this.log(`debug/${this.rule.name}`, () => ({
                    msg: 'Dropping event message',
                    event: utils.stringify(event)
                }));
            }
            return optionIndex;
        } catch (e) {
            this.log(`error/${this.rule.name}`, e);
            return -1;
        }
    }

    _sampleLog(event, request, res) {
        const sampleLog = () => ({
            message: 'Processed event sample',
            event: utils.stringify(event),
            request,
            response: {
                status: res.status,
                headers: res.headers
            }
        });
        // Don't include the full body in the log
        if (res.status >= 400) {
            sampleLog.response.body = BaseExecutor.decodeError(res).body;
        }
        this.hyper.log('trace/sample', sampleLog);
    }

    _exec(origEvent, handler, statDelayStartTime, retryEvent) {
        const rule = this.rule;
        const startTime = Date.now();
        const expander = {
            message: origEvent,
            match: handler.expand(origEvent)
        };

        this.log(`trace/${rule.name}`, () => ({
            msg: 'Event message received',
            event: utils.stringify(origEvent) }));

        // latency from the original event creation time to execution time
        this.hyper.metrics.endTiming([`${this.statName}_delay`],
            statDelayStartTime || new Date(origEvent.meta.dt));

        return P.each(handler.exec, (tpl) => {
            const request = tpl.expand(expander);
            request.headers = Object.assign(request.headers, {
                'x-request-id': origEvent.meta.request_id,
                'x-triggered-by': utils.triggeredBy(retryEvent || origEvent)
            });
            return this.hyper.request(request)
            .tap((res) => {
                if (res.status === 301) {
                    // As we don't follow redirects, and we must always use normalized titles,
                    // receiving 301 indicates some error. Log a warning.
                    this.log(`warn/${this.rule.name}`, () => ({
                        message: '301 redirect received, used a non-normalized title',
                        rule: this.rule.name,
                        event: utils.stringify(origEvent)
                    }));
                }
            })
            .tap(this._sampleLog.bind(this, retryEvent || origEvent, request))
            .catch((e) => {
                this._sampleLog(retryEvent || origEvent, request, e);
                throw e;
            });
        })
        .finally(() => {
            this.hyper.metrics.endTiming([`${this.statName}_exec`], startTime);
        });
    }

    _retryTopicName() {
        return `change-prop.retry.${this.rule.topic}`;
    }

    _emitterId() {
        return `change-prop#${this.rule.name}`;
    }

    _constructRetryMessage(event, errorRes, retriesLeft, retryEvent) {
        return {
            meta: {
                topic: this._retryTopicName(),
                schema_uri: 'retry/1',
                uri: event.meta.uri,
                request_id: event.meta.request_id,
                id: undefined, // will be filled later
                dt: undefined, // will be filled later
                domain: event.meta.domain
            },
            triggered_by: utils.triggeredBy(retryEvent || event),
            emitter_id: this._emitterId(),
            retries_left: retriesLeft === undefined ? this.rule.spec.retry_limit : retriesLeft,
            original_event: event,
            reason: errorRes && errorRes.body && errorRes.body.title
        };
    }

    /**
     * Checks whether retry limit for this rule is exceeded.
     *
     * @param {Object} message a retry message to check
     * @param {Error} [e] optional Error that caused a retry
     * @returns {boolean}
     * @private
     */
    _isLimitExceeded(message, e) {
        if (message.retries_left <= 0) {
            this.log(`info/${this.rule.name}`, () => ({
                message: 'Retry count exceeded',
                event: utils.stringify(message),
                description: `${e}`
            }));
            return true;
        }
        return false;
    }

    _catch(message, retryMessage, e) {
        const reportError = () => this.hyper.post({
            uri: '/sys/queue/events',
            body: [this._constructErrorMessage(e, message)]
        });

        if (e.constructor.name !== 'HTTPError') {
            // We've got an error, but it's not from the update request, it's
            // some bug in change-prop. Log and send a fatal error message.
            this.hyper.log(`error/${this.rule.name}`, () => ({
                message: 'Internal error in change-prop',
                description: `${e}`,
                stack: e.stack,
                event: utils.stringify(message)
            }));
            return reportError();
        }

        if (!this.rule.shouldIgnoreError(e)) {
            if (this.rule.shouldRetry(e)
                && !this._isLimitExceeded(retryMessage, e)) {
                return this.hyper.post({
                    uri: '/sys/queue/events',
                    body: [ retryMessage ]
                });
            }
            return reportError();
        }
    }

    /**
     * Create an error message for a special Kafka topic
     *
     * @param {Error} e an exception that caused a failure
     * @param {string|Object} event an original event. In case JSON parsing failed - it's a string.
     */
    _constructErrorMessage(e, event) {
        const eventUri = typeof event === 'string' ? '/error/uri' : event.meta.uri;
        const domain = typeof event === 'string' ? 'unknown' : event.meta.domain;
        const now = new Date();
        const errorEvent = {
            meta: {
                topic: 'change-prop.error',
                schema_uri: 'error/1',
                uri: eventUri,
                request_id: typeof event === 'string' ? undefined : event.meta.request_id,
                id: uuid.fromDate(now),
                dt: now.toISOString(),
                domain
            },
            emitter_id: `change-prop#${this.rule.name}`,
            raw_event: typeof event === 'string' ? event : JSON.stringify(event),
            message: e.message,
            stack: e.stack
        };
        if (e.constructor === HTTPError) {
            errorEvent.details = {
                status: e.status,
                headers: e.headers,
                body: e.body
            };
        }
        return errorEvent;
    }

    close() {
        this.consumer.disconnect();
    }

    static decodeError(e) {
        if (Buffer.isBuffer(e.body)) {
            e.body = e.body.toString();
            try {
                e.body = JSON.parse(e.body);
            } catch (err) {
                // Not a JSON error
            }
        }
        return e;
    }
}

module.exports = BaseExecutor;
<commit_msg>Fixed sample log generation<commit_after>"use strict";

const P = require('bluebird');
const uuid = require('cassandra-uuid').TimeUuid;
const HTTPError = require('hyperswitch').HTTPError;
const kafka = require('node-rdkafka');

const utils = require('./utils');

/**
 * The default number of tasks that could be run concurrently
 *
 * @type {number}
 * @const
 */
const DEFAULT_CONCURRENCY = 30;

/**
 * The default maximum delay to commit the offsets
 *
 * @const
 * @type {number}
 */
const DEFAULT_COMMIT_INTERVAL = 500;

class BaseExecutor {

    /**
     * Creates a new instance of a rule executor
     *
     * @param {Rule} rule
     * @param {KafkaFactory} kafkaFactory
     * @param {Object} hyper
     * @param {function} log
     * @param {Object} options
     * @constructor
     */
    constructor(rule, kafkaFactory, hyper, log, options) {
        if (this.constructor.name === 'BaseExecutor') {
            throw new Error('BaseService is abstract. Create Master or Worker instance.');
        }

        this.rule = rule;
        this.kafkaFactory = kafkaFactory;
        this.hyper = hyper;
        this.log = log;
        this.concurrency = rule.spec.concurrency || options.concurrency || DEFAULT_CONCURRENCY;

        this._commitTimeout = null;
        this._pendingMsgs = [];
        this._toCommit = undefined;
        this._consuming = false;
    }

    subscribe() {
        return this.kafkaFactory.createConsumer(
            `change-prop-${this.rule.name}`,
            this.subscribeTopic,
            this.hyper.metrics
        )
        .then((consumer) => {
            this.consumer = consumer;
            P.delay(this.kafkaFactory.startup_delay).then(() => this._consume());
        });
    }

    _safeParse(payload) {
        try {
            return JSON.parse(payload);
        } catch (e) {
            this.log(`error/${this.rule.name}`, e);
            this.hyper.post({
                uri: '/sys/queue/events',
                body: [ this._constructErrorMessage(e, payload) ]
            });
        }
    }

    _consume() {
        this._consuming = true;
        this.consumer.consumeAsync()
        .then((msg) => {
            this.hyper.metrics.increment(`${this.statName}_dequeue`, 1, 0.1);

            const message = this._safeParse(msg.value.toString('utf8'));
            const handler = this.getHandler(message);
            if (handler) {
                // We're pushing it to pending messages only if it matched so that items
                // that don't match don't count against the concurrency limit.
                this._pendingMsgs.push(msg);
                // Note: we don't return the promise here since we wanna process messages
                // asynchronously from consuming them to be able to fill up the pendingMsg
                // queue and achieve the level of concurrency we want.
                this.processMessage(message, handler)
                .finally(() => {
                    this._notifyFinished(msg);
                    if (this._pendingMsgs.length < this.concurrency && !this._consuming) {
                        this._consume();
                    }
                });
            }
        })
        .catch((e) => {
            // This errors must come from the KafkaConsumer
            // since the actual handler must never throw errors
            /* eslint-disable indent */
            switch (e.code) {
                case kafka.CODES.ERRORS.ERR__PARTITION_EOF:
                case kafka.CODES.ERRORS.ERR__TIMED_OUT:
                    // We're reading to fast, nothing is there, slow down a little bit
                    return P.delay(100);

                default:
                    this.log(`error/consumer/${this.rule.name}`, e);
            }
            /* eslint-enable indent */
        })
        .finally(() => {
            if (this._pendingMsgs.length < this.concurrency) {
                this._consume();
            } else {
                this._consuming = false;
            }
        });
    }

    _notifyFinished(finishedMsg) {
        this._pendingMsgs = this._pendingMsgs.filter((o) => o.offset !== finishedMsg.offset);
        if (this._pendingMsgs.length) {
            this._toCommit = this._pendingMsgs.sort((msg1, msg2) => {
                return msg1.offset - msg2.offset;
            })[0];
        } else {
            this._toCommit = finishedMsg;
            // We need to commit the offset of the next message we want to consume
            this._toCommit.offset += 1;
        }

        if (!this._commitTimeout) {
            this._commitTimeout = setTimeout(() => {
                this._commitTimeout = null;
                if (this._toCommit) {
                    const committing = this._toCommit;
                    return this.consumer.commitAsync(committing)
                    .then(() => {
                        if (this._toCommit && this._toCommit.offset === committing.offset) {
                            // Don't commit what we've just committed
                            this._toCommit = undefined;
                        }
                    })
                    .catch((e) => {
                        this.log(`error/commit/${this.rule.name}`, {
                            msg: 'Commit failed',
                            offset: committing.offset,
                            raw_event: committing.value.toString(),
                            description: e.toString()
                        });
                    });
                }
            }, DEFAULT_COMMIT_INTERVAL);
        }
    }

    /** Private methods */

    _test(event) {
        try {
            const optionIndex = this.rule.test(event);
            if (optionIndex === -1) {
                // no match, drop the message
                this.log(`debug/${this.rule.name}`, () => ({
                    msg: 'Dropping event message',
                    event: utils.stringify(event)
                }));
            }
            return optionIndex;
        } catch (e) {
            this.log(`error/${this.rule.name}`, e);
            return -1;
        }
    }

    _sampleLog(event, request, res) {
        const sampleLog = () => {
            const log = {
                message: 'Processed event sample',
                event: utils.stringify(event),
                request,
                response: {
                    status: res.status,
                    headers: res.headers
                }
            };
            // Don't include the full body in the log
            if (res.status >= 400) {
                sampleLog.response.body = BaseExecutor.decodeError(res).body;
            }
            return log;
        };
        this.hyper.log('trace/sample', sampleLog);
    }

    _exec(origEvent, handler, statDelayStartTime, retryEvent) {
        const rule = this.rule;
        const startTime = Date.now();
        const expander = {
            message: origEvent,
            match: handler.expand(origEvent)
        };

        this.log(`trace/${rule.name}`, () => ({
            msg: 'Event message received',
            event: utils.stringify(origEvent) }));

        // latency from the original event creation time to execution time
        this.hyper.metrics.endTiming([`${this.statName}_delay`],
            statDelayStartTime || new Date(origEvent.meta.dt));

        return P.each(handler.exec, (tpl) => {
            const request = tpl.expand(expander);
            request.headers = Object.assign(request.headers, {
                'x-request-id': origEvent.meta.request_id,
                'x-triggered-by': utils.triggeredBy(retryEvent || origEvent)
            });
            return this.hyper.request(request)
            .tap((res) => {
                if (res.status === 301) {
                    // As we don't follow redirects, and we must always use normalized titles,
                    // receiving 301 indicates some error. Log a warning.
                    this.log(`warn/${this.rule.name}`, () => ({
                        message: '301 redirect received, used a non-normalized title',
                        rule: this.rule.name,
                        event: utils.stringify(origEvent)
                    }));
                }
            })
            .tap(this._sampleLog.bind(this, retryEvent || origEvent, request))
            .catch((e) => {
                this._sampleLog(retryEvent || origEvent, request, e);
                throw e;
            });
        })
        .finally(() => {
            this.hyper.metrics.endTiming([`${this.statName}_exec`], startTime);
        });
    }

    _retryTopicName() {
        return `change-prop.retry.${this.rule.topic}`;
    }

    _emitterId() {
        return `change-prop#${this.rule.name}`;
    }

    _constructRetryMessage(event, errorRes, retriesLeft, retryEvent) {
        return {
            meta: {
                topic: this._retryTopicName(),
                schema_uri: 'retry/1',
                uri: event.meta.uri,
                request_id: event.meta.request_id,
                id: undefined, // will be filled later
                dt: undefined, // will be filled later
                domain: event.meta.domain
            },
            triggered_by: utils.triggeredBy(retryEvent || event),
            emitter_id: this._emitterId(),
            retries_left: retriesLeft === undefined ? this.rule.spec.retry_limit : retriesLeft,
            original_event: event,
            reason: errorRes && errorRes.body && errorRes.body.title
        };
    }

    /**
     * Checks whether retry limit for this rule is exceeded.
     *
     * @param {Object} message a retry message to check
     * @param {Error} [e] optional Error that caused a retry
     * @returns {boolean}
     * @private
     */
    _isLimitExceeded(message, e) {
        if (message.retries_left <= 0) {
            this.log(`info/${this.rule.name}`, () => ({
                message: 'Retry count exceeded',
                event: utils.stringify(message),
                description: `${e}`
            }));
            return true;
        }
        return false;
    }

    _catch(message, retryMessage, e) {
        const reportError = () => this.hyper.post({
            uri: '/sys/queue/events',
            body: [this._constructErrorMessage(e, message)]
        });

        if (e.constructor.name !== 'HTTPError') {
            // We've got an error, but it's not from the update request, it's
            // some bug in change-prop. Log and send a fatal error message.
            this.hyper.log(`error/${this.rule.name}`, () => ({
                message: 'Internal error in change-prop',
                description: `${e}`,
                stack: e.stack,
                event: utils.stringify(message)
            }));
            return reportError();
        }

        if (!this.rule.shouldIgnoreError(e)) {
            if (this.rule.shouldRetry(e)
                && !this._isLimitExceeded(retryMessage, e)) {
                return this.hyper.post({
                    uri: '/sys/queue/events',
                    body: [ retryMessage ]
                });
            }
            return reportError();
        }
    }

    /**
     * Create an error message for a special Kafka topic
     *
     * @param {Error} e an exception that caused a failure
     * @param {string|Object} event an original event. In case JSON parsing failed - it's a string.
     */
    _constructErrorMessage(e, event) {
        const eventUri = typeof event === 'string' ? '/error/uri' : event.meta.uri;
        const domain = typeof event === 'string' ? 'unknown' : event.meta.domain;
        const now = new Date();
        const errorEvent = {
            meta: {
                topic: 'change-prop.error',
                schema_uri: 'error/1',
                uri: eventUri,
                request_id: typeof event === 'string' ? undefined : event.meta.request_id,
                id: uuid.fromDate(now),
                dt: now.toISOString(),
                domain
            },
            emitter_id: `change-prop#${this.rule.name}`,
            raw_event: typeof event === 'string' ? event : JSON.stringify(event),
            message: e.message,
            stack: e.stack
        };
        if (e.constructor === HTTPError) {
            errorEvent.details = {
                status: e.status,
                headers: e.headers,
                body: e.body
            };
        }
        return errorEvent;
    }

    close() {
        this.consumer.disconnect();
    }

    static decodeError(e) {
        if (Buffer.isBuffer(e.body)) {
            e.body = e.body.toString();
            try {
                e.body = JSON.parse(e.body);
            } catch (err) {
                // Not a JSON error
            }
        }
        return e;
    }
}

module.exports = BaseExecutor;
<|endoftext|>


<commit_before>// Batch Request

var _ = require('lodash'),
    check = require('validator').check,
    methods = require('methods'),
    Promise = require('bluebird'),
    request = Promise.promisify(require('request')),
    url = require('url');

function getFinalUrl(req, r) {
    // Accept either uri or url (this is what request does, we just mirror)
    r.url = r.url || r.uri;

    // Convert relative paths to full paths
    if (typeof r.url === 'string' && /^\//.test(r.url) === true) {
        return req.protocol + '://' + req.get('host') + r.url;
    }

    return r.url;
}

var batchRequest = function(params) {

    // Set default option values
    params = params || {};
    params.localOnly = (typeof params.localOnly === 'undefined') ? true : params.localOnly;
    params.httpsAlways = (typeof params.localOnly === 'undefined') ? false : params.localOnly;
    params.max = (typeof params.max === 'undefined') ? 20 : params.max;
    params.validateRespond = (typeof params.validateRespond === 'undefined') ? true : params.validateRespond;
    params.allowedHosts = (typeof params.allowedHosts === 'undefined') ? null : params.allowedHosts;
    params.defaultHeaders = (typeof params.defaultHeaders === 'undefined') ? {} : params.defaultHeaders;
    params.forwardHeaders = (typeof params.forwardHeaders === 'undefined') ? [] : params.forwardHeaders;

    var _prepareRequest = function( req, r, rp ) {
        r.headers = r.headers || {};

        r.url = getFinalUrl(req, r);
        if( rp ) {
            var dependency_parameter_regex = /{([A-Za-z]+)}/g;

            var match = dependency_parameter_regex.exec( r.url );
            while (match != null) {

                var url_pre_replacement = r.url.substring( 0, match.index );
                var url_replacement = rp.body[ match[1] ];
                var url_post_replacement = r.url.substring( match.index + match[0].length );

                r.uri = r.url = url_pre_replacement + url_replacement + url_post_replacement;

                match = dependency_parameter_regex.exec( rp.body );
            }
        }

        _.each(params.defaultHeaders, function(headerV, headerK) {
            if (!(headerK in r.headers)) { // copy defaults if not already exposed
                r.headers[headerK] = headerV;
            }
        });
        _.each(params.forwardHeaders, function(headerK) {
            if (!(headerK in r.headers) && headerK in req.headers) { // copy forward if not already exposed
                var forwardValue = req.headers[headerK];
                r.headers[headerK] = forwardValue;
            }
        });

        return r;
    };

    var batch = function(req, res, next) {
        // Here we assume it the request has already been validated, either by
        // our included middleware or otherwise by the app developer.

        // We also assume it has been run through some middleware like
        // express.bodyParser() or express.json() to parse the requests

        var requests = req.body;

        // First, let's fire off all calls without any dependencies, accumulate their promises
        var requestPromises = _.reduce(requests, function(promises, r, key) {
            if (!r.dependency || r.dependency === 'none') {
                _prepareRequest( req, r );

                promises[key] = request(r).spread(function(response, body) {
                    return {
                        'statusCode': response.statusCode,
                        'body': body,
                        'headers': response.headers
                    };
                });
            }
            // And while we do that, build the dependency object with those items as keys
            // { key: Promise }
            return promises;
        }, {});

        // Then recursively iterate over all items with dependencies, resolving some at each pass
        var recurseDependencies = function (reqs) {
            // End state hit when the number of promises we have matches the number
            // of request objects we started with.
            if (_.size(requestPromises) >= _.size(reqs)) {
                return;
            } else {
                _.each(requestPromises, function(rp, key) {
                    var dependentKey = null;
                    var dependent = _.find(reqs, function(request, dKey) {
                        dependentKey = dKey;
                        return request.dependency === key && (typeof requestPromises[dKey] === 'undefined');
                    });
                    if (dependent) {
                        requestPromises[dependentKey] = rp.then(function( response ) {
                            if( response.statusCode !== 200 ) {
                                dependent = _prepareRequest( req, dependent );
                            } else {
                                dependent = _prepareRequest( req, dependent, response );
                            }
                            return request(dependent);
                        }).spread(function(response, body) {
                            return response;
                        });
                    }
                });
                recurseDependencies(reqs);
            }
        };

        // Recurse dependencies
        recurseDependencies(requests);

        // Wait for all to complete before responding
        Promise
            .props(requestPromises).then(function(result) {

                // remove all properties, except status, body, and headers
                var output = {};
                for(var prop in result){
                    output[prop] = { statusCode: result[prop].statusCode, body: result[prop].body, headers: result[prop].headers};
                }
                res.json(output);
                // next(); // this line is causing the response to be 0
            })
            .catch(function(error) {
                next(error);
            });
    };

    batch.validate = function(req, res, next) {
        var err = null,
            requests = req.body,
            requestHost;

        // Validation on Request object as a whole
        try {
            check(_.size(requests), 'Cannot make a batch request with an empty request object').min(1);
            check(_.size(requests), 'Over the max request limit. Please limit batches to ' + params.max + ' requests').max(params.max);
            if (req.method === 'POST' && !req.is('json')) {
                throw new Error('Batch Request will only accept body as json');
            }
        } catch (e) {
            err = {
                error: {
                    'message': e.message,
                    'type': 'ValidationError'
                }
            };
        }

        // Validation on each request object
        _.each(requests, function(r, key) {

            // If no method provided, default to GET
            r.method = (typeof r.method === 'string') ? r.method.toLowerCase() : 'get';

            r.url = getFinalUrl(req, r);

            try {
                check(r.url, 'Invalid URL').isUrl();
                check(r.method, 'Invalid method').isIn(methods);
                if (r.body !== undefined) {
                    check(r.method.toLowerCase(), 'Request body not allowed for this method').isIn(['put', 'post', 'options']);
                }
            } catch (e) {
                err = {
                    error: {
                        'message': e.message,
                        'request': key,
                        'type': 'ValidationError'
                    }
                };
            }

            if (params.allowedHosts !== null) {
                requestHost = url.parse(r.url).host;
                if (params.allowedHosts.indexOf(requestHost) === -1) {
                    err = {
                        error: {
                            'message': 'Cannot make batch request to a host which is not allowed',
                            'host': requestHost,
                            'type': 'ValidationError'
                        }
                    };
                }
            }
        });

        if (err !== null) {
            res.send(400, err);
            next(err);
        } else {
            next();
        }
    };

    return batch;
};

module.exports = batchRequest;
<commit_msg>Made variable names camel case<commit_after>// Batch Request

var _ = require('lodash'),
    check = require('validator').check,
    methods = require('methods'),
    Promise = require('bluebird'),
    request = Promise.promisify(require('request')),
    url = require('url');

function getFinalUrl(req, r) {
    // Accept either uri or url (this is what request does, we just mirror)
    r.url = r.url || r.uri;

    // Convert relative paths to full paths
    if (typeof r.url === 'string' && /^\//.test(r.url) === true) {
        return req.protocol + '://' + req.get('host') + r.url;
    }

    return r.url;
}

var batchRequest = function(params) {

    // Set default option values
    params = params || {};
    params.localOnly = (typeof params.localOnly === 'undefined') ? true : params.localOnly;
    params.httpsAlways = (typeof params.localOnly === 'undefined') ? false : params.localOnly;
    params.max = (typeof params.max === 'undefined') ? 20 : params.max;
    params.validateRespond = (typeof params.validateRespond === 'undefined') ? true : params.validateRespond;
    params.allowedHosts = (typeof params.allowedHosts === 'undefined') ? null : params.allowedHosts;
    params.defaultHeaders = (typeof params.defaultHeaders === 'undefined') ? {} : params.defaultHeaders;
    params.forwardHeaders = (typeof params.forwardHeaders === 'undefined') ? [] : params.forwardHeaders;

    var _prepareRequest = function( req, r, rp ) {
        r.headers = r.headers || {};

        r.url = getFinalUrl(req, r);
        if( rp ) {
            var dependencyParameterRegex = /{([A-Za-z]+)}/g;

            var match = dependencyParameterRegex.exec( r.url );
            while (match != null) {

                var urlPreReplacement = r.url.substring( 0, match.index );
                var urlReplacement = rp.body[ match[1] ];
                var urlPostReplacement = r.url.substring( match.index + match[0].length );

                r.uri = r.url = urlPreReplacement + urlReplacement + urlPostReplacement;

                match = dependencyParameterRegex.exec( rp.body );
            }
        }

        _.each(params.defaultHeaders, function(headerV, headerK) {
            if (!(headerK in r.headers)) { // copy defaults if not already exposed
                r.headers[headerK] = headerV;
            }
        });
        _.each(params.forwardHeaders, function(headerK) {
            if (!(headerK in r.headers) && headerK in req.headers) { // copy forward if not already exposed
                var forwardValue = req.headers[headerK];
                r.headers[headerK] = forwardValue;
            }
        });

        return r;
    };

    var batch = function(req, res, next) {
        // Here we assume it the request has already been validated, either by
        // our included middleware or otherwise by the app developer.

        // We also assume it has been run through some middleware like
        // express.bodyParser() or express.json() to parse the requests

        var requests = req.body;

        // First, let's fire off all calls without any dependencies, accumulate their promises
        var requestPromises = _.reduce(requests, function(promises, r, key) {
            if (!r.dependency || r.dependency === 'none') {
                _prepareRequest( req, r );

                promises[key] = request(r).spread(function(response, body) {
                    return {
                        'statusCode': response.statusCode,
                        'body': body,
                        'headers': response.headers
                    };
                });
            }
            // And while we do that, build the dependency object with those items as keys
            // { key: Promise }
            return promises;
        }, {});

        // Then recursively iterate over all items with dependencies, resolving some at each pass
        var recurseDependencies = function (reqs) {
            // End state hit when the number of promises we have matches the number
            // of request objects we started with.
            if (_.size(requestPromises) >= _.size(reqs)) {
                return;
            } else {
                _.each(requestPromises, function(rp, key) {
                    var dependentKey = null;
                    var dependent = _.find(reqs, function(request, dKey) {
                        dependentKey = dKey;
                        return request.dependency === key && (typeof requestPromises[dKey] === 'undefined');
                    });
                    if (dependent) {
                        requestPromises[dependentKey] = rp.then(function( response ) {
                            if( response.statusCode !== 200 ) {
                                dependent = _prepareRequest( req, dependent );
                            } else {
                                dependent = _prepareRequest( req, dependent, response );
                            }
                            return request(dependent);
                        }).spread(function(response, body) {
                            return response;
                        });
                    }
                });
                recurseDependencies(reqs);
            }
        };

        // Recurse dependencies
        recurseDependencies(requests);

        // Wait for all to complete before responding
        Promise
            .props(requestPromises).then(function(result) {

                // remove all properties, except status, body, and headers
                var output = {};
                for(var prop in result){
                    output[prop] = { statusCode: result[prop].statusCode, body: result[prop].body, headers: result[prop].headers};
                }
                res.json(output);
                // next(); // this line is causing the response to be 0
            })
            .catch(function(error) {
                next(error);
            });
    };

    batch.validate = function(req, res, next) {
        var err = null,
            requests = req.body,
            requestHost;

        // Validation on Request object as a whole
        try {
            check(_.size(requests), 'Cannot make a batch request with an empty request object').min(1);
            check(_.size(requests), 'Over the max request limit. Please limit batches to ' + params.max + ' requests').max(params.max);
            if (req.method === 'POST' && !req.is('json')) {
                throw new Error('Batch Request will only accept body as json');
            }
        } catch (e) {
            err = {
                error: {
                    'message': e.message,
                    'type': 'ValidationError'
                }
            };
        }

        // Validation on each request object
        _.each(requests, function(r, key) {

            // If no method provided, default to GET
            r.method = (typeof r.method === 'string') ? r.method.toLowerCase() : 'get';

            r.url = getFinalUrl(req, r);

            try {
                check(r.url, 'Invalid URL').isUrl();
                check(r.method, 'Invalid method').isIn(methods);
                if (r.body !== undefined) {
                    check(r.method.toLowerCase(), 'Request body not allowed for this method').isIn(['put', 'post', 'options']);
                }
            } catch (e) {
                err = {
                    error: {
                        'message': e.message,
                        'request': key,
                        'type': 'ValidationError'
                    }
                };
            }

            if (params.allowedHosts !== null) {
                requestHost = url.parse(r.url).host;
                if (params.allowedHosts.indexOf(requestHost) === -1) {
                    err = {
                        error: {
                            'message': 'Cannot make batch request to a host which is not allowed',
                            'host': requestHost,
                            'type': 'ValidationError'
                        }
                    };
                }
            }
        });

        if (err !== null) {
            res.send(400, err);
            next(err);
        } else {
            next();
        }
    };

    return batch;
};

module.exports = batchRequest;
<|endoftext|>