<commit_before>// Copyright (c) 2016, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

import * as Sentry from '@sentry/node';
import fs from 'fs-extra';
import httpProxy from 'http-proxy';
import path from 'path';
import PromClient from 'prom-client';
import temp from 'temp';
import _ from 'underscore';
import which from 'which';

import { getCompilerTypeByKey } from '../compilers';
import { logger } from '../logger';
import * as utils from '../utils';

temp.track();

let hasSetUpAutoClean = false;

function initialise(compilerEnv) {
    if (hasSetUpAutoClean) return;
    hasSetUpAutoClean = true;
    const tempDirCleanupSecs = compilerEnv.ceProps('tempDirCleanupSecs', 600);
    logger.info(`Cleaning temp dirs every ${tempDirCleanupSecs} secs`);

    let cyclesBusy = 0;
    setInterval(() => {
        const status = compilerEnv.compilationQueue.status();
        if (status.busy) {
            cyclesBusy++;
            logger.warn(
                `temp cleanup skipped, pending: ${status.pending}, waiting: ${status.size}, cycles: ${cyclesBusy}`);
            return;
        }

        cyclesBusy = 0;

        temp.cleanup((err, stats) => {
            if (err) logger.error('temp cleanup error', err);
            if (stats) logger.debug('temp cleanup stats', stats);
        });
    }, tempDirCleanupSecs * 1000);
}

export class CompileHandler {
    constructor(compilationEnvironment, awsProps) {
        this.compilersById = {};
        this.compilerEnv = compilationEnvironment;
        this.textBanner = this.compilerEnv.ceProps('textBanner');
        this.proxy = httpProxy.createProxyServer({});
        this.awsProps = awsProps;
        initialise(this.compilerEnv);

        this.compileCounter = new PromClient.Counter({
            name: 'ce_compilations_total',
            help: 'Number of compilations',
            labelNames: ['language'],
        });

        this.executeCounter = new PromClient.Counter({
            name: 'ce_executions_total',
            help: 'Number of executions',
            labelNames: ['language'],
        });

        // Mostly cribbed from
        // https://github.com/nodejitsu/node-http-proxy/blob/master/examples/middleware/bodyDecoder-middleware.js
        // We just keep the body as-is though: no encoding using queryString.stringify(), as we don't use a form
        // decoding middleware.
        this.proxy.on('proxyReq', function (proxyReq, req) {
            if (!req.body || Object.keys(req.body).length === 0) {
                return;
            }

            const contentType = proxyReq.getHeader('Content-Type');
            let bodyData;

            if (contentType === 'application/json') {
                bodyData = JSON.stringify(req.body);
            }

            if (contentType === 'application/x-www-form-urlencoded') {
                bodyData = req.body;
            }

            if (bodyData) {
                proxyReq.setHeader('Content-Length', Buffer.byteLength(bodyData));
                proxyReq.write(bodyData);
            }
        });
    }

    async create(compiler) {
        const type = compiler.compilerType || 'default';
        const compilerClass = getCompilerTypeByKey(type);

        // attempt to resolve non absolute exe paths
        if (compiler.exe && !path.isAbsolute(compiler.exe)) {
            const exe = await which(compiler.exe).catch(() => null);
            if (exe) {
                logger.debug(`Resolved '${compiler.exe}' to path '${exe}'`);
            } else {
                // errors resolving to absolute path are not fatal for backwards compatibility sake
                logger.error(`Unable to resolve '${compiler.exe}'`);
            }
        }

        if (compiler.exe && path.isAbsolute(compiler.exe)) {
            // Try stat'ing the compiler to cache its mtime and only re-run it if it
            // has changed since the last time.
            try {
                const res = await fs.stat(compiler.exe);
                const cached = this.findCompiler(compiler.lang, compiler.id);
                if (cached && cached.mtime.getTime() === res.mtime.getTime()) {
                    logger.debug(`${compiler.id} is unchanged`);
                    return cached;
                }
                const compilerObj = new compilerClass(compiler, this.compilerEnv);
                return compilerObj.initialise(res.mtime);
            } catch (err) {
                logger.warn(`Unable to stat ${compiler.id} compiler binary: `, err);
                return null;
            }
        } else {
            return new compilerClass(compiler, this.compilerEnv);
        }
    }

    async setCompilers(compilers) {
        // Be careful not to update this.compilersById until we can replace it entirely.
        const compilersById = {};
        try {
            const createdCompilers = _.compact(await Promise.all(_.map(compilers, this.create, this)));
            for (const compiler of createdCompilers) {
                const langId = compiler.compiler.lang;
                if (!compilersById[langId]) compilersById[langId] = {};
                compilersById[langId][compiler.compiler.id] = compiler;
            }
            if (this.awsProps) {
                logger.info('Fetching possible arguments from storage');
                await Promise.all(createdCompilers.map(
                    compiler => compiler.possibleArguments.loadFromStorage(this.awsProps)));
            }
            this.compilersById = compilersById;
            return createdCompilers.map(compiler => compiler.getInfo());
        } catch (err) {
            logger.error('Exception while processing compilers:', err);
        }
    }

    findCompiler(langId, compilerId) {
        if (!compilerId) return;

        const langCompilers = this.compilersById[langId];
        if (langCompilers && langCompilers[compilerId]) {
            return langCompilers[compilerId];
        }
        // If the lang is bad, try to find it in every language
        let response;
        _.each(this.compilersById, compilerInLang => {
            if (response === undefined) {
                _.each(compilerInLang, compiler => {
                    if (response === undefined &&
                        (compiler.compiler.id === compilerId || compiler.compiler.alias === compilerId)) {

                        response = compiler;
                    }
                });
            }
        });
        return response;
    }

    compilerFor(req) {
        if (req.is('json')) {
            const compiler = this.findCompiler(req.lang || req.body.lang, req.params.compiler);
            if (!compiler) {
                const withoutBody = _.extend({}, req.body, {source: '<removed>'});
                logger.warn(`Unable to find compiler with lang ${req.lang} for request`, withoutBody);
            }
            return compiler;
        } else if (req.body && req.body.compiler) {
            const compiler = this.findCompiler(req.body.lang, req.body.compiler);
            if (!compiler) {
                const withoutBody = _.extend({}, req.body, {source: '<removed>'});
                logger.warn(`Unable to find compiler with lang ${req.body.lang} for request`, withoutBody);
            }
            return compiler;
        } else {
            const compiler = this.findCompiler(req.lang, req.params.compiler);
            if (!compiler) {
                logger.warn(`Unable to find compiler with lang ${req.lang} for request params`, req.params);
            }
            return compiler;
        }
    }

    parseRequest(req, compiler) {
        let source, options, backendOptions = {}, filters, bypassCache = false, tools, executionParameters = {};
        let libraries = [];
        // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION!
        if (req.is('json')) {
            // JSON-style request
            const requestOptions = req.body.options;
            source = req.body.source;
            if (req.body.bypassCache)
                bypassCache = true;
            options = requestOptions.userArguments;
            const execParams = requestOptions.executeParameters || {};
            executionParameters.args = execParams.args;
            executionParameters.stdin = execParams.stdin;
            backendOptions = requestOptions.compilerOptions || {};
            filters = requestOptions.filters || compiler.getDefaultFilters();
            tools = requestOptions.tools;
            libraries = requestOptions.libraries || [];
        } else if (req.body && req.body.compiler) {
            source = req.body.source;
            if (req.body.bypassCache)
                bypassCache = true;
            options = req.body.userArguments;
            executionParameters.args = req.body.executeParametersArgs;
            executionParameters.stdin = req.body.executeParametersStdin;

            filters = compiler.getDefaultFilters();
            _.each(filters, (value, item) => {
                filters[item] = (req.body[item] === 'true');
            });

            backendOptions.skipAsm = req.body.skipAsm === 'true';
        } else {
            // API-style
            source = req.body;
            options = req.query.options;
            // By default we get the default filters.
            filters = compiler.getDefaultFilters();
            // If specified exactly, we'll take that with ?filters=a,b,c
            if (req.query.filters) {
                filters = _.object(_.map(req.query.filters.split(','), filter => [filter, true]));
            }
            // Add a filter. ?addFilters=binary
            _.each((req.query.addFilters || '').split(','), filter => {
                if (filter) filters[filter] = true;
            });
            // Remove a filter. ?removeFilter=intel
            _.each((req.query.removeFilters || '').split(','), filter => {
                if (filter) delete filters[filter];
            });
            // Ask for asm not to be returned
            backendOptions.skipAsm = req.query.skipAsm === 'true';

        }
        options = utils.splitArguments(options);
        if (!Array.isArray(executionParameters.args)) {
            executionParameters.args = utils.splitArguments(executionParameters.args);
        }

        tools = tools || [];
        tools.forEach((tool) => {
            tool.args = utils.splitArguments(tool.args);
        });
        return {source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries};
    }

    handlePopularArguments(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) return next();
        res.send(compiler.possibleArguments.getPopularArguments(this.getUsedOptions(req)));
    }

    handleOptimizationArguments(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) return next();
        res.send(compiler.possibleArguments.getOptimizationArguments(this.getUsedOptions(req)));
    }

    getUsedOptions(req) {
        if (req.body) {
            const data = (typeof req.body === 'string') ? JSON.parse(req.body) : req.body;

            if (data.presplit) {
                return data.usedOptions;
            } else {
                return utils.splitArguments(data.usedOptions);
            }
        }
        return false;
    }

    handle(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) {
            return next();
        }
        const {
            source, options, backendOptions, filters,
            bypassCache, tools, executionParameters, libraries,
        } = this.parseRequest(req, compiler);
        const remote = compiler.getRemote();
        if (remote) {
            req.url = remote.path;
            this.proxy.web(req, res, {target: remote.target, changeOrigin: true}, e => {
                logger.error('Proxy error: ', e);
                next(e);
            });
            return;
        }

        if (source === undefined) {
            logger.warn('No body found in request', req);
            return next(new Error('Bad request'));
        }

        function textify(array) {
            return _.pluck(array || [], 'text').join('\n');
        }

        this.compileCounter.inc({language: compiler.lang.id});
        // eslint-disable-next-line promise/catch-or-return
        compiler.compile(source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries)
            .then(
                result => {
                    if (req.accepts(['text', 'json']) === 'json') {
                        res.send(result);
                    } else {
                        res.set('Content-Type', 'text/plain');
                        try {
                            if (!_.isEmpty(this.textBanner)) res.write('# ' + this.textBanner + '\n');
                            res.write(textify(result.asm));
                            if (result.code !== 0) res.write('\n# Compiler exited with result code ' + result.code);
                            if (!_.isEmpty(result.stdout)) res.write('\nStandard out:\n' + textify(result.stdout));
                            if (!_.isEmpty(result.stderr)) res.write('\nStandard error:\n' + textify(result.stderr));

                            if (result.execResult) {
                                this.executeCounter.inc({language: compiler.lang.id});
                                res.write('\n\n# Execution result with exit code ' + result.execResult.code + '\n');
                                if (!_.isEmpty(result.execResult.stdout)) {
                                    res.write('# Standard out:\n' + textify(result.execResult.stdout));
                                }
                                if (!_.isEmpty(result.execResult.stderr)) {
                                    res.write('\n# Standard error:\n' + textify(result.execResult.stderr));
                                }
                            }
                        } catch (ex) {
                            Sentry.captureException(ex);
                            res.write(`Error handling request: ${ex}`);
                        }
                        res.end('\n');
                    }
                },
                error => {
                    if (typeof (error) !== 'string') {
                        if (error.stack) {
                            logger.error('Error during compilation: ', error);
                            Sentry.captureException(error);
                        } else if (error.code) {
                            logger.error('Error during compilation: ', error.code);
                            if (typeof (error.stderr) === 'string') {
                                error.stdout = utils.parseOutput(error.stdout);
                                error.stderr = utils.parseOutput(error.stderr);
                            }
                            res.end(JSON.stringify(error));
                            return;
                        } else {
                            logger.error('Error during compilation: ', error);
                        }

                        error = `Internal Compiler Explorer error: ${error.stack || error}`;
                    } else {
                        logger.error('Error during compilation: ', {error});
                    }
                    res.end(JSON.stringify({code: -1, stderr: [{text: error}]}));
                });
    }
}

export function SetTestMode() {
    hasSetUpAutoClean = true;
}
<commit_msg>move execution counting spot<commit_after>// Copyright (c) 2016, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

import * as Sentry from '@sentry/node';
import fs from 'fs-extra';
import httpProxy from 'http-proxy';
import path from 'path';
import PromClient from 'prom-client';
import temp from 'temp';
import _ from 'underscore';
import which from 'which';

import { getCompilerTypeByKey } from '../compilers';
import { logger } from '../logger';
import * as utils from '../utils';

temp.track();

let hasSetUpAutoClean = false;

function initialise(compilerEnv) {
    if (hasSetUpAutoClean) return;
    hasSetUpAutoClean = true;
    const tempDirCleanupSecs = compilerEnv.ceProps('tempDirCleanupSecs', 600);
    logger.info(`Cleaning temp dirs every ${tempDirCleanupSecs} secs`);

    let cyclesBusy = 0;
    setInterval(() => {
        const status = compilerEnv.compilationQueue.status();
        if (status.busy) {
            cyclesBusy++;
            logger.warn(
                `temp cleanup skipped, pending: ${status.pending}, waiting: ${status.size}, cycles: ${cyclesBusy}`);
            return;
        }

        cyclesBusy = 0;

        temp.cleanup((err, stats) => {
            if (err) logger.error('temp cleanup error', err);
            if (stats) logger.debug('temp cleanup stats', stats);
        });
    }, tempDirCleanupSecs * 1000);
}

export class CompileHandler {
    constructor(compilationEnvironment, awsProps) {
        this.compilersById = {};
        this.compilerEnv = compilationEnvironment;
        this.textBanner = this.compilerEnv.ceProps('textBanner');
        this.proxy = httpProxy.createProxyServer({});
        this.awsProps = awsProps;
        initialise(this.compilerEnv);

        this.compileCounter = new PromClient.Counter({
            name: 'ce_compilations_total',
            help: 'Number of compilations',
            labelNames: ['language'],
        });

        this.executeCounter = new PromClient.Counter({
            name: 'ce_executions_total',
            help: 'Number of executions',
            labelNames: ['language'],
        });

        // Mostly cribbed from
        // https://github.com/nodejitsu/node-http-proxy/blob/master/examples/middleware/bodyDecoder-middleware.js
        // We just keep the body as-is though: no encoding using queryString.stringify(), as we don't use a form
        // decoding middleware.
        this.proxy.on('proxyReq', function (proxyReq, req) {
            if (!req.body || Object.keys(req.body).length === 0) {
                return;
            }

            const contentType = proxyReq.getHeader('Content-Type');
            let bodyData;

            if (contentType === 'application/json') {
                bodyData = JSON.stringify(req.body);
            }

            if (contentType === 'application/x-www-form-urlencoded') {
                bodyData = req.body;
            }

            if (bodyData) {
                proxyReq.setHeader('Content-Length', Buffer.byteLength(bodyData));
                proxyReq.write(bodyData);
            }
        });
    }

    async create(compiler) {
        const type = compiler.compilerType || 'default';
        const compilerClass = getCompilerTypeByKey(type);

        // attempt to resolve non absolute exe paths
        if (compiler.exe && !path.isAbsolute(compiler.exe)) {
            const exe = await which(compiler.exe).catch(() => null);
            if (exe) {
                logger.debug(`Resolved '${compiler.exe}' to path '${exe}'`);
            } else {
                // errors resolving to absolute path are not fatal for backwards compatibility sake
                logger.error(`Unable to resolve '${compiler.exe}'`);
            }
        }

        if (compiler.exe && path.isAbsolute(compiler.exe)) {
            // Try stat'ing the compiler to cache its mtime and only re-run it if it
            // has changed since the last time.
            try {
                const res = await fs.stat(compiler.exe);
                const cached = this.findCompiler(compiler.lang, compiler.id);
                if (cached && cached.mtime.getTime() === res.mtime.getTime()) {
                    logger.debug(`${compiler.id} is unchanged`);
                    return cached;
                }
                const compilerObj = new compilerClass(compiler, this.compilerEnv);
                return compilerObj.initialise(res.mtime);
            } catch (err) {
                logger.warn(`Unable to stat ${compiler.id} compiler binary: `, err);
                return null;
            }
        } else {
            return new compilerClass(compiler, this.compilerEnv);
        }
    }

    async setCompilers(compilers) {
        // Be careful not to update this.compilersById until we can replace it entirely.
        const compilersById = {};
        try {
            const createdCompilers = _.compact(await Promise.all(_.map(compilers, this.create, this)));
            for (const compiler of createdCompilers) {
                const langId = compiler.compiler.lang;
                if (!compilersById[langId]) compilersById[langId] = {};
                compilersById[langId][compiler.compiler.id] = compiler;
            }
            if (this.awsProps) {
                logger.info('Fetching possible arguments from storage');
                await Promise.all(createdCompilers.map(
                    compiler => compiler.possibleArguments.loadFromStorage(this.awsProps)));
            }
            this.compilersById = compilersById;
            return createdCompilers.map(compiler => compiler.getInfo());
        } catch (err) {
            logger.error('Exception while processing compilers:', err);
        }
    }

    findCompiler(langId, compilerId) {
        if (!compilerId) return;

        const langCompilers = this.compilersById[langId];
        if (langCompilers && langCompilers[compilerId]) {
            return langCompilers[compilerId];
        }
        // If the lang is bad, try to find it in every language
        let response;
        _.each(this.compilersById, compilerInLang => {
            if (response === undefined) {
                _.each(compilerInLang, compiler => {
                    if (response === undefined &&
                        (compiler.compiler.id === compilerId || compiler.compiler.alias === compilerId)) {

                        response = compiler;
                    }
                });
            }
        });
        return response;
    }

    compilerFor(req) {
        if (req.is('json')) {
            const compiler = this.findCompiler(req.lang || req.body.lang, req.params.compiler);
            if (!compiler) {
                const withoutBody = _.extend({}, req.body, {source: '<removed>'});
                logger.warn(`Unable to find compiler with lang ${req.lang} for request`, withoutBody);
            }
            return compiler;
        } else if (req.body && req.body.compiler) {
            const compiler = this.findCompiler(req.body.lang, req.body.compiler);
            if (!compiler) {
                const withoutBody = _.extend({}, req.body, {source: '<removed>'});
                logger.warn(`Unable to find compiler with lang ${req.body.lang} for request`, withoutBody);
            }
            return compiler;
        } else {
            const compiler = this.findCompiler(req.lang, req.params.compiler);
            if (!compiler) {
                logger.warn(`Unable to find compiler with lang ${req.lang} for request params`, req.params);
            }
            return compiler;
        }
    }

    parseRequest(req, compiler) {
        let source, options, backendOptions = {}, filters, bypassCache = false, tools, executionParameters = {};
        let libraries = [];
        // IF YOU MODIFY ANYTHING HERE PLEASE UPDATE THE DOCUMENTATION!
        if (req.is('json')) {
            // JSON-style request
            const requestOptions = req.body.options;
            source = req.body.source;
            if (req.body.bypassCache)
                bypassCache = true;
            options = requestOptions.userArguments;
            const execParams = requestOptions.executeParameters || {};
            executionParameters.args = execParams.args;
            executionParameters.stdin = execParams.stdin;
            backendOptions = requestOptions.compilerOptions || {};
            filters = requestOptions.filters || compiler.getDefaultFilters();
            tools = requestOptions.tools;
            libraries = requestOptions.libraries || [];
        } else if (req.body && req.body.compiler) {
            source = req.body.source;
            if (req.body.bypassCache)
                bypassCache = true;
            options = req.body.userArguments;
            executionParameters.args = req.body.executeParametersArgs;
            executionParameters.stdin = req.body.executeParametersStdin;

            filters = compiler.getDefaultFilters();
            _.each(filters, (value, item) => {
                filters[item] = (req.body[item] === 'true');
            });

            backendOptions.skipAsm = req.body.skipAsm === 'true';
        } else {
            // API-style
            source = req.body;
            options = req.query.options;
            // By default we get the default filters.
            filters = compiler.getDefaultFilters();
            // If specified exactly, we'll take that with ?filters=a,b,c
            if (req.query.filters) {
                filters = _.object(_.map(req.query.filters.split(','), filter => [filter, true]));
            }
            // Add a filter. ?addFilters=binary
            _.each((req.query.addFilters || '').split(','), filter => {
                if (filter) filters[filter] = true;
            });
            // Remove a filter. ?removeFilter=intel
            _.each((req.query.removeFilters || '').split(','), filter => {
                if (filter) delete filters[filter];
            });
            // Ask for asm not to be returned
            backendOptions.skipAsm = req.query.skipAsm === 'true';

        }
        options = utils.splitArguments(options);
        if (!Array.isArray(executionParameters.args)) {
            executionParameters.args = utils.splitArguments(executionParameters.args);
        }

        tools = tools || [];
        tools.forEach((tool) => {
            tool.args = utils.splitArguments(tool.args);
        });
        return {source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries};
    }

    handlePopularArguments(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) return next();
        res.send(compiler.possibleArguments.getPopularArguments(this.getUsedOptions(req)));
    }

    handleOptimizationArguments(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) return next();
        res.send(compiler.possibleArguments.getOptimizationArguments(this.getUsedOptions(req)));
    }

    getUsedOptions(req) {
        if (req.body) {
            const data = (typeof req.body === 'string') ? JSON.parse(req.body) : req.body;

            if (data.presplit) {
                return data.usedOptions;
            } else {
                return utils.splitArguments(data.usedOptions);
            }
        }
        return false;
    }

    handle(req, res, next) {
        const compiler = this.compilerFor(req);
        if (!compiler) {
            return next();
        }
        const {
            source, options, backendOptions, filters,
            bypassCache, tools, executionParameters, libraries,
        } = this.parseRequest(req, compiler);
        const remote = compiler.getRemote();
        if (remote) {
            req.url = remote.path;
            this.proxy.web(req, res, {target: remote.target, changeOrigin: true}, e => {
                logger.error('Proxy error: ', e);
                next(e);
            });
            return;
        }

        if (source === undefined) {
            logger.warn('No body found in request', req);
            return next(new Error('Bad request'));
        }

        function textify(array) {
            return _.pluck(array || [], 'text').join('\n');
        }

        this.compileCounter.inc({language: compiler.lang.id});
        // eslint-disable-next-line promise/catch-or-return
        compiler.compile(source, options, backendOptions, filters, bypassCache, tools, executionParameters, libraries)
            .then(
                result => {
                    if (result.execResult)
                        this.executeCounter.inc({language: compiler.lang.id});
                    if (req.accepts(['text', 'json']) === 'json') {
                        res.send(result);
                    } else {
                        res.set('Content-Type', 'text/plain');
                        try {
                            if (!_.isEmpty(this.textBanner)) res.write('# ' + this.textBanner + '\n');
                            res.write(textify(result.asm));
                            if (result.code !== 0) res.write('\n# Compiler exited with result code ' + result.code);
                            if (!_.isEmpty(result.stdout)) res.write('\nStandard out:\n' + textify(result.stdout));
                            if (!_.isEmpty(result.stderr)) res.write('\nStandard error:\n' + textify(result.stderr));

                            if (result.execResult) {
                                res.write('\n\n# Execution result with exit code ' + result.execResult.code + '\n');
                                if (!_.isEmpty(result.execResult.stdout)) {
                                    res.write('# Standard out:\n' + textify(result.execResult.stdout));
                                }
                                if (!_.isEmpty(result.execResult.stderr)) {
                                    res.write('\n# Standard error:\n' + textify(result.execResult.stderr));
                                }
                            }
                        } catch (ex) {
                            Sentry.captureException(ex);
                            res.write(`Error handling request: ${ex}`);
                        }
                        res.end('\n');
                    }
                },
                error => {
                    if (typeof (error) !== 'string') {
                        if (error.stack) {
                            logger.error('Error during compilation: ', error);
                            Sentry.captureException(error);
                        } else if (error.code) {
                            logger.error('Error during compilation: ', error.code);
                            if (typeof (error.stderr) === 'string') {
                                error.stdout = utils.parseOutput(error.stdout);
                                error.stderr = utils.parseOutput(error.stderr);
                            }
                            res.end(JSON.stringify(error));
                            return;
                        } else {
                            logger.error('Error during compilation: ', error);
                        }

                        error = `Internal Compiler Explorer error: ${error.stack || error}`;
                    } else {
                        logger.error('Error during compilation: ', {error});
                    }
                    res.end(JSON.stringify({code: -1, stderr: [{text: error}]}));
                });
    }
}

export function SetTestMode() {
    hasSetUpAutoClean = true;
}
<|endoftext|>


<commit_before>// Enable expanding/folding folders in TracBrowser

(function($){
  var FOLDERID_COUNTER = 0;
  var SUBFOLDER_INDENT = 20;

  // enableExpandDir adds the capability to ''folder'' rows in a table
  // to be expanded and folded.
  //
  // It also teach the rows about their ancestors. It expects:
  //  - `parent_tr`, the logical parent row (`null` if there's no ancestor)
  //  - a `rows` jQuery object matching the newly created entry rows
  //  - `qargs`, additional parameters to send to the server when expanding
  //  - `autoexpand`, an optional array corresponding to a splitted sub-path
  //    of entries that will be expanded automatically.

  window.enableExpandDir = function(parent_tr, rows, qargs, autoexpand) {
    // the ancestors folder ids are present in the parent_tr class attribute
    var ancestor_folderids = [];

    if (parent_tr) // rows are logical children of the parent_tr row
      ancestor_folderids = $.grep(parent_tr.attr("class").split(" "),
                                  function(c) { return c.match(/^f\d+$/)});
    else { // rows are toplevel rows, this is the initial call
      var anchor = window.location.hash.substr(1);
      if (anchor)
        autoexpand = anchor.split("/");
    }

    var autoexpand_expander = null;
    rows.each(function () {
      var a = $(this).find("a.dir");

      if (a.length) { // then the entry is a folder
        // create new folder id
        var folderid = "f" + FOLDERID_COUNTER++;
        this.id = folderid;
        $(this).addClass(folderid);

        // add the expander icon
        var expander = $(' ')
          .attr("title", _("Expand sub-directory in place"))
          .click(function() { toggleDir($(this), qargs); })
        a.wrap('

').before(expander);
        if (autoexpand && a.text() == autoexpand[0])
          autoexpand_expander = expander;
      }

      // tie that row to ancestor folders
      if (parent_tr)
        $(this).addClass(ancestor_folderids.join(" "));
    });

    if ( autoexpand_expander )
      toggleDir(autoexpand_expander, qargs, autoexpand.slice(1));
  }

  // handler for click event on the expander icons
  window.toggleDir = function(expander, qargs, autoexpand) {
    var tr = expander.parents("tr:first");
    var folderid = tr.get(0).id;

    if ( tr.hasClass("expanded") ) { // then *fold*
      tr.removeClass("expanded");
      if (tr.next().hasClass("error")) {
        tr.next().remove();
      } else {
        tr.addClass("collapsed");
        tr.siblings("tr."+folderid).hide();
      }
      expander.attr("title", _("Re-expand directory"));
      return;
    }

    // update location, unless autoexpand in progress
    var a = expander.next("a");
    if ( !autoexpand )
      window.location.hash = a.attr("href")
        .substr(window.location.pathname.length+1)
        .replace(/([^?]*)(\?.*)?$/, '$1');

    // update sort links in column headers
    tr.parents("table:first").find("thead tr:first").find("a").each(function(){
      var href = $(this).attr("href").replace(/#.*$/, '');
      $(this).attr("href", href+window.location.hash);
     });

    if ( tr.hasClass("collapsed") ) { // then *expand*
      tr.removeClass("collapsed").addClass("expanded");
      tr.siblings("tr."+folderid).show();
      // Note that the above will show all the already fetched subtrees,
      // so we have to fold again the folders which were already collapsed.
      tr.siblings("tr.collapsed").each(function() {
        tr.siblings("tr."+this.id).not(this).hide();
      });
    } else {                                // then *fetch*
      var td = expander.parents("td:first");
      var td_class = td.attr("class");
      var depth =
        parseFloat(td.css("padding-left").replace(/^(\d*\.\d*).*$/, "$1")) +
        SUBFOLDER_INDENT;

      tr.addClass("expanded");
      // insert "Loading ..." row
      var loading_row = $($.htmlFormat(
        '
' +
        ' ' +
        '  ${loading}' +
        ' 	' +
        '
', {
        td_class: td_class,
        colspan: tr.children("td").length,
        depth: depth,
        loading: babel.format(_("Loading %(entry)s..."), {entry: a.text()})
      }));
      tr.after(loading_row);

      // XHR for getting the rows corresponding to the folder entries
      $.ajax({
        type: "GET",
        url: a.attr("href"),
        data: qargs,
        dataType: "html",
        success: function(data) {
          // Safari 3.1.1 has some trouble reconstructing HTML snippets
          // bigger than 50k - splitting in rows before building DOM nodes
          var rows = data.replace(/^<!DOCTYPE[^>]+>/, "").split("
");
          if (rows.length) {
            // insert entry rows
            $(rows).each(function() {
              row = $(this+"
");
              row.children("td."+td_class).css("padding-left", depth);
              // make all entry rows collapsible but only subdir rows expandable
              loading_row.before(row);
              enableExpandDir(tr, row, qargs, autoexpand);
            });
            // remove "Loading ..." row
            loading_row.remove();
          } else {
            loading_row.find("span.loading")
              .text("")
              .append("" + _("(empty)") + "")
              .removeClass("loading");
            enableExpandDir(tr, loading_row, qargs); // make it collapsible
          }
        },
        error: function(req, err, exc) {
          loading_row.find("span.loading")
            .text("")
            .append("" + _("(error)") + "")
            .removeClass("loading");
          loading_row.addClass("error");
          enableExpandDir(tr, loading_row, qargs); // make it collapsible
        }
      });
    }
    expander.attr("title", _("Fold directory"));
  }

})(jQuery);
<commit_msg>0.12.1dev: Fixed the update of the page location when expanding a directory in the source browser, in the case where the URL has trailing slashes.<commit_after>// Enable expanding/folding folders in TracBrowser

(function($){
  var FOLDERID_COUNTER = 0;
  var SUBFOLDER_INDENT = 20;

  // enableExpandDir adds the capability to ''folder'' rows in a table
  // to be expanded and folded.
  //
  // It also teach the rows about their ancestors. It expects:
  //  - `parent_tr`, the logical parent row (`null` if there's no ancestor)
  //  - a `rows` jQuery object matching the newly created entry rows
  //  - `qargs`, additional parameters to send to the server when expanding
  //  - `autoexpand`, an optional array corresponding to a splitted sub-path
  //    of entries that will be expanded automatically.

  window.enableExpandDir = function(parent_tr, rows, qargs, autoexpand) {
    // the ancestors folder ids are present in the parent_tr class attribute
    var ancestor_folderids = [];

    if (parent_tr) // rows are logical children of the parent_tr row
      ancestor_folderids = $.grep(parent_tr.attr("class").split(" "),
                                  function(c) { return c.match(/^f\d+$/)});
    else { // rows are toplevel rows, this is the initial call
      var anchor = window.location.hash.substr(1);
      if (anchor)
        autoexpand = anchor.split("/");
    }

    var autoexpand_expander = null;
    rows.each(function () {
      var a = $(this).find("a.dir");

      if (a.length) { // then the entry is a folder
        // create new folder id
        var folderid = "f" + FOLDERID_COUNTER++;
        this.id = folderid;
        $(this).addClass(folderid);

        // add the expander icon
        var expander = $(' ')
          .attr("title", _("Expand sub-directory in place"))
          .click(function() { toggleDir($(this), qargs); })
        a.wrap('

').before(expander);
        if (autoexpand && a.text() == autoexpand[0])
          autoexpand_expander = expander;
      }

      // tie that row to ancestor folders
      if (parent_tr)
        $(this).addClass(ancestor_folderids.join(" "));
    });

    if ( autoexpand_expander )
      toggleDir(autoexpand_expander, qargs, autoexpand.slice(1));
  }

  // handler for click event on the expander icons
  window.toggleDir = function(expander, qargs, autoexpand) {
    var tr = expander.parents("tr:first");
    var folderid = tr.get(0).id;

    if ( tr.hasClass("expanded") ) { // then *fold*
      tr.removeClass("expanded");
      if (tr.next().hasClass("error")) {
        tr.next().remove();
      } else {
        tr.addClass("collapsed");
        tr.siblings("tr."+folderid).hide();
      }
      expander.attr("title", _("Re-expand directory"));
      return;
    }

    // update location, unless autoexpand in progress
    var a = expander.next("a");
    if ( !autoexpand )
      window.location.hash = a.attr("href")
        .substr(window.location.pathname.replace(/\/+$/, '').length + 1)
        .replace(/([^?]*)(\?.*)?$/, '$1');

    // update sort links in column headers
    tr.parents("table:first").find("thead tr:first").find("a").each(function(){
      var href = $(this).attr("href").replace(/#.*$/, '');
      $(this).attr("href", href+window.location.hash);
     });

    if ( tr.hasClass("collapsed") ) { // then *expand*
      tr.removeClass("collapsed").addClass("expanded");
      tr.siblings("tr."+folderid).show();
      // Note that the above will show all the already fetched subtrees,
      // so we have to fold again the folders which were already collapsed.
      tr.siblings("tr.collapsed").each(function() {
        tr.siblings("tr."+this.id).not(this).hide();
      });
    } else {                                // then *fetch*
      var td = expander.parents("td:first");
      var td_class = td.attr("class");
      var depth =
        parseFloat(td.css("padding-left").replace(/^(\d*\.\d*).*$/, "$1")) +
        SUBFOLDER_INDENT;

      tr.addClass("expanded");
      // insert "Loading ..." row
      var loading_row = $($.htmlFormat(
        '
' +
        ' ' +
        '  ${loading}' +
        ' 	' +
        '
', {
        td_class: td_class,
        colspan: tr.children("td").length,
        depth: depth,
        loading: babel.format(_("Loading %(entry)s..."), {entry: a.text()})
      }));
      tr.after(loading_row);

      // XHR for getting the rows corresponding to the folder entries
      $.ajax({
        type: "GET",
        url: a.attr("href"),
        data: qargs,
        dataType: "html",
        success: function(data) {
          // Safari 3.1.1 has some trouble reconstructing HTML snippets
          // bigger than 50k - splitting in rows before building DOM nodes
          var rows = data.replace(/^<!DOCTYPE[^>]+>/, "").split("
");
          if (rows.length) {
            // insert entry rows
            $(rows).each(function() {
              row = $(this+"
");
              row.children("td."+td_class).css("padding-left", depth);
              // make all entry rows collapsible but only subdir rows expandable
              loading_row.before(row);
              enableExpandDir(tr, row, qargs, autoexpand);
            });
            // remove "Loading ..." row
            loading_row.remove();
          } else {
            loading_row.find("span.loading")
              .text("")
              .append("" + _("(empty)") + "")
              .removeClass("loading");
            enableExpandDir(tr, loading_row, qargs); // make it collapsible
          }
        },
        error: function(req, err, exc) {
          loading_row.find("span.loading")
            .text("")
            .append("" + _("(error)") + "")
            .removeClass("loading");
          loading_row.addClass("error");
          enableExpandDir(tr, loading_row, qargs); // make it collapsible
        }
      });
    }
    expander.attr("title", _("Fold directory"));
  }

})(jQuery);
<|endoftext|>


<commit_before>
module.exports = function create_device_obscurity (env) {
  function obscure_device (req, res, next) {
    if (res.entries && env.settings.obscureDeviceProvenance) {
      for (var i = 0; i < res.entries.length; i++) {
        res.entries[i].device = env.settings.obscureDeviceProvenance;
      }
    }
    next( );
  }
  return obscure_device;
}
<commit_msg>ensure obfuscation only affects rendered values<commit_after>var _ = require('lodash');

module.exports = function create_device_obscurity (env) {
  function obscure_device (req, res, next) {
    if (res.entries && env.settings.obscureDeviceProvenance) {
      var entries = _.cloneDeep(res.entries);
      for (var i = 0; i < entries.length; i++) {
        entries[i].device = env.settings.obscureDeviceProvenance;
      }
      res.entries = entries;
    }
    next( );
  }
  return obscure_device;
}
<|endoftext|>


<commit_before>'use strict';
/**
 * @file Retrieve mails from the account
 */

var Imap = require('imap');
var async = require('async');
var MailParser = require('mailparser').MailParser;
var xoauth2 = require('xoauth2');
var inspect = require('util').inspect;

var gmailUrl = require('./gmail-url.js');
var config = require('../../config/configuration.js');

// Store total number of mail to exit on last mail
var totalNumberOfMail = -1;

/**
 * Retrieve the All email box from Gmail not depending of the location
 */
var getAllInbox = function(imap, cb) {

  var prefix = "[Gmail]";
  imap.getBoxes(prefix + "/", function(err, boxes) {
    if(err) {
      return cb(err);
    }

    for(var key in boxes[prefix].children) {
      if(listBoxGmail[key].attribs.indexOf("\\All") !== -1) {
        return cb(null, prefix + "/" + key);
      }
    }
  });
};

// Retrieve all mails associated with this user account,
// using refresh_token
// starting from message #`fromUid`.
// For each message, `mailCb` will be called with datas as parameters.
// then calls `finalCb` when everything is finished.
var retrieveGmail = function(refreshToken, googleAccount, fromUid, mailCb, finalCb) {
  // Gmail connection
  var imap = null;

  // Keep trace of the last uid for the next call of this function
  var lastUid = 0;

  // Function called when the retrieve of the email block.
  // This is just a call to retrieveGmail with the current lastUid
  var retryAfterTimeout = function() {
    imap.destroy();
    return retrieveGmail(refreshToken, googleAccount, lastUid, mailCb, finalCb);
  };

  var idTimer = null;

  // Function called when a message is retrieved from Gmail
  // This function get the body and the attributs of the mail and retun a
  // task variable with all the information of the email
  var onMessage = function(msg, seqno, cb) {
    console.log('Message #%d', seqno);

    clearTimeout(idTimer);

    // Build a buffer containing all datas from the mail
    var attrs = null;
    var mailObject = null;

    var parser = new MailParser();

    var prefix = '(#' + seqno + '/' + totalNumberOfMail + ') ';

    async.parallel([
      function(cb) {
        msg.once('body', function(stream) {
          console.log(prefix + 'Body');

          stream.pipe(parser);

          cb();
        });
      },
      function(cb) {
        msg.once('attributes', function(a) {
          attrs = a;

          console.log(prefix + 'Attributes: %s', inspect(attrs));
          cb();
        });
      },
      function(cb) {
        parser.on("end", function(mail_object) {
          mailObject = mail_object;
          cb();
        });
      },
      function(cb) {
        msg.once('end', function() {
          console.log(prefix + 'Finished');

          clearTimeout(idTimer);
          // Create a new timer to ensure we won't get stuck waiting for imap new message
          idTimer = setTimeout(retryAfterTimeout, config.imap_timeout);
          cb();
        });
      }
    ],
    function() {
      var url = gmailUrl(googleAccount, attrs['x-gm-thrid']);
      var task = {
        url: url,
        id: attrs['x-gm-msgid'],
        uid: attrs.uid,
        threadid: attrs['x-gm-thrid'],
        from: mailObject.from,
        to: mailObject.to,
        subject: mailObject.subject,
        text: mailObject.text,
        html: mailObject.html,
        labels: attrs['x-gm-labels'],
      };

      cb(task);
    });
  };

  // Generate a XOAUTH2.0 token
  // See https://developers.google.com/gmail/xoauth2_protocol
  // And https://github.com/andris9/xoauth2
  var xoauth2gen = xoauth2.createXOAuth2Generator( {
    user: googleAccount,
    clientId: config.google_id,
    clientSecret: config.google_secret,
    refreshToken: refreshToken
  });

  async.waterfall([
    // Create a token from refresh_token
    function(cb) {
      xoauth2gen.getToken(cb);
    },
    // Open imap connection
    function(token, accessToken, cb) {
      // Create a new Imap connection
      // See https://github.com/mscdex/node-imap
      console.log("Start of the imap");
      imap = new Imap( {
        xoauth2: token,
        host: 'imap.gmail.com',
        port: 993,
        tls: true,
        tlsOptions: { rejectUnauthorized: false }
      });

      imap.once('ready', function(err) {
        if(err) {
          return finalCb(err);
        }
        cb();
      }
    },
    function(cb) {
      getAllInbox(imap, function(err, boxName) {
        if (err) {
          return finalCb(err);
        }
        cb(boxName);
      }
    },
    function(boxName, cb) {
      imap.openBox(boxName, true, function(err) {
        if (err) {
          return finalCb(err);
        }
        cb();
      }
    },
    function(cb) {
      imap.search([['UID', fromUid + ':*']], function(err, results) {
        if(totalNumberOfMail === -1) {
          totalNumberOfMail = results.length;
        }
        if(totalNumberOfMail === 0) {
          return finalCb();
        }
        if(process.env.NODE_ENV !== 'test') {
          console.log(results.length + " mails for " + googleAccount);
        }
        console.log("Start of the fetch");
        var f = imap.fetch(results, {  bodies: '' , size : true });

        // Set the timeout
        idTimer = setTimeout(retryAfterTimeout, config.imap_timeout);

        f.on('message', function(msg, seqno){
          onMessage(msg, seqno, function(task) {
            lastUid = task.uid + 1;
            mailCb(task);

            // If we're done fetching all the emails
            if(seqno === totalNumberOfMail) {
              console.log('Done fetching all messages!');
              cb();
              return finalCb(null, lastUid);
            }
          });
        });

        f.on('error', function(err) {
          console.log('Fetch error: ' + err);
          return finalCb(err);
        });
      });

      imap.connect();
    }],
    function() {
    console.log("End of the fetch of the message");
  });
};

module.exports = function(refreshToken, googleAccount, fromUid, mailCb, finalCb) {
  var onMailReceived = function(task) {
    var uid = task.uid;
    if(!mails[uid]) {
      mailCb(task);
      mails.push(task.uid);
    }
  };

  var mails = [];
  retrieveGmail(refreshToken, googleAccount, fromUid, onMailReceived, finalCb);
};
<commit_msg>Fix waterfall<commit_after>'use strict';
/**
 * @file Retrieve mails from the account
 */

var Imap = require('imap');
var async = require('async');
var MailParser = require('mailparser').MailParser;
var xoauth2 = require('xoauth2');
var inspect = require('util').inspect;

var gmailUrl = require('./gmail-url.js');
var config = require('../../config/configuration.js');

/**
 * Retrieve the All email box from Gmail not depending of the location
 */
var getAllInbox = function(imap, cb) {

  var prefix = "[Gmail]";
  imap.getBoxes(prefix + "/", function(err, boxes) {
    if(err) {
      return cb(err);
    }

    for(var key in boxes[prefix].children) {
      if(boxes[prefix].children[key].attribs.indexOf("\\All") !== -1) {
        return cb(null, prefix + "/" + key);
      }
    }
  });
};

// Retrieve all mails associated with this user account,
// using refresh_token
// starting from message #`fromUid`.
// For each message, `mailCb` will be called with datas as parameters.
// then calls `finalCb` when everything is finished.
var retrieveGmail = function(refreshToken, googleAccount, fromUid, mailCb, finalCb) {
  // Gmail connection
  var imap = null;

  // Store total number of mail to exit on last mail
  var totalNumberOfMail;

  // Keep trace of the last uid for the next call of this function
  var lastUid = 0;

  // Function called when the retrieve of the email block.
  // This is just a call to retrieveGmail with the current lastUid
  var retryAfterTimeout = function() {
    imap.destroy();
    return retrieveGmail(refreshToken, googleAccount, lastUid, mailCb, finalCb);
  };

  var idTimer = null;

  // Function called when a message is retrieved from Gmail
  // This function get the body and the attributs of the mail and retun a
  // task variable with all the information of the email
  var onMessage = function(msg, seqno, cb) {
    console.log('Message #%d', seqno);

    clearTimeout(idTimer);

    // Build a buffer containing all datas from the mail
    var attrs = null;
    var mailObject = null;

    var parser = new MailParser();

    var prefix = '(#' + seqno + '/' + totalNumberOfMail + ') ';

    async.parallel([
      function(cb) {
        msg.once('body', function(stream) {
          console.log(prefix + 'Body');

          stream.pipe(parser);

          cb();
        });
      },
      function(cb) {
        msg.once('attributes', function(a) {
          attrs = a;

          console.log(prefix + 'Attributes: %s', inspect(attrs));
          cb();
        });
      },
      function(cb) {
        parser.on("end", function(mail_object) {
          mailObject = mail_object;
          cb();
        });
      },
      function(cb) {
        msg.once('end', function() {
          console.log(prefix + 'Finished');

          clearTimeout(idTimer);
          // Create a new timer to ensure we won't get stuck waiting for imap new message
          idTimer = setTimeout(retryAfterTimeout, config.imap_timeout);
          cb();
        });
      }
    ],
    function() {
      var url = gmailUrl(googleAccount, attrs['x-gm-thrid']);
      var task = {
        url: url,
        id: attrs['x-gm-msgid'],
        uid: attrs.uid,
        threadid: attrs['x-gm-thrid'],
        from: mailObject.from,
        to: mailObject.to,
        subject: mailObject.subject,
        text: mailObject.text,
        html: mailObject.html,
        labels: attrs['x-gm-labels'],
      };

      cb(task);
    });
  };

  // Generate a XOAUTH2.0 token
  // See https://developers.google.com/gmail/xoauth2_protocol
  // And https://github.com/andris9/xoauth2
  var xoauth2gen = xoauth2.createXOAuth2Generator( {
    user: googleAccount,
    clientId: config.google_id,
    clientSecret: config.google_secret,
    refreshToken: refreshToken
  });

  async.waterfall([
    // Create a token from refresh_token
    function(cb) {
      xoauth2gen.getToken(cb);
    },
    function(token, accessToken, cb) {
      // Create a new Imap connection
      // See https://github.com/mscdex/node-imap
      console.log("Start of the imap");
      imap = new Imap( {
        xoauth2: token,
        host: 'imap.gmail.com',
        port: 993,
        tls: true,
        tlsOptions: { rejectUnauthorized: false }
      });

      imap.once('ready', cb);

      imap.connect();
    },
    function(cb) {
      getAllInbox(imap, cb);
    },
    function(boxName, cb) {
      imap.openBox(boxName, true, cb);
    },
    function(mailbox, cb) {
      imap.search([['UID', fromUid + ':*']], function(err, results) {
        totalNumberOfMail = results.length;
        if(totalNumberOfMail === 0) {
          return cb();
        }
        if(process.env.NODE_ENV !== 'test') {
          console.log(results.length + " mails for " + googleAccount);
        }
        console.log("Start of the fetch");
        var f = imap.fetch(results, {  bodies: '' , size : true });

        // Set the timeout
        idTimer = setTimeout(retryAfterTimeout, config.imap_timeout);

        f.on('message', function(msg, seqno){
          onMessage(msg, seqno, function(task) {
            lastUid = task.uid + 1;
            mailCb(task);

            // If we're done fetching all the emails
            if(seqno === totalNumberOfMail) {
              console.log('Done fetching all messages!');
              return cb();
            }
          });
        });

        f.on('error', function(err) {
          console.log('Fetch error: ' + err);
          cb(err);
        });
      });

    }
  ], function(err) {
      finalCb(err, lastUid)
      console.log("End of the fetch of the message");
  });
};

module.exports = function(refreshToken, googleAccount, fromUid, mailCb, finalCb) {
  var onMailReceived = function(task) {
    var uid = task.uid;
    if(!mails[uid]) {
      mailCb(task);
      mails.push(task.uid);
    }
  };

  var mails = [];
  retrieveGmail(refreshToken, googleAccount, fromUid, onMailReceived, finalCb);
};
<|endoftext|>


<commit_before>'use strict';

const { isObject, isString, asArray, asSingle } = require('../../utils/objectUtils');
const { Type: ValidationErrorType } = require('../../model/ValidationError');
const { ModelGraphNode } = require('./ModelGraphNode');
const { ModelGraphEdge } = require('./ModelGraphEdge');

class ModelGraphBuilder {
  constructor() {
    this.nodes = [];
    this.edges = [];
  }

  static buildGraph(rootModelClass, roots) {
    const builder = new this();
    builder._buildGraph(rootModelClass, roots);

    return builder;
  }

  _buildGraph(rootModelClass, roots) {
    if (roots) {
      if (Array.isArray(roots)) {
        this._buildNodes(rootModelClass, roots);
      } else if (isObject(roots)) {
        this._buildNode(rootModelClass, roots);
      } else {
        throw createNotModelError(rootModelClass, roots);
      }
    }

    this._buildReferences();
  }

  _buildNodes(modelClass, objs, parentNode = null, relation = null) {
    asArray(objs).forEach((obj, index) => {
      this._buildNode(modelClass, obj, parentNode, relation, index);
    });
  }

  _buildNode(modelClass, obj, parentNode = null, relation = null, index = null) {
    if (!isObject(obj) || !obj.$isObjectionModel) {
      throw createNotModelError(modelClass, obj);
    }

    const node = new ModelGraphNode(modelClass, asSingle(obj));
    this.nodes.push(node);

    if (parentNode) {
      const edge = new ModelGraphEdge(
        ModelGraphEdge.Type.Relation,
        parentNode,
        node,
        relation,
        index
      );

      node.parentEdge = edge;
      this._addEdge(parentNode, node, edge);
    }

    this._buildRelationNodes(node);
  }

  _buildRelationNodes(node) {
    for (const relation of node.modelClass.getRelationArray()) {
      const relatedObjects = node.obj[relation.name];

      if (!relatedObjects) {
        continue;
      }

      if (relation.isOneToOne()) {
        this._buildNode(relation.relatedModelClass, relatedObjects, node, relation);
      } else {
        this._buildNodes(relation.relatedModelClass, relatedObjects, node, relation);
      }
    }
  }

  _buildReferences() {
    const nodesByUid = this._nodesByUid();

    this._buildObjectReferences(nodesByUid);
    this._buildPropertyReferences(nodesByUid);
  }

  _nodesByUid() {
    const nodesByUid = new Map();

    for (const node of this.nodes) {
      const uid = node.uid;

      if (uid === undefined) {
        continue;
      }

      nodesByUid.set(uid, node);
    }

    return nodesByUid;
  }

  _buildObjectReferences(nodesByUid) {
    for (const node of this.nodes) {
      const ref = node.reference;

      if (ref === undefined) {
        continue;
      }

      const refNode = nodesByUid.get(ref);

      if (!refNode) {
        throw createReferenceFoundError(node, ref);
      }

      const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode);
      edge.refType = ModelGraphEdge.ReferenceType.Object;

      this._addEdge(node, refNode, edge);
    }
  }

  _buildPropertyReferences(nodesByUid) {
    for (const node of this.nodes) {
      const relations = node.modelClass.getRelations();

      for (const prop of Object.keys(node.obj)) {
        if (relations[prop]) {
          continue;
        }

        this._buildPropertyReference(nodesByUid, node, prop);
      }
    }
  }

  _buildPropertyReference(nodesByUid, node, prop) {
    visitStrings(node.obj[prop], [prop], (str, path) => {
      forEachMatch(node.modelClass.propRefRegex, str, match => {
        const [_, ref, refPath] = match;
        const refNode = nodesByUid.get(ref);

        if (!refNode) {
          throw createReferenceFoundError(node, ref);
        }

        const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode);

        edge.refType = ModelGraphEdge.ReferenceType.Property;
        edge.refMatch = match[0];
        edge.refOwnerDataPath = path.slice();
        edge.refRelatedDataPath = refPath.split('.');

        this._addEdge(node, refNode, edge);
      });
    });
  }

  _addEdge(ownerNode, relatedNode, edge) {
    this.edges.push(edge);

    ownerNode.edges.push(edge);
    relatedNode.edges.push(edge);

    if (edge.type === ModelGraphEdge.Type.Reference) {
      ownerNode.refEdges.push(edge);
      relatedNode.refEdges.push(edge);
    }
  }
}

function visitStrings(value, path, visit) {
  if (Array.isArray(value)) {
    visitStringsInArray(value, path, visit);
  } else if (isObject(value)) {
    visitStringsInObject(value, path, visit);
  } else if (isString(value)) {
    visit(value, path);
  }
}

function visitStringsInArray(value, path, visit) {
  for (let i = 0; i < value.length; ++i) {
    path.push(i);
    visitStrings(value[i], path, visit);
    path.pop();
  }
}

function visitStringsInObject(value, path, visit) {
  for (const prop of Object.keys(value)) {
    path.push(prop);
    visitStrings(value[prop], path, visit);
    path.pop();
  }
}

function forEachMatch(regex, str, cb) {
  let matchResult = regex.exec(str);

  while (matchResult) {
    cb(matchResult);
    matchResult = regex.exec(str);
  }
}

function createReferenceFoundError(node, ref) {
  return new Error('no reference found');
}

function createNotModelError(modelClass, value) {
  throw modelClass.createValidationError({
    type: ValidationErrorType.InvalidGraph,
    message: `expected value "${value}" to be an instance of ${modelClass.name}`
  });
}

module.exports = {
  ModelGraphBuilder,
  createNotModelError
};
<commit_msg>fix regression with arrays in belongsToOne + upsertGraph<commit_after>'use strict';

const { isObject, isString, asArray, asSingle } = require('../../utils/objectUtils');
const { Type: ValidationErrorType } = require('../../model/ValidationError');
const { ModelGraphNode } = require('./ModelGraphNode');
const { ModelGraphEdge } = require('./ModelGraphEdge');

class ModelGraphBuilder {
  constructor() {
    this.nodes = [];
    this.edges = [];
  }

  static buildGraph(rootModelClass, roots) {
    const builder = new this();
    builder._buildGraph(rootModelClass, roots);

    return builder;
  }

  _buildGraph(rootModelClass, roots) {
    if (roots) {
      if (Array.isArray(roots)) {
        this._buildNodes(rootModelClass, roots);
      } else if (isObject(roots)) {
        this._buildNode(rootModelClass, roots);
      } else {
        throw createNotModelError(rootModelClass, roots);
      }
    }

    this._buildReferences();
  }

  _buildNodes(modelClass, objs, parentNode = null, relation = null) {
    objs = asArray(objs);

    objs.forEach((obj, index) => {
      this._buildNode(modelClass, obj, parentNode, relation, index);
    });
  }

  _buildNode(modelClass, obj, parentNode = null, relation = null, index = null) {
    obj = asSingle(obj);

    if (!isObject(obj) || !obj.$isObjectionModel) {
      throw createNotModelError(modelClass, obj);
    }

    const node = new ModelGraphNode(modelClass, obj);
    this.nodes.push(node);

    if (parentNode) {
      const edge = new ModelGraphEdge(
        ModelGraphEdge.Type.Relation,
        parentNode,
        node,
        relation,
        index
      );

      node.parentEdge = edge;
      this._addEdge(parentNode, node, edge);
    }

    this._buildRelationNodes(node);
  }

  _buildRelationNodes(node) {
    for (const relation of node.modelClass.getRelationArray()) {
      const relatedObjects = node.obj[relation.name];

      if (!relatedObjects) {
        continue;
      }

      if (relation.isOneToOne()) {
        this._buildNode(relation.relatedModelClass, relatedObjects, node, relation);
      } else {
        this._buildNodes(relation.relatedModelClass, relatedObjects, node, relation);
      }
    }
  }

  _buildReferences() {
    const nodesByUid = this._nodesByUid();

    this._buildObjectReferences(nodesByUid);
    this._buildPropertyReferences(nodesByUid);
  }

  _nodesByUid() {
    const nodesByUid = new Map();

    for (const node of this.nodes) {
      const uid = node.uid;

      if (uid === undefined) {
        continue;
      }

      nodesByUid.set(uid, node);
    }

    return nodesByUid;
  }

  _buildObjectReferences(nodesByUid) {
    for (const node of this.nodes) {
      const ref = node.reference;

      if (ref === undefined) {
        continue;
      }

      const refNode = nodesByUid.get(ref);

      if (!refNode) {
        throw createReferenceFoundError(node, ref);
      }

      const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode);
      edge.refType = ModelGraphEdge.ReferenceType.Object;

      this._addEdge(node, refNode, edge);
    }
  }

  _buildPropertyReferences(nodesByUid) {
    for (const node of this.nodes) {
      const relations = node.modelClass.getRelations();

      for (const prop of Object.keys(node.obj)) {
        if (relations[prop]) {
          continue;
        }

        this._buildPropertyReference(nodesByUid, node, prop);
      }
    }
  }

  _buildPropertyReference(nodesByUid, node, prop) {
    visitStrings(node.obj[prop], [prop], (str, path) => {
      forEachMatch(node.modelClass.propRefRegex, str, match => {
        const [_, ref, refPath] = match;
        const refNode = nodesByUid.get(ref);

        if (!refNode) {
          throw createReferenceFoundError(node, ref);
        }

        const edge = new ModelGraphEdge(ModelGraphEdge.Type.Reference, node, refNode);

        edge.refType = ModelGraphEdge.ReferenceType.Property;
        edge.refMatch = match[0];
        edge.refOwnerDataPath = path.slice();
        edge.refRelatedDataPath = refPath.split('.');

        this._addEdge(node, refNode, edge);
      });
    });
  }

  _addEdge(ownerNode, relatedNode, edge) {
    this.edges.push(edge);

    ownerNode.edges.push(edge);
    relatedNode.edges.push(edge);

    if (edge.type === ModelGraphEdge.Type.Reference) {
      ownerNode.refEdges.push(edge);
      relatedNode.refEdges.push(edge);
    }
  }
}

function visitStrings(value, path, visit) {
  if (Array.isArray(value)) {
    visitStringsInArray(value, path, visit);
  } else if (isObject(value)) {
    visitStringsInObject(value, path, visit);
  } else if (isString(value)) {
    visit(value, path);
  }
}

function visitStringsInArray(value, path, visit) {
  for (let i = 0; i < value.length; ++i) {
    path.push(i);
    visitStrings(value[i], path, visit);
    path.pop();
  }
}

function visitStringsInObject(value, path, visit) {
  for (const prop of Object.keys(value)) {
    path.push(prop);
    visitStrings(value[prop], path, visit);
    path.pop();
  }
}

function forEachMatch(regex, str, cb) {
  let matchResult = regex.exec(str);

  while (matchResult) {
    cb(matchResult);
    matchResult = regex.exec(str);
  }
}

function createReferenceFoundError(node, ref) {
  return new Error('no reference found');
}

function createNotModelError(modelClass, value) {
  throw modelClass.createValidationError({
    type: ValidationErrorType.InvalidGraph,
    message: `expected value "${value}" to be an instance of ${modelClass.name}`
  });
}

module.exports = {
  ModelGraphBuilder,
  createNotModelError
};
<|endoftext|>


<commit_before>module.exports = function (sails) {

  /**
   * Module dependencies.
   */

  var async     = require('async');

  return function startServer (cb) {

    // Used to warn about possible issues if starting the server is taking a long time
    var liftAbortTimer;
    var liftTimeout = sails.config.liftTimeout || 4000;

    async.auto({

      // Start Express server
      start: function (cb) {

        var explicitHost = sails.config.explicitHost;

        // If host is explicitly declared, include it in express's listen() call
        if (explicitHost) {
          sails.log.verbose('Restricting access to explicit host: '+explicitHost);
          sails.hooks.http.server.listen(sails.config.port, explicitHost, cb);
        }
        else {
          // Listen for error events that may be emitted as the server attempts to start
          sails.hooks.http.server.on('error', failedToStart);
          sails.hooks.http.server.listen(sails.config.port, function(err) {
            // Remove the error listener so future error events emitted by the server
            // don't get handled by the "failedToStart" function below.
            sails.hooks.http.server.removeListener('error', failedToStart);
            cb(err);
          });
        }

        // Start timer in case this takes suspiciously long...
        liftAbortTimer = setTimeout(failedToStart, liftTimeout);

        // If the server fails to start because of an error, or if it's just taking
        // too long, show some troubleshooting notes and bail out.
        function failedToStart(err) {

          // If this was called because of an actual error, clear the timeout
          // so failedToStart doesn't get called again.
          if (err) {
            clearTimeout(liftAbortTimer);
          }

          // If sails is exiting already, don't worry about the timer going off.
          if (sails._exiting) {return;}

          // Figure out if this user is on Windows
          var isWin = !!process.platform.match(/^win/);

          // If server isn't starting, provide general troubleshooting information,
          // sharpened with a few simple heuristics:
          console.log('');
          if (err) {
            sails.log.error('Server failed to start.');
            if (err.code) {
              sails.log.error('(received error: ' + err.code + ')');
            }
          } else {
            sails.log.error('Server is taking a while to start up (it\'s been 4 seconds).');
          }

          sails.log.error();
          sails.log.error('Troubleshooting tips:');
          sails.log.error();

          // 0. Just a slow Grunt task
          if (sails.hooks.grunt && ! (err && err.code == 'EADDRINUSE')) {
            if (process.env.NODE_ENV === 'production') {
              sails.log.error(
                ' -> Do you have a slow Grunt task?  You are running in production mode where, by default, tasks are configured to minify the JavaScript and CSS/LESS files in your assets/ directory.  Sometimes, these processes can be slow, particularly if you have lots of these types of files.'
              );
            }
            else {
              sails.log.error(
                ' -> Do you have a slow Grunt task, or lots of assets?'
              );
            }
            sails.log.error();
          }

          // 1. Unauthorized
          if (sails.config.port < 1024) {
            sails.log.error(
            ' -> Do you have permission to use port ' + sails.config.port + ' on this system?',
            // Don't mention `sudo` to Windows users-- I hear you guys get touchy about that sort of thing :)
            (isWin) ? '' : '(you might try `sudo`)'
            );

            sails.log.error();
          }

          // 2. Invalid or unauthorized explicitHost configuration.
          if (explicitHost) {
            sails.log.error(
            ' -> You might remove your explicit host configuration and try lifting again (you specified',
            '`'+explicitHost+'`',
            '.)');

            sails.log.error();
          }

          // 3. Something else is running on this port
          sails.log.error(
          ' -> Is something else already running on port', sails.config.port,
          (explicitHost ? (' with hostname ' + explicitHost) : '') + '?'
          );
          sails.log.error();

          // 4. invalid explicitHost
          if (!explicitHost) {
            sails.log.error(
            ' -> Are you deploying on a platform that requires an explicit hostname,',
            'like OpenShift?');
            sails.log.error(
            '    (Try setting the `explicitHost` config to the hostname where the server will be accessible.)'
            );
            sails.log.error(
            '    (e.g. `mydomain.com` or `183.24.244.42`)'
            );
          }
          console.log('');

          // Lower Sails to do any necessary cleanup
          sails.lower(function(){
            // Exit with a non-zero value to indicate an error
            process.exit(1);
          });
        }
      },

      verify: ['start', function (cb) {
        var explicitHost = sails.config.explicitHost;

        // Check for port conflicts
        // Ignore this check if explicit host is set, since other more complicated things might be going on.
        if( !explicitHost && !sails.hooks.http.server.address() ) {
          var portBusyError = '';
          portBusyError += 'Trying to start server on port ' + sails.config.port + ' but can\'t...';
          portBusyError += 'Something else is probably running on that port!' + '\n';
          portBusyError += 'Please disable the other server, or choose a different port and try again.';
          sails.log.error(portBusyError);
          throw new Error(portBusyError);
        }

        cb();
      }]

    }, function expressListening (err) {
      clearTimeout(liftAbortTimer);

      if (err) return cb(err);

      // Announce that express is now listening on a port
      sails.emit('hook:http:listening');

      cb && cb(err);
    });
  };

};
<commit_msg>Trivial.<commit_after>/**
 * Module dependencies.
 */

var async = require('async');

module.exports = function (sails) {

  return function startServer (cb) {

    // Used to warn about possible issues if starting the server is taking a long time
    var liftAbortTimer;
    var liftTimeout = sails.config.liftTimeout || 4000;

    async.auto({

      // Start Express server
      start: function (cb) {

        var explicitHost = sails.config.explicitHost;

        // If host is explicitly declared, include it in express's listen() call
        if (explicitHost) {
          sails.log.verbose('Restricting access to explicit host: '+explicitHost);
          sails.hooks.http.server.listen(sails.config.port, explicitHost, cb);
        }
        else {
          // Listen for error events that may be emitted as the server attempts to start
          sails.hooks.http.server.on('error', failedToStart);
          sails.hooks.http.server.listen(sails.config.port, function(err) {
            // Remove the error listener so future error events emitted by the server
            // don't get handled by the "failedToStart" function below.
            sails.hooks.http.server.removeListener('error', failedToStart);
            cb(err);
          });
        }

        // Start timer in case this takes suspiciously long...
        liftAbortTimer = setTimeout(failedToStart, liftTimeout);

        // If the server fails to start because of an error, or if it's just taking
        // too long, show some troubleshooting notes and bail out.
        function failedToStart(err) {

          // If this was called because of an actual error, clear the timeout
          // so failedToStart doesn't get called again.
          if (err) {
            clearTimeout(liftAbortTimer);
          }

          // If sails is exiting already, don't worry about the timer going off.
          if (sails._exiting) {return;}

          // Figure out if this user is on Windows
          var isWin = !!process.platform.match(/^win/);

          // If server isn't starting, provide general troubleshooting information,
          // sharpened with a few simple heuristics:
          console.log('');
          if (err) {
            sails.log.error('Server failed to start.');
            if (err.code) {
              sails.log.error('(received error: ' + err.code + ')');
            }
          } else {
            sails.log.error('Server is taking a while to start up (it\'s been 4 seconds).');
          }

          sails.log.error();
          sails.log.error('Troubleshooting tips:');
          sails.log.error();

          // 0. Just a slow Grunt task
          if (sails.hooks.grunt && ! (err && err.code == 'EADDRINUSE')) {
            if (process.env.NODE_ENV === 'production') {
              sails.log.error(
                ' -> Do you have a slow Grunt task?  You are running in production mode where, by default, tasks are configured to minify the JavaScript and CSS/LESS files in your assets/ directory.  Sometimes, these processes can be slow, particularly if you have lots of these types of files.'
              );
            }
            else {
              sails.log.error(
                ' -> Do you have a slow Grunt task, or lots of assets?'
              );
            }
            sails.log.error();
          }

          // 1. Unauthorized
          if (sails.config.port < 1024) {
            sails.log.error(
            ' -> Do you have permission to use port ' + sails.config.port + ' on this system?',
            // Don't mention `sudo` to Windows users-- I hear you guys get touchy about that sort of thing :)
            (isWin) ? '' : '(you might try `sudo`)'
            );

            sails.log.error();
          }

          // 2. Invalid or unauthorized explicitHost configuration.
          if (explicitHost) {
            sails.log.error(
            ' -> You might remove your explicit host configuration and try lifting again (you specified',
            '`'+explicitHost+'`',
            '.)');

            sails.log.error();
          }

          // 3. Something else is running on this port
          sails.log.error(
          ' -> Is something else already running on port', sails.config.port,
          (explicitHost ? (' with hostname ' + explicitHost) : '') + '?'
          );
          sails.log.error();

          // 4. invalid explicitHost
          if (!explicitHost) {
            sails.log.error(
            ' -> Are you deploying on a platform that requires an explicit hostname,',
            'like OpenShift?');
            sails.log.error(
            '    (Try setting the `explicitHost` config to the hostname where the server will be accessible.)'
            );
            sails.log.error(
            '    (e.g. `mydomain.com` or `183.24.244.42`)'
            );
          }
          console.log('');

          // Lower Sails to do any necessary cleanup
          sails.lower(function(){
            // Exit with a non-zero value to indicate an error
            process.exit(1);
          });
        }
      },

      verify: ['start', function (cb) {
        var explicitHost = sails.config.explicitHost;

        // Check for port conflicts
        // Ignore this check if explicit host is set, since other more complicated things might be going on.
        if( !explicitHost && !sails.hooks.http.server.address() ) {
          var portBusyError = '';
          portBusyError += 'Trying to start server on port ' + sails.config.port + ' but can\'t...';
          portBusyError += 'Something else is probably running on that port!' + '\n';
          portBusyError += 'Please disable the other server, or choose a different port and try again.';
          sails.log.error(portBusyError);
          throw new Error(portBusyError);
        }

        cb();
      }]

    }, function expressListening (err) {
      clearTimeout(liftAbortTimer);

      if (err) return cb(err);

      // Announce that express is now listening on a port
      sails.emit('hook:http:listening');

      cb && cb(err);
    });
  };

};
<|endoftext|>


<commit_before>import {TextBuffer} from 'atom';

export default class MultiFilePatch {
  constructor(buffer = null, layers = {}, filePatches = []) {
    this.buffer = buffer;

    this.patchLayer = layers.patch;
    this.hunkLayer = layers.hunk;
    this.unchangedLayer = layers.unchanged;
    this.additionLayer = layers.addition;
    this.deletionLayer = layers.deletion;
    this.noNewlineLayer = layers.noNewline;

    this.filePatches = filePatches;

    this.filePatchesByMarker = new Map();
    this.hunksByMarker = new Map();

    for (const filePatch of this.filePatches) {
      this.filePatchesByMarker.set(filePatch.getMarker(), filePatch);
      for (const hunk of filePatch.getHunks()) {
        this.hunksByMarker.set(hunk.getMarker(), hunk);
      }
    }
  }

  getBuffer() {
    return this.buffer;
  }

  getHunkLayer() {
    return this.hunkLayer;
  }

  getUnchangedLayer() {
    return this.unchangedLayer;
  }

  getAdditionLayer() {
    return this.additionLayer;
  }

  getDeletionLayer() {
    return this.deletionLayer;
  }

  getNoNewlineLayer() {
    return this.noNewlineLayer;
  }

  getFilePatches() {
    return this.filePatches;
  }

  getFilePatchAt(bufferRow) {
    const [marker] = this.patchLayer.findMarkers({intersectsRow: bufferRow});
    return this.filePatchesByMarker.get(marker);
  }

  getHunkAt(bufferRow) {
    const [marker] = this.hunkLayer.findMarkers({intersectsRow: bufferRow});
    return this.hunksByMarker.get(marker);
  }

  getStagePatchForLines(selectedLineSet) {
    const nextLayeredBuffer = this.buildLayeredBuffer();
    const nextFilePatches = Array.from(this.getFilePatchesContaining(selectedLineSet), fp => {
      return fp.buildStagePatchForLines(this.getBuffer(), nextLayeredBuffer, selectedLineSet);
    });

    return new MultiFilePatch(
      nextLayeredBuffer.buffer,
      nextLayeredBuffer.layers,
      nextFilePatches,
    );
  }

  getStagePatchForHunk(hunk) {
    return this.getStagePatchForLines(new Set(hunk.getBufferRows()));
  }

  getUnstagePatchForLines(selectedLineSet) {
    const nextLayeredBuffer = this.buildLayeredBuffer();
    const nextFilePatches = this.getFilePatchesContaining(selectedLineSet).map(fp => {
      return fp.buildUnstagePatchForLines(this.getBuffer(), nextLayeredBuffer, selectedLineSet);
    });

    return new MultiFilePatch(
      nextLayeredBuffer.buffer,
      nextLayeredBuffer.layers,
      nextFilePatches,
    );
  }

  getUnstagePatchForHunk(hunk) {
    return this.getUnstagePatchForLines(new Set(hunk.getBufferRows()));
  }

  getNextSelectionRange(lastMultiFilePatch, lastSelectedRows) {
    if (lastSelectedRows.size === 0) {
      const [firstPatch] = this.getFilePatches();
      if (!firstPatch) {
        return [[0, 0], [0, 0]];
      }

      return firstPatch.getFirstChangeRange();
    }

    const lastMax = Math.max(...lastSelectedRows);

    let lastSelectionIndex = 0;
    for (const lastFilePatch of lastMultiFilePatch.getFilePatches()) {
      for (const hunk of lastFilePatch.getHunks()) {
        let includesMax = false;
        let hunkSelectionOffset = 0;

        changeLoop: for (const change of hunk.getChanges()) {
          for (const {intersection, gap} of change.intersectRows(lastSelectedRows, true)) {
          // Only include a partial range if this intersection includes the last selected buffer row.
            includesMax = intersection.intersectsRow(lastMax);
            const delta = includesMax ? lastMax - intersection.start.row + 1 : intersection.getRowCount();

            if (gap) {
            // Range of unselected changes.
              hunkSelectionOffset += delta;
            }

            if (includesMax) {
              break changeLoop;
            }
          }
        }

        lastSelectionIndex += hunkSelectionOffset;

        if (includesMax) {
          break;
        }
      }
    }

    let newSelectionRow = 0;
    patchLoop: for (const filePatch of this.getFilePatches()) {
      for (const hunk of filePatch.getHunks()) {
        for (const change of hunk.getChanges()) {
          if (lastSelectionIndex < change.bufferRowCount()) {
            newSelectionRow = change.getStartBufferRow() + lastSelectionIndex;
            break patchLoop;
          } else {
            lastSelectionIndex -= change.bufferRowCount();
          }
        }
      }
    }

    return [[newSelectionRow, 0], [newSelectionRow, Infinity]];
  }

  adoptBufferFrom(lastMultiFilePatch) {
    lastMultiFilePatch.getHunkLayer().clear();
    lastMultiFilePatch.getUnchangedLayer().clear();
    lastMultiFilePatch.getAdditionLayer().clear();
    lastMultiFilePatch.getDeletionLayer().clear();
    lastMultiFilePatch.getNoNewlineLayer().clear();

    const nextBuffer = lastMultiFilePatch.getBuffer();
    nextBuffer.setText(this.getBuffer().getText());

    for (const patch of this.getFilePatches()) {
      for (const hunk of patch.getHunks()) {
        hunk.reMarkOn(lastMultiFilePatch.getHunkLayer());
        for (const region of hunk.getRegions()) {
          const target = region.when({
            unchanged: () => lastMultiFilePatch.getUnchangedLayer(),
            addition: () => lastMultiFilePatch.getAdditionLayer(),
            deletion: () => lastMultiFilePatch.getDeletionLayer(),
            nonewline: () => lastMultiFilePatch.getNoNewlineLayer(),
          });
          region.reMarkOn(target);
        }
      }
    }

    this.filePatchesByMarker.clear();
    this.hunksByMarker.clear();

    for (const filePatch of this.filePatches) {
      this.filePatchesByMarker.set(filePatch.getMarker(), filePatch);
      for (const hunk of filePatch.getHunks()) {
        this.hunksByMarker.set(hunk.getMarker(), hunk);
      }
    }

    this.patchLayer = lastMultiFilePatch.getPatchLayer();
    this.hunkLayer = lastMultiFilePatch.getHunkLayer();
    this.unchangedLayer = lastMultiFilePatch.getUnchangedLayer();
    this.additionLayer = lastMultiFilePatch.getAdditionLayer();
    this.deletionLayer = lastMultiFilePatch.getDeletionLayer();
    this.noNewlineLayer = lastMultiFilePatch.getNoNewlineLayer();

    this.buffer = nextBuffer;
  }

  buildLayeredBuffer() {
    const buffer = new TextBuffer();
    buffer.retain();

    return {
      buffer,
      layers: {
        patch: buffer.addMarkerLayer(),
        hunk: buffer.addMarkerLayer(),
        unchanged: buffer.addMarkerLayer(),
        addition: buffer.addMarkerLayer(),
        deletion: buffer.addMarkerLayer(),
        noNewline: buffer.addMarkerLayer(),
      },
    };
  }

  /*
   * Efficiently locate the FilePatch instances that contain at least one row from a Set.
   */
  getFilePatchesContaining(rowSet) {
    const sortedRowSet = Array.from(rowSet);
    sortedRowSet.sort((a, b) => b - a);

    const filePatches = new Set();
    let lastFilePatch = null;
    for (const row in sortedRowSet) {
      // Because the rows are sorted, consecutive rows will almost certainly belong to the same patch, so we can save
      // many avoidable marker index lookups by comparing with the last.
      if (lastFilePatch && lastFilePatch.containsRow(row)) {
        continue;
      }

      lastFilePatch = this.getFilePatchAt(row);
      filePatches.add(lastFilePatch);
    }

    return filePatches;
  }

  anyPresent() {
    return this.buffer !== null;
  }

  didAnyChangeExecutableMode() {
    for (const filePatch of this.getFilePatches()) {
      if (filePatch.didChangeExecutableMode()) {
        return true;
      }
    }
    return false;
  }

  anyHaveSymlink() {
    for (const filePatch of this.getFilePatches()) {
      if (filePatch.hasSymlink()) {
        return true;
      }
    }
    return false;
  }

  getMaxLineNumberWidth() {
    return this.getFilePatches().reduce((maxWidth, filePatch) => {
      const width = filePatch.getMaxLineNumberWidth();
      return maxWidth >= width ? maxWidth : width;
    }, 0);
  }

  /*
   * Construct an apply-able patch String.
   */
  toString() {
    return this.filePatches.map(fp => fp.toStringIn(this.buffer)).join('');
  }
}
<commit_msg>Catch that other "getFilePatchesContaining() returns a Set" call<commit_after>import {TextBuffer} from 'atom';

export default class MultiFilePatch {
  constructor(buffer = null, layers = {}, filePatches = []) {
    this.buffer = buffer;

    this.patchLayer = layers.patch;
    this.hunkLayer = layers.hunk;
    this.unchangedLayer = layers.unchanged;
    this.additionLayer = layers.addition;
    this.deletionLayer = layers.deletion;
    this.noNewlineLayer = layers.noNewline;

    this.filePatches = filePatches;

    this.filePatchesByMarker = new Map();
    this.hunksByMarker = new Map();

    for (const filePatch of this.filePatches) {
      this.filePatchesByMarker.set(filePatch.getMarker(), filePatch);
      for (const hunk of filePatch.getHunks()) {
        this.hunksByMarker.set(hunk.getMarker(), hunk);
      }
    }
  }

  getBuffer() {
    return this.buffer;
  }

  getHunkLayer() {
    return this.hunkLayer;
  }

  getUnchangedLayer() {
    return this.unchangedLayer;
  }

  getAdditionLayer() {
    return this.additionLayer;
  }

  getDeletionLayer() {
    return this.deletionLayer;
  }

  getNoNewlineLayer() {
    return this.noNewlineLayer;
  }

  getFilePatches() {
    return this.filePatches;
  }

  getFilePatchAt(bufferRow) {
    const [marker] = this.patchLayer.findMarkers({intersectsRow: bufferRow});
    return this.filePatchesByMarker.get(marker);
  }

  getHunkAt(bufferRow) {
    const [marker] = this.hunkLayer.findMarkers({intersectsRow: bufferRow});
    return this.hunksByMarker.get(marker);
  }

  getStagePatchForLines(selectedLineSet) {
    const nextLayeredBuffer = this.buildLayeredBuffer();
    const nextFilePatches = Array.from(this.getFilePatchesContaining(selectedLineSet), fp => {
      return fp.buildStagePatchForLines(this.getBuffer(), nextLayeredBuffer, selectedLineSet);
    });

    return new MultiFilePatch(
      nextLayeredBuffer.buffer,
      nextLayeredBuffer.layers,
      nextFilePatches,
    );
  }

  getStagePatchForHunk(hunk) {
    return this.getStagePatchForLines(new Set(hunk.getBufferRows()));
  }

  getUnstagePatchForLines(selectedLineSet) {
    const nextLayeredBuffer = this.buildLayeredBuffer();
    const nextFilePatches = Array.from(this.getFilePatchesContaining(selectedLineSet), fp => {
      return fp.buildUnstagePatchForLines(this.getBuffer(), nextLayeredBuffer, selectedLineSet);
    });

    return new MultiFilePatch(
      nextLayeredBuffer.buffer,
      nextLayeredBuffer.layers,
      nextFilePatches,
    );
  }

  getUnstagePatchForHunk(hunk) {
    return this.getUnstagePatchForLines(new Set(hunk.getBufferRows()));
  }

  getNextSelectionRange(lastMultiFilePatch, lastSelectedRows) {
    if (lastSelectedRows.size === 0) {
      const [firstPatch] = this.getFilePatches();
      if (!firstPatch) {
        return [[0, 0], [0, 0]];
      }

      return firstPatch.getFirstChangeRange();
    }

    const lastMax = Math.max(...lastSelectedRows);

    let lastSelectionIndex = 0;
    for (const lastFilePatch of lastMultiFilePatch.getFilePatches()) {
      for (const hunk of lastFilePatch.getHunks()) {
        let includesMax = false;
        let hunkSelectionOffset = 0;

        changeLoop: for (const change of hunk.getChanges()) {
          for (const {intersection, gap} of change.intersectRows(lastSelectedRows, true)) {
          // Only include a partial range if this intersection includes the last selected buffer row.
            includesMax = intersection.intersectsRow(lastMax);
            const delta = includesMax ? lastMax - intersection.start.row + 1 : intersection.getRowCount();

            if (gap) {
            // Range of unselected changes.
              hunkSelectionOffset += delta;
            }

            if (includesMax) {
              break changeLoop;
            }
          }
        }

        lastSelectionIndex += hunkSelectionOffset;

        if (includesMax) {
          break;
        }
      }
    }

    let newSelectionRow = 0;
    patchLoop: for (const filePatch of this.getFilePatches()) {
      for (const hunk of filePatch.getHunks()) {
        for (const change of hunk.getChanges()) {
          if (lastSelectionIndex < change.bufferRowCount()) {
            newSelectionRow = change.getStartBufferRow() + lastSelectionIndex;
            break patchLoop;
          } else {
            lastSelectionIndex -= change.bufferRowCount();
          }
        }
      }
    }

    return [[newSelectionRow, 0], [newSelectionRow, Infinity]];
  }

  adoptBufferFrom(lastMultiFilePatch) {
    lastMultiFilePatch.getHunkLayer().clear();
    lastMultiFilePatch.getUnchangedLayer().clear();
    lastMultiFilePatch.getAdditionLayer().clear();
    lastMultiFilePatch.getDeletionLayer().clear();
    lastMultiFilePatch.getNoNewlineLayer().clear();

    const nextBuffer = lastMultiFilePatch.getBuffer();
    nextBuffer.setText(this.getBuffer().getText());

    for (const patch of this.getFilePatches()) {
      for (const hunk of patch.getHunks()) {
        hunk.reMarkOn(lastMultiFilePatch.getHunkLayer());
        for (const region of hunk.getRegions()) {
          const target = region.when({
            unchanged: () => lastMultiFilePatch.getUnchangedLayer(),
            addition: () => lastMultiFilePatch.getAdditionLayer(),
            deletion: () => lastMultiFilePatch.getDeletionLayer(),
            nonewline: () => lastMultiFilePatch.getNoNewlineLayer(),
          });
          region.reMarkOn(target);
        }
      }
    }

    this.filePatchesByMarker.clear();
    this.hunksByMarker.clear();

    for (const filePatch of this.filePatches) {
      this.filePatchesByMarker.set(filePatch.getMarker(), filePatch);
      for (const hunk of filePatch.getHunks()) {
        this.hunksByMarker.set(hunk.getMarker(), hunk);
      }
    }

    this.patchLayer = lastMultiFilePatch.getPatchLayer();
    this.hunkLayer = lastMultiFilePatch.getHunkLayer();
    this.unchangedLayer = lastMultiFilePatch.getUnchangedLayer();
    this.additionLayer = lastMultiFilePatch.getAdditionLayer();
    this.deletionLayer = lastMultiFilePatch.getDeletionLayer();
    this.noNewlineLayer = lastMultiFilePatch.getNoNewlineLayer();

    this.buffer = nextBuffer;
  }

  buildLayeredBuffer() {
    const buffer = new TextBuffer();
    buffer.retain();

    return {
      buffer,
      layers: {
        patch: buffer.addMarkerLayer(),
        hunk: buffer.addMarkerLayer(),
        unchanged: buffer.addMarkerLayer(),
        addition: buffer.addMarkerLayer(),
        deletion: buffer.addMarkerLayer(),
        noNewline: buffer.addMarkerLayer(),
      },
    };
  }

  /*
   * Efficiently locate the FilePatch instances that contain at least one row from a Set.
   */
  getFilePatchesContaining(rowSet) {
    const sortedRowSet = Array.from(rowSet);
    sortedRowSet.sort((a, b) => b - a);

    const filePatches = new Set();
    let lastFilePatch = null;
    for (const row in sortedRowSet) {
      // Because the rows are sorted, consecutive rows will almost certainly belong to the same patch, so we can save
      // many avoidable marker index lookups by comparing with the last.
      if (lastFilePatch && lastFilePatch.containsRow(row)) {
        continue;
      }

      lastFilePatch = this.getFilePatchAt(row);
      filePatches.add(lastFilePatch);
    }

    return filePatches;
  }

  anyPresent() {
    return this.buffer !== null;
  }

  didAnyChangeExecutableMode() {
    for (const filePatch of this.getFilePatches()) {
      if (filePatch.didChangeExecutableMode()) {
        return true;
      }
    }
    return false;
  }

  anyHaveSymlink() {
    for (const filePatch of this.getFilePatches()) {
      if (filePatch.hasSymlink()) {
        return true;
      }
    }
    return false;
  }

  getMaxLineNumberWidth() {
    return this.getFilePatches().reduce((maxWidth, filePatch) => {
      const width = filePatch.getMaxLineNumberWidth();
      return maxWidth >= width ? maxWidth : width;
    }, 0);
  }

  /*
   * Construct an apply-able patch String.
   */
  toString() {
    return this.filePatches.map(fp => fp.toStringIn(this.buffer)).join('');
  }
}
<|endoftext|>