<commit_before>var logger = require('../logger'),
    utils = require('../utils'),
    argParse = require('../args').parse,
    couchdb = require('../couchdb'),
    kansorc = require('../kansorc');
    json = require('../data-stream'),
    async = require('async'),
    urlParse = require('url').parse,
    urlFormat = require('url').format,
    events = require('events'),
    fs = require('fs');

exports.summary = 'Upload a file or directory of JSON files to DB';
exports.usage = '' +
'kanso upload [OPTIONS] PATH [DB]\n' +
'\n' +
'Parameters:\n' +
'  PATH   A file or directory containing JSON documents to upload\n' +
'  DB     The CouchDB database to upload the documents to, if not specified\n' +
'         it will use the "default" env in your .kansorc if available\n' +
'\n'+
'Options:\n' +
'  -f, --force      Ignore document conflict errors and upload anyway';

exports.run = function (settings, args, commands) {
    var a = argParse(args, {
        'force': {match: ['--force','-f']}
    });
    if (!a.positional[0]) {
        logger.error('No data file or directory specified');
        logger.info('Usage: ' + exports.usage);
        return;
    }
    var path = a.positional[0] || '.';
    var url = a.positional[1];

    kansorc.extend(settings, '.kansorc', function (err, settings) {
        if (err) {
            return logger.error(err);
        }
        url = utils.argToEnv(settings, url).db;
        utils.completeAuth(url, false, function (err, url) {
            if (err) {
                return logger.error(err);
            }
            var db = couchdb(url);
            utils.catchAuthError(
                exports.pushDocs, url, [path, a.options],
                function (err) {
                    // errors now reported per-file
                }
            );
        });
    });
};

exports.pushDoc = function (url, path, doc, i, options, callback) {
    var db = couchdb(url);
    // TODO: add option which allows pushing docs without ids?
    if (!doc._id) {
        return logger.error('No _id defined for ' + path);
    }
    db.save(doc._id, doc, {force: options.force}, function (err, res) {
        if (!err) {
            logger.info('Saved',
                (res._id || res.id) + ' (' + path + ', entry: ' + i + ')'
            );
        }
        else {
            logger.error('Error saving ' + path +
                (doc._id ? ' (' + doc._id + ')': '(entry: ' + i + ')')
            );
            logger.error(err);
        }
        callback(err);
    });
};

exports.pushDocs = function (url, path, options, callback) {
    var db = couchdb(url);
    db.ensureDB(function (err) {
        if (err) {
            return callback(err);
        }
        console.log('Uploading docs to ' + utils.noAuthURL(url));
        exports.find(path, function (err, files) {
            if (err) {
                return callback(err);
            }
            var all_successful = 0;
            var all_total = 0;

            async.forEachSeries(files, function (f, cb) {
                logger.info('Reading', f);
                exports.readFile(f, url, options, function (err, succ, total) {
                    if (err) {
                        cb(err);
                    }
                    all_successful += succ;
                    all_total += total;
                    if (succ % 100 !== 0) {
                        logger.info('Uploaded ' + succ+ ' docs');
                    }
                    cb();
                });
            },
            function (err) {
                if (err) {
                    return callback(err);
                }
                if (all_total === all_successful) {
                    logger.end(
                        'Uploaded ' + all_successful + ' of ' +
                        all_total + ' docs'
                    );
                }
                else {
                    logger.error(
                        'Uploaded ' + all_successful + ' of ' +
                        all_total + ' docs'
                    );
                }
                callback();
            });
        });
    });
}

exports.readFile = function (path, url, options, callback) {
    var pstream = json.createParseStream();
    var rstream = fs.createReadStream(path);
    rstream.pause();

    var total = 0;
    var successful = 0;
    var waiting = 0;
    var ev = new events.EventEmitter();

    pstream.on('doc', function (doc) {
        rstream.pause();
        waiting++;
        total++;
        exports.pushDoc(url, path, doc, total, options, function (err) {
            if (err) {
                logger.error(err);
            }
            else {
                successful++;
                if (successful % 100 === 0 && successful != 0) {
                    logger.info('Uploaded ' + successful + ' docs');
                }
            }
            waiting--;
            ev.emit('doc_pushed');
            rstream.resume();
        });
    });
    pstream.on('error', function (err) {
        logger.error(err);
    });
    pstream.on('end', function () {
        if (waiting <= 0) {
            callback(null, successful, total);
        }
        else {
            ev.on('doc_pushed', function () {
                if (waiting <= 0) {
                    callback(null, successful, total);
                }
            });
        }
    });
    rstream.pipe(pstream);
    rstream.resume();
};

/**
 * Find all .json files below a given path, recursing through subdirectories
 *
 * @param {String} p - the path to search
 * @param {Function} callback
 */

exports.find = function (p, callback) {
    utils.find(p, exports.filenameFilter(p), callback);
};

/**
 * Creates a filter used when searching for data files. This function tests
 * for a .json extension and omits hidden dot-preceeded filenames.
 *
 * @param {String} p - the path to the directory being searched
 * @returns {Function}
 */

exports.filenameFilter = function (p) {
    return function (f) {
        if (f === p) {
            return true;
        }
        var relpath = utils.relpath(f, p);
        // should not start with a '.'
        if (/^\.[^\/]?/.test(relpath)) {
            return false;
        }
        // should not contain a file or folder starting with a '.'
        if (/\/\./.test(relpath)) {
            return false;
        }
        // should have a .js extension
        if (!/\.json$/.test(f)) {
            return false;
        }
        return true;
    };
};
<commit_msg>add option to skip conflicts when uploading<commit_after>var logger = require('../logger'),
    utils = require('../utils'),
    argParse = require('../args').parse,
    couchdb = require('../couchdb'),
    kansorc = require('../kansorc');
    json = require('../data-stream'),
    async = require('async'),
    urlParse = require('url').parse,
    urlFormat = require('url').format,
    events = require('events'),
    fs = require('fs');

exports.summary = 'Upload a file or directory of JSON files to DB';
exports.usage = '' +
'kanso upload [OPTIONS] PATH [DB]\n' +
'\n' +
'Parameters:\n' +
'  PATH   A file or directory containing JSON documents to upload\n' +
'  DB     The CouchDB database to upload the documents to, if not specified\n' +
'         it will use the "default" env in your .kansorc if available\n' +
'\n'+
'Options:\n' +
'  -f, --force      Ignore document conflict errors and upload anyway\n' +
'  -s, --skip       Skip document conflict errors and do not retry';

exports.run = function (settings, args, commands) {
    var a = argParse(args, {
        'force': {match: ['--force','-f']},
        'skip': {match: ['--skip', '-s']}
    });
    if (!a.positional[0]) {
        logger.error('No data file or directory specified');
        logger.info('Usage: ' + exports.usage);
        return;
    }
    var path = a.positional[0] || '.';
    var url = a.positional[1];

    kansorc.extend(settings, '.kansorc', function (err, settings) {
        if (err) {
            return logger.error(err);
        }
        url = utils.argToEnv(settings, url).db;
        utils.completeAuth(url, false, function (err, url) {
            if (err) {
                return logger.error(err);
            }
            var db = couchdb(url);
            utils.catchAuthError(
                exports.pushDocs, url, [path, a.options],
                function (err) {
                    // errors now reported per-file
                }
            );
        });
    });
};

exports.pushDoc = function (url, path, doc, i, options, callback) {
    var db = couchdb(url);
    // TODO: add option which allows pushing docs without ids?
    if (!doc._id) {
        return logger.error('No _id defined for ' + path);
    }
    db.save(doc._id, doc, {force: options.force}, function (err, res) {
        var skipped = false;
        if (!err) {
            logger.info('Saved',
                (res._id || res.id) + ' (' + path + ', entry: ' + i + ')'
            );
        }
        else if (err.error === 'conflict') {
            err = null;
            skipped = true;
            logger.info('Skipped',
                  (doc._id || doc.id) + ' (' + path + ', entry: ' + i + ')'
            );
        }
        else {
            logger.error('Error saving ' + path +
                (doc._id ? ' (' + doc._id + ')': '(entry: ' + i + ')')
            );
            logger.error(err);
        }
        callback(err, skipped);
    });
};

exports.pushDocs = function (url, path, options, callback) {
    var db = couchdb(url);
    db.ensureDB(function (err) {
        if (err) {
            return callback(err);
        }
        console.log('Uploading docs to ' + utils.noAuthURL(url));
        exports.find(path, function (err, files) {
            if (err) {
                return callback(err);
            }
            var all_successful = 0;
            var all_skipped = 0;
            var all_total = 0;

            async.forEachSeries(files, function (f, cb) {
                logger.info('Reading', f);
                exports.readFile(f, url, options, function (err, succ, skip, total) {
                    if (err) {
                        cb(err);
                    }
                    all_successful += succ;
                    all_skipped += skip;
                    all_total += total;
                    if (succ % 100 !== 0) {
                        logger.info('Uploaded ' + (succ - skip) + ' docs');
                        logger.info('Skipped ' + skip + ' docs');
                    }
                    cb();
                });
            },
            function (err) {
                if (err) {
                    return callback(err);
                }
                if (all_total === all_successful) {
                    logger.end(
                        'Uploaded ' + (all_successful - all_skipped) + ' of ' +
                        all_total + ' docs'
                    );
                }
                else {
                    logger.error(
                        'Uploaded ' + (all_successful - all_skipped) + ' of ' +
                        all_total + ' docs'
                    );
                }
                callback();
            });
        });
    });
}

exports.readFile = function (path, url, options, callback) {
    var pstream = json.createParseStream();
    var rstream = fs.createReadStream(path);
    rstream.pause();

    var total = 0;
    var successful = 0;
    var skipped = 0;
    var waiting = 0;
    var ev = new events.EventEmitter();

    pstream.on('doc', function (doc) {
        rstream.pause();
        waiting++;
        total++;
        exports.pushDoc(url, path, doc, total, options, function (err, skip) {
            if (err) {
                logger.error(err);
            }
            else {
                if(skip) skipped++;
                successful++;
                if (successful % 100 === 0 && successful != 0) {
                    logger.info('Uploaded ' + successful + ' docs');
                }
            }
            waiting--;
            ev.emit('doc_pushed');
            rstream.resume();
        });
    });
    pstream.on('error', function (err) {
        logger.error(err);
    });
    pstream.on('end', function () {
        if (waiting <= 0) {
            callback(null, successful, skipped, total);
        }
        else {
            ev.on('doc_pushed', function () {
                if (waiting <= 0) {
                    callback(null, successful, skipped, total);
                }
            });
        }
    });
    rstream.pipe(pstream);
    rstream.resume();
};

/**
 * Find all .json files below a given path, recursing through subdirectories
 *
 * @param {String} p - the path to search
 * @param {Function} callback
 */

exports.find = function (p, callback) {
    utils.find(p, exports.filenameFilter(p), callback);
};

/**
 * Creates a filter used when searching for data files. This function tests
 * for a .json extension and omits hidden dot-preceeded filenames.
 *
 * @param {String} p - the path to the directory being searched
 * @returns {Function}
 */

exports.filenameFilter = function (p) {
    return function (f) {
        if (f === p) {
            return true;
        }
        var relpath = utils.relpath(f, p);
        // should not start with a '.'
        if (/^\.[^\/]?/.test(relpath)) {
            return false;
        }
        // should not contain a file or folder starting with a '.'
        if (/\/\./.test(relpath)) {
            return false;
        }
        // should have a .js extension
        if (!/\.json$/.test(f)) {
            return false;
        }
        return true;
    };
};
<|endoftext|>


<commit_before>// Copyright (c) 2018, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

import http from 'http';
import https from 'https';
import path from 'path';
import { promisify } from 'util';

import fs from 'fs-extra';
import _ from 'underscore';
import urljoin from 'url-join';

import { InstanceFetcher } from './aws';
import { logger } from './logger';

const sleep = promisify(setTimeout);

/***
 * Finds and initializes the compilers stored on the properties files
 */
export class CompilerFinder {
    /***
     * @param {CompileHandler} compileHandler
     * @param {CompilerProps} compilerProps
     * @param {propsFor} awsProps
     * @param {Object} args
     * @param {Object} optionsHandler
     */
    constructor(compileHandler, compilerProps, awsProps, args, optionsHandler) {
        this.compilerProps = compilerProps.get.bind(compilerProps);
        this.ceProps = compilerProps.ceProps;
        this.awsProps = awsProps;
        this.args = args;
        this.compileHandler = compileHandler;
        this.languages = compilerProps.languages;
        this.awsPoller = null;
        this.optionsHandler = optionsHandler;
    }

    awsInstances() {
        if (!this.awsPoller) this.awsPoller = new InstanceFetcher(this.awsProps);
        return this.awsPoller.getInstances();
    }

    async fetchRemote(host, port, uriBase, props, langId) {
        const requestLib = port === 443 ? https : http;
        const uriSchema = port === 443 ? 'https' : 'http';
        const uri = urljoin(`${uriSchema}://${host}:${port}`, uriBase);
        const apiPath = urljoin('/', uriBase || '', 'api/compilers', langId || '', '?fields=all');
        logger.info(`Fetching compilers from remote source ${uri}`);
        return this.retryPromise(
            () => {
                return new Promise((resolve, reject) => {
                    const request = requestLib.get({
                        hostname: host,
                        port: port,
                        path: apiPath,
                        headers: {
                            Accept: 'application/json',
                        },
                    }, res => {
                        let error;
                        const {statusCode, headers: {'content-type': contentType}} = res;

                        if (statusCode !== 200) {
                            error = new Error('Failed fetching remote compilers from ' +
                                `${uriSchema}://${host}:${port}${apiPath}\n` +
                                `Status Code: ${statusCode}`);
                        } else if (!/^application\/json/.test(contentType)) {
                            error = new Error('Invalid content-type.\n' +
                                `Expected application/json but received ${contentType}`);
                        }
                        if (error) {
                            logger.error(error.message);
                            // consume response data to free up memory
                            res.resume();
                            reject(error);
                            return;
                        }
                        let str = '';
                        res.on('data', chunk => {
                            str += chunk;
                        });
                        res.on('end', () => {
                            try {
                                const compilers = JSON.parse(str).map(compiler => {
                                    // Fix up old upstream implementations of Compiler Explorer
                                    // e.g. https://www.godbolt.ms
                                    // (see https://github.com/compiler-explorer/compiler-explorer/issues/1768)
                                    if (!compiler.alias) compiler.alias = [];
                                    if (typeof compiler.alias == 'string') compiler.alias = [compiler.alias];
                                    // End fixup
                                    compiler.exe = null;
                                    compiler.remote = {
                                        target: `${uriSchema}://${host}:${port}`,
                                        path: urljoin('/', uriBase, 'api/compiler', compiler.id, 'compile'),
                                    };
                                    return compiler;
                                });
                                resolve(compilers);
                            } catch (e) {
                                logger.error(`Error parsing response from ${uri} '${str}': ${e.message}`);
                                reject(e);
                            }
                        });
                    })
                        .on('error', reject)
                        .on('timeout', () => reject('timeout'));
                    request.setTimeout(this.awsProps('proxyTimeout', 1000));
                });
            },
            `${host}:${port}`,
            props('proxyRetries', 5),
            props('proxyRetryMs', 500),
        ).catch(() => {
            logger.warn(`Unable to contact ${host}:${port}; skipping`);
            return [];
        });
    }

    async fetchAws() {
        logger.info('Fetching instances from AWS');
        const instances = await this.awsInstances();
        return Promise.all(instances.map(instance => {
            logger.info('Checking instance ' + instance.InstanceId);
            const address = this.awsProps('externalTestMode', false) ? instance.PublicDnsName : instance.PrivateDnsName;
            return this.fetchRemote(address, this.args.port, '', this.awsProps, null);
        }));
    }

    async compilerConfigFor(langId, compilerName, parentProps) {
        const base = `compiler.${compilerName}.`;

        function props(propName, def) {
            const propsForCompiler = parentProps(langId, base + propName);
            if (propsForCompiler !== undefined) return propsForCompiler;
            return parentProps(langId, propName, def);
        }

        const ceToolsPath = props('ceToolsPath', './');

        const supportsBinary = !!props('supportsBinary', true);
        const interpreted = !!props('interpreted', false);
        const supportsExecute = (interpreted || supportsBinary) && !!props('supportsExecute', true);
        const executionWrapper = props('executionWrapper', '');
        const supportsLibraryCodeFilter = !!props('supportsLibraryCodeFilter', true);

        const group = props('group', '');

        const demanglerProp = props('demangler', '');
        const demangler = demanglerProp ? path.normalize(demanglerProp.replace('${ceToolsPath}', ceToolsPath)) : '';

        const isSemVer = props('isSemVer', false);
        const baseName = props('baseName', null);
        const semverVer = props('semver', '');

        const name = props('name', compilerName);

        const baseOptions = props('baseOptions', '');
        const options = props('options', '');
        const actualOptions = _.compact([baseOptions, options]).join(' ');

        const envVars = (() => {
            const envVarsString = props('envVars', '');
            if (envVarsString === '') {
                return [];
            }
            const arr = [];
            for (const el of envVarsString.split(':')) {
                const [env, setting] = el.split('=');
                arr.push([env, setting]);
            }
            return arr;
        })();
        const exe = props('exe', compilerName);
        const exePath = path.dirname(exe);
        const compilerInfo = {
            id: compilerName,
            exe: exe,
            name: isSemVer && baseName ? `${baseName} ${semverVer}` : name,
            alias: _.filter(props('alias', '').split(':'), (a) => a !== ''),
            options: actualOptions,
            versionFlag: props('versionFlag'),
            versionRe: props('versionRe'),
            explicitVersion: props('explicitVersion'),
            compilerType: props('compilerType', ''),
            demangler: demangler,
            demanglerType: props('demanglerType', ''),
            objdumper: props('objdumper', ''),
            objdumperType: props('objdumperType', ''),
            intelAsm: props('intelAsm', ''),
            instructionSet: props('instructionSet', ''),
            needsMulti: !!props('needsMulti', true),
            supportsDemangle: !!demangler,
            supportsBinary,
            interpreted,
            supportsExecute,
            executionWrapper,
            supportsLibraryCodeFilter: supportsLibraryCodeFilter,
            postProcess: props('postProcess', '').split('|'),
            lang: langId,
            group: group,
            groupName: props('groupName', ''),
            includeFlag: props('includeFlag', '-isystem'),
            includePath: props('includePath', ''),
            linkFlag: props('linkFlag', '-l'),
            rpathFlag: props('rpathFlag', '-Wl,-rpath,'),
            libpathFlag: props('libpathFlag', '-L'),
            libPath: props('libPath', ''),
            ldPath: props('ldPath', '')
                .split('|')
                .map(x => path.normalize(x.replace('${exePath}', exePath))),
            envVars: envVars,
            notification: props('notification', ''),
            isSemVer: isSemVer,
            semver: semverVer,
            libsArr: this.getSupportedLibrariesArr(props, langId),
            tools: _.omit(
                this.optionsHandler.get().tools[langId],
                tool => tool.isCompilerExcluded(compilerName, props)),
            unwiseOptions: props('unwiseOptions', '').split('|'),
            hidden: props('hidden', false),
            buildenvsetup: {
                id: props('buildenvsetup', ''),
                props: (name, def) => {
                    return props(`buildenvsetup.${name}`, def);
                },
            },
        };

        if (props('demanglerClassFile') !== undefined) {
            logger.error(`Error in compiler.${compilerName}: ` +
                'demanglerClassFile is no longer supported, please use demanglerType');
            return [];
        }

        if (process.platform === 'win32') {
            compilerInfo.libPath = compilerInfo.libPath.split(';').filter((p) => p !== '');
        } else {
            compilerInfo.libPath = compilerInfo.libPath.split(':').filter((p) => p !== '');
        }

        logger.debug('Found compiler', compilerInfo);
        return compilerInfo;
    }

    getSupportedLibrariesArr(props) {
        return _.filter(props('supportsLibraries', '').split(':'), (a) => a !== '');
    }

    async recurseGetCompilers(langId, compilerName, parentProps) {
        // Don't treat @ in paths as remote addresses if requested
        if (this.args.fetchCompilersFromRemote && compilerName.includes('@')) {
            const bits = compilerName.split('@');
            const host = bits[0];
            const pathParts = bits[1].split('/');
            const port = parseInt(pathParts.shift());
            const path = pathParts.join('/');
            return this.fetchRemote(host, port, path, this.ceProps, langId);
        }
        if (compilerName.indexOf('&') === 0) {
            const groupName = compilerName.substr(1);

            const props = (langId, name, def) => {
                if (name === 'group') {
                    return groupName;
                }
                return this.compilerProps(langId, `group.${groupName}.${name}`, parentProps(langId, name, def));
            };
            const exes = _.compact(this.compilerProps(langId, `group.${groupName}.compilers`, '').split(':'));
            logger.debug(`Processing compilers from group ${groupName}`);
            return Promise.all(exes.map(compiler => this.recurseGetCompilers(langId, compiler, props)));
        }
        if (compilerName === 'AWS') return this.fetchAws();
        return this.compilerConfigFor(langId, compilerName, parentProps);
    }

    async getCompilers() {
        const compilers = [];
        _.each(this.getExes(), (exs, langId) => {
            _.each(exs, exe => compilers.push(this.recurseGetCompilers(langId, exe, this.compilerProps)));
        });
        return Promise.all(compilers);
    }

    ensureDistinct(compilers) {
        const ids = {};
        let foundClash = false;
        _.each(compilers, compiler => {
            if (!ids[compiler.id]) ids[compiler.id] = [];
            ids[compiler.id].push(compiler);
        });
        _.each(ids, (list, id) => {
            if (list.length !== 1) {
                foundClash = true;
                logger.error(`Compiler ID clash for '${id}' - used by ${
                    _.map(list, o => `lang:${o.lang} name:${o.name}`).join(', ')}`);
            }
        });
        return {compilers, foundClash};
    }

    async retryPromise(promiseFunc, name, maxFails, retryMs) {
        for (let fails = 0; fails < maxFails; ++fails) {
            try {
                return await promiseFunc();
            } catch (e) {
                if (fails < (maxFails - 1)) {
                    logger.warn(`Failed ${name} : ${e}, retrying`);
                    await sleep(retryMs);
                } else {
                    logger.error(`Too many retries for ${name} : ${e}`);
                    throw e;
                }
            }
        }
    }

    getExes() {
        const langToCompilers = this.compilerProps(this.languages, 'compilers', '', exs => _.compact(exs.split(':')));
        this.addNdkExes(langToCompilers);
        logger.info('Exes found:', langToCompilers);
        return langToCompilers;
    }

    addNdkExes(langToCompilers) {
        const ndkPaths = this.compilerProps(this.languages, 'androidNdk');
        _.each(ndkPaths, (ndkPath, langId) => {
            if (ndkPath) {
                const toolchains = fs.readdirSync(`${ndkPath}/toolchains`);
                for (const [version, index] of toolchains) {
                    const path = `${ndkPath}/toolchains/${version}/prebuilt/linux-x86_64/bin/`;
                    if (fs.existsSync(path)) {
                        const cc = fs.readdirSync(path).find(filename => filename.includes('g++'));
                        toolchains[index] = path + cc;
                    } else {
                        toolchains[index] = null;
                    }
                }
                langToCompilers[langId].push(toolchains.filter(x => x !== null));
            }
        });
    }

    async find() {
        let compilers = (await this.getCompilers()).flat(Infinity);
        compilers = await this.compileHandler.setCompilers(compilers, this.optionsHandler.get());
        const result = this.ensureDistinct(_.compact(compilers));
        return {foundClash: result.foundClash, compilers: _.sortBy(result.compilers, 'name')};
    }

    async loadPrediscovered(compilers) {
        for (const compiler of compilers) {
            if (compiler.buildenvsetup) {
                const langId = compiler.lang;

                compiler.buildenvsetup.props = (propName, def) => {
                    return this.compilerProps(langId, 'buildenvsetup.' + propName, def);
                };

                compiler.tools = _.omit(
                    this.optionsHandler.get().tools[langId],
                    tool => tool.isCompilerExcluded(compiler.id, this.compilerProps));
            }
        }
        return this.compileHandler.setCompilers(compilers, this.optionsHandler.get());
    }
}
<commit_msg>fix props lambda<commit_after>// Copyright (c) 2018, Compiler Explorer Authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.

import http from 'http';
import https from 'https';
import path from 'path';
import { promisify } from 'util';

import fs from 'fs-extra';
import _ from 'underscore';
import urljoin from 'url-join';

import { InstanceFetcher } from './aws';
import { logger } from './logger';

const sleep = promisify(setTimeout);

/***
 * Finds and initializes the compilers stored on the properties files
 */
export class CompilerFinder {
    /***
     * @param {CompileHandler} compileHandler
     * @param {CompilerProps} compilerProps
     * @param {propsFor} awsProps
     * @param {Object} args
     * @param {Object} optionsHandler
     */
    constructor(compileHandler, compilerProps, awsProps, args, optionsHandler) {
        this.compilerProps = compilerProps.get.bind(compilerProps);
        this.ceProps = compilerProps.ceProps;
        this.awsProps = awsProps;
        this.args = args;
        this.compileHandler = compileHandler;
        this.languages = compilerProps.languages;
        this.awsPoller = null;
        this.optionsHandler = optionsHandler;
    }

    awsInstances() {
        if (!this.awsPoller) this.awsPoller = new InstanceFetcher(this.awsProps);
        return this.awsPoller.getInstances();
    }

    async fetchRemote(host, port, uriBase, props, langId) {
        const requestLib = port === 443 ? https : http;
        const uriSchema = port === 443 ? 'https' : 'http';
        const uri = urljoin(`${uriSchema}://${host}:${port}`, uriBase);
        const apiPath = urljoin('/', uriBase || '', 'api/compilers', langId || '', '?fields=all');
        logger.info(`Fetching compilers from remote source ${uri}`);
        return this.retryPromise(
            () => {
                return new Promise((resolve, reject) => {
                    const request = requestLib.get({
                        hostname: host,
                        port: port,
                        path: apiPath,
                        headers: {
                            Accept: 'application/json',
                        },
                    }, res => {
                        let error;
                        const {statusCode, headers: {'content-type': contentType}} = res;

                        if (statusCode !== 200) {
                            error = new Error('Failed fetching remote compilers from ' +
                                `${uriSchema}://${host}:${port}${apiPath}\n` +
                                `Status Code: ${statusCode}`);
                        } else if (!/^application\/json/.test(contentType)) {
                            error = new Error('Invalid content-type.\n' +
                                `Expected application/json but received ${contentType}`);
                        }
                        if (error) {
                            logger.error(error.message);
                            // consume response data to free up memory
                            res.resume();
                            reject(error);
                            return;
                        }
                        let str = '';
                        res.on('data', chunk => {
                            str += chunk;
                        });
                        res.on('end', () => {
                            try {
                                const compilers = JSON.parse(str).map(compiler => {
                                    // Fix up old upstream implementations of Compiler Explorer
                                    // e.g. https://www.godbolt.ms
                                    // (see https://github.com/compiler-explorer/compiler-explorer/issues/1768)
                                    if (!compiler.alias) compiler.alias = [];
                                    if (typeof compiler.alias == 'string') compiler.alias = [compiler.alias];
                                    // End fixup
                                    compiler.exe = null;
                                    compiler.remote = {
                                        target: `${uriSchema}://${host}:${port}`,
                                        path: urljoin('/', uriBase, 'api/compiler', compiler.id, 'compile'),
                                    };
                                    return compiler;
                                });
                                resolve(compilers);
                            } catch (e) {
                                logger.error(`Error parsing response from ${uri} '${str}': ${e.message}`);
                                reject(e);
                            }
                        });
                    })
                        .on('error', reject)
                        .on('timeout', () => reject('timeout'));
                    request.setTimeout(this.awsProps('proxyTimeout', 1000));
                });
            },
            `${host}:${port}`,
            props('proxyRetries', 5),
            props('proxyRetryMs', 500),
        ).catch(() => {
            logger.warn(`Unable to contact ${host}:${port}; skipping`);
            return [];
        });
    }

    async fetchAws() {
        logger.info('Fetching instances from AWS');
        const instances = await this.awsInstances();
        return Promise.all(instances.map(instance => {
            logger.info('Checking instance ' + instance.InstanceId);
            const address = this.awsProps('externalTestMode', false) ? instance.PublicDnsName : instance.PrivateDnsName;
            return this.fetchRemote(address, this.args.port, '', this.awsProps, null);
        }));
    }

    async compilerConfigFor(langId, compilerName, parentProps) {
        const base = `compiler.${compilerName}.`;

        function props(propName, def) {
            const propsForCompiler = parentProps(langId, base + propName);
            if (propsForCompiler !== undefined) return propsForCompiler;
            return parentProps(langId, propName, def);
        }

        const ceToolsPath = props('ceToolsPath', './');

        const supportsBinary = !!props('supportsBinary', true);
        const interpreted = !!props('interpreted', false);
        const supportsExecute = (interpreted || supportsBinary) && !!props('supportsExecute', true);
        const executionWrapper = props('executionWrapper', '');
        const supportsLibraryCodeFilter = !!props('supportsLibraryCodeFilter', true);

        const group = props('group', '');

        const demanglerProp = props('demangler', '');
        const demangler = demanglerProp ? path.normalize(demanglerProp.replace('${ceToolsPath}', ceToolsPath)) : '';

        const isSemVer = props('isSemVer', false);
        const baseName = props('baseName', null);
        const semverVer = props('semver', '');

        const name = props('name', compilerName);

        const baseOptions = props('baseOptions', '');
        const options = props('options', '');
        const actualOptions = _.compact([baseOptions, options]).join(' ');

        const envVars = (() => {
            const envVarsString = props('envVars', '');
            if (envVarsString === '') {
                return [];
            }
            const arr = [];
            for (const el of envVarsString.split(':')) {
                const [env, setting] = el.split('=');
                arr.push([env, setting]);
            }
            return arr;
        })();
        const exe = props('exe', compilerName);
        const exePath = path.dirname(exe);
        const compilerInfo = {
            id: compilerName,
            exe: exe,
            name: isSemVer && baseName ? `${baseName} ${semverVer}` : name,
            alias: _.filter(props('alias', '').split(':'), (a) => a !== ''),
            options: actualOptions,
            versionFlag: props('versionFlag'),
            versionRe: props('versionRe'),
            explicitVersion: props('explicitVersion'),
            compilerType: props('compilerType', ''),
            demangler: demangler,
            demanglerType: props('demanglerType', ''),
            objdumper: props('objdumper', ''),
            objdumperType: props('objdumperType', ''),
            intelAsm: props('intelAsm', ''),
            instructionSet: props('instructionSet', ''),
            needsMulti: !!props('needsMulti', true),
            supportsDemangle: !!demangler,
            supportsBinary,
            interpreted,
            supportsExecute,
            executionWrapper,
            supportsLibraryCodeFilter: supportsLibraryCodeFilter,
            postProcess: props('postProcess', '').split('|'),
            lang: langId,
            group: group,
            groupName: props('groupName', ''),
            includeFlag: props('includeFlag', '-isystem'),
            includePath: props('includePath', ''),
            linkFlag: props('linkFlag', '-l'),
            rpathFlag: props('rpathFlag', '-Wl,-rpath,'),
            libpathFlag: props('libpathFlag', '-L'),
            libPath: props('libPath', ''),
            ldPath: props('ldPath', '')
                .split('|')
                .map(x => path.normalize(x.replace('${exePath}', exePath))),
            envVars: envVars,
            notification: props('notification', ''),
            isSemVer: isSemVer,
            semver: semverVer,
            libsArr: this.getSupportedLibrariesArr(props, langId),
            tools: _.omit(
                this.optionsHandler.get().tools[langId],
                tool => tool.isCompilerExcluded(compilerName, props)),
            unwiseOptions: props('unwiseOptions', '').split('|'),
            hidden: props('hidden', false),
            buildenvsetup: {
                id: props('buildenvsetup', ''),
                props: (name, def) => {
                    return props(`buildenvsetup.${name}`, def);
                },
            },
        };

        if (props('demanglerClassFile') !== undefined) {
            logger.error(`Error in compiler.${compilerName}: ` +
                'demanglerClassFile is no longer supported, please use demanglerType');
            return [];
        }

        if (process.platform === 'win32') {
            compilerInfo.libPath = compilerInfo.libPath.split(';').filter((p) => p !== '');
        } else {
            compilerInfo.libPath = compilerInfo.libPath.split(':').filter((p) => p !== '');
        }

        logger.debug('Found compiler', compilerInfo);
        return compilerInfo;
    }

    getSupportedLibrariesArr(props) {
        return _.filter(props('supportsLibraries', '').split(':'), (a) => a !== '');
    }

    async recurseGetCompilers(langId, compilerName, parentProps) {
        // Don't treat @ in paths as remote addresses if requested
        if (this.args.fetchCompilersFromRemote && compilerName.includes('@')) {
            const bits = compilerName.split('@');
            const host = bits[0];
            const pathParts = bits[1].split('/');
            const port = parseInt(pathParts.shift());
            const path = pathParts.join('/');
            return this.fetchRemote(host, port, path, this.ceProps, langId);
        }
        if (compilerName.indexOf('&') === 0) {
            const groupName = compilerName.substr(1);

            const props = (langId, name, def) => {
                if (name === 'group') {
                    return groupName;
                }
                return this.compilerProps(langId, `group.${groupName}.${name}`, parentProps(langId, name, def));
            };
            const exes = _.compact(this.compilerProps(langId, `group.${groupName}.compilers`, '').split(':'));
            logger.debug(`Processing compilers from group ${groupName}`);
            return Promise.all(exes.map(compiler => this.recurseGetCompilers(langId, compiler, props)));
        }
        if (compilerName === 'AWS') return this.fetchAws();
        return this.compilerConfigFor(langId, compilerName, parentProps);
    }

    async getCompilers() {
        const compilers = [];
        _.each(this.getExes(), (exs, langId) => {
            _.each(exs, exe => compilers.push(this.recurseGetCompilers(langId, exe, this.compilerProps)));
        });
        return Promise.all(compilers);
    }

    ensureDistinct(compilers) {
        const ids = {};
        let foundClash = false;
        _.each(compilers, compiler => {
            if (!ids[compiler.id]) ids[compiler.id] = [];
            ids[compiler.id].push(compiler);
        });
        _.each(ids, (list, id) => {
            if (list.length !== 1) {
                foundClash = true;
                logger.error(`Compiler ID clash for '${id}' - used by ${
                    _.map(list, o => `lang:${o.lang} name:${o.name}`).join(', ')}`);
            }
        });
        return {compilers, foundClash};
    }

    async retryPromise(promiseFunc, name, maxFails, retryMs) {
        for (let fails = 0; fails < maxFails; ++fails) {
            try {
                return await promiseFunc();
            } catch (e) {
                if (fails < (maxFails - 1)) {
                    logger.warn(`Failed ${name} : ${e}, retrying`);
                    await sleep(retryMs);
                } else {
                    logger.error(`Too many retries for ${name} : ${e}`);
                    throw e;
                }
            }
        }
    }

    getExes() {
        const langToCompilers = this.compilerProps(this.languages, 'compilers', '', exs => _.compact(exs.split(':')));
        this.addNdkExes(langToCompilers);
        logger.info('Exes found:', langToCompilers);
        return langToCompilers;
    }

    addNdkExes(langToCompilers) {
        const ndkPaths = this.compilerProps(this.languages, 'androidNdk');
        _.each(ndkPaths, (ndkPath, langId) => {
            if (ndkPath) {
                const toolchains = fs.readdirSync(`${ndkPath}/toolchains`);
                for (const [version, index] of toolchains) {
                    const path = `${ndkPath}/toolchains/${version}/prebuilt/linux-x86_64/bin/`;
                    if (fs.existsSync(path)) {
                        const cc = fs.readdirSync(path).find(filename => filename.includes('g++'));
                        toolchains[index] = path + cc;
                    } else {
                        toolchains[index] = null;
                    }
                }
                langToCompilers[langId].push(toolchains.filter(x => x !== null));
            }
        });
    }

    async find() {
        let compilers = (await this.getCompilers()).flat(Infinity);
        compilers = await this.compileHandler.setCompilers(compilers, this.optionsHandler.get());
        const result = this.ensureDistinct(_.compact(compilers));
        return {foundClash: result.foundClash, compilers: _.sortBy(result.compilers, 'name')};
    }

    async loadPrediscovered(compilers) {
        for (const compiler of compilers) {
            if (compiler.buildenvsetup) {
                const langId = compiler.lang;

                compiler.buildenvsetup.props = (propName, def) => {
                    return this.compilerProps(langId, 'buildenvsetup.' + propName, def);
                };

                const fullOptions = this.optionsHandler.get();

                compiler.tools = _.omit(
                    fullOptions.tools[langId],
                    tool => tool.isCompilerExcluded(compiler.id, (propName, def) => {
                        return this.compilerProps(langId, propName, def);
                    }));
            }
        }
        return this.compileHandler.setCompilers(compilers, this.optionsHandler.get());
    }
}
<|endoftext|>


<commit_before>/* global Blob,URL,requestAnimationFrame */
import React from 'react';
import Terminal from 'xterm';
import {PureComponent} from '../base-components';
import terms from '../terms';
import returnKey from '../utils/keymaps';
import CommandRegistry from '../command-registry';

// map old hterm constants to xterm.js
const CURSOR_STYLES = {
  BEAM: 'bar',
  UNDERLINE: 'underline',
  BLOCK: 'block'
};

export default class Term extends PureComponent {
  constructor(props) {
    super(props);
    props.ref_(props.uid, this);
    this.termRef = null;
    this.termWrapperRef = null;
    this.termRect = null;
    this.onOpen = this.onOpen.bind(this);
    this.onWindowResize = this.onWindowResize.bind(this);
    this.onTermRef = this.onTermRef.bind(this);
    this.onTermWrapperRef = this.onTermWrapperRef.bind(this);
  }

  componentDidMount() {
    const {props} = this;

    // we need to use this hack to retain the term reference
    // as we move the term around splits, until xterm adds
    // support for getState / setState
    if (props.term) {
      this.term = props.term;
      this.termRef.appendChild(this.term.element);
      this.onOpen();
    } else {
      this.term =
        props.term ||
        new Terminal({
          cursorStyle: CURSOR_STYLES[props.cursorShape],
          cursorBlink: props.cursorBlink
        });
      this.term.attachCustomKeyEventHandler(this.keyboardHandler);
      this.term.on('open', this.onOpen);
      this.term.open(this.termRef, {
        focus: false
      });
    }

    if (props.onTitle) {
      this.term.on('title', props.onTitle);
    }

    if (props.onActive) {
      this.term.on('focus', () => {
        // xterm@2 emits this event 2 times. Will be fixed in xterm@3.
        if (!this.props.isTermActive) {
          props.onActive();
        }
      });
    }

    if (props.onData) {
      this.term.on('data', props.onData);
    }

    if (props.onResize) {
      this.term.on('resize', ({cols, rows}) => {
        props.onResize(cols, rows);
      });
    }

    window.addEventListener('resize', this.onWindowResize, {
      passive: true
    });

    terms[this.props.uid] = this;
  }

  onOpen() {
    // we need to delay one frame so that aphrodite styles
    // get applied and we can make an accurate measurement
    // of the container width and height
    requestAnimationFrame(() => {
      // at this point it would make sense for character
      // measurement to have taken place but it seems that
      // xterm.js might be doing this asynchronously, so
      // we force it instead
      this.term.charMeasure.measure();
      this.fitResize();
    });
  }

  getTermDocument() {
    // eslint-disable-next-line no-console
    console.error('unimplemented');
  }

  onWindowResize() {
    this.fitResize();
  }

  write(data) {
    this.term.write(data);
  }

  focus() {
    this.term.focus();
  }

  clear() {
    this.term.clear();
  }

  reset() {
    this.term.reset();
  }

  resize(cols, rows) {
    this.term.resize(cols, rows);
  }

  fitResize() {
    const termRect = this.termWrapperRef.getBoundingClientRect();
    const cols = Math.floor(termRect.width / this.term.charMeasure.width);
    const rows = Math.floor(termRect.height / this.term.charMeasure.height);
    this.resize(cols, rows);
  }

  keyboardHandler(e) {
    // test key from keymaps before moving forward with actions
    const key = returnKey(e);
    if (key) {
      if (CommandRegistry.getCommand(key)) {
        CommandRegistry.exec(key, e);
      }
      return false;
    }
  }

  componentWillReceiveProps(nextProps) {
    if (!this.props.cleared && nextProps.cleared) {
      this.clear();
    }

    if (!this.props.isTermActive && nextProps.isTermActive) {
      requestAnimationFrame(() => {
        this.term.charMeasure.measure();
        this.fitResize();
      });
    }

    if (this.props.fontSize !== nextProps.fontSize || this.props.fontFamily !== nextProps.fontFamily) {
      // invalidate xterm cache about how wide each
      // character is
      this.term.charMeasure.measure();

      // resize to fit the container
      this.fitResize();
    }

    if (nextProps.rows !== this.props.rows || nextProps.cols !== this.props.cols) {
      this.resize(nextProps.cols, nextProps.rows);
    }
  }

  onTermWrapperRef(component) {
    this.termWrapperRef = component;
  }

  onTermRef(component) {
    this.termRef = component;
  }

  componentWillUnmount() {
    terms[this.props.uid] = null;
    this.props.ref_(this.props.uid, null);

    // to clean up the terminal, we remove the listeners
    // instead of invoking `destroy`, since it will make the
    // term insta un-attachable in the future (which we need
    // to do in case of splitting, see `componentDidMount`
    this.term._events = {};

    window.removeEventListener('resize', this.onWindowResize, {
      passive: true
    });
  }

  template(css) {
    return (

        {this.props.customChildrenBefore}

        {this.props.customChildren}

    );
  }

  styles() {
    return {
      fit: {
        display: 'block',
        width: '100%',
        height: '100%'
      },
      wrapper: {
        // TODO: decide whether to keep this or not based on
        // understanding what xterm-selection is for
        overflow: 'hidden'
      },
      term: {}
    };
  }
}
<commit_msg>add BC `getTermDocument` and a note about deprecation<commit_after>/* global Blob,URL,requestAnimationFrame */
import React from 'react';
import Terminal from 'xterm';
import {PureComponent} from '../base-components';
import terms from '../terms';
import returnKey from '../utils/keymaps';
import CommandRegistry from '../command-registry';

// map old hterm constants to xterm.js
const CURSOR_STYLES = {
  BEAM: 'bar',
  UNDERLINE: 'underline',
  BLOCK: 'block'
};

export default class Term extends PureComponent {
  constructor(props) {
    super(props);
    props.ref_(props.uid, this);
    this.termRef = null;
    this.termWrapperRef = null;
    this.termRect = null;
    this.onOpen = this.onOpen.bind(this);
    this.onWindowResize = this.onWindowResize.bind(this);
    this.onTermRef = this.onTermRef.bind(this);
    this.onTermWrapperRef = this.onTermWrapperRef.bind(this);
  }

  componentDidMount() {
    const {props} = this;

    // we need to use this hack to retain the term reference
    // as we move the term around splits, until xterm adds
    // support for getState / setState
    if (props.term) {
      this.term = props.term;
      this.termRef.appendChild(this.term.element);
      this.onOpen();
    } else {
      this.term =
        props.term ||
        new Terminal({
          cursorStyle: CURSOR_STYLES[props.cursorShape],
          cursorBlink: props.cursorBlink
        });
      this.term.attachCustomKeyEventHandler(this.keyboardHandler);
      this.term.on('open', this.onOpen);
      this.term.open(this.termRef, {
        focus: false
      });
    }

    if (props.onTitle) {
      this.term.on('title', props.onTitle);
    }

    if (props.onActive) {
      this.term.on('focus', () => {
        // xterm@2 emits this event 2 times. Will be fixed in xterm@3.
        if (!this.props.isTermActive) {
          props.onActive();
        }
      });
    }

    if (props.onData) {
      this.term.on('data', props.onData);
    }

    if (props.onResize) {
      this.term.on('resize', ({cols, rows}) => {
        props.onResize(cols, rows);
      });
    }

    window.addEventListener('resize', this.onWindowResize, {
      passive: true
    });

    terms[this.props.uid] = this;
  }

  onOpen() {
    // we need to delay one frame so that aphrodite styles
    // get applied and we can make an accurate measurement
    // of the container width and height
    requestAnimationFrame(() => {
      // at this point it would make sense for character
      // measurement to have taken place but it seems that
      // xterm.js might be doing this asynchronously, so
      // we force it instead
      this.term.charMeasure.measure();
      this.fitResize();
    });
  }

  getTermDocument() {
    // eslint-disable-next-line no-console
    console.warn(
      'The underlying terminal engine of Hyper no longer ' +
        'uses iframes with individual `document` objects for each ' +
        'terminal instance. This method call is retained for ' +
        "backwards compatibility reasons. It's ok to attach direclty" +
        'to the `document` object of the main `window`.'
    );
    return document;
  }

  onWindowResize() {
    this.fitResize();
  }

  write(data) {
    this.term.write(data);
  }

  focus() {
    this.term.focus();
  }

  clear() {
    this.term.clear();
  }

  reset() {
    this.term.reset();
  }

  resize(cols, rows) {
    this.term.resize(cols, rows);
  }

  fitResize() {
    const termRect = this.termWrapperRef.getBoundingClientRect();
    const cols = Math.floor(termRect.width / this.term.charMeasure.width);
    const rows = Math.floor(termRect.height / this.term.charMeasure.height);
    this.resize(cols, rows);
  }

  keyboardHandler(e) {
    // test key from keymaps before moving forward with actions
    const key = returnKey(e);
    if (key) {
      if (CommandRegistry.getCommand(key)) {
        CommandRegistry.exec(key, e);
      }
      return false;
    }
  }

  componentWillReceiveProps(nextProps) {
    if (!this.props.cleared && nextProps.cleared) {
      this.clear();
    }

    if (!this.props.isTermActive && nextProps.isTermActive) {
      requestAnimationFrame(() => {
        this.term.charMeasure.measure();
        this.fitResize();
      });
    }

    if (this.props.fontSize !== nextProps.fontSize || this.props.fontFamily !== nextProps.fontFamily) {
      // invalidate xterm cache about how wide each
      // character is
      this.term.charMeasure.measure();

      // resize to fit the container
      this.fitResize();
    }

    if (nextProps.rows !== this.props.rows || nextProps.cols !== this.props.cols) {
      this.resize(nextProps.cols, nextProps.rows);
    }
  }

  onTermWrapperRef(component) {
    this.termWrapperRef = component;
  }

  onTermRef(component) {
    this.termRef = component;
  }

  componentWillUnmount() {
    terms[this.props.uid] = null;
    this.props.ref_(this.props.uid, null);

    // to clean up the terminal, we remove the listeners
    // instead of invoking `destroy`, since it will make the
    // term insta un-attachable in the future (which we need
    // to do in case of splitting, see `componentDidMount`
    this.term._events = {};

    window.removeEventListener('resize', this.onWindowResize, {
      passive: true
    });
  }

  template(css) {
    return (

        {this.props.customChildrenBefore}

        {this.props.customChildren}

    );
  }

  styles() {
    return {
      fit: {
        display: 'block',
        width: '100%',
        height: '100%'
      },
      wrapper: {
        // TODO: decide whether to keep this or not based on
        // understanding what xterm-selection is for
        overflow: 'hidden'
      },
      term: {}
    };
  }
}
<|endoftext|>


<commit_before>const
util = require('util'),
path = require('path'),
child_process = require('child_process'),
events = require('events');

function ComputeCluster(options) {
  if (!options || typeof options.module !== 'string') {
    throw "missing required 'module' argument";
  }
  // is module a file?
  if (!path.existsSync(options.module)) {
    throw "module doesn't exist: " + options.module;
  }

  events.EventEmitter.call(this);

  // an array of child processes
  this._kids = {};
  this._MAX_KIDS = (options.num || Math.ceil(require('os').cpus().length * 1.25));
  this._work_q = [];
  this._exiting = false;
  this._exit_cb;
  this._module = options.module;
};

util.inherits(ComputeCluster, events.EventEmitter);

ComputeCluster.prototype._onWorkerExit = function(pid) {
  var self = this;
  return function (code) {
    // if _exiting is false, we don't expect to be shutting down!
    if (!self._exiting) {
      self.emit('error',
                "compute process (" + pid + ") dies with code: " + code);
    }

    delete self._kids[pid];
    if (self._exit_cb) {
      if (Object.keys(self._kids).length === 0) {
        self._exit_cb(null);
        _exit_cb = undefined;
      }
    }
  }
}

ComputeCluster.prototype._getFreeWorker = function() {
  var self = this;

  for (var i in this._kids) {
    if (!this._kids[i].job) return this._kids[i];
  }

  // no workers!  can we spawn one?
  if (Object.keys(this._kids).length < this._MAX_KIDS) {
    var k = { worker: child_process.fork(this._module) };
    k.worker.on('exit', this._onWorkerExit(k.worker.pid));
    this._kids[k.worker.pid] = k;
    return k;
  }
};

ComputeCluster.prototype._runWorkOnWorker = function(work, worker) {
  var self = this;
  worker.worker.once('message', function(m) {
    if (worker.job.cb) worker.job.cb(null, m);
    delete worker.job;
    self._assignWork();
  });
  worker.worker.send(work.job);
  worker.job = work;
};

  // assign as many work units from work_q as possible to avialable
  // compute processes
ComputeCluster.prototype._assignWork = function() {
  while (this._work_q.length > 0) {
    var worker = this._getFreeWorker();
    if (!worker) break;

    this._runWorkOnWorker(this._work_q.shift(), worker);
  }
};

ComputeCluster.prototype.enqueue = function(args, cb) {
  this._work_q.push({ job: args, cb: cb });
  this._assignWork();
  return this;
};

ComputeCluster.prototype.exit = function(cb) {
  if (Object.keys(this._kids).length === 0) {
    setTimeout(function() { cb(null); }, 0);
  } else {
    this._exiting = true;
    this._exit_cb = cb;
    for (var i in this._kids) {
      this._kids[i].worker.kill();
    }
  }
  return this;
};

module.exports = ComputeCluster;

<commit_msg>formalize the max_processes parameter a bit<commit_after>const
util = require('util'),
path = require('path'),
child_process = require('child_process'),
events = require('events');

function ComputeCluster(options) {
  if (!options || typeof options.module !== 'string') {
    throw "missing required 'module' argument";
  }
  // is module a file?
  if (!path.existsSync(options.module)) {
    throw "module doesn't exist: " + options.module;
  }
  if (options.max_processes &&
      (typeof max_processes != 'number' || max_processes < 1)) {
    throw "when provided, max_processes must be an integer greater than one";
  }

  events.EventEmitter.call(this);

  // an array of child processes
  this._kids = {};
  this._MAX_KIDS = (options.max_processes || Math.ceil(require('os').cpus().length * 1.25));
  this._work_q = [];
  this._exiting = false;
  this._exit_cb;
  this._module = options.module;
};

util.inherits(ComputeCluster, events.EventEmitter);

ComputeCluster.prototype._onWorkerExit = function(pid) {
  var self = this;
  return function (code) {
    // if _exiting is false, we don't expect to be shutting down!
    if (!self._exiting) {
      self.emit('error',
                "compute process (" + pid + ") dies with code: " + code);
    }

    delete self._kids[pid];
    if (self._exit_cb) {
      if (Object.keys(self._kids).length === 0) {
        self._exit_cb(null);
        _exit_cb = undefined;
      }
    }
  }
}

ComputeCluster.prototype._getFreeWorker = function() {
  var self = this;

  for (var i in this._kids) {
    if (!this._kids[i].job) return this._kids[i];
  }

  // no workers!  can we spawn one?
  if (Object.keys(this._kids).length < this._MAX_KIDS) {
    var k = { worker: child_process.fork(this._module) };
    k.worker.on('exit', this._onWorkerExit(k.worker.pid));
    this._kids[k.worker.pid] = k;
    return k;
  }
};

ComputeCluster.prototype._runWorkOnWorker = function(work, worker) {
  var self = this;
  worker.worker.once('message', function(m) {
    if (worker.job.cb) worker.job.cb(null, m);
    delete worker.job;
    self._assignWork();
  });
  worker.worker.send(work.job);
  worker.job = work;
};

  // assign as many work units from work_q as possible to avialable
  // compute processes
ComputeCluster.prototype._assignWork = function() {
  while (this._work_q.length > 0) {
    var worker = this._getFreeWorker();
    if (!worker) break;

    this._runWorkOnWorker(this._work_q.shift(), worker);
  }
};

ComputeCluster.prototype.enqueue = function(args, cb) {
  this._work_q.push({ job: args, cb: cb });
  this._assignWork();
  return this;
};

ComputeCluster.prototype.exit = function(cb) {
  if (Object.keys(this._kids).length === 0) {
    setTimeout(function() { cb(null); }, 0);
  } else {
    this._exiting = true;
    this._exit_cb = cb;
    for (var i in this._kids) {
      this._kids[i].worker.kill();
    }
  }
  return this;
};

module.exports = ComputeCluster;

<|endoftext|>


<commit_before>

const debug    = require('debug')('kuss:couchdb:wrapper')
const R        = require('ramda')
const GotCouch = require('got-couch')
const Response = require('./response.js')
const CouchDBError = require('./error.js')

const DEFAULT_OPTIONS = {}

const handler = R.compose(
  R.tap((x) => debug('RESPONSE: %o', x))
, Response.getBodyOrThrowError
)

const wrapper = (fn) => R.curryN(
  fn.length
, (...args) => fn.apply(fn, args).then(handler)
)

const find = wrapper((couch, bucket, query) =>
  couch.query(bucket, query, DEFAULT_OPTIONS)
)

const insert = wrapper((couch, bucket, doc) =>
  couch.insert(bucket, doc)
)

const create = wrapper((couch, bucket, id, doc) =>
  couch.create(bucket, id, doc)
)

const get = wrapper((couch, bucket, id) =>
  couch.get(bucket, id)
)

const list = wrapper((couch, bucket, params) =>
  couch.list(bucket, params)
)

module.exports = (config) => GotCouch(config).then((couch) => ({
  query: find(couch)
, insert: insert(couch)
, create: create(couch)
, get: get(couch)
, list: list(couch)
}))
<commit_msg>remove unused import<commit_after>

const debug    = require('debug')('kuss:couchdb:wrapper')
const R        = require('ramda')
const GotCouch = require('got-couch')
const Response = require('./response.js')

const DEFAULT_OPTIONS = {}

const handler = R.compose(
  R.tap((x) => debug('RESPONSE: %o', x))
, Response.getBodyOrThrowError
)

const wrapper = (fn) => R.curryN(
  fn.length
, (...args) => fn.apply(fn, args).then(handler)
)

const find = wrapper((couch, bucket, query) =>
  couch.query(bucket, query, DEFAULT_OPTIONS)
)

const insert = wrapper((couch, bucket, doc) =>
  couch.insert(bucket, doc)
)

const create = wrapper((couch, bucket, id, doc) =>
  couch.create(bucket, id, doc)
)

const get = wrapper((couch, bucket, id) =>
  couch.get(bucket, id)
)

const list = wrapper((couch, bucket, params) =>
  couch.list(bucket, params)
)

module.exports = (config) => GotCouch(config).then((couch) => ({
  query: find(couch)
, insert: insert(couch)
, create: create(couch)
, get: get(couch)
, list: list(couch)
}))
<|endoftext|>


<commit_before>var url = require('url');
var fs = require('fs');

var debug = require('./debug');
var createRequest = require('./create_request');
var LimitTable = require('./limit_table');

var limitTable = exports.limitTable = new LimitTable();
exports.debug = debug;

var dispatch = function(opt, callback){
    var select = null;
    switch(opt.protocol){
    case "http:":
        opt.agent = false;
        select = createRequest.createHttpProtocol(opt);
        break;
    case "https:":
        opt.agent = false;
        select = createRequest.createHttpsProtocol(opt);
        break;
    }
    if(select){
        select(callback);
    }else{
        callback(new Error('unknown protocol:' + opt.protocol), null);
    }
}

var limit_request = function(urlstr, callback){
    var opt = url.parse(urlstr);
    var check = limitTable.get(opt.hostname);
    debug.emit('limit_request', 'try', {url:urlstr});
    if(check.func()){
        debug.emit('limit_request', 'request', {url:urlstr});
        dispatch(opt, function(err, val){
            debug.emit('limit_request', 'response', {url:urlstr});
            callback(err, val);
        });
    }else{
        debug.emit('limit_request', 'wait', {url:urlstr});
        setTimeout(function(){
            limit_request(urlstr, callback);
        }, check.time * 1000);
    }
}

var request_get_raw = exports.request_get_raw = function(urlstr, callback){
    limit_request(urlstr, callback);
};
var request_get_json = exports.request_get_json = function(urlstr, callback){
    limit_request(urlstr, function(err, rawbody){
        if(err){
            debug.emit('request_get_json', 'request_error', {url:urlstr});
            return callback(err, rawbody);
        }
        var obj = null;
        try{
            obj = JSON.parse(rawbody);
        }catch(e){
            debug.emit('request_get_json', 'json_error', {url:urlstr});
            err = e;
        }
        callback(err, obj);
    });
};
var readfile_raw = exports.readfile_raw = function(file, calllback){
    fs.readFile(file, calllback);
};
var readfile_json = exports.readfile_json = function(file, callback){
    fs.readFile(file, 'utf8', function(err, raw){
        if(err){
            return callback(err, raw);
        }
        var obj = null;
        try{
            obj = JSON.parse(raw);
        }catch(e){
            err = e;
        }
        callback(err, obj);
    });
};

var child_process = require('child_process');
var exec_command = exports.exec_command = function(cmd, callback){
    child_process.exec(cmd, callback);
};

var createRedis = exports.redis = require('./create_redis');
<commit_msg>modify cmd result raw and json<commit_after>var url = require('url');
var fs = require('fs');

var debug = require('./debug');
var createRequest = require('./create_request');
var LimitTable = require('./limit_table');

var limitTable = exports.limitTable = new LimitTable();
exports.debug = debug;

var dispatch = function(opt, callback){
    var select = null;
    switch(opt.protocol){
    case "http:":
        opt.agent = false;
        select = createRequest.createHttpProtocol(opt);
        break;
    case "https:":
        opt.agent = false;
        select = createRequest.createHttpsProtocol(opt);
        break;
    }
    if(select){
        select(callback);
    }else{
        callback(new Error('unknown protocol:' + opt.protocol), null);
    }
}

var limit_request = function(urlstr, callback){
    var opt = url.parse(urlstr);
    var check = limitTable.get(opt.hostname);
    debug.emit('limit_request', 'try', {url:urlstr});
    if(check.func()){
        debug.emit('limit_request', 'request', {url:urlstr});
        dispatch(opt, function(err, val){
            debug.emit('limit_request', 'response', {url:urlstr});
            callback(err, val);
        });
    }else{
        debug.emit('limit_request', 'wait', {url:urlstr});
        setTimeout(function(){
            limit_request(urlstr, callback);
        }, check.time * 1000);
    }
}

var request_get_raw = exports.request_get_raw = function(urlstr, callback){
    limit_request(urlstr, callback);
};
var request_get_json = exports.request_get_json = function(urlstr, callback){
    limit_request(urlstr, function(err, rawbody){
        if(err){
            debug.emit('request_get_json', 'request_error', {url:urlstr});
            return callback(err, rawbody);
        }
        var obj = null;
        try{
            obj = JSON.parse(rawbody);
        }catch(e){
            debug.emit('request_get_json', 'json_error', {url:urlstr});
            err = e;
        }
        callback(err, obj);
    });
};
var readfile_raw = exports.readfile_raw = function(file, calllback){
    fs.readFile(file, calllback);
};
var readfile_json = exports.readfile_json = function(file, callback){
    fs.readFile(file, 'utf8', function(err, raw){
        if(err){
            return callback(err, raw);
        }
        var obj = null;
        try{
            obj = JSON.parse(raw);
        }catch(e){
            err = e;
        }
        callback(err, obj);
    });
};

var child_process = require('child_process');
var exec_command_raw = exports.exec_command_raw = function(cmd, callback){
    child_process.exec(cmd, callback);
};
var exec_command_json = exports.exec_command_json = function(cmd, callback){
    child_process.exec(cmd, function(err, stdout, stderr){
        if(err){
            return callback(err, stdout);
        }
        var obj = null;
        try{
            obj = JSON.parse(stdout);
        }catch(e){
            err = e;
        }
        callback(err, obj);
    });
};

var createRedis = exports.redis = require('./create_redis');
<|endoftext|>


<commit_before>'use strict';

var _ = require('lodash');
var async = require('async');
var times = require('../times');
var fitTreatmentsToBGCurve = require('./treatmenttocurve');

var ONE_DAY = 86400000
  , TWO_DAYS = 172800000;

function uniq(a) {
  var seen = {};
  return a.filter(function (item) {
    return seen.hasOwnProperty(item.mills) ? false : (seen[item.mills] = true);
  });
}

function init(env, ctx) {

  var dataloader = { };

  dataloader.update = function update(ddata, done) {

    console.log('running data.update');
    ddata.lastUpdated = Date.now();

    function loadComplete (err, result) {
      ddata.treatments = _.uniq(ddata.treatments, false, function (item) { return item._id.toString(); });
      //sort treatments so the last is the most recent
      ddata.treatments = _.sortBy(ddata.treatments, function (item) { return item.mills; });
      fitTreatmentsToBGCurve(ddata, env.settings);
      if (err) {
        console.error(err);
      }
      done(err, result);
    }

    // clear treatments, we're going to merge from more queries
    ddata.treatments = [];

    async.parallel([
      loadEntries.bind(null, ddata, ctx)
      , loadTreatments.bind(null, ddata, ctx)
      , loadProfileSwitchTreatments.bind(null, ddata, ctx)
      , loadSensorTreatments.bind(null, ddata, ctx)
      , loadProfile.bind(null, ddata, ctx)
      , loadDeviceStatus.bind(null, ddata, env, ctx)
    ], loadComplete);

  };

  return dataloader;

}

function loadEntries (ddata, ctx, callback) {
  var q = {
    find: {
      date: {
        $gte: ddata.lastUpdated - TWO_DAYS
      }
    }
    , sort: {date: 1}
  };

  ctx.entries.list(q, function (err, results) {
    if (!err && results) {
      var mbgs = [];
      var sgvs = [];
      var cals = [];
      results.forEach(function (element) {
        if (element) {
          if (element.mbg) {
            mbgs.push({
              mgdl: Number(element.mbg), mills: element.date, device: element.device
            });
          } else if (element.sgv) {
            sgvs.push({
              mgdl: Number(element.sgv), mills: element.date, device: element.device, direction: element.direction, filtered: element.filtered, unfiltered: element.unfiltered, noise: element.noise, rssi: element.rssi
            });
          } else if (element.type === 'cal') {
            cals.push({
              mills: element.date, scale: element.scale, intercept: element.intercept, slope: element.slope
            });
          }
        }
      });
      ddata.mbgs = uniq(mbgs);
      ddata.sgvs = uniq(sgvs);
      ddata.cals = uniq(cals);
    }
    callback();
  });
}

function mergeToTreatments (ddata, results) {
  var filtered = _.filter(results, function hasId (treatment) {
    return _.isObject(treatment._id);
  });

  var treatments = _.map(filtered, function update (treatment) {
    treatment.mills = new Date(treatment.created_at).getTime();
    return treatment;
  });

  //filter out temps older than a day and an hour ago since we don't display them
  var oneDayAgo = ddata.lastUpdated - ONE_DAY - times.hour().msecs;
  treatments = _.filter(treatments, function noOldTemps (treatment) {
    return !treatment.eventType || treatment.eventType.indexOf('Temp Basal') === -1 || treatment.mills > oneDayAgo;
  });

  ddata.treatments = _.union(ddata.treatments, treatments);
}

function loadTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 8)).toISOString()
      }
    }
    , sort: {created_at: 1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    callback();
  });
}

function loadProfileSwitchTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      eventType: {
        $eq: 'Profile Switch'
      }
      , created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 31 * 12)).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    // Store last profile switch
    if (results) {
      ddata.lastProfileFromSwitch = null;
      var now = new Date().getTime();
      for (var p = 0; p < results.length; p++ ) {
        var pdate = new Date(results[p].created_at).getTime();
        if (pdate < now) {
          ddata.lastProfileFromSwitch = results[p].profile;
          break;
        }
      }
    }

    callback();
  });
}

function loadSensorTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      eventType: {
        $in: [ 'Sensor Start', 'Sensor Change']
      }
      , created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 32)).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    callback();
  });
}

function loadProfile (ddata, ctx, callback) {
  ctx.profile.last(function (err, results) {
    if (!err && results) {
      var profiles = [];
      results.forEach(function (element) {
        if (element) {
            profiles[0] = element;
        }
      });
      ddata.profiles = profiles;
    }
    callback();
  });
}

function loadDeviceStatus (ddata, env, ctx, callback) {
  var opts = {
    find: {
      created_at: {
        $gte: new Date(ddata.lastUpdated - ONE_DAY).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  if (env.extendedSettings.devicestatus && env.extendedSettings.devicestatus.advanced) {
    //not adding count: 1 restriction
  } else {
    opts.count = 1;
  }

  ctx.devicestatus.list(opts, function (err, results) {
    if (!err && results) {
      ddata.devicestatus = _.map(results, function eachStatus (result) {
        result.mills = new Date(result.created_at).getTime();
        if ('uploaderBattery' in result) {
          result.uploader = {
            battery: result.uploaderBattery
          };
          delete result.uploaderBattery;
        }
        return result;
      }).reverse();
    } else {
      ddata.devicestatus = [];
    }
    callback();
  });
}

module.exports = init;<commit_msg>Fixes IAGE > 7 days<commit_after>'use strict';

var _ = require('lodash');
var async = require('async');
var times = require('../times');
var fitTreatmentsToBGCurve = require('./treatmenttocurve');

var ONE_DAY = 86400000
  , TWO_DAYS = 172800000;

function uniq(a) {
  var seen = {};
  return a.filter(function (item) {
    return seen.hasOwnProperty(item.mills) ? false : (seen[item.mills] = true);
  });
}

function init(env, ctx) {

  var dataloader = { };

  dataloader.update = function update(ddata, done) {

    console.log('running data.update');
    ddata.lastUpdated = Date.now();

    function loadComplete (err, result) {
      ddata.treatments = _.uniq(ddata.treatments, false, function (item) { return item._id.toString(); });
      //sort treatments so the last is the most recent
      ddata.treatments = _.sortBy(ddata.treatments, function (item) { return item.mills; });
      fitTreatmentsToBGCurve(ddata, env.settings);
      if (err) {
        console.error(err);
      }
      done(err, result);
    }

    // clear treatments, we're going to merge from more queries
    ddata.treatments = [];

    async.parallel([
      loadEntries.bind(null, ddata, ctx)
      , loadTreatments.bind(null, ddata, ctx)
      , loadProfileSwitchTreatments.bind(null, ddata, ctx)
      , loadSensorAndInsulinTreatments.bind(null, ddata, ctx)
      , loadProfile.bind(null, ddata, ctx)
      , loadDeviceStatus.bind(null, ddata, env, ctx)
    ], loadComplete);

  };

  return dataloader;

}

function loadEntries (ddata, ctx, callback) {
  var q = {
    find: {
      date: {
        $gte: ddata.lastUpdated - TWO_DAYS
      }
    }
    , sort: {date: 1}
  };

  ctx.entries.list(q, function (err, results) {
    if (!err && results) {
      var mbgs = [];
      var sgvs = [];
      var cals = [];
      results.forEach(function (element) {
        if (element) {
          if (element.mbg) {
            mbgs.push({
              mgdl: Number(element.mbg), mills: element.date, device: element.device
            });
          } else if (element.sgv) {
            sgvs.push({
              mgdl: Number(element.sgv), mills: element.date, device: element.device, direction: element.direction, filtered: element.filtered, unfiltered: element.unfiltered, noise: element.noise, rssi: element.rssi
            });
          } else if (element.type === 'cal') {
            cals.push({
              mills: element.date, scale: element.scale, intercept: element.intercept, slope: element.slope
            });
          }
        }
      });
      ddata.mbgs = uniq(mbgs);
      ddata.sgvs = uniq(sgvs);
      ddata.cals = uniq(cals);
    }
    callback();
  });
}

function mergeToTreatments (ddata, results) {
  var filtered = _.filter(results, function hasId (treatment) {
    return _.isObject(treatment._id);
  });

  var treatments = _.map(filtered, function update (treatment) {
    treatment.mills = new Date(treatment.created_at).getTime();
    return treatment;
  });

  //filter out temps older than a day and an hour ago since we don't display them
  var oneDayAgo = ddata.lastUpdated - ONE_DAY - times.hour().msecs;
  treatments = _.filter(treatments, function noOldTemps (treatment) {
    return !treatment.eventType || treatment.eventType.indexOf('Temp Basal') === -1 || treatment.mills > oneDayAgo;
  });

  ddata.treatments = _.union(ddata.treatments, treatments);
}

function loadTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 8)).toISOString()
      }
    }
    , sort: {created_at: 1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    callback();
  });
}

function loadProfileSwitchTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      eventType: {
        $eq: 'Profile Switch'
      }
      , created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 31 * 12)).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    // Store last profile switch
    if (results) {
      ddata.lastProfileFromSwitch = null;
      var now = new Date().getTime();
      for (var p = 0; p < results.length; p++ ) {
        var pdate = new Date(results[p].created_at).getTime();
        if (pdate < now) {
          ddata.lastProfileFromSwitch = results[p].profile;
          break;
        }
      }
    }

    callback();
  });
}

function loadSensorAndInsulinTreatments (ddata, ctx, callback) {
  var tq = {
    find: {
      eventType: {
        $in: [ 'Sensor Start', 'Sensor Change','Insulin Change']
      }
      , created_at: {
        $gte: new Date(ddata.lastUpdated - (ONE_DAY * 32)).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  ctx.treatments.list(tq, function (err, results) {
    if (!err && results) {
      mergeToTreatments(ddata, results);
    }

    callback();
  });
}

function loadProfile (ddata, ctx, callback) {
  ctx.profile.last(function (err, results) {
    if (!err && results) {
      var profiles = [];
      results.forEach(function (element) {
        if (element) {
            profiles[0] = element;
        }
      });
      ddata.profiles = profiles;
    }
    callback();
  });
}

function loadDeviceStatus (ddata, env, ctx, callback) {
  var opts = {
    find: {
      created_at: {
        $gte: new Date(ddata.lastUpdated - ONE_DAY).toISOString()
      }
    }
    , sort: {created_at: -1}
  };

  if (env.extendedSettings.devicestatus && env.extendedSettings.devicestatus.advanced) {
    //not adding count: 1 restriction
  } else {
    opts.count = 1;
  }

  ctx.devicestatus.list(opts, function (err, results) {
    if (!err && results) {
      ddata.devicestatus = _.map(results, function eachStatus (result) {
        result.mills = new Date(result.created_at).getTime();
        if ('uploaderBattery' in result) {
          result.uploader = {
            battery: result.uploaderBattery
          };
          delete result.uploaderBattery;
        }
        return result;
      }).reverse();
    } else {
      ddata.devicestatus = [];
    }
    callback();
  });
}

module.exports = init;<|endoftext|>