<commit_before>const { Constants } = require('../core/constants');
const { Context } = require('../core/context');
const fs = require('fs');
const path = require('path');
const { OperationResult } = require('../core/operation_result');
const { PluginManager } = require('../plugins/plugin_manager');
const { StorageService } = require('../core/storage_service');
const { Util } = require('../core/util');
const { Worker } = require('./worker');

// This implementation is convoluted and messy...

/**
 * The Uploader performs the upload operations for a job.
 * It catches events from the underlying network plugin,
 * wraps them in {@link JobStatus} objects, and writes them
 * to STDOUT and STDERR to communicate with the parent process.
 *
 * param {Job} - The job to run. The job object must contain at
 * at least one {@link UploadOperation}, or the upload will do
 * nothing.
 *
 * @param {Job}
 *
 */
class Uploader extends Worker {

    constructor(job) {
        super('upload')
        this.job = job;
    }

    /**
     * This runs all of the job's upload operations.
     * It returns an array of promises. This runs all uploads
     * in parallel.
     *
     * @returns {Array<Promise>}
     */
    run() {
        let uploader = this;
        let errors = this.validateParams();
        if (errors.length > 0) {
            return Promise.all([new Promise(function(resolve, reject) {
                reject(uploader.validationError(errors));
            })]);
        }
        let promises = [];
        for (let op of this.job.uploadOps) {
            promises = promises.concat(this.doUpload(op));
        }
        return Promise.all(promises);
    }

    /**
     * doUpload initiates a single upload operation and returns
     * an array of promises. If a single operation includes multiple
     * source files, it will upload them in parallel. This returns
     * one promise per source file.
     *
     * Because each upload resets the exitCode, the caller should
     * ignore the final exitCode and check the results of all of the
     * promises instead. [This needs a more elegant fix.]
     *
     * @param {UploadOperation} - An upload operation containing one
     * or more source files.
     *
     * @returns {Array<Promise>}
     */
    doUpload(uploadOp) {
        let uploader = this;
        let ss = StorageService.find(uploadOp.storageServiceId);
        if (!ss) {
            uploadOp.results[0] = new OperationResult('upload', 'none');
            uploadOp.results[0].start();
            uploadOp.results[0].finish(Context.y18n.__('Cannot find StorageService record'));
            return new Promise(function(resolve, reject) {
                reject(uploader.validationError(uploadOp.results[0].errors));
            });

        }
        let providerClass = this.getProvider(ss.protocol);
        let promises = [];
        for (let filepath of uploadOp.sourceFiles) {
            let provider = new providerClass(ss);
            var promise = new Promise(function(resolve, reject) {
                let lastPercentComplete = 0;
                provider.on('start', function(result) {
                    // Note: percentComplete is -1 because we don't
                    // yet have a way of getting that info.
                    uploader.info('start',
                                  Constants.OP_IN_PROGRESS,
                                  ss.name,
                                  -1,
                                  false);
                });
                provider.on('finish', function(result) {
                    uploadOp.results.push(result);
                    if (result.errors.length > 0) {
                        uploader.completedWithError(result.errors);
                    } else {
                        uploader.completedSuccess(ss.name, false);
                    }
                    resolve(result);
                });
                provider.on('error', function(result) {
                    // Reject causes the entire Promise.all chain
                    // to fail. We want to let other pending promises
                    // complete instead of stopping the chain. We will
                    // handle retries elsewhere.
                    uploadOp.results.push(result);
                    uploader.runtimeError('completed', result.errors);
                    resolve(result);
                    provider = null;
                });
                provider.on('warning', function(result) {
                    // Note: percentComplete is -1 because we don't
                    // yet have a way of getting that info.
                    uploader.info('upload', Constants.OP_IN_PROGRESS,
                                  result.warning, -1, false);
                });
                provider.on('status', function(xfer) {
                    // Uploader reads faster than it writes, so fudge this.
                    // Only write for changes >= 1%, otherwise, on large
                    // uploads, this can write thousands of lines into the
                    // logs.
                    let pctComplete = xfer.percentComplete() * 0.985;
                    if (pctComplete - lastPercentComplete > 1) {
                        uploader.info('status', Constants.OP_IN_PROGRESS,
                                      `${ss.name } - ${pctComplete.toFixed(2)}%`,
                                      pctComplete, false);
                        lastPercentComplete = pctComplete;
                    }
                });
            });
            promises.push(promise);
            provider.upload(filepath, path.basename(filepath));
        }
        return promises;
    }

    /**
     * This checks to ensure that the {@link UploadOperation} includes a
     * target and at least one source file. It also ensures that the
     * specified source files exist. It returns an array of strings decribing
     * the validation errors.
     *
     * @return {Array<string>}
     */
    validateParams() {
        let errors = [];
        for (let op of this.job.uploadOps) {
            let opErrors = []
            if (Util.isEmpty(op.storageServiceId)) {
                opErrors.push(Context.y18n.__('Specify where you want to upload the file.'));
            }
            if (!op.sourceFiles || Util.isEmptyStringArray(op.sourceFiles)) {
                opErrors.push(Context.y18n.__('Specify at least one file to upload.'));
            }
            for (let f of op.sourceFiles) {
                if (!fs.existsSync(f)) {
                    opErrors.push(Context.y18n.__(`File to be uploaded does not exist: %s.`, f));
                }
            }
            if (opErrors.length > 0) {
                let providerDesc = 'Unknown upload provider';
                try {
                    let ss = StorageService.find(op.storageServiceId);
                    let providerClass = this.getProvider(ss.protocol)
                    providerDesc = providerClass.description().name;
                } catch (err) {
                    providerDesc += ` ${err.message}`;
                }
                let result = new OperationResult('upload', providerDesc);
                result.start();
                result.finish(opErrors.join(' '));
                op.results.push(result);
                errors = errors.concat(opErrors);
            }
        }
        return errors;
    }

    /**
     * This returns the first network provider plugin that implements the
     * S3 protocol. If it can't find a plugin for the S3 protocol, it throws
     * an exception.
     *
     * @returns {Plugin}
     */
    getProvider(protocol) {
        let providers = PluginManager.implementsProtocol(protocol);
        if (providers.length == 0) {
            throw `Cannot find a plugin that implements the ${protocol} protocol.`
        }
        return providers[0];
    }
}

module.exports.Uploader = Uploader;
<commit_msg>Clear prior upload results before each new attempt<commit_after>const { Constants } = require('../core/constants');
const { Context } = require('../core/context');
const fs = require('fs');
const path = require('path');
const { OperationResult } = require('../core/operation_result');
const { PluginManager } = require('../plugins/plugin_manager');
const { StorageService } = require('../core/storage_service');
const { Util } = require('../core/util');
const { Worker } = require('./worker');

// This implementation is convoluted and messy...

/**
 * The Uploader performs the upload operations for a job.
 * It catches events from the underlying network plugin,
 * wraps them in {@link JobStatus} objects, and writes them
 * to STDOUT and STDERR to communicate with the parent process.
 *
 * param {Job} - The job to run. The job object must contain at
 * at least one {@link UploadOperation}, or the upload will do
 * nothing.
 *
 * @param {Job}
 *
 */
class Uploader extends Worker {

    constructor(job) {
        super('upload')
        this.job = job;
    }

    /**
     * This runs all of the job's upload operations.
     * It returns an array of promises. This runs all uploads
     * in parallel.
     *
     * @returns {Array<Promise>}
     */
    run() {
        let uploader = this;
        let errors = this.validateParams();
        if (errors.length > 0) {
            return Promise.all([new Promise(function(resolve, reject) {
                reject(uploader.validationError(errors));
            })]);
        }
        let promises = [];
        for (let op of this.job.uploadOps) {
            promises = promises.concat(this.doUpload(op));
        }
        return Promise.all(promises);
    }

    /**
     * doUpload initiates a single upload operation and returns
     * an array of promises. If a single operation includes multiple
     * source files, it will upload them in parallel. This returns
     * one promise per source file.
     *
     * Because each upload resets the exitCode, the caller should
     * ignore the final exitCode and check the results of all of the
     * promises instead. [This needs a more elegant fix.]
     *
     * @param {UploadOperation} - An upload operation containing one
     * or more source files.
     *
     * @returns {Array<Promise>}
     */
    doUpload(uploadOp) {
        let uploader = this;
        // Clear prior upload results. They may cause the UI to
        // report successful upload as failed, or vice-versa.
        uploadOp.results = [];
        let ss = StorageService.find(uploadOp.storageServiceId);
        if (!ss) {
            uploadOp.results[0] = new OperationResult('upload', 'none');
            uploadOp.results[0].start();
            uploadOp.results[0].finish(Context.y18n.__('Cannot find StorageService record'));
            return new Promise(function(resolve, reject) {
                reject(uploader.validationError(uploadOp.results[0].errors));
            });

        }
        let providerClass = this.getProvider(ss.protocol);
        let promises = [];
        for (let filepath of uploadOp.sourceFiles) {
            let provider = new providerClass(ss);
            var promise = new Promise(function(resolve, reject) {
                let lastPercentComplete = 0;
                provider.on('start', function(result) {
                    // Note: percentComplete is -1 because we don't
                    // yet have a way of getting that info.
                    uploader.info('start',
                                  Constants.OP_IN_PROGRESS,
                                  ss.name,
                                  -1,
                                  false);
                });
                provider.on('finish', function(result) {
                    uploadOp.results.push(result);
                    if (result.errors.length > 0) {
                        uploader.completedWithError(result.errors);
                    } else {
                        uploader.completedSuccess(ss.name, false);
                    }
                    resolve(result);
                });
                provider.on('error', function(result) {
                    // Reject causes the entire Promise.all chain
                    // to fail. We want to let other pending promises
                    // complete instead of stopping the chain. We will
                    // handle retries elsewhere.
                    uploadOp.results.push(result);
                    uploader.runtimeError('completed', result.errors);
                    resolve(result);
                    provider = null;
                });
                provider.on('warning', function(result) {
                    // Note: percentComplete is -1 because we don't
                    // yet have a way of getting that info.
                    uploader.info('upload', Constants.OP_IN_PROGRESS,
                                  result.warning, -1, false);
                });
                provider.on('status', function(xfer) {
                    // Uploader reads faster than it writes, so fudge this.
                    // Only write for changes >= 1%, otherwise, on large
                    // uploads, this can write thousands of lines into the
                    // logs.
                    let pctComplete = xfer.percentComplete() * 0.985;
                    if (pctComplete - lastPercentComplete > 1) {
                        uploader.info('status', Constants.OP_IN_PROGRESS,
                                      `${ss.name } - ${pctComplete.toFixed(2)}%`,
                                      pctComplete, false);
                        lastPercentComplete = pctComplete;
                    }
                });
            });
            promises.push(promise);
            provider.upload(filepath, path.basename(filepath));
        }
        return promises;
    }

    /**
     * This checks to ensure that the {@link UploadOperation} includes a
     * target and at least one source file. It also ensures that the
     * specified source files exist. It returns an array of strings decribing
     * the validation errors.
     *
     * @return {Array<string>}
     */
    validateParams() {
        let errors = [];
        for (let op of this.job.uploadOps) {
            let opErrors = []
            if (Util.isEmpty(op.storageServiceId)) {
                opErrors.push(Context.y18n.__('Specify where you want to upload the file.'));
            }
            if (!op.sourceFiles || Util.isEmptyStringArray(op.sourceFiles)) {
                opErrors.push(Context.y18n.__('Specify at least one file to upload.'));
            }
            for (let f of op.sourceFiles) {
                if (!fs.existsSync(f)) {
                    opErrors.push(Context.y18n.__(`File to be uploaded does not exist: %s.`, f));
                }
            }
            if (opErrors.length > 0) {
                let providerDesc = 'Unknown upload provider';
                try {
                    let ss = StorageService.find(op.storageServiceId);
                    let providerClass = this.getProvider(ss.protocol)
                    providerDesc = providerClass.description().name;
                } catch (err) {
                    providerDesc += ` ${err.message}`;
                }
                let result = new OperationResult('upload', providerDesc);
                result.start();
                result.finish(opErrors.join(' '));
                op.results.push(result);
                errors = errors.concat(opErrors);
            }
        }
        return errors;
    }

    /**
     * This returns the first network provider plugin that implements the
     * S3 protocol. If it can't find a plugin for the S3 protocol, it throws
     * an exception.
     *
     * @returns {Plugin}
     */
    getProvider(protocol) {
        let providers = PluginManager.implementsProtocol(protocol);
        if (providers.length == 0) {
            throw `Cannot find a plugin that implements the ${protocol} protocol.`
        }
        return providers[0];
    }
}

module.exports.Uploader = Uploader;
<|endoftext|>


<commit_before>/*global define*/
define([
        '../Core/defaultValue',
        '../Core/Cartographic',
        '../Core/DeveloperError',
        '../Core/Event',
        '../Core/loadXML',
        '../Core/writeTextToCanvas',
        '../Core/Extent',
        '../Core/RuntimeError',
        './ImageryProvider',
        './WebMercatorTilingScheme',
        './GeographicTilingScheme'
    ], function(
        defaultValue,
        Cartographic,
        DeveloperError,
        Event,
        loadXML,
        writeTextToCanvas,
        Extent,
        RuntimeError,
        ImageryProvider,
        WebMercatorTilingScheme,
        GeographicTilingScheme) {
    "use strict";

    var trailingSlashRegex = /\/$/;

    /**
     * Provides tiled imagery as generated by MapTiler / GDDAL2Tiles etc.
     *
     * @alias TileMapServiceImageryProvider
     * @constructor
     *
     * @param {String} [description.url='.'] Path to image tiles on server.
     * @param {String} [description.fileExtension='png'] The file extension for images on the server.
     * @param {Object} [description.proxy] A proxy to use for requests. This object is expected to have a getURL function which returns the proxied URL.
     * @param {String} [description.credit=''] A string crediting the data source, which is displayed on the canvas.
     * @param {Number} [description.maximumLevel=18] The maximum level-of-detail supported by the imagery provider.
     * @param {Extent} [description.extent=Extent.MAX_VALUE] The extent, in radians, covered by the image.
     * @param {TilingScheme} [description.tilingScheme] The tiling scheme specifying how the ellipsoidal
     * surface is broken into tiles.  If this parameter is not provided, a {@link WebMercatorTilingScheme}
     * is used.
     * @param {Number} [description.tileWidth=256] Pixel width of image tiles.
     * @param {Number} [description.tileHeight=256] Pixel height of image tiles.
     *
     * @see ArcGisMapServerImageryProvider
     * @see BingMapsImageryProvider
     * @see OpenStreetMapImageryProvider
     * @see SingleTileImageryProvider
     * @see WebMapServiceImageryProvider
     *
     * @see MapTiler
     * @see GDDAL2Tiles
     * @see Cross-Origin Resource Sharing
     *
     * @example
     * // TileMapService tile provider
     * var tms = new TileMapServiceImageryProvider({
     *    url : '../images/cesium_maptiler/Cesium_Logo_Color',
     *    fileExtension: 'png',
     *    maximumLevel: 4,
     *    extent: new Cesium.Extent(
     *        Cesium.Math.toRadians(-120.0),
     *        Cesium.Math.toRadians(20.0),
     *        Cesium.Math.toRadians(-60.0),
     *        Cesium.Math.toRadians(40.0))
     * });
     */
    var TileMapServiceImageryProvider = function TileMapServiceImageryProvider(description) {
        description = defaultValue(description, {});

        if (typeof description.url === 'undefined') {
            throw new DeveloperError('description.url is required.');
        }

        var url = description.url;

        if (!trailingSlashRegex.test(url)) {
            url = url + '/';
        }

        this._url = url;
        this._ready = false;

        this._proxy = description.proxy;
        this._tileDiscardPolicy = description.tileDiscardPolicy;

        this._errorEvent = new Event();

        var credit = description.credit;
        if (typeof credit !== 'undefined') {
            this._logo = writeTextToCanvas(credit, {
                font : '12px sans-serif'
            });
        }

        var that = this;

        // Try to load remaining parameters from XML
        loadXML(url + 'tilemapresource.xml').then(function(xml) {
            // Allowing description properties to override XML values
            var format = xml.getElementsByTagName('TileFormat')[0];
            that._fileExtension = defaultValue(description.fileExtension, format.getAttribute('extension'));
            that._tileWidth = defaultValue(description.tileWidth, parseInt(format.getAttribute('width'), 10));
            that._tileHeight = defaultValue(description.tileHeight, parseInt(format.getAttribute('height'), 10));
            var tilesets = xml.getElementsByTagName('TileSet');
            that._minimumLevel = defaultValue(description.minimumLevel, parseInt(tilesets[0].getAttribute('order'), 10));
            that._maximumLevel = defaultValue(description.maximumLevel, parseInt(tilesets[tilesets.length - 1].getAttribute('order'), 10));

            // extent handling
            that._extent = description.extent;
            if (typeof that._extent === 'undefined') {
                var bbox = xml.getElementsByTagName('BoundingBox')[0];
                var sw = Cartographic.fromDegrees(parseFloat(bbox.getAttribute('miny')), parseFloat(bbox.getAttribute('minx')));
                var ne = Cartographic.fromDegrees(parseFloat(bbox.getAttribute('maxy')), parseFloat(bbox.getAttribute('maxx')));
                that._extent = new Extent(sw.longitude, sw.latitude, ne.longitude, ne.latitude);
            } else {
                that._extent = that._extent.clone();
            }

            // tiling scheme handling
            var tilingScheme = description.tilingScheme;
            if (typeof tilingScheme === 'undefined') {
                var tilingSchemeName = xml.getElementsByTagName('TileSets')[0].getAttribute('profile');
                tilingScheme = tilingSchemeName === 'geodetic' ? new GeographicTilingScheme() : new WebMercatorTilingScheme();
            }

            // The extent must not be outside the bounds allowed by the tiling scheme.
            if (that._extent.west < tilingScheme.getExtent().west) {
                that._extent.west = tilingScheme.getExtent().west;
            }
            if (that._extent.east > tilingScheme.getExtent().east) {
                that._extent.east = tilingScheme.getExtent().east;
            }
            if (that._extent.south < tilingScheme.getExtent().south) {
                that._extent.south = tilingScheme.getExtent().south;
            }
            if (that._extent.north > tilingScheme.getExtent().north) {
                that._extent.north = tilingScheme.getExtent().north;
            }

            // Check the number of tiles at the minimum level and throw if it's too big.
            var swTile = tilingScheme.positionToTileXY(that._extent.getSouthwest(), that._minimumLevel);
            var neTile = tilingScheme.positionToTileXY(that._extent.getNortheast(), that._minimumLevel);
            var tileCount = (Math.abs(neTile.x - swTile.x) + 1) * (Math.abs(neTile.y - swTile.y) + 1);
            if (tileCount > 4) {
                var message = 'The Tile Map Service (TMS) imagery at ' + that._url + ' cannot be used because it has too many tiles (' + tileCount + ') at the root level.';
                console.log(message);
                throw new RuntimeError(message);
            }

            that._tilingScheme = tilingScheme;
            that._ready = true;
        }, function(error) {
            // Can't load XML, still allow description and defaults
            that._fileExtension = defaultValue(description.fileExtension, 'png');
            that._tileWidth = defaultValue(description.tileWidth, 256);
            that._tileHeight = defaultValue(description.tileHeight, 256);
            that._minimumLevel = defaultValue(description.minimumLevel, 0);
            that._maximumLevel = defaultValue(description.maximumLevel, 18);
            that._tilingScheme = defaultValue(description.tilingScheme, new WebMercatorTilingScheme());
            that._extent = defaultValue(description.extent, that._tilingScheme.getExtent());
            that._ready = true;
        });

    };

    function buildImageUrl(imageryProvider, x, y, level) {
        var yTiles = imageryProvider._tilingScheme.getNumberOfYTilesAtLevel(level);
        var url = imageryProvider._url + level + '/' + x + '/' + (yTiles - y - 1) + '.' + imageryProvider._fileExtension;

        var proxy = imageryProvider._proxy;
        if (typeof proxy !== 'undefined') {
            url = proxy.getURL(url);
        }

        return url;
    }

    /**
     * Gets the URL of the service hosting the imagery.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {String} The URL.
     */
    TileMapServiceImageryProvider.prototype.getUrl = function() {
        return this._url;
    };

    /**
     * Gets the proxy used by this provider.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Proxy} The proxy.
     *
     * @see DefaultProxy
     */
    TileMapServiceImageryProvider.prototype.getProxy = function() {
        return this._proxy;
    };

    /**
     * Gets the width of each tile, in pixels.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The width.
     */
    TileMapServiceImageryProvider.prototype.getTileWidth = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileWidth must not be called before the imagery provider is ready.');
        }
        return this._tileWidth;
    };

    /**
     * Gets the height of each tile, in pixels.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The height.
     */
    TileMapServiceImageryProvider.prototype.getTileHeight = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileHeight must not be called before the imagery provider is ready.');
        }
        return this._tileHeight;
    };

    /**
     * Gets the maximum level-of-detail that can be requested.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The maximum level.
     */
    TileMapServiceImageryProvider.prototype.getMinimumLevel = function() {
        if (!this._ready) {
            throw new DeveloperError('getMinimumLevel must not be called before the imagery provider is ready.');
        }
        return this._minimumLevel;
    };

    /**
     * Gets the maximum level-of-detail that can be requested.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The maximum level.
     */
    TileMapServiceImageryProvider.prototype.getMaximumLevel = function() {
        if (!this._ready) {
            throw new DeveloperError('getMaximumLevel must not be called before the imagery provider is ready.');
        }
        return this._maximumLevel;
    };

    /**
     * Gets the tiling scheme used by this provider.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {TilingScheme} The tiling scheme.
     * @see WebMercatorTilingScheme
     * @see GeographicTilingScheme
     */
    TileMapServiceImageryProvider.prototype.getTilingScheme = function() {
        if (!this._ready) {
            throw new DeveloperError('getTilingScheme must not be called before the imagery provider is ready.');
        }
        return this._tilingScheme;
    };

    /**
     * Gets the extent, in radians, of the imagery provided by this instance.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Extent} The extent.
     */
    TileMapServiceImageryProvider.prototype.getExtent = function() {
        if (!this._ready) {
            throw new DeveloperError('getExtent must not be called before the imagery provider is ready.');
        }
        return this._extent;
    };

    /**
     * Gets the tile discard policy.  If not undefined, the discard policy is responsible
     * for filtering out "missing" tiles via its shouldDiscardImage function.  If this function
     * returns undefined, no tiles are filtered.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {TileDiscardPolicy} The discard policy.
     *
     * @see DiscardMissingTileImagePolicy
     * @see NeverTileDiscardPolicy
     */
    TileMapServiceImageryProvider.prototype.getTileDiscardPolicy = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileDiscardPolicy must not be called before the imagery provider is ready.');
        }
        return this._tileDiscardPolicy;
    };

    /**
     * Gets an event that is raised when the imagery provider encounters an asynchronous error.  By subscribing
     * to the event, you will be notified of the error and can potentially recover from it.  Event listeners
     * are passed an instance of {@link TileProviderError}.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Event} The event.
     */
    TileMapServiceImageryProvider.prototype.getErrorEvent = function() {
        return this._errorEvent;
    };

    /**
     * Gets a value indicating whether or not the provider is ready for use.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Boolean} True if the provider is ready to use; otherwise, false.
     */
    TileMapServiceImageryProvider.prototype.isReady = function() {
        return this._ready;
    };

    /**
     * Requests the image for a given tile.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @param {Number} x The tile X coordinate.
     * @param {Number} y The tile Y coordinate.
     * @param {Number} level The tile level.
     *
     * @returns {Promise} A promise for the image that will resolve when the image is available, or
     *          undefined if there are too many active requests to the server, and the request
     *          should be retried later.  The resolved image may be either an
     *          Image or a Canvas DOM object.
     */
    TileMapServiceImageryProvider.prototype.requestImage = function(x, y, level) {
        if (!this._ready) {
            throw new DeveloperError('requestImage must not be called before the imagery provider is ready.');
        }
        var url = buildImageUrl(this, x, y, level);
        return ImageryProvider.loadImage(this, url);
    };

    /**
     * Gets the logo to display when this imagery provider is active.  Typically this is used to credit
     * the source of the imagery.  This function should not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Image|Canvas} A canvas or image containing the log to display, or undefined if there is no logo.
     */
    TileMapServiceImageryProvider.prototype.getLogo = function() {
        return this._logo;
    };

    return TileMapServiceImageryProvider;
});<commit_msg>Use previous behavior when minimum level is no good.<commit_after>/*global define*/
define([
        '../Core/defaultValue',
        '../Core/Cartographic',
        '../Core/DeveloperError',
        '../Core/Event',
        '../Core/loadXML',
        '../Core/writeTextToCanvas',
        '../Core/Extent',
        '../Core/RuntimeError',
        './ImageryProvider',
        './WebMercatorTilingScheme',
        './GeographicTilingScheme'
    ], function(
        defaultValue,
        Cartographic,
        DeveloperError,
        Event,
        loadXML,
        writeTextToCanvas,
        Extent,
        RuntimeError,
        ImageryProvider,
        WebMercatorTilingScheme,
        GeographicTilingScheme) {
    "use strict";

    var trailingSlashRegex = /\/$/;

    /**
     * Provides tiled imagery as generated by MapTiler / GDDAL2Tiles etc.
     *
     * @alias TileMapServiceImageryProvider
     * @constructor
     *
     * @param {String} [description.url='.'] Path to image tiles on server.
     * @param {String} [description.fileExtension='png'] The file extension for images on the server.
     * @param {Object} [description.proxy] A proxy to use for requests. This object is expected to have a getURL function which returns the proxied URL.
     * @param {String} [description.credit=''] A string crediting the data source, which is displayed on the canvas.
     * @param {Number} [description.maximumLevel=18] The maximum level-of-detail supported by the imagery provider.
     * @param {Extent} [description.extent=Extent.MAX_VALUE] The extent, in radians, covered by the image.
     * @param {TilingScheme} [description.tilingScheme] The tiling scheme specifying how the ellipsoidal
     * surface is broken into tiles.  If this parameter is not provided, a {@link WebMercatorTilingScheme}
     * is used.
     * @param {Number} [description.tileWidth=256] Pixel width of image tiles.
     * @param {Number} [description.tileHeight=256] Pixel height of image tiles.
     *
     * @see ArcGisMapServerImageryProvider
     * @see BingMapsImageryProvider
     * @see OpenStreetMapImageryProvider
     * @see SingleTileImageryProvider
     * @see WebMapServiceImageryProvider
     *
     * @see MapTiler
     * @see GDDAL2Tiles
     * @see Cross-Origin Resource Sharing
     *
     * @example
     * // TileMapService tile provider
     * var tms = new TileMapServiceImageryProvider({
     *    url : '../images/cesium_maptiler/Cesium_Logo_Color',
     *    fileExtension: 'png',
     *    maximumLevel: 4,
     *    extent: new Cesium.Extent(
     *        Cesium.Math.toRadians(-120.0),
     *        Cesium.Math.toRadians(20.0),
     *        Cesium.Math.toRadians(-60.0),
     *        Cesium.Math.toRadians(40.0))
     * });
     */
    var TileMapServiceImageryProvider = function TileMapServiceImageryProvider(description) {
        description = defaultValue(description, {});

        if (typeof description.url === 'undefined') {
            throw new DeveloperError('description.url is required.');
        }

        var url = description.url;

        if (!trailingSlashRegex.test(url)) {
            url = url + '/';
        }

        this._url = url;
        this._ready = false;

        this._proxy = description.proxy;
        this._tileDiscardPolicy = description.tileDiscardPolicy;

        this._errorEvent = new Event();

        var credit = description.credit;
        if (typeof credit !== 'undefined') {
            this._logo = writeTextToCanvas(credit, {
                font : '12px sans-serif'
            });
        }

        var that = this;

        // Try to load remaining parameters from XML
        loadXML(url + 'tilemapresource.xml').then(function(xml) {
            // Allowing description properties to override XML values
            var format = xml.getElementsByTagName('TileFormat')[0];
            that._fileExtension = defaultValue(description.fileExtension, format.getAttribute('extension'));
            that._tileWidth = defaultValue(description.tileWidth, parseInt(format.getAttribute('width'), 10));
            that._tileHeight = defaultValue(description.tileHeight, parseInt(format.getAttribute('height'), 10));
            var tilesets = xml.getElementsByTagName('TileSet');
            that._minimumLevel = defaultValue(description.minimumLevel, parseInt(tilesets[0].getAttribute('order'), 10));
            that._maximumLevel = defaultValue(description.maximumLevel, parseInt(tilesets[tilesets.length - 1].getAttribute('order'), 10));

            // extent handling
            that._extent = description.extent;
            if (typeof that._extent === 'undefined') {
                var bbox = xml.getElementsByTagName('BoundingBox')[0];
                var sw = Cartographic.fromDegrees(parseFloat(bbox.getAttribute('miny')), parseFloat(bbox.getAttribute('minx')));
                var ne = Cartographic.fromDegrees(parseFloat(bbox.getAttribute('maxy')), parseFloat(bbox.getAttribute('maxx')));
                that._extent = new Extent(sw.longitude, sw.latitude, ne.longitude, ne.latitude);
            } else {
                that._extent = that._extent.clone();
            }

            // tiling scheme handling
            var tilingScheme = description.tilingScheme;
            if (typeof tilingScheme === 'undefined') {
                var tilingSchemeName = xml.getElementsByTagName('TileSets')[0].getAttribute('profile');
                tilingScheme = tilingSchemeName === 'geodetic' ? new GeographicTilingScheme() : new WebMercatorTilingScheme();
            }

            // The extent must not be outside the bounds allowed by the tiling scheme.
            if (that._extent.west < tilingScheme.getExtent().west) {
                that._extent.west = tilingScheme.getExtent().west;
            }
            if (that._extent.east > tilingScheme.getExtent().east) {
                that._extent.east = tilingScheme.getExtent().east;
            }
            if (that._extent.south < tilingScheme.getExtent().south) {
                that._extent.south = tilingScheme.getExtent().south;
            }
            if (that._extent.north > tilingScheme.getExtent().north) {
                that._extent.north = tilingScheme.getExtent().north;
            }

            // Check the number of tiles at the minimum level and throw if it's too big.
            var swTile = tilingScheme.positionToTileXY(that._extent.getSouthwest(), that._minimumLevel);
            var neTile = tilingScheme.positionToTileXY(that._extent.getNortheast(), that._minimumLevel);
            var tileCount = (Math.abs(neTile.x - swTile.x) + 1) * (Math.abs(neTile.y - swTile.y) + 1);
            if (tileCount > 4) {
                that._minimumLevel = 0;
            }

            that._tilingScheme = tilingScheme;
            that._ready = true;
        }, function(error) {
            // Can't load XML, still allow description and defaults
            that._fileExtension = defaultValue(description.fileExtension, 'png');
            that._tileWidth = defaultValue(description.tileWidth, 256);
            that._tileHeight = defaultValue(description.tileHeight, 256);
            that._minimumLevel = defaultValue(description.minimumLevel, 0);
            that._maximumLevel = defaultValue(description.maximumLevel, 18);
            that._tilingScheme = defaultValue(description.tilingScheme, new WebMercatorTilingScheme());
            that._extent = defaultValue(description.extent, that._tilingScheme.getExtent());
            that._ready = true;
        });

    };

    function buildImageUrl(imageryProvider, x, y, level) {
        var yTiles = imageryProvider._tilingScheme.getNumberOfYTilesAtLevel(level);
        var url = imageryProvider._url + level + '/' + x + '/' + (yTiles - y - 1) + '.' + imageryProvider._fileExtension;

        var proxy = imageryProvider._proxy;
        if (typeof proxy !== 'undefined') {
            url = proxy.getURL(url);
        }

        return url;
    }

    /**
     * Gets the URL of the service hosting the imagery.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {String} The URL.
     */
    TileMapServiceImageryProvider.prototype.getUrl = function() {
        return this._url;
    };

    /**
     * Gets the proxy used by this provider.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Proxy} The proxy.
     *
     * @see DefaultProxy
     */
    TileMapServiceImageryProvider.prototype.getProxy = function() {
        return this._proxy;
    };

    /**
     * Gets the width of each tile, in pixels.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The width.
     */
    TileMapServiceImageryProvider.prototype.getTileWidth = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileWidth must not be called before the imagery provider is ready.');
        }
        return this._tileWidth;
    };

    /**
     * Gets the height of each tile, in pixels.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The height.
     */
    TileMapServiceImageryProvider.prototype.getTileHeight = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileHeight must not be called before the imagery provider is ready.');
        }
        return this._tileHeight;
    };

    /**
     * Gets the maximum level-of-detail that can be requested.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The maximum level.
     */
    TileMapServiceImageryProvider.prototype.getMinimumLevel = function() {
        if (!this._ready) {
            throw new DeveloperError('getMinimumLevel must not be called before the imagery provider is ready.');
        }
        return this._minimumLevel;
    };

    /**
     * Gets the maximum level-of-detail that can be requested.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Number} The maximum level.
     */
    TileMapServiceImageryProvider.prototype.getMaximumLevel = function() {
        if (!this._ready) {
            throw new DeveloperError('getMaximumLevel must not be called before the imagery provider is ready.');
        }
        return this._maximumLevel;
    };

    /**
     * Gets the tiling scheme used by this provider.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {TilingScheme} The tiling scheme.
     * @see WebMercatorTilingScheme
     * @see GeographicTilingScheme
     */
    TileMapServiceImageryProvider.prototype.getTilingScheme = function() {
        if (!this._ready) {
            throw new DeveloperError('getTilingScheme must not be called before the imagery provider is ready.');
        }
        return this._tilingScheme;
    };

    /**
     * Gets the extent, in radians, of the imagery provided by this instance.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Extent} The extent.
     */
    TileMapServiceImageryProvider.prototype.getExtent = function() {
        if (!this._ready) {
            throw new DeveloperError('getExtent must not be called before the imagery provider is ready.');
        }
        return this._extent;
    };

    /**
     * Gets the tile discard policy.  If not undefined, the discard policy is responsible
     * for filtering out "missing" tiles via its shouldDiscardImage function.  If this function
     * returns undefined, no tiles are filtered.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {TileDiscardPolicy} The discard policy.
     *
     * @see DiscardMissingTileImagePolicy
     * @see NeverTileDiscardPolicy
     */
    TileMapServiceImageryProvider.prototype.getTileDiscardPolicy = function() {
        if (!this._ready) {
            throw new DeveloperError('getTileDiscardPolicy must not be called before the imagery provider is ready.');
        }
        return this._tileDiscardPolicy;
    };

    /**
     * Gets an event that is raised when the imagery provider encounters an asynchronous error.  By subscribing
     * to the event, you will be notified of the error and can potentially recover from it.  Event listeners
     * are passed an instance of {@link TileProviderError}.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Event} The event.
     */
    TileMapServiceImageryProvider.prototype.getErrorEvent = function() {
        return this._errorEvent;
    };

    /**
     * Gets a value indicating whether or not the provider is ready for use.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Boolean} True if the provider is ready to use; otherwise, false.
     */
    TileMapServiceImageryProvider.prototype.isReady = function() {
        return this._ready;
    };

    /**
     * Requests the image for a given tile.  This function should
     * not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @param {Number} x The tile X coordinate.
     * @param {Number} y The tile Y coordinate.
     * @param {Number} level The tile level.
     *
     * @returns {Promise} A promise for the image that will resolve when the image is available, or
     *          undefined if there are too many active requests to the server, and the request
     *          should be retried later.  The resolved image may be either an
     *          Image or a Canvas DOM object.
     */
    TileMapServiceImageryProvider.prototype.requestImage = function(x, y, level) {
        if (!this._ready) {
            throw new DeveloperError('requestImage must not be called before the imagery provider is ready.');
        }
        var url = buildImageUrl(this, x, y, level);
        return ImageryProvider.loadImage(this, url);
    };

    /**
     * Gets the logo to display when this imagery provider is active.  Typically this is used to credit
     * the source of the imagery.  This function should not be called before {@link TileMapServiceImageryProvider#isReady} returns true.
     *
     * @memberof TileMapServiceImageryProvider
     *
     * @returns {Image|Canvas} A canvas or image containing the log to display, or undefined if there is no logo.
     */
    TileMapServiceImageryProvider.prototype.getLogo = function() {
        return this._logo;
    };

    return TileMapServiceImageryProvider;
});<|endoftext|>


<commit_before>import React from 'react'
import { Link, graphql } from 'gatsby'
import Img from 'gatsby-image'
import Bio from '../components/Bio'
import Layout from '../components/Layout'
import AMPSEO from '../components/AMPSEO'
import { Tag } from '../components/Tag'
import { BeforeReading } from '../components/BeforeReading'
import { rhythm } from '../utils/typography'

class BlogPostTemplate extends React.Component {
  render() {
    const post = this.props.data.markdownRemark
    const siteTitle = this.props.data.site.siteMetadata.title
    const { previous, next, amp = false } = this.props.pageContext

    return (
      <Layout location={this.props.location} title={siteTitle} headerDimmed>
        <AMPSEO
          amp={amp}
          title={post.frontmatter.title}
          description={post.excerpt}
          slug={this.props.pageContext.slug}
          date={new Date(post.frontmatter.date)}
          featuredImageName={
            post.frontmatter.featuredImage
              ? post.frontmatter.featuredImage.base
              : null
          }
        />

{post.frontmatter.title}

        <BeforeReading
          publishedAtStr={post.frontmatter.date}
          timeToRead={post.timeToRead}
        />
        {post.frontmatter.featuredImage &&
          post.frontmatter.featuredImage.childImageSharp && (

          )}

        {post.frontmatter.tags && (
          <small
            style={{
              display: `block`,
              marginBottom: rhythm(0.5),
            }}
          >
            Tags:
            {post.frontmatter.tags.map(tagName => (
              <Tag key={tagName} tagName={tagName} />
            ))}

        )}

        <ul
          style={{
            display: `flex`,
            flexWrap: `wrap`,
            justifyContent: `space-between`,
            listStyle: `none`,
            padding: 0,
          }}
        >

            {previous && (
              <Link to={`post${previous.fields.slug}`} rel="prev">
                ← {previous.frontmatter.title}
              </Link>
            )}

            {next && (
              <Link to={`post${next.fields.slug}`} rel="next">
                {next.frontmatter.title} →
              </Link>
            )}

      </Layout>
    )
  }
}

export default BlogPostTemplate

export const pageQuery = graphql`
  query BlogPostBySlugAMP($slug: String!) {
    site {
      siteMetadata {
        title
        author
      }
    }
    markdownRemark(fields: { slug: { eq: $slug } }) {
      id
      excerpt(pruneLength: 160)
      html
      timeToRead
      frontmatter {
        title
        tags
        date(formatString: "MMMM DD, YYYY")
        featuredImage {
          base
          childImageSharp {
            fluid(maxWidth: 700) {
              ...GatsbyImageSharpFluid
            }
          }
        }
      }
    }
  }
`
<commit_msg>fix lint errors<commit_after>import React from 'react'
import { Link, graphql } from 'gatsby'
import Img from 'gatsby-image'
import Bio from '../components/Bio'
import Layout from '../components/Layout'
import AMPSEO from '../components/AMPSEO'
import { Tag } from '../components/Tag'
import { BeforeReading } from '../components/BeforeReading'
import { rhythm } from '../utils/typography'

class BlogPostTemplate extends React.Component {
  render() {
    const post = this.props.data.markdownRemark
    const siteTitle = this.props.data.site.siteMetadata.title
    const { previous, next, amp = false } = this.props.pageContext

    return (
      <Layout location={this.props.location} title={siteTitle} headerDimmed>
        <AMPSEO
          amp={amp}
          title={post.frontmatter.title}
          description={post.excerpt}
          slug={this.props.pageContext.slug}
          date={new Date(post.frontmatter.date)}
          featuredImageName={
            post.frontmatter.featuredImage
              ? post.frontmatter.featuredImage.base
              : null
          }
        />

{post.frontmatter.title}

        <BeforeReading
          publishedAtStr={post.frontmatter.date}
          timeToRead={post.timeToRead}
        />
        {post.frontmatter.featuredImage &&
          post.frontmatter.featuredImage.childImageSharp && (

          )}

        {post.frontmatter.tags && (
          <small
            style={{
              display: `block`,
              marginBottom: rhythm(0.5),
            }}
          >
            Tags:
            {post.frontmatter.tags.map(tagName => (
              <Tag key={tagName} tagName={tagName} />
            ))}

        )}

        <Bio />

        <ul
          style={{
            display: `flex`,
            flexWrap: `wrap`,
            justifyContent: `space-between`,
            listStyle: `none`,
            padding: 0,
          }}
        >

            {previous && (
              <Link to={`post${previous.fields.slug}`} rel="prev">
                ← {previous.frontmatter.title}
              </Link>
            )}

            {next && (
              <Link to={`post${next.fields.slug}`} rel="next">
                {next.frontmatter.title} →
              </Link>
            )}

      </Layout>
    )
  }
}

export default BlogPostTemplate

export const pageQuery = graphql`
  query BlogPostBySlugAMP($slug: String!) {
    site {
      siteMetadata {
        title
        author
      }
    }
    markdownRemark(fields: { slug: { eq: $slug } }) {
      id
      excerpt(pruneLength: 160)
      html
      timeToRead
      frontmatter {
        title
        tags
        date(formatString: "MMMM DD, YYYY")
        featuredImage {
          base
          childImageSharp {
            fluid(maxWidth: 700) {
              ...GatsbyImageSharpFluid
            }
          }
        }
      }
    }
  }
`
<|endoftext|>


<commit_before>import * as tabs from '../background/tabs';
import * as histories from '../background/histories';
import * as consoleActions from './console';

const normalizeUrl = (string) => {
  try {
    return new URL(string).href;
  } catch (e) {
    return 'http://' + string;
  }
};

const openCommand = (url) => {
  return browser.tabs.query({
    active: true, currentWindow: true
  }).then((gotTabs) => {
    if (gotTabs.length > 0) {
      return browser.tabs.update(gotTabs[0].id, { url: url });
    }
  });
};

const tabopenCommand = (url) => {
  return browser.tabs.create({ url: url });
};

const bufferCommand = (keywords) => {
  return browser.tabs.query({
    active: true, currentWindow: true
  }).then((gotTabs) => {
    if (gotTabs.length > 0) {
      if (isNaN(keywords)) {
        return tabs.selectByKeyword(gotTabs[0], keywords);
      }
      let index = parseInt(keywords, 10) - 1;
      return tabs.selectAt(index);
    }
  });
};

const doCommand = (name, remaining) => {
  switch (name) {
  case 'o':
  case 'open':
    // TODO use search engined and pass keywords to them
    return openCommand(normalizeUrl(remaining));
  case 't':
  case 'tabopen':
    return tabopenCommand(normalizeUrl(remaining));
  case 'b':
  case 'buffer':
    return bufferCommand(remaining);
  }
  throw new Error(name + ' command is not defined');
};

const getCompletions = (command, keywords) => {
  switch (command) {
  case 'o':
  case 'open':
  case 't':
  case 'tabopen':
    return histories.getCompletions(keywords).then((pages) => {
      let items = pages.map((page) => {
        return {
          caption: page.title,
          content: page.url,
          url: page.url
        };
      });
      return [
        {
          name: 'History',
          items
        }
      ];
    });
  case 'b':
  case 'buffer':
    return tabs.getCompletions(keywords).then((gotTabs) => {
      let items = gotTabs.map((tab) => {
        return {
          caption: tab.title,
          content: tab.title,
          url: tab.url,
          icon: tab.favIconUrl
        };
      });
      return [
        {
          name: 'Buffers',
          items: items
        }
      ];
    });
  }
  return Promise.resolve([]);
};

const exec = (line) => {
  let name = line.split(' ')[0];
  let remaining = line.replace(name + ' ', '');
  return doCommand(name, remaining).then(() => {
    return consoleActions.hide();
  });
};

const complete = (line) => {
  let command = line.split(' ', 1)[0];
  let keywords = line.replace(command + ' ', '');
  return getCompletions(command, keywords).then(consoleActions.setCompletions);
};

export { exec, complete };
<commit_msg>search with engines<commit_after>import * as tabs from '../background/tabs';
import * as histories from '../background/histories';
import * as consoleActions from './console';

const DEFAULT_SEARCH_ENGINES = {
  default: 'google',
  engines: {
    'google': 'https://google.com/search?q={}',
    'yahoo': 'https://search.yahoo.com/search?p={}',
    'bing': 'https://www.bing.com/search?q={}',
    'duckduckgo': 'https://duckduckgo.com/?q={}',
    'twitter': 'https://twitter.com/search?q={}',
    'wikipedia': 'https://en.wikipedia.org/w/index.php?search={}'
  }
};

const normalizeUrl = (string, searchConfig) => {
  try {
    return new URL(string).href;
  } catch (e) {
    if (string.includes('.') && !string.includes(' ')) {
      return 'http://' + string;
    }
    let query = encodeURI(string);
    let template = searchConfig.engines[
      searchConfig.default
    ];
    for (let key in searchConfig.engines) {
      if (string.startsWith(key + ' ')) {
        query = encodeURI(string.replace(key + ' ', ''));
        template = searchConfig.engines[key];
      }
    }
    return template.replace('{}', query);
  }
};

const openCommand = (url) => {
  return browser.tabs.query({
    active: true, currentWindow: true
  }).then((gotTabs) => {
    if (gotTabs.length > 0) {
      return browser.tabs.update(gotTabs[0].id, { url: url });
    }
  });
};

const tabopenCommand = (url) => {
  return browser.tabs.create({ url: url });
};

const bufferCommand = (keywords) => {
  return browser.tabs.query({
    active: true, currentWindow: true
  }).then((gotTabs) => {
    if (gotTabs.length > 0) {
      if (isNaN(keywords)) {
        return tabs.selectByKeyword(gotTabs[0], keywords);
      }
      let index = parseInt(keywords, 10) - 1;
      return tabs.selectAt(index);
    }
  });
};

const doCommand = (name, remaining) => {
  switch (name) {
  case 'o':
  case 'open':
    // TODO use search engined and pass keywords to them
    return openCommand(normalizeUrl(remaining, DEFAULT_SEARCH_ENGINES));
  case 't':
  case 'tabopen':
    return tabopenCommand(normalizeUrl(remaining, DEFAULT_SEARCH_ENGINES));
  case 'b':
  case 'buffer':
    return bufferCommand(remaining);
  }
  throw new Error(name + ' command is not defined');
};

const getCompletions = (command, keywords) => {
  switch (command) {
  case 'o':
  case 'open':
  case 't':
  case 'tabopen':
    return histories.getCompletions(keywords).then((pages) => {
      let items = pages.map((page) => {
        return {
          caption: page.title,
          content: page.url,
          url: page.url
        };
      });
      return [
        {
          name: 'History',
          items
        }
      ];
    });
  case 'b':
  case 'buffer':
    return tabs.getCompletions(keywords).then((gotTabs) => {
      let items = gotTabs.map((tab) => {
        return {
          caption: tab.title,
          content: tab.title,
          url: tab.url,
          icon: tab.favIconUrl
        };
      });
      return [
        {
          name: 'Buffers',
          items: items
        }
      ];
    });
  }
  return Promise.resolve([]);
};

const exec = (line) => {
  let name = line.split(' ')[0];
  let remaining = line.replace(name + ' ', '');
  return doCommand(name, remaining).then(() => {
    return consoleActions.hide();
  });
};

const complete = (line) => {
  let command = line.split(' ', 1)[0];
  let keywords = line.replace(command + ' ', '');
  return getCompletions(command, keywords).then(consoleActions.setCompletions);
};

export { exec, complete };
<|endoftext|>


<commit_before>// actions.js
//

var stream = require('stream');
var app = require('../app.js'),
    consts = require('../consts.js'),
    loader = require('../core/loader.js');
var metadata = require('./metadata.js');

var _actionSets = null;

function resolveAction(route) {
  var action = null;

  var actionSet = _actionSets[route.params.actionSet];
  if (actionSet) {
    action = actionSet.actions[route.params.action || route.method];
  }

  if (action && (action.httpMethod == route.method)) {
    if (!actionSet.thisObject || (typeof(actionSet.thisObject) != 'object')) {
      actionSet.thisObject = loader.loadObject(actionSet.thisObject);
    }

    if (typeof(action.script) != 'function') {
      action.script = loader.loadFunction(action.script, 'action', [ 'request' ],
                                          actionSet.thisObject);
    }
  }

  return action;
}

function requestHandler(route, request, response) {
  var requestObject = {
    _httpRequest: request,
    _httpResponse: response,
    id: request.id,
    url: request.url,
    path: route.path,
    qualifier: route.params.qualifier,
    params: request.query,
    data: request.body,
    log: request.log
  };

  var statusCode = 200;
  var result;
  var error;
  try {
    var action = resolveAction(route);
    if (action) {
      result = action.script(app.require, console, requestObject);
      if (result === undefined) {
        result = requestObject.result;
      }
    }
    else {
      statusCode = 404;
    }
  }
  catch (e) {
    error = e;
    statusCode = 500;
  }

  var content = '';
  var contentType = 'text/plain';
  var contentLength = 0;
  var contentStream = false;

  if (statusCode != 200) {
    if (error) {
      content = error.stack;
      if (!content) {
        content = error.toString();
      }
    }
  }
  else if ((result === undefined) || (result === null)) {
    statusCode = 204;
  }
  else if (typeof(result) != 'object') {
    // Handles number, string, function
    content = result.toString();
  }
  else if (result instanceof Buffer) {
    content = result;
    contentType = result.mimeType || 'application/octet-stream';
    contentLength = result.length;
  }
  else if (result instanceof stream.Readable) {
    content = result;
    contentType = result.mimeType || 'application/octet-stream';
    contentStream = true;
  }
  else {
    content = JSON.stringify(result);
    contentType = 'application/json';
  }

  if (content && !contentStream && !contentLength) {
    contentLength = Buffer.byteLength(content);
  }

  if (!contentStream) {
    response.writeHead(statusCode, { 'Content-Type': contentType, 'Content-Length': contentLength });
    if (statusCode != 204) {
      response.write(content);
    }
    response.end();
  }
  else {
    response.writeHead(statusCode, { 'Content-Type': contentType });
    content.pipe(response).on('close', function() {
      response.end();
    });
  }
}

requestHandler.route = consts.routes.actions;
requestHandler.initialize = function() {
  _actionSets = metadata.load();
}

module.exports = requestHandler;

<commit_msg>Some more refactoring<commit_after>// actions.js
//

var stream = require('stream');
var app = require('../app.js'),
    consts = require('../consts.js'),
    loader = require('../core/loader.js');
var metadata = require('./metadata.js');

var _actionSets = null;

function resolveAction(route) {
  var action = null;

  var actionSet = _actionSets[route.params.actionSet];
  if (actionSet) {
    action = actionSet.actions[route.params.action || route.method];
  }

  if (action && (action.httpMethod == route.method)) {
    if (!actionSet.thisObject || (typeof(actionSet.thisObject) != 'object')) {
      actionSet.thisObject = loader.loadObject(actionSet.thisObject);
    }

    if (typeof(action.script) != 'function') {
      action.script = loader.loadFunction(action.script, 'action', [ 'request' ],
                                          actionSet.thisObject);
    }
  }

  return action;
}

function createRequestApi(route, request, response) {
  return {
    _httpRequest: request,
    _httpResponse: response,
    id: request.id,
    url: request.url,
    path: route.path,
    qualifier: route.params.qualifier,
    params: request.query,
    data: request.body,
    log: request.log
  };
}

function requestHandler(route, request, response) {
  var statusCode = 200;
  var result;
  var error;
  try {
    var action = resolveAction(route);
    if (action) {
      var requestApi = createRequestApi(route, request, response);

      result = action.script(app.require, console, requestApi);
      if (result === undefined) {
        result = requestApi.result;
      }
    }
    else {
      statusCode = 404;
    }
  }
  catch (e) {
    error = e;
    statusCode = 500;
  }

  var content = '';
  var contentType = 'text/plain';
  var contentLength = 0;
  var contentStream = false;

  if (statusCode != 200) {
    if (error) {
      content = error.stack;
      if (!content) {
        content = error.toString();
      }
    }
  }
  else if ((result === undefined) || (result === null)) {
    statusCode = 204;
  }
  else if (typeof(result) != 'object') {
    // Handles number, string, function
    content = result.toString();
  }
  else if (result instanceof Buffer) {
    content = result;
    contentType = result.mimeType || 'application/octet-stream';
    contentLength = result.length;
  }
  else if (result instanceof stream.Readable) {
    content = result;
    contentType = result.mimeType || 'application/octet-stream';
    contentStream = true;
  }
  else {
    content = JSON.stringify(result);
    contentType = 'application/json';
  }

  if (content && !contentStream && !contentLength) {
    contentLength = Buffer.byteLength(content);
  }

  if (!contentStream) {
    response.writeHead(statusCode, { 'Content-Type': contentType, 'Content-Length': contentLength });
    if (statusCode != 204) {
      response.write(content);
    }
    response.end();
  }
  else {
    response.writeHead(statusCode, { 'Content-Type': contentType });
    content.pipe(response).on('close', function() {
      response.end();
    });
  }
}

requestHandler.route = consts.routes.actions;
requestHandler.initialize = function() {
  _actionSets = metadata.load();
}

module.exports = requestHandler;

<|endoftext|>


<commit_before>var bidfactory = require('../bidfactory.js');
var bidmanager = require('../bidmanager.js');
var adloader = require('../adloader');

var CriteoAdapter = function CriteoAdapter() {

  var _publisherTagUrl = window.location.protocol + '//static.criteo.net/js/ld/publishertag.js';
  var _bidderCode = 'criteo';
  var _profileId = 125;

  function _callBids(params) {
    if (!window.criteo_pubtag || window.criteo_pubtag instanceof Array) {
      // publisherTag not loaded yet

      _pushBidRequestEvent(params);
      adloader.loadScript(
        _publisherTagUrl,
        function () {},
        true
      );
    }
    else {
      // publisherTag already loaded
      _pushBidRequestEvent(params);
    }
  }

  // send bid request to criteo direct bidder handler
  function _pushBidRequestEvent(params) {

    // if we want to be fully asynchronous, we must first check window.criteo_pubtag in case publishertag.js is not loaded yet.
    window.Criteo = window.Criteo || {};
    window.Criteo.events = window.Criteo.events || [];

    // generate the bidding event
    var biddingEventFunc = function () {

      var bids = params.bids || [];

      var slots = [];

      var isAudit = false;

      // build slots before sending one multi-slots bid request
      for (var i = 0; i < bids.length; i++) {
        var bid = bids[i];
        slots.push(
          new Criteo.PubTag.DirectBidding.DirectBiddingSlot(
            bid.placementCode,
            bid.params.zoneId
          )
        );

        isAudit |= bid.params.audit !== undefined;
      }

      var biddingEvent = new Criteo.PubTag.DirectBidding.DirectBiddingEvent(
        _profileId,
        new Criteo.PubTag.DirectBidding.DirectBiddingUrlBuilder(isAudit),
        slots,
        _callbackSuccess(slots),
        _callbackError(slots),
        _callbackError(slots) // timeout handled as error
      );

      // process the event as soon as possible
      window.criteo_pubtag.push(biddingEvent);
    };

    window.Criteo.events.push(biddingEventFunc);

  }

  function parseBidResponse(bidsResponse) {
    try {
      return JSON.parse(bidsResponse);
    }
    catch (error) {
      return {};
    }
  }

  function isNoBidResponse(jsonbidsResponse) {
    return jsonbidsResponse.slots === undefined;
  }

  function _callbackSuccess(slots) {
    return function (bidsResponse) {
      var jsonbidsResponse = parseBidResponse(bidsResponse);

      if (isNoBidResponse(jsonbidsResponse))
        return _callbackError(slots)();

      for (var i = 0; i < slots.length; i++) {
        var bidResponse = null;

        // look for the matching bid response
        for (var j = 0; j < jsonbidsResponse.slots.length; j++) {
          if (jsonbidsResponse.slots[j] && jsonbidsResponse.slots[j].impid === slots[i].impId) {
            bidResponse = jsonbidsResponse.slots.splice(j, 1)[0];
            break;
          }
        }

        // register the bid response
        var bidObject;
        if (bidResponse) {
          bidObject = bidfactory.createBid(1);
          bidObject.bidderCode = _bidderCode;
          bidObject.cpm = bidResponse.cpm;
          bidObject.ad = bidResponse.creative;
          bidObject.width = bidResponse.width;
          bidObject.height = bidResponse.height;
        }
        else {
          bidObject = _invalidBidResponse();
        }
        bidmanager.addBidResponse(slots[i].impId, bidObject);
      }
    };
  }

  function _callbackError(slots) {
    return function () {
      for (var i = 0; i < slots.length; i++) {
        bidmanager.addBidResponse(slots[i].impId, _invalidBidResponse());
      }
    };
  }

  function _invalidBidResponse() {
    var bidObject = bidfactory.createBid(2);
    bidObject.bidderCode = _bidderCode;
    return bidObject;
  }

  return {
    callBids: _callBids
  };
};

module.exports = CriteoAdapter;<commit_msg>Send transactionID to Criteo Services (#1113)<commit_after>var bidfactory = require('../bidfactory.js');
var bidmanager = require('../bidmanager.js');
var adloader = require('../adloader');

var CriteoAdapter = function CriteoAdapter() {

  var _publisherTagUrl = window.location.protocol + '//static.criteo.net/js/ld/publishertag.js';
  var _bidderCode = 'criteo';
  var _profileId = 125;

  function _callBids(params) {
    if (!window.criteo_pubtag || window.criteo_pubtag instanceof Array) {
      // publisherTag not loaded yet

      _pushBidRequestEvent(params);
      adloader.loadScript(
        _publisherTagUrl,
        function () {},
        true
      );
    }
    else {
      // publisherTag already loaded
      _pushBidRequestEvent(params);
    }
  }

  // send bid request to criteo direct bidder handler
  function _pushBidRequestEvent(params) {

    // if we want to be fully asynchronous, we must first check window.criteo_pubtag in case publishertag.js is not loaded yet.
    window.Criteo = window.Criteo || {};
    window.Criteo.events = window.Criteo.events || [];

    // generate the bidding event
    var biddingEventFunc = function () {

      var bids = params.bids || [];

      var slots = [];

      var isAudit = false;

      // build slots before sending one multi-slots bid request
      for (var i = 0; i < bids.length; i++) {
        var bid = bids[i];
        slots.push(
          new Criteo.PubTag.DirectBidding.DirectBiddingSlot(
            bid.placementCode,
            bid.params.zoneId,
            undefined,
            bid.transactionId
          )
        );

        isAudit |= bid.params.audit !== undefined;
      }

      var biddingEvent = new Criteo.PubTag.DirectBidding.DirectBiddingEvent(
        _profileId,
        new Criteo.PubTag.DirectBidding.DirectBiddingUrlBuilder(isAudit),
        slots,
        _callbackSuccess(slots),
        _callbackError(slots),
        _callbackError(slots) // timeout handled as error
      );

      // process the event as soon as possible
      window.criteo_pubtag.push(biddingEvent);
    };

    window.Criteo.events.push(biddingEventFunc);

  }

  function parseBidResponse(bidsResponse) {
    try {
      return JSON.parse(bidsResponse);
    }
    catch (error) {
      return {};
    }
  }

  function isNoBidResponse(jsonbidsResponse) {
    return jsonbidsResponse.slots === undefined;
  }

  function _callbackSuccess(slots) {
    return function (bidsResponse) {
      var jsonbidsResponse = parseBidResponse(bidsResponse);

      if (isNoBidResponse(jsonbidsResponse))
        return _callbackError(slots)();

      for (var i = 0; i < slots.length; i++) {
        var bidResponse = null;

        // look for the matching bid response
        for (var j = 0; j < jsonbidsResponse.slots.length; j++) {
          if (jsonbidsResponse.slots[j] && jsonbidsResponse.slots[j].impid === slots[i].impId) {
            bidResponse = jsonbidsResponse.slots.splice(j, 1)[0];
            break;
          }
        }

        // register the bid response
        var bidObject;
        if (bidResponse) {
          bidObject = bidfactory.createBid(1);
          bidObject.bidderCode = _bidderCode;
          bidObject.cpm = bidResponse.cpm;
          bidObject.ad = bidResponse.creative;
          bidObject.width = bidResponse.width;
          bidObject.height = bidResponse.height;
        }
        else {
          bidObject = _invalidBidResponse();
        }
        bidmanager.addBidResponse(slots[i].impId, bidObject);
      }
    };
  }

  function _callbackError(slots) {
    return function () {
      for (var i = 0; i < slots.length; i++) {
        bidmanager.addBidResponse(slots[i].impId, _invalidBidResponse());
      }
    };
  }

  function _invalidBidResponse() {
    var bidObject = bidfactory.createBid(2);
    bidObject.bidderCode = _bidderCode;
    return bidObject;
  }

  return {
    callBids: _callBids
  };
};

module.exports = CriteoAdapter;<|endoftext|>


<commit_before>"use strict";

var caching = {};

var dottemplates = require("./dottemplates.js"),
    pkg          = require("./../../package.json"),
    paths        = require("./paths.js").get();

var async        = require("async"),
    autoprefixer = require("autoprefixer-core"),
    cleanCSS     = require("clean-css"),
    crypto       = require("crypto"),
    fs           = require("graceful-fs"),
    htmlMinifier = require("html-minifier"),
    mime         = require("mime"),
    path         = require("path"),
    uglify       = require("uglify-js"),
    zlib         = require("zlib");

var doMinify, modesByMime,
    etag         = crypto.createHash("md5").update(String(Date.now())).digest("hex"),
    themesPath   = path.join(paths.mod, "/node_modules/codemirror/theme"),
    modesPath    = path.join(paths.mod, "/node_modules/codemirror/mode");

var minfierOptions = {
    get uglify() {
        return {
            fromString: true,
            compress: {
                unsafe: true,
                screw_ie8: true
            }
        };
    },
    get cleanCSS() {
        return {
            advanced : true,
            keepSpecialComments : 0
        };
    },
    get htmlMinifier() {
        return {
            removeComments: true,
            collapseWhitespace: true,
            collapseBooleanAttributes: true,
            removeRedundantAttributes: true,
            caseSensitive: true,
            minifyCSS: false
        };
    }
};

cleanCSS = new cleanCSS(minfierOptions.cleanCSS);

caching.files = {
        css: [
            "node_modules/codemirror/lib/codemirror.css",
            "client/style.css",
            "client/sprites.css"
        ],
        js: [
            "node_modules/jquery/dist/jquery.js",
            "client/lib/jquery.customSelect.min.js",
            "node_modules/draggabilly/dist/draggabilly.pkgd.min.js",
            "node_modules/pretty-bytes/pretty-bytes.js",
            "node_modules/codemirror/lib/codemirror.js",
            "node_modules/codemirror/addon/selection/active-line.js",
            "node_modules/codemirror/addon/selection/mark-selection.js",
            "node_modules/codemirror/addon/search/searchcursor.js",
            "node_modules/codemirror/addon/edit/matchbrackets.js",
            "node_modules/codemirror/keymap/sublime.js",
            "client/client.js"
        ],
        html: [
            "client/html/base.html",
            "client/html/auth.html",
            "client/html/main.html"
        ],
        templates: [
            "client/templates/views/directory.dotjs",
            "client/templates/views/document.dotjs",
            "client/templates/views/media.dotjs",
            "client/templates/options.dotjs"
        ],
        other: [
            "client/Roboto.woff",
            "client/images/logo.svg",
            "client/images/logo16.png",
            "client/images/logo32.png",
            "client/images/logo128.png",
            "client/images/logo152.png",
            "client/images/logo180.png",
            "client/images/logo192.png",
            "client/images/favicon.ico"
        ]
    };

// On-demand loadable libs, preferably minified. Will be available as ?!/[property value]
var libs = {
    "node_modules/video.js/dist/video-js/video.js"         : "video.js/vjs.js",
    "node_modules/video.js/dist/video-js/video-js.min.css" : "video.js/vjs.css",
    "node_modules/video.js/dist/video-js/video-js.swf"     : "video.js/vjs.swf",
    "node_modules/video.js/dist/video-js/font/vjs.eot"     : "video.js/font/vjs.eot",
    "node_modules/video.js/dist/video-js/font/vjs.svg"     : "video.js/font/vjs.svg",
    "node_modules/video.js/dist/video-js/font/vjs.ttf"     : "video.js/font/vjs.ttf",
    "node_modules/video.js/dist/video-js/font/vjs.woff"    : "video.js/font/vjs.woff"
};

caching.init = function init(minify, mimes, callback) {
    modesByMime = mimes;
    doMinify = minify;
    async.series([
        compileResources,
        readThemes,
        readModes,
        readLibs
    ], function (err, results) {
        if (err) return callback(err);
        var cache = { res: results[0], themes: {}, modes: {}, lib: {} };

        Object.keys(results[1]).forEach(function (theme) {
            cache.themes[theme] = {data: results[1][theme], etag: etag, mime: mime.lookup("css")};
        });

        Object.keys(results[2]).forEach(function (mode) {
            cache.modes[mode] = {data: results[2][mode], etag: etag, mime: mime.lookup("js")};
        });

        Object.keys(results[3]).forEach(function (file) {
            cache.lib[file] = {data: results[3][file], etag: etag, mime: mime.lookup(path.basename(file))};
        });

        addGzip(cache, function (err, cache) {
            cache.etags = {};
            callback(err, cache);
        });
    });
};

// Create gzip compressed data
function addGzip(cache, callback) {
    var types = Object.keys(cache), funcs = [];
    types.forEach(function (type) {
        funcs.push(function (cb) {
            gzipMap(cache[type], cb);
        });
    });
    async.parallel(funcs, function (err, results) {
        if (err) return callback(err);
        types.forEach(function (type, index) {
            cache[type] = results[index];
        });
        callback(null, cache);
    });
}

function gzipMap(map, callback) {
    var names = Object.keys(map), funcs = [];
    names.forEach(function (name) {
        funcs.push(function (cb) {
            gzip(map[name].data, cb);
        });
    });
    async.parallel(funcs, function (err, results) {
        if (err) return callback(err);
        names.forEach(function (name, index) {
            map[name].gzip = results[index];
        });
        callback(null, map);
    });
}

function gzip(data, callback) {
    zlib.gzip(data, function (err, gzipped) {
        if (err) return callback(err);
        callback(null, gzipped);
    });
}

function readThemes(callback) {
    var themes = {};
    fs.readdir(themesPath, function (err, filenames) {
        if (err) return callback(err);

        var files = filenames.map(function (name) {
            return path.join(themesPath, name);
        });

        async.map(files, fs.readFile, function (err, data) {
            if (err) return callback(err);

            filenames.forEach(function (name, index) {
                var css = String(data[index]);
                themes[name.replace(/\.css$/, "")] = doMinify ? cleanCSS.minify(css).styles : css;
            });

            callback(err, themes);
        });
    });
}

function readModes(callback) {
    var modes = {};
    Object.keys(modesByMime).forEach(function (mime) {
        var mode = modesByMime[mime];
        if (!modes[mode]) modes[mode] = "";
    });

    var cbDue = 0, cbFired = 0, ret = {};
    Object.keys(modes).forEach(function (mode) {
        cbDue++;
        fs.readFile(path.join(modesPath, mode, mode + ".js"), function (err, data) {
            if (err) callback(err);
            cbFired++;

            if (doMinify)
                ret[mode] = uglify.minify(data.toString(), {fromString: true, compress: {unsafe: true, screw_ie8: true}}).code;
            else
                ret[mode] = data.toString();

            if (cbFired === cbDue) callback(null, ret);
        });
    });
}

function readLibs(callback) {
    var ret = {};
    async.each(Object.keys(libs), function (p, cb) {
        fs.readFile(path.join(paths.mod, p), function (err, data) {
            ret[libs[p]] = data;
            cb(err);
        });
    }, function (err) {
        callback(err, ret);
    });
}

function compileResources(callback) {
    var resData  = {}, resCache = {},
        out      = { css : "", js  : "" };

    // Read resources
    Object.keys(caching.files).forEach(function (type) {
        resData[type] = caching.files[type].map(function read(file) {
            var data;
            try {
                data = fs.readFileSync(path.join(paths.mod, file)).toString("utf8");
            } catch (error) {
                return callback(error);
            }
            return data;
        });
    });

    // Concatenate CSS
    resData.css.forEach(function (data) {
        out.css += data;
    });

    // Concatenate JS
    resData.js.forEach(function (data) {
        out.js += data + ";";
    });

    // Add SVG object
    var svgDir = paths.svg, svgData = {};
    fs.readdirSync(svgDir).forEach(function (name) {
        svgData[name.slice(0, name.length - 4)] = fs.readFileSync(path.join(svgDir, name), "utf8");
    });
    out.js = out.js.replace("/* {{ svg }} */", "droppy.svg = " + JSON.stringify(svgData) + ";");

    // Insert Templates Code
    var templateCode = "var t = {fn:{},views:{}};";
    resData.templates.forEach(function (data, index) {
        // Produce the doT functions
        templateCode += dottemplates
            .produceFunction("t." + caching.files.templates[index].replace(/\.dotjs$/, "")
            .split("/").slice(2).join("."), data);
    });
    templateCode += ";";
    out.js = out.js.replace("/* {{ templates }} */", templateCode);

    // Insert CM mime modes
    out.js = out.js.replace("/* {{ droppy.mimeModes }} */", "droppy.mimeModes = " + JSON.stringify(modesByMime) + ";\n");

    // Add CSS vendor prefixes
    try {
        out.css = autoprefixer.process(out.css).css;
    } catch (e) {
        return callback(e);
    }

    // Minify JS and CSS
    if (doMinify) {
        out.js  = uglify.minify(out.js, minfierOptions.uglify).code;
        out.css = cleanCSS.minify(out.css).styles;
    }

    // Read and minifiy HTML files
    while (caching.files.html.length) {
        var name = path.basename(caching.files.html.pop()),
            data = resData.html.pop()
                .replace(/\{\{version\}\}/gm, pkg.version)
                .replace(/\{\{name\}\}/gm, pkg.name);

        if (doMinify) {
            data = htmlMinifier.minify(data, minfierOptions.htmlMinifier);
        }

        resCache[name] = {data: data, etag: etag, mime: mime.lookup("html")};
    }

    resCache["client.js"] = {data: out.js, etag: etag, mime: mime.lookup("js")};
    resCache["style.css"] = {data: out.css, etag: etag, mime: mime.lookup("css")};

    // Read misc files
    caching.files.other.forEach(function (file) {
        var data, date,
            name     = path.basename(file),
            fullPath = path.join(paths.mod, file);

        try {
            data = fs.readFileSync(fullPath);
            date = fs.statSync(fullPath).mtime;
        } catch (err) {
            callback(err);
        }

        resCache[name] = {
            data: data,
            etag: crypto.createHash("md5").update(String(date)).digest("hex"),
            mime: mime.lookup(name)
        };
    });
    callback(null, resCache);
}

exports = module.exports = caching;
<commit_msg>Backend: Store all cached resources as buffers<commit_after>"use strict";

var caching = {};

var dottemplates = require("./dottemplates.js"),
    pkg          = require("./../../package.json"),
    paths        = require("./paths.js").get();

var async        = require("async"),
    autoprefixer = require("autoprefixer-core"),
    cleanCSS     = require("clean-css"),
    crypto       = require("crypto"),
    fs           = require("graceful-fs"),
    htmlMinifier = require("html-minifier"),
    mime         = require("mime"),
    path         = require("path"),
    uglify       = require("uglify-js"),
    zlib         = require("zlib");

var doMinify, modesByMime,
    etag         = crypto.createHash("md5").update(String(Date.now())).digest("hex"),
    themesPath   = path.join(paths.mod, "/node_modules/codemirror/theme"),
    modesPath    = path.join(paths.mod, "/node_modules/codemirror/mode");

var minfierOptions = {
    get uglify() {
        return {
            fromString: true,
            compress: {
                unsafe: true,
                screw_ie8: true
            }
        };
    },
    get cleanCSS() {
        return {
            advanced : true,
            keepSpecialComments : 0
        };
    },
    get htmlMinifier() {
        return {
            removeComments: true,
            collapseWhitespace: true,
            collapseBooleanAttributes: true,
            removeRedundantAttributes: true,
            caseSensitive: true,
            minifyCSS: false
        };
    }
};

cleanCSS = new cleanCSS(minfierOptions.cleanCSS);

caching.files = {
        css: [
            "node_modules/codemirror/lib/codemirror.css",
            "client/style.css",
            "client/sprites.css"
        ],
        js: [
            "node_modules/jquery/dist/jquery.js",
            "client/lib/jquery.customSelect.min.js",
            "node_modules/draggabilly/dist/draggabilly.pkgd.min.js",
            "node_modules/pretty-bytes/pretty-bytes.js",
            "node_modules/codemirror/lib/codemirror.js",
            "node_modules/codemirror/addon/selection/active-line.js",
            "node_modules/codemirror/addon/selection/mark-selection.js",
            "node_modules/codemirror/addon/search/searchcursor.js",
            "node_modules/codemirror/addon/edit/matchbrackets.js",
            "node_modules/codemirror/keymap/sublime.js",
            "client/client.js"
        ],
        html: [
            "client/html/base.html",
            "client/html/auth.html",
            "client/html/main.html"
        ],
        templates: [
            "client/templates/views/directory.dotjs",
            "client/templates/views/document.dotjs",
            "client/templates/views/media.dotjs",
            "client/templates/options.dotjs"
        ],
        other: [
            "client/Roboto.woff",
            "client/images/logo.svg",
            "client/images/logo16.png",
            "client/images/logo32.png",
            "client/images/logo128.png",
            "client/images/logo152.png",
            "client/images/logo180.png",
            "client/images/logo192.png",
            "client/images/favicon.ico"
        ]
    };

// On-demand loadable libs, preferably minified. Will be available as ?!/[property value]
var libs = {
    "node_modules/video.js/dist/video-js/video.js"         : "video.js/vjs.js",
    "node_modules/video.js/dist/video-js/video-js.min.css" : "video.js/vjs.css",
    "node_modules/video.js/dist/video-js/video-js.swf"     : "video.js/vjs.swf",
    "node_modules/video.js/dist/video-js/font/vjs.eot"     : "video.js/font/vjs.eot",
    "node_modules/video.js/dist/video-js/font/vjs.svg"     : "video.js/font/vjs.svg",
    "node_modules/video.js/dist/video-js/font/vjs.ttf"     : "video.js/font/vjs.ttf",
    "node_modules/video.js/dist/video-js/font/vjs.woff"    : "video.js/font/vjs.woff"
};

caching.init = function init(minify, mimes, callback) {
    modesByMime = mimes;
    doMinify = minify;
    async.series([
        compileResources,
        readThemes,
        readModes,
        readLibs
    ], function (err, results) {
        if (err) return callback(err);
        var cache = { res: results[0], themes: {}, modes: {}, lib: {} };

        Object.keys(results[1]).forEach(function (theme) {
            cache.themes[theme] = {data: results[1][theme], etag: etag, mime: mime.lookup("css")};
        });

        Object.keys(results[2]).forEach(function (mode) {
            cache.modes[mode] = {data: results[2][mode], etag: etag, mime: mime.lookup("js")};
        });

        Object.keys(results[3]).forEach(function (file) {
            cache.lib[file] = {data: results[3][file], etag: etag, mime: mime.lookup(path.basename(file))};
        });

        addGzip(cache, function (err, cache) {
            cache.etags = {};
            callback(err, cache);
        });
    });
};

// Create gzip compressed data
function addGzip(cache, callback) {
    var types = Object.keys(cache), funcs = [];
    types.forEach(function (type) {
        funcs.push(function (cb) {
            gzipMap(cache[type], cb);
        });
    });
    async.parallel(funcs, function (err, results) {
        if (err) return callback(err);
        types.forEach(function (type, index) {
            cache[type] = results[index];
        });
        callback(null, cache);
    });
}

function gzipMap(map, callback) {
    var names = Object.keys(map), funcs = [];
    names.forEach(function (name) {
        funcs.push(function (cb) {
            gzip(map[name].data, cb);
        });
    });
    async.parallel(funcs, function (err, results) {
        if (err) return callback(err);
        names.forEach(function (name, index) {
            map[name].gzip = results[index];
        });
        callback(null, map);
    });
}

function gzip(data, callback) {
    zlib.gzip(data, function (err, gzipped) {
        if (err) return callback(err);
        callback(null, gzipped);
    });
}

function readThemes(callback) {
    var themes = {};
    fs.readdir(themesPath, function (err, filenames) {
        if (err) return callback(err);

        var files = filenames.map(function (name) {
            return path.join(themesPath, name);
        });

        async.map(files, fs.readFile, function (err, data) {
            if (err) return callback(err);

            filenames.forEach(function (name, index) {
                var css = String(data[index]);
                themes[name.replace(/\.css$/, "")] = doMinify ? new Buffer(cleanCSS.minify(css).styles) : new Buffer(css);
            });

            callback(err, themes);
        });
    });
}

function readModes(callback) {
    var modes = {};
    Object.keys(modesByMime).forEach(function (mime) {
        var mode = modesByMime[mime];
        if (!modes[mode]) modes[mode] = "";
    });

    var cbDue = 0, cbFired = 0, ret = {};
    Object.keys(modes).forEach(function (mode) {
        cbDue++;
        fs.readFile(path.join(modesPath, mode, mode + ".js"), function (err, data) {
            if (err) callback(err);
            cbFired++;

            if (doMinify)
                ret[mode] = new Buffer(uglify.minify(data.toString(), minfierOptions.uglify).code);
            else
                ret[mode] = new Buffer(data.toString());

            if (cbFired === cbDue) callback(null, ret);
        });
    });
}

function readLibs(callback) {
    var ret = {};
    async.each(Object.keys(libs), function (p, cb) {
        fs.readFile(path.join(paths.mod, p), function (err, data) {
            ret[libs[p]] = data;
            cb(err);
        });
    }, function (err) {
        callback(err, ret);
    });
}

function compileResources(callback) {
    var resData  = {}, resCache = {},
        out      = { css : "", js  : "" };

    // Read resources
    Object.keys(caching.files).forEach(function (type) {
        resData[type] = caching.files[type].map(function read(file) {
            var data;
            try {
                data = fs.readFileSync(path.join(paths.mod, file)).toString("utf8");
            } catch (error) {
                return callback(error);
            }
            return data;
        });
    });

    // Concatenate CSS
    resData.css.forEach(function (data) {
        out.css += data;
    });

    // Concatenate JS
    resData.js.forEach(function (data) {
        out.js += data + ";";
    });

    // Add SVG object
    var svgDir = paths.svg, svgData = {};
    fs.readdirSync(svgDir).forEach(function (name) {
        svgData[name.slice(0, name.length - 4)] = fs.readFileSync(path.join(svgDir, name), "utf8");
    });
    out.js = out.js.replace("/* {{ svg }} */", "droppy.svg = " + JSON.stringify(svgData) + ";");

    // Insert Templates Code
    var templateCode = "var t = {fn:{},views:{}};";
    resData.templates.forEach(function (data, index) {
        // Produce the doT functions
        templateCode += dottemplates
            .produceFunction("t." + caching.files.templates[index].replace(/\.dotjs$/, "")
            .split("/").slice(2).join("."), data);
    });
    templateCode += ";";
    out.js = out.js.replace("/* {{ templates }} */", templateCode);

    // Insert CM mime modes
    out.js = out.js.replace("/* {{ droppy.mimeModes }} */", "droppy.mimeModes = " + JSON.stringify(modesByMime) + ";\n");

    // Add CSS vendor prefixes
    try {
        out.css = autoprefixer.process(out.css).css;
    } catch (e) {
        return callback(e);
    }

    // Minify JS and CSS
    if (doMinify) {
        out.js  = uglify.minify(out.js, minfierOptions.uglify).code;
        out.css = cleanCSS.minify(out.css).styles;
    }

    // Read and minifiy HTML files
    while (caching.files.html.length) {
        var name = path.basename(caching.files.html.pop()),
            data = resData.html.pop()
                .replace(/\{\{version\}\}/gm, pkg.version)
                .replace(/\{\{name\}\}/gm, pkg.name);

        if (doMinify) {
            data = htmlMinifier.minify(data, minfierOptions.htmlMinifier);
        }

        resCache[name] = {data: new Buffer(data), etag: etag, mime: mime.lookup("html")};
    }

    resCache["client.js"] = {data: new Buffer(out.js), etag: etag, mime: mime.lookup("js")};
    resCache["style.css"] = {data: new Buffer(out.css), etag: etag, mime: mime.lookup("css")};

    // Read misc files
    caching.files.other.forEach(function (file) {
        var data, date,
            name     = path.basename(file),
            fullPath = path.join(paths.mod, file);

        try {
            data = fs.readFileSync(fullPath);
            date = fs.statSync(fullPath).mtime;
        } catch (err) {
            callback(err);
        }

        resCache[name] = {
            data: data,
            etag: crypto.createHash("md5").update(String(date)).digest("hex"),
            mime: mime.lookup(name)
        };
    });
    callback(null, resCache);
}

exports = module.exports = caching;
<|endoftext|>