<commit_before>var assert = require('assert');

var _ = require('underscore');
var PSQL = require('cartodb-psql');
var queue = require('queue-async');
var step = require('step');

var Datasource = require('../models/datasource');
var MapConfig = require('../models/mapconfig');
var MapValidatorBackend = require('./map_validator');
var RendererParams = require('../renderers/renderer_params');
var Timer = require('../stats/timer');

function MapBackend(renderCache, mapStore, options) {
    options = options || {};

    this._renderCache = renderCache;
    this._mapStore = mapStore;

    // called before a layergroup configuration is created, it allows to modify the configuration
    // @param req request (body is the map configuration)
    // @param requestMapConfig layergroup map configuration
    // @param callback to be called with "err" as first argument
    this.beforeLayergroupCreate = options.beforeLayergroupCreate || function(req, requestMapConfig, callback) {
        return callback(null, requestMapConfig, Datasource.EmptyDatasource());
    };
    // called after a layergroup configuration is created
    // @param req request (body is the map configuration)
    // @param layergroup map configuration
    // @param response response object, can be modified
    // @param callback to be called with "err" as first argument
    this.afterLayergroupCreate = options.afterLayergroupCreate || function(req, layergroup, response, callback) {
        return callback(null);
    };

    this._mapValidatorBackend = new MapValidatorBackend(this);
}

module.exports = MapBackend;

MapBackend.prototype.createLayergroup = function(requestMapConfig, req, callback) {
    var self = this;

    req.profiler.start('createLayergroup');

    var response = {};

    // Inject db parameters into the configuration
    // to ensure getting different identifiers for
    // maps created against different databases
    // or users. See
    // https://github.com/CartoDB/Windshaft/issues/163
    requestMapConfig.dbparams = {
        name: req.params.dbname,
        user: req.params.dbuser
    };

    var mapConfig;

    step(
        function preLayerCreate() {
            self.beforeLayergroupCreate(req, requestMapConfig, this);
        },
        function initLayergroup(err, layergroupMapConfig, datasource) {
            assert.ifError(err);

            mapConfig = new MapConfig(layergroupMapConfig, datasource || Datasource.EmptyDatasource());

            // will save only if successful
            self._mapStore.save(mapConfig, this);
        },
        function handleMapConfigSave(err, mapConfigId, known) {
            req.profiler.done('mapSave');

            assert.ifError(err);

            response.layergroupid = mapConfig.id();

            if (known) {
                return true;
            } else {
                var next = this;
                self._mapValidatorBackend.validate(req, mapConfig, function(err, isValid) {
                    req.profiler.done('validate');
                    if (isValid) {
                        return next(err);
                    }
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            console.error("Failed to delete MapConfig '" + mapConfig.id() + "' after: " + err);
                        }
                        return next(err);
                    });
                });
            }
        },
        function fetchLayersMetadata(err) {
            assert.ifError(err);

            var next = this;

            self.getLayersMetadata(req, mapConfig, function(err, layersMetadata) {
                if (err) {
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            console.error("Failed to delete MapConfig '" + mapConfig.id() + " after: " + err);
                        }
                        return next(err);
                    });
                } else {
                    if (layersMetadata) {
                        response.metadata = response.metadata || {};
                        response.metadata.layers = layersMetadata;

                        // backwards compatibility for torque
                        var torqueMetadata = layersMetadata.reduce(function(acc, layer, layerId) {
                            if (layer.type === 'torque') {
                                acc[layerId] = layer.meta;
                            }
                            return acc;
                        }, {});
                        if (Object.keys(torqueMetadata).length) {
                            response.metadata.torque = torqueMetadata;
                        }
                    }
                    return next(err);
                }
            });
        },
        function posLayerCreate(err) {
            assert.ifError(err);
            self.afterLayergroupCreate(req, mapConfig.obj(), response, this);
        },
        function finish(err) {
            if (!err) {
                req.profiler.done('afterLayergroupCreate');
            }
            req.profiler.end();
            callback(err, response);
        }
    );
};

// Gets a tile for a given set of tile ZXY coords. (OSM style)
// Call with .png for images, or .grid.json for UTFGrid tiles
//
// query string arguments:
//
// * sql - use SQL to filter displayed results or perform operations pre-render
// * style - assign a per tile style using carto
// * interactivity - specify which columns to represent in the UTFGrid
// * cache_buster - specify to ensure a new renderer is used
// * geom_type - specify default style to use if no style present
MapBackend.prototype.getTileOrGrid = function(params, callback) {

    var self = this;

    var timer = new Timer();

    var renderer;

    timer.start('getTileOrGrid');
    var extraHeaders = {};
    step(
        function() {
            if (params.format === 'grid.json' && !params.interactivity) {
                if ( ! params.token ) { // token embeds interactivity
                    throw new Error("Missing interactivity parameter");
                }
            }
            timer.start('getRenderer');
            self._renderCache.getRenderer(params, this);
        },
        function(err, r, is_cached) {
            timer.end('getRenderer');
            renderer = r;
            if ( is_cached ) {
                extraHeaders['X-Windshaft-Cache'] = Date.now() - renderer.ctime;
            }
            assert.ifError(err);
            timer.start('render-' + params.format.replace('.','-'));
            renderer.getTile(+params.z, +params.x, +params.y, this);
        },
        function(err, tile, headers, stats) {
            timer.end('render-' + params.format.replace('.','-'));

            if ( renderer ) {
                renderer.release();
            }
            timer.end('getTileOrGrid');
            callback(err, tile, _.extend(extraHeaders, headers), _.extend(timer.getTimes(), stats));
        }
    );
};

/// Gets attributes for a given layer feature
//
/// Calls req2params, then expects parameters:
///
/// * token - MapConfig identifier
/// * layer - Layer number
/// * fid   - Feature identifier
///
/// The referenced layer must have been configured
/// to allow for attributes fetching.
/// See https://github.com/CartoDB/Windshaft/wiki/MapConfig-1.1.0
///
/// @param testMode if true generates a call returning requested
///                 columns plus the fid column of the first record
///                 it is only meant to check validity of configuration
///
MapBackend.prototype.getFeatureAttributes = function (params, testMode, callback) {
    var self = this;

    var timer = new Timer();

    var mapConfig;
    step(
        function getMapConfig() {
            timer.start('MapStore.load');
            self._mapStore.load(params.token, this);
        },
        function getPGClient(err, data) {
            assert.ifError(err);

            timer.end('MapStore.load');
            mapConfig = data;

            var dbParams = RendererParams.dbParamsFromReqParams(params);
            _.extend(dbParams, mapConfig.getLayerDatasource(params.layer));
            return new PSQL(dbParams);
        },
        function getAttributes(err, pg) {
            timer.start('getAttributes');

            assert.ifError(err);

            var layer = mapConfig.getLayer(params.layer);
            if ( ! layer ) {
                throw new Error("Map " + params.token +
                    " has no layer number " + params.layer);
            }
            var attributes = layer.options.attributes;
            if ( ! attributes ) {
                throw new Error("Layer " + params.layer +
                    " has no exposed attributes");
            }

            // NOTE: we're assuming that the presence of "attributes"
            //       means it is well-formed (should be checked at
            //       MapConfig construction/validation time).

            var fid_col = attributes.id;
            var att_cols = attributes.columns;

            // prepare columns with double quotes
            var quoted_att_cols = _.map(att_cols, function(n) {
                return pg.quoteIdentifier(n);
            }).join(',');

            if ( testMode ) {
                quoted_att_cols += ',' + pg.quoteIdentifier(fid_col);
            }

            var sql = 'select ' + quoted_att_cols + ' from ( ' + layer.options.sql + ' ) as _windshaft_subquery ';
            if ( ! testMode ) {
                sql += ' WHERE ' + pg.quoteIdentifier(fid_col) + ' = ' + params.fid;
            } else {
                sql += ' LIMIT 1';
            }

            // console.log("SQL:  " + sql);

            pg.query(sql, this, true); // use read-only transaction
        },
        function formatAttributes(err, data) {
            timer.end('getAttributes');
            assert.ifError(err);

            if ( data.rows.length !== 1 ) {
                if ( testMode ) {
                    return null;
                }
                else {
                    var rowsLengthError = new Error(data.rows.length +
                        " features in layer " + params.layer +
                        " of map " + params.token +
                        " are identified by fid " + params.fid);
                    if ( ! data.rows.length ) {
                        rowsLengthError.http_status = 404;
                    }
                    throw rowsLengthError;
                }
            }
            return data.rows[0];
        },
        function returnCallback(err, tile) {
            return callback(err, tile, timer.getTimes());
        }
    );
};

MapBackend.prototype.getLayersMetadata = function(req, mapConfig, callback) {
    var self = this;

    var metadata = [];

    var torqueLayers = [];

    mapConfig.getLayers().forEach(function(layer, layerId) {
        var layerType = mapConfig.layerType(layerId);
        metadata[layerId] = {
            type: layerType,
            meta: {}
        };

        if (layerType === 'torque') {
            torqueLayers.push(layerId);
        }
    });

    if ( ! torqueLayers.length ) {
        return callback(null, metadata);
    }

    var metadataQueue = queue(torqueLayers.length);

    torqueLayers.forEach(function(layerId) {
        metadataQueue.defer(function(params, token, rendererType, layerId, done) {
            self.getLayerMetadata(params, token, rendererType, layerId, done);
        }, req.params, mapConfig.id(), 'json.torque', layerId);
    });

    function metadataQueueFinish(err, results) {
        if (err) {
            return callback(err, results);
        }
        if (!results) {
            return callback(null, null);
        }

        torqueLayers.forEach(function(layerId, i) {
            metadata[layerId] = {
                type: 'torque',
                meta: results[i]
            };
        });
        return callback(err, metadata);
    }

    metadataQueue.awaitAll(metadataQueueFinish);
};

/// Fetch metadata for a tileset in a MapConfig
//
/// @param rendererType a MapConfig renderer type (layer.type in MapConfig spec)
/// @param layerId layer index within the mapConfig
/// @param callback function(err, metadata) where metadata format
///
MapBackend.prototype.getLayerMetadata = function(params, token, format, layerId, callback) {
    var self = this;

    params = _.extend({}, params, {
        token: token,
        format: format,
        layer: layerId
    });

    var renderer;

    step(
        function(){
            self._renderCache.getRenderer(params, this);
        },
        function(err, r) {
            assert.ifError(err);
            renderer = r;
            renderer.getMetadata(this);
        },
        function(err, meta) {
            if ( renderer ) {
                renderer.release();
            }
            callback(err, meta);
        }
    );
};
<commit_msg>getLayersMetadata now uses params instead of request<commit_after>var assert = require('assert');

var _ = require('underscore');
var PSQL = require('cartodb-psql');
var queue = require('queue-async');
var step = require('step');

var Datasource = require('../models/datasource');
var MapConfig = require('../models/mapconfig');
var MapValidatorBackend = require('./map_validator');
var RendererParams = require('../renderers/renderer_params');
var Timer = require('../stats/timer');

function MapBackend(renderCache, mapStore, options) {
    options = options || {};

    this._renderCache = renderCache;
    this._mapStore = mapStore;

    // called before a layergroup configuration is created, it allows to modify the configuration
    // @param req request (body is the map configuration)
    // @param requestMapConfig layergroup map configuration
    // @param callback to be called with "err" as first argument
    this.beforeLayergroupCreate = options.beforeLayergroupCreate || function(req, requestMapConfig, callback) {
        return callback(null, requestMapConfig, Datasource.EmptyDatasource());
    };
    // called after a layergroup configuration is created
    // @param req request (body is the map configuration)
    // @param layergroup map configuration
    // @param response response object, can be modified
    // @param callback to be called with "err" as first argument
    this.afterLayergroupCreate = options.afterLayergroupCreate || function(req, layergroup, response, callback) {
        return callback(null);
    };

    this._mapValidatorBackend = new MapValidatorBackend(this);
}

module.exports = MapBackend;

MapBackend.prototype.createLayergroup = function(requestMapConfig, req, callback) {
    var self = this;

    req.profiler.start('createLayergroup');

    var response = {};

    // Inject db parameters into the configuration
    // to ensure getting different identifiers for
    // maps created against different databases
    // or users. See
    // https://github.com/CartoDB/Windshaft/issues/163
    requestMapConfig.dbparams = {
        name: req.params.dbname,
        user: req.params.dbuser
    };

    var mapConfig;

    step(
        function preLayerCreate() {
            self.beforeLayergroupCreate(req, requestMapConfig, this);
        },
        function initLayergroup(err, layergroupMapConfig, datasource) {
            assert.ifError(err);

            mapConfig = new MapConfig(layergroupMapConfig, datasource || Datasource.EmptyDatasource());

            // will save only if successful
            self._mapStore.save(mapConfig, this);
        },
        function handleMapConfigSave(err, mapConfigId, known) {
            req.profiler.done('mapSave');

            assert.ifError(err);

            response.layergroupid = mapConfig.id();

            if (known) {
                return true;
            } else {
                var next = this;
                self._mapValidatorBackend.validate(req, mapConfig, function(err, isValid) {
                    req.profiler.done('validate');
                    if (isValid) {
                        return next(err);
                    }
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            console.error("Failed to delete MapConfig '" + mapConfig.id() + "' after: " + err);
                        }
                        return next(err);
                    });
                });
            }
        },
        function fetchLayersMetadata(err) {
            assert.ifError(err);

            var next = this;

            self.getLayersMetadata(req.params, mapConfig, function(err, layersMetadata) {
                if (err) {
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            console.error("Failed to delete MapConfig '" + mapConfig.id() + " after: " + err);
                        }
                        return next(err);
                    });
                } else {
                    if (layersMetadata) {
                        response.metadata = response.metadata || {};
                        response.metadata.layers = layersMetadata;

                        // backwards compatibility for torque
                        var torqueMetadata = layersMetadata.reduce(function(acc, layer, layerId) {
                            if (layer.type === 'torque') {
                                acc[layerId] = layer.meta;
                            }
                            return acc;
                        }, {});
                        if (Object.keys(torqueMetadata).length) {
                            response.metadata.torque = torqueMetadata;
                        }
                    }
                    return next(err);
                }
            });
        },
        function posLayerCreate(err) {
            assert.ifError(err);
            self.afterLayergroupCreate(req, mapConfig.obj(), response, this);
        },
        function finish(err) {
            if (!err) {
                req.profiler.done('afterLayergroupCreate');
            }
            req.profiler.end();
            callback(err, response);
        }
    );
};

// Gets a tile for a given set of tile ZXY coords. (OSM style)
// Call with .png for images, or .grid.json for UTFGrid tiles
//
// query string arguments:
//
// * sql - use SQL to filter displayed results or perform operations pre-render
// * style - assign a per tile style using carto
// * interactivity - specify which columns to represent in the UTFGrid
// * cache_buster - specify to ensure a new renderer is used
// * geom_type - specify default style to use if no style present
MapBackend.prototype.getTileOrGrid = function(params, callback) {

    var self = this;

    var timer = new Timer();

    var renderer;

    timer.start('getTileOrGrid');
    var extraHeaders = {};
    step(
        function() {
            if (params.format === 'grid.json' && !params.interactivity) {
                if ( ! params.token ) { // token embeds interactivity
                    throw new Error("Missing interactivity parameter");
                }
            }
            timer.start('getRenderer');
            self._renderCache.getRenderer(params, this);
        },
        function(err, r, is_cached) {
            timer.end('getRenderer');
            renderer = r;
            if ( is_cached ) {
                extraHeaders['X-Windshaft-Cache'] = Date.now() - renderer.ctime;
            }
            assert.ifError(err);
            timer.start('render-' + params.format.replace('.','-'));
            renderer.getTile(+params.z, +params.x, +params.y, this);
        },
        function(err, tile, headers, stats) {
            timer.end('render-' + params.format.replace('.','-'));

            if ( renderer ) {
                renderer.release();
            }
            timer.end('getTileOrGrid');
            callback(err, tile, _.extend(extraHeaders, headers), _.extend(timer.getTimes(), stats));
        }
    );
};

/// Gets attributes for a given layer feature
//
/// Calls req2params, then expects parameters:
///
/// * token - MapConfig identifier
/// * layer - Layer number
/// * fid   - Feature identifier
///
/// The referenced layer must have been configured
/// to allow for attributes fetching.
/// See https://github.com/CartoDB/Windshaft/wiki/MapConfig-1.1.0
///
/// @param testMode if true generates a call returning requested
///                 columns plus the fid column of the first record
///                 it is only meant to check validity of configuration
///
MapBackend.prototype.getFeatureAttributes = function (params, testMode, callback) {
    var self = this;

    var timer = new Timer();

    var mapConfig;
    step(
        function getMapConfig() {
            timer.start('MapStore.load');
            self._mapStore.load(params.token, this);
        },
        function getPGClient(err, data) {
            assert.ifError(err);

            timer.end('MapStore.load');
            mapConfig = data;

            var dbParams = RendererParams.dbParamsFromReqParams(params);
            _.extend(dbParams, mapConfig.getLayerDatasource(params.layer));
            return new PSQL(dbParams);
        },
        function getAttributes(err, pg) {
            timer.start('getAttributes');

            assert.ifError(err);

            var layer = mapConfig.getLayer(params.layer);
            if ( ! layer ) {
                throw new Error("Map " + params.token +
                    " has no layer number " + params.layer);
            }
            var attributes = layer.options.attributes;
            if ( ! attributes ) {
                throw new Error("Layer " + params.layer +
                    " has no exposed attributes");
            }

            // NOTE: we're assuming that the presence of "attributes"
            //       means it is well-formed (should be checked at
            //       MapConfig construction/validation time).

            var fid_col = attributes.id;
            var att_cols = attributes.columns;

            // prepare columns with double quotes
            var quoted_att_cols = _.map(att_cols, function(n) {
                return pg.quoteIdentifier(n);
            }).join(',');

            if ( testMode ) {
                quoted_att_cols += ',' + pg.quoteIdentifier(fid_col);
            }

            var sql = 'select ' + quoted_att_cols + ' from ( ' + layer.options.sql + ' ) as _windshaft_subquery ';
            if ( ! testMode ) {
                sql += ' WHERE ' + pg.quoteIdentifier(fid_col) + ' = ' + params.fid;
            } else {
                sql += ' LIMIT 1';
            }

            // console.log("SQL:  " + sql);

            pg.query(sql, this, true); // use read-only transaction
        },
        function formatAttributes(err, data) {
            timer.end('getAttributes');
            assert.ifError(err);

            if ( data.rows.length !== 1 ) {
                if ( testMode ) {
                    return null;
                }
                else {
                    var rowsLengthError = new Error(data.rows.length +
                        " features in layer " + params.layer +
                        " of map " + params.token +
                        " are identified by fid " + params.fid);
                    if ( ! data.rows.length ) {
                        rowsLengthError.http_status = 404;
                    }
                    throw rowsLengthError;
                }
            }
            return data.rows[0];
        },
        function returnCallback(err, tile) {
            return callback(err, tile, timer.getTimes());
        }
    );
};

MapBackend.prototype.getLayersMetadata = function(params, mapConfig, callback) {
    var self = this;

    var metadata = [];

    var torqueLayers = [];

    mapConfig.getLayers().forEach(function(layer, layerId) {
        var layerType = mapConfig.layerType(layerId);
        metadata[layerId] = {
            type: layerType,
            meta: {}
        };

        if (layerType === 'torque') {
            torqueLayers.push(layerId);
        }
    });

    if ( ! torqueLayers.length ) {
        return callback(null, metadata);
    }

    var metadataQueue = queue(torqueLayers.length);

    torqueLayers.forEach(function(layerId) {
        metadataQueue.defer(function(params, token, rendererType, layerId, done) {
            self.getLayerMetadata(params, token, rendererType, layerId, done);
        }, params, mapConfig.id(), 'json.torque', layerId);
    });

    function metadataQueueFinish(err, results) {
        if (err) {
            return callback(err, results);
        }
        if (!results) {
            return callback(null, null);
        }

        torqueLayers.forEach(function(layerId, i) {
            metadata[layerId] = {
                type: 'torque',
                meta: results[i]
            };
        });
        return callback(err, metadata);
    }

    metadataQueue.awaitAll(metadataQueueFinish);
};

/// Fetch metadata for a tileset in a MapConfig
//
/// @param rendererType a MapConfig renderer type (layer.type in MapConfig spec)
/// @param layerId layer index within the mapConfig
/// @param callback function(err, metadata) where metadata format
///
MapBackend.prototype.getLayerMetadata = function(params, token, format, layerId, callback) {
    var self = this;

    params = _.extend({}, params, {
        token: token,
        format: format,
        layer: layerId
    });

    var renderer;

    step(
        function(){
            self._renderCache.getRenderer(params, this);
        },
        function(err, r) {
            assert.ifError(err);
            renderer = r;
            renderer.getMetadata(this);
        },
        function(err, meta) {
            if ( renderer ) {
                renderer.release();
            }
            callback(err, meta);
        }
    );
};
<|endoftext|>


<commit_before>var assert = require('assert');

var _ = require('underscore');
var queue = require('queue-async');
var step = require('step');
var debug = require('debug')('windshaft:backend:map');
var PSQL = require('cartodb-psql');
var RendererParams = require('../renderers/renderer_params');
var queryUtils = require('../utils/query_utils');

var DummyMapConfigProvider = require('../models/providers/dummy_mapconfig_provider');
var Timer = require('../stats/timer');

/**
 * @param {RendererCache} rendererCache
 * @param {MapStore} mapStore
 * @param {MapValidatorBackend} mapValidatorBackend
 * @constructor
 */
function MapBackend(rendererCache, mapStore, mapValidatorBackend) {
    this._rendererCache = rendererCache;
    this._mapStore = mapStore;
    this._mapValidatorBackend = mapValidatorBackend;
}

module.exports = MapBackend;

MapBackend.prototype.createLayergroup = function(mapConfig, params, validatorMapConfigProvider, callback) {
    var self = this;

    var timer = new Timer();
    timer.start('createLayergroup');

    var response = {};

    // Inject db parameters into the configuration
    // to ensure getting different identifiers for
    // maps created against different databases
    // or users. See
    // https://github.com/CartoDB/Windshaft/issues/163
    mapConfig.setDbParams({
        name: params.dbname,
        user: params.dbuser
    });

    if (params.filters) {
        try {
            mapConfig.setFiltersParams(JSON.parse(params.filters));
        } catch (e) {
            return callback(e);
        }
    }

    step(
        function initLayergroup() {
            timer.start('mapSave');
            // will save only if successful
            self._mapStore.save(mapConfig, this);
        },
        function handleMapConfigSave(err, mapConfigId, known) {
            timer.end('mapSave');

            assert.ifError(err);

            response.layergroupid = mapConfig.id();

            if (known) {
                return true;
            } else {
                var next = this;
                timer.start('validate');
                self._mapValidatorBackend.validate(validatorMapConfigProvider, function(err, isValid) {
                    timer.end('validate');
                    if (isValid) {
                        return next(err);
                    }
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            debug("Failed to delete MapConfig '" + mapConfig.id() + "' after: " + err);
                        }
                        return next(err);
                    });
                });
            }
        },
        function fetchLayersMetadata(err) {
            assert.ifError(err);

            var next = this;

            getLayersMetadata(self._rendererCache, params, mapConfig, function(err, layersMetadata) {
                if (err) {
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            debug("Failed to delete MapConfig '" + mapConfig.id() + " after: " + err);
                        }
                        return next(err);
                    });
                } else {
                    if (layersMetadata) {
                        response.metadata = response.metadata || {};
                        response.metadata.layers = layersMetadata;

                        // backwards compatibility for torque
                        var torqueMetadata = layersMetadata.reduce(function(acc, layer, layerId) {
                            if (layer.type === 'torque') {
                                acc[layerId] = layer.meta;
                            }
                            return acc;
                        }, {});
                        if (Object.keys(torqueMetadata).length) {
                            response.metadata.torque = torqueMetadata;
                        }
                    }
                    return next();
                }
            });
        },
        function finish(err) {
            timer.end('createLayergroup');
            callback(err, response, timer.getTimes());
        }
    );
};

function getLayersMetadata(rendererCache, params, mapConfig, callback) {
    var metadata = [];

    var metaQueue = queue(mapConfig.getLayers().length);

    mapConfig.getLayers().forEach(function(layer, layerId) {
        var layerType = mapConfig.layerType(layerId);

        function getFakeLayerMetadada(callback) {
            process.nextTick(function() {
                callback(null, {
                    stats: []
                });
            });
        }

        if (layerType === 'torque') {
            metaQueue.defer(getTorqueLayerMetadata, rendererCache, mapConfig, params, 'json.torque', layerId);
        } else if (layerType === 'mapnik') {
            metaQueue.defer(getMapnikLayerMetadata, layer, mapConfig, params);
        } else {
            metaQueue.defer(getFakeLayerMetadada);
        }
    });

    metaQueue.awaitAll(function (err, results) {
        if (err) {
            return callback(err);
        }

        if (!results) {
            return callback(null, null);
        }

        mapConfig.getLayers().forEach(function(layer, layerId) {
            var layerType = mapConfig.layerType(layerId);

            metadata[layerId] = {
                type: layerType,
                meta: results[layerId]
            };

            var widgets = mapConfig.getWidgets(layerId);

            if (_.size(widgets)) {
                metadata[layerId].widgets = widgets;
            }
        });

        return callback(err, metadata);
    });
}

/// Fetch metadata for a tileset in a MapConfig
//
/// @param rendererType a MapConfig renderer type (layer.type in MapConfig spec)
/// @param layerId layer index within the mapConfig
/// @param callback function(err, metadata) where metadata format
///
function getTorqueLayerMetadata(rendererCache, mapConfig, params, format, layerId, callback) {
    params = _.extend({}, params, {
        token: mapConfig.id(),
        format: format,
        layer: layerId
    });

    var renderer;

    step(
        function(){
            rendererCache.getRenderer(new DummyMapConfigProvider(mapConfig, params), this);
        },
        function(err, _renderer) {
            assert.ifError(err);
            renderer = _renderer;
            renderer.getMetadata(this);
        },
        function(err, meta) {
            if ( renderer ) {
                renderer.release();
            }
            callback(err, meta);
        }
    );
}

function getMapnikLayerMetadata(layer, mapConfig, params, callback) {
    step(
        function getPGClient(err) {
            assert.ifError(err);

            var dbParams = RendererParams.dbParamsFromReqParams(params);
            _.extend(dbParams, mapConfig.getLayerDatasource(params.layer));

            return new PSQL(dbParams);
        },
        function fetchLayerTablenames(err, pg) {
            assert.ifError(err);

            var next = this;
            var query = queryUtils.extractTableNames(layer.options.sql);

            pg.query(query, function (err, result) {
                if (err) {
                    return next(err);
                }

                next(null, pg, result.rows[0].tablenames);
            });
        },

        function getTableStats(err, pg, tablenames) {
            assert.ifError(err);

            var tableQueue = queue(tablenames.length);

            tablenames.forEach(function (table) {
                tableQueue.defer(function (pg, table, done) {
                    var tableStats = {
                        type: 'table',
                        name: table,
                    };
                    var query = queryUtils.getTableStats(table);

                    pg.query(query, function (err, resultTable) {
                        if (err) {
                            tableStats.features = -1;
                        } else {
                            try {
                                tableStats.features = JSON.parse(resultTable.rows[0].result.stats).table_features;
                            } catch (e) {
                                tableStats.features = -1;
                            }
                        }
                        done(null, tableStats);
                    });
                }, pg, table);
            });

            tableQueue.awaitAll(this);
        },
        function finnish(err, result) {
            if (err) {
                return callback(err);
            }

            callback(null, {
                stats: result
            });
        }
    );
}
<commit_msg>Moved clousured function to a better place<commit_after>var assert = require('assert');

var _ = require('underscore');
var queue = require('queue-async');
var step = require('step');
var debug = require('debug')('windshaft:backend:map');
var PSQL = require('cartodb-psql');
var RendererParams = require('../renderers/renderer_params');
var queryUtils = require('../utils/query_utils');

var DummyMapConfigProvider = require('../models/providers/dummy_mapconfig_provider');
var Timer = require('../stats/timer');

function getFakeLayerMetadada(callback) {
    process.nextTick(function() {
        callback(null, {
            stats: []
        });
    });
}

/**
 * @param {RendererCache} rendererCache
 * @param {MapStore} mapStore
 * @param {MapValidatorBackend} mapValidatorBackend
 * @constructor
 */
function MapBackend(rendererCache, mapStore, mapValidatorBackend) {
    this._rendererCache = rendererCache;
    this._mapStore = mapStore;
    this._mapValidatorBackend = mapValidatorBackend;
}

module.exports = MapBackend;

MapBackend.prototype.createLayergroup = function(mapConfig, params, validatorMapConfigProvider, callback) {
    var self = this;

    var timer = new Timer();
    timer.start('createLayergroup');

    var response = {};

    // Inject db parameters into the configuration
    // to ensure getting different identifiers for
    // maps created against different databases
    // or users. See
    // https://github.com/CartoDB/Windshaft/issues/163
    mapConfig.setDbParams({
        name: params.dbname,
        user: params.dbuser
    });

    if (params.filters) {
        try {
            mapConfig.setFiltersParams(JSON.parse(params.filters));
        } catch (e) {
            return callback(e);
        }
    }

    step(
        function initLayergroup() {
            timer.start('mapSave');
            // will save only if successful
            self._mapStore.save(mapConfig, this);
        },
        function handleMapConfigSave(err, mapConfigId, known) {
            timer.end('mapSave');

            assert.ifError(err);

            response.layergroupid = mapConfig.id();

            if (known) {
                return true;
            } else {
                var next = this;
                timer.start('validate');
                self._mapValidatorBackend.validate(validatorMapConfigProvider, function(err, isValid) {
                    timer.end('validate');
                    if (isValid) {
                        return next(err);
                    }
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            debug("Failed to delete MapConfig '" + mapConfig.id() + "' after: " + err);
                        }
                        return next(err);
                    });
                });
            }
        },
        function fetchLayersMetadata(err) {
            assert.ifError(err);

            var next = this;

            getLayersMetadata(self._rendererCache, params, mapConfig, function(err, layersMetadata) {
                if (err) {
                    self._mapStore.del(mapConfig.id(), function(delErr) {
                        if (delErr) {
                            debug("Failed to delete MapConfig '" + mapConfig.id() + " after: " + err);
                        }
                        return next(err);
                    });
                } else {
                    if (layersMetadata) {
                        response.metadata = response.metadata || {};
                        response.metadata.layers = layersMetadata;

                        // backwards compatibility for torque
                        var torqueMetadata = layersMetadata.reduce(function(acc, layer, layerId) {
                            if (layer.type === 'torque') {
                                acc[layerId] = layer.meta;
                            }
                            return acc;
                        }, {});
                        if (Object.keys(torqueMetadata).length) {
                            response.metadata.torque = torqueMetadata;
                        }
                    }
                    return next();
                }
            });
        },
        function finish(err) {
            timer.end('createLayergroup');
            callback(err, response, timer.getTimes());
        }
    );
};

function getLayersMetadata(rendererCache, params, mapConfig, callback) {
    var metadata = [];

    var metaQueue = queue(mapConfig.getLayers().length);

    mapConfig.getLayers().forEach(function(layer, layerId) {
        var layerType = mapConfig.layerType(layerId);

        if (layerType === 'torque') {
            metaQueue.defer(getTorqueLayerMetadata, rendererCache, mapConfig, params, 'json.torque', layerId);
        } else if (layerType === 'mapnik') {
            metaQueue.defer(getMapnikLayerMetadata, layer, mapConfig, params);
        } else {
            metaQueue.defer(getFakeLayerMetadada);
        }
    });

    metaQueue.awaitAll(function (err, results) {
        if (err) {
            return callback(err);
        }

        if (!results) {
            return callback(null, null);
        }

        mapConfig.getLayers().forEach(function(layer, layerId) {
            var layerType = mapConfig.layerType(layerId);

            metadata[layerId] = {
                type: layerType,
                meta: results[layerId]
            };

            var widgets = mapConfig.getWidgets(layerId);

            if (_.size(widgets)) {
                metadata[layerId].widgets = widgets;
            }
        });

        return callback(err, metadata);
    });
}

/// Fetch metadata for a tileset in a MapConfig
//
/// @param rendererType a MapConfig renderer type (layer.type in MapConfig spec)
/// @param layerId layer index within the mapConfig
/// @param callback function(err, metadata) where metadata format
///
function getTorqueLayerMetadata(rendererCache, mapConfig, params, format, layerId, callback) {
    params = _.extend({}, params, {
        token: mapConfig.id(),
        format: format,
        layer: layerId
    });

    var renderer;

    step(
        function(){
            rendererCache.getRenderer(new DummyMapConfigProvider(mapConfig, params), this);
        },
        function(err, _renderer) {
            assert.ifError(err);
            renderer = _renderer;
            renderer.getMetadata(this);
        },
        function(err, meta) {
            if ( renderer ) {
                renderer.release();
            }
            callback(err, meta);
        }
    );
}

function getMapnikLayerMetadata(layer, mapConfig, params, callback) {
    step(
        function getPGClient(err) {
            assert.ifError(err);

            var dbParams = RendererParams.dbParamsFromReqParams(params);
            _.extend(dbParams, mapConfig.getLayerDatasource(params.layer));

            return new PSQL(dbParams);
        },
        function fetchLayerTablenames(err, pg) {
            assert.ifError(err);

            var next = this;
            var query = queryUtils.extractTableNames(layer.options.sql);

            pg.query(query, function (err, result) {
                if (err) {
                    return next(err);
                }

                next(null, pg, result.rows[0].tablenames);
            });
        },

        function getTableStats(err, pg, tablenames) {
            assert.ifError(err);

            var tableQueue = queue(tablenames.length);

            tablenames.forEach(function (table) {
                tableQueue.defer(function (pg, table, done) {
                    var tableStats = {
                        type: 'table',
                        name: table,
                    };
                    var query = queryUtils.getTableStats(table);

                    pg.query(query, function (err, resultTable) {
                        if (err) {
                            tableStats.features = -1;
                        } else {
                            try {
                                tableStats.features = JSON.parse(resultTable.rows[0].result.stats).table_features;
                            } catch (e) {
                                tableStats.features = -1;
                            }
                        }
                        done(null, tableStats);
                    });
                }, pg, table);
            });

            tableQueue.awaitAll(this);
        },
        function finnish(err, result) {
            if (err) {
                return callback(err);
            }

            callback(null, {
                stats: result
            });
        }
    );
}
<|endoftext|>


<commit_before>// caches Mapnik render objects, purging after 60 seconds of inactivity:
var   _                     = require('underscore')
    , CacheEntry            = require('./cache/cache_entry')
    , Step                  = require('step')
    , tilelive              = require('tilelive')
    , TorqueRendererFactory = require('./renderers/torque.js');

require('tilelive-mapnik').registerProtocols(tilelive);

module.exports = function(timeout, mml_store, map_store, mapnik_opts) {

    // Configure bases for cache keys suitable for string interpolation
    var baseKey   = "<%= dbname %>:<%= table %>:";
    var renderKey = baseKey + "<%= dbuser %>:<%= format %>:<%= geom_type %>:<%= sql %>:<%= layer %>:<%= interactivity %>:<%= style %>:<%= style_version %>";

    // Set default mapnik options
    mapnik_opts = _.defaults( mapnik_opts ? mapnik_opts : {}, {

      // Metatile is the number of tiles-per-side that are going
      // to be rendered at once. If all of them will be requested
      // we'd have saved time. If only one will be used, we'd have
      // wasted time.
      //
      // Defaults to 2 as of tilelive-mapnik@0.3.2
      //
      // We'll assume an average of a 4x4 viewport
      metatile: 4,

      // Buffer size is the tickness in pixel of a buffer
      // around the rendered (meta?)tile.
      //
      // This is important for labels and other marker that overlap tile boundaries.
      // Setting to 128 ensures no render artifacts.
      // 64 may have artifacts but is faster.
      // Less important if we can turn metatiling on.
      //
      // defaults to 128 as of tilelive-mapnik@0.3.2
      //
      bufferSize: 64
    });

    var torqueFactory = new TorqueRendererFactory(); // TODO: specify options

    var me = {
        serial: 0,
        renderers: {},
        timeout: timeout || 60000,
        mapnik_opts: mapnik_opts
    };

    me.dumpStats = function() {
      var now = Date.now();
      var renderers = me.renderers;
      var timeout = me.timeout;
      var itemkeys = _.keys(renderers);
      console.log("RenderCache items " + itemkeys.length + " timeout " + timeout + " next gc in " + ( timeout - ( now - me.lastCacheClearRun ) ) );
      _.each(itemkeys, function(key){
        var item = renderers[key];
        console.log(" ItemKey " + key + " ttl " + ( timeout - item.timeSinceLastAccess(now) ) + " cb " + item.cache_buster );
      });
    };

    me.lastCacheClearRun = Date.now();
    me.cacheClearInterval = setInterval( function() {
      var now = Date.now();
      var that = me;
      var itemkeys = _.keys(that.renderers);
      _.each(itemkeys, function(key){
          var cache_entry = that.renderers[key];
          if ( cache_entry.timeSinceLastAccess(now) > that.timeout ) {
            that.del(key);
          }
      });
      // Only dump if verbose ?
      console.log("RenderCache items: " + _.keys(that.renderers).length + "/" + itemkeys.length);
      me.lastCacheClearRun = now;
    }, me.timeout );

    // Create a string ID/key from a set of params
    me.createKey = function(params, base) {
        var opts =  _.extend({}, params); // as params is a weird arrayobj right here
        delete opts.x;
        delete opts.y;
        delete opts.z;
        delete opts.callback;
        if ( ! opts.table ) {
          opts.table = opts.token;
          // interactivity is encoded in token
          delete opts.interactivity;
        }
        _.defaults(opts, {
          dbname:'', dbuser:'', table:'',
          format:'', geom_type:'', sql:'',
          interactivity:'', layer:'', style:'', style_version:''
        });
        return _.template(base ? baseKey : renderKey, opts);
    };

    // If renderer cache entry exists at req-derived key, return it,
    // else generate a new one and save at key.
    //
    // Caches lifetime is driven by the timeout passed at RendererCache
    // construction time.
    //
    //
    // @param callback will be called with (err, renderer)
    //        If `err` is null the renderer should be
    //        ready for you to use (calling getTile or getGrid).
    //        Note that the object is a proxy to the actual TileStore
    //        so you won't get the whole TileLive interface available.
    //        If you need that, use the .get() function.
    //        In order to reduce memory usage call renderer.release()
    //        when you're sure you won't need it anymore.
    //
    //
    me.getRenderer = function(req, callback) {

        var params = req.params;
        var cache_buster = this.getCacheBusterValue(params.cache_buster);

        // setup
        var key  = this.createKey(params);

        var cache_entry = this.renderers[key];

        if (this.shouldRecreateRenderer(cache_entry, cache_buster)) {

            cache_entry = this.renderers[key] = new CacheEntry();
            cache_entry.cache_buster = cache_buster;
            cache_entry.key = key; // TODO: drop, is unused
            cache_entry._addRef(); // we add another ref for this.renderers[key]

            var that = this;

            cache_entry.on('error', function(err) {
              console.log("Removing RendererCache " + key + " on error " + err);
              that.del(key);
            });

            Step(
              function invokeHook () {
                if ( ! params.processRendererCache ) return true;
                params.processRendererCache(cache_entry, req, this);
              },
              function makeRenderer (err) {
                if ( err ) { callback(err); return; }
                that.makeRenderer(req, function(err, renderer) {
                  if ( req.profiler ) req.profiler.done('makeRenderer-'+req.params.format.replace('.','_'));
                  cache_entry.setReady(err, renderer);
                });
              }
            );
        }

        cache_entry.pushCallback(callback);
    };

    me.getCacheBusterValue = function(cache_buster) {
        if (_.isUndefined(cache_buster)) {
            return 0;
        }
        if (_.isNumber(cache_buster)) {
            return Math.min(this._getMaxCacheBusterValue(), cache_buster);
        }
        return cache_buster;
    };

    me._getMaxCacheBusterValue = function() {
        return Date.now();
    };

    me.shouldRecreateRenderer = function(cacheEntry, cacheBuster) {
        if (cacheEntry) {
            var entryCacheBuster = parseFloat(cacheEntry.cache_buster);
                requestCacheBuster = parseFloat(cacheBuster);

            if (isNaN(entryCacheBuster) || isNaN(requestCacheBuster)) {
                return cacheEntry.cache_buster != cacheBuster;
            }
            return requestCacheBuster > entryCacheBuster;
        }
        return true;
    }

    me.dbParamsFromReqParams = function(params) {
        var dbParams = {};
        if ( params.dbuser ) dbParams.user = params.dbuser;
        if ( params.dbpassword ) dbParams.pass = params.dbpassword;
        if ( params.dbhost ) dbParams.host = params.dbhost;
        if ( params.dbport ) dbParams.port = params.dbport;
        if ( params.dbname ) dbParams.dbname = params.dbname;
        return dbParams;
    };

    me.mapConfigToMMLBuilderConfig = function(mapConfig) {
        var cfg = mapConfig.obj();
        var sql = [];
        var style = [];
        var interactivity = [];
        var style_version = cfg.hasOwnProperty('global_cartocss_version') ? cfg.global_cartocss_version : [];
        for ( var i=0; i<cfg.layers.length; ++i ) {
          var lyr = cfg.layers[i];
          if ( mapConfig.layerType(i) != 'mapnik' ) continue;
          if ( ! lyr.hasOwnProperty('options') )
            throw new Error("Missing options from layer " + i + " of layergroup config");
          var lyropt = lyr.options;
          if ( ! lyropt.hasOwnProperty('sql') )
            throw new Error("Missing sql for layer " + i + " options");
          // Wrap SQL requests in mapnik format if sent
          sql.push( "(" + lyropt.sql.replace(/;\s*$/, '') + ") as cdbq");
          if ( ! lyropt.hasOwnProperty('cartocss') )
            throw new Error("Missing cartocss for layer " + i + " options");
          style.push(lyropt.cartocss);
          if ( _.isArray(style_version) ) {
            if ( ! lyropt.hasOwnProperty('cartocss_version') ) {
              throw new Error("Missing cartocss_version for layer " + i + " options");
            }
            style_version.push(lyropt.cartocss_version);
          }
          // NOTE: interactivity used to be a string as of version 1.0.0
          if ( _.isArray(lyropt.interactivity) ) {
            lyropt.interactivity = lyropt.interactivity.join(',');
          }
          interactivity.push(lyropt.interactivity);
        }
        if ( ! sql.length ) throw new Error("No 'mapnik' layers in MapConfig");
        return { sql: sql, style: style, style_version: style_version, interactivity: interactivity, ttl: 0 }
    };

    function TileliveAdaptor(renderer, format) {
      this.renderer = renderer;
      this.close = this.renderer.close.bind(this.renderer);
      this.get = function() { return renderer; };
      if ( format == 'png' ) {
        this.getTile = this.renderer.getTile.bind(this.renderer);
      }
      else if ( format == 'grid.json' ) {
        this.getTile = this.renderer.getGrid.bind(this.renderer);
      }
      else throw new Error("Unsupported format " + format);
    }

    // controls the instantiation of mapnik renderer objects from mapnik XML
    me.makeRendererMapnik = function(req, callback){
        var that = this;
        var params = _.extend({}, req.params);
        var mml_builder;

        // returns a tilelive renderer by:
        // 1. generating or retrieving mapnik XML
        // 2. configuring a full mml document with mapnik XML, interactivity and other elements set
        Step(
            function getConfig(){
              if ( ! params.token ) return null;
              map_store.load(params.token, this);
            },
            function initBuilder(err, mapConfig) {
              if (err) throw err;
              if ( params.token ) {
                delete params.interactivity; // will be rewritten from MapConfig
                params = _.defaults(params, that.mapConfigToMMLBuilderConfig(mapConfig));
                delete params.token;
              }
              // create an mapnik mml builder object
              mml_builder = mml_store.mml_builder(params, this);
            },
            function generateXML(err){
                if (err) throw err;
                mml_builder.toXML(this);
            },
            function loadMapnik(err, data){
                if (err) throw err;

                var query = {
                    // TODO: document what `base` is
                    base: that.createKey(params) + '/xia',
                    metatile: that.mapnik_opts.metatile,
                    bufferSize: that.mapnik_opts.bufferSize,
                    internal_cache: false
                };

                // build full document to pass to tilelive
                var uri = {
                    query: query,
                    protocol: 'mapnik:',
                    slashes: true,
                    xml: data,
                    strict: !!params.strict, // force boolean
                    mml: {
                        format: params.format
                    }
                };

                // hand off to tilelive to create a renderer
                tilelive.load(uri, this);
            },
            function makeAdaptor(err, source){
                if ( err ) throw err;
                return new TileliveAdaptor(source, params.format);
            },
            function returnCallback(err, source){
                callback(err, source);
            }
        );
    };

    function TorqueAdaptor(renderer) {
      this.renderer = renderer;
      this.close = function() { /* nothing to do */ };
      this.get = function() { return renderer; };
      this.getTile = this.renderer.getTile.bind(this.renderer);
    }

    // controls the instantiation of mapnik renderer objects from mapnik XML
    me.makeRendererTorque = function(req, callback){
        var that = this;
        var params = _.extend({}, req.params);
        if ( ! params.token || ! params.hasOwnProperty('layer') ) {
          callback(new Error("Torque renderer can only be initialized with map token and layer id"));
          return;
        }

        Step(
            function getConfig(){
              map_store.load(params.token, this);
            },
            function initRenderer(err, mapConfig) {
              if (err) throw err;
              var dbParams = that.dbParamsFromReqParams(params);
              var format = params.format;
              var layer = params.layer;
              torqueFactory.getRenderer(mapConfig.obj(), dbParams, format, layer, this);
            },
            function makeAdaptor(err, renderer){
                if ( err ) throw err;
                return new TorqueAdaptor(renderer);
            },
            function returnCallback(err, renderer){
                callback(err, renderer);
            }
        );
    };

    me.makeRenderer = function(req, callback){
      var params = req.params;
      if ( params.format.match(/^(png|grid\.json)$/) ) {
        me.makeRendererMapnik(req, callback);
      }
      else if ( params.format.match(/torque/) ) {
        me.makeRendererTorque(req, callback);
      }
      else {
        callback(new Error("Unsupported format " + params.format));
      }
    };

    // delete all renderers in cache
    me.purge = function(){
        var that = this;
        _.each(_.keys(that.renderers), function(key){
            that.del(key);
        });
    };

    // Clears out all renderers related to a given table, regardless of other arguments
    me.reset = function(req){
        var base_key = this.createKey(req.params, true);
        var regex = new RegExp('^' + base_key + '.*');
        var that = this;

        _.each(_.keys(this.renderers), function(key){
            if(key.match(regex)){
                that.del(key);
            }
        });
    };

    // drain render pools, remove renderer and associated timeout calls
    me.del = function(id){
        var cache_entry = this.renderers[id];
        delete this.renderers[id];
        cache_entry.release();
    };

    return me;
};
<commit_msg>CDB-2570 checkstyle<commit_after>// caches Mapnik render objects, purging after 60 seconds of inactivity:
var   _                     = require('underscore')
    , CacheEntry            = require('./cache/cache_entry')
    , Step                  = require('step')
    , tilelive              = require('tilelive')
    , TorqueRendererFactory = require('./renderers/torque.js');

require('tilelive-mapnik').registerProtocols(tilelive);

module.exports = function(timeout, mml_store, map_store, mapnik_opts) {

    // Configure bases for cache keys suitable for string interpolation
    var baseKey   = "<%= dbname %>:<%= table %>:";
    var renderKey = baseKey + "<%= dbuser %>:<%= format %>:<%= geom_type %>:<%= sql %>:<%= layer %>:<%= interactivity %>:<%= style %>:<%= style_version %>";

    // Set default mapnik options
    mapnik_opts = _.defaults( mapnik_opts ? mapnik_opts : {}, {

      // Metatile is the number of tiles-per-side that are going
      // to be rendered at once. If all of them will be requested
      // we'd have saved time. If only one will be used, we'd have
      // wasted time.
      //
      // Defaults to 2 as of tilelive-mapnik@0.3.2
      //
      // We'll assume an average of a 4x4 viewport
      metatile: 4,

      // Buffer size is the tickness in pixel of a buffer
      // around the rendered (meta?)tile.
      //
      // This is important for labels and other marker that overlap tile boundaries.
      // Setting to 128 ensures no render artifacts.
      // 64 may have artifacts but is faster.
      // Less important if we can turn metatiling on.
      //
      // defaults to 128 as of tilelive-mapnik@0.3.2
      //
      bufferSize: 64
    });

    var torqueFactory = new TorqueRendererFactory(); // TODO: specify options

    var me = {
        serial: 0,
        renderers: {},
        timeout: timeout || 60000,
        mapnik_opts: mapnik_opts
    };

    me.dumpStats = function() {
      var now = Date.now();
      var renderers = me.renderers;
      var timeout = me.timeout;
      var itemkeys = _.keys(renderers);
      console.log("RenderCache items " + itemkeys.length + " timeout " + timeout + " next gc in " + ( timeout - ( now - me.lastCacheClearRun ) ) );
      _.each(itemkeys, function(key){
        var item = renderers[key];
        console.log(" ItemKey " + key + " ttl " + ( timeout - item.timeSinceLastAccess(now) ) + " cb " + item.cache_buster );
      });
    };

    me.lastCacheClearRun = Date.now();
    me.cacheClearInterval = setInterval( function() {
      var now = Date.now();
      var that = me;
      var itemkeys = _.keys(that.renderers);
      _.each(itemkeys, function(key){
          var cache_entry = that.renderers[key];
          if ( cache_entry.timeSinceLastAccess(now) > that.timeout ) {
            that.del(key);
          }
      });
      // Only dump if verbose ?
      console.log("RenderCache items: " + _.keys(that.renderers).length + "/" + itemkeys.length);
      me.lastCacheClearRun = now;
    }, me.timeout );

    // Create a string ID/key from a set of params
    me.createKey = function(params, base) {
        var opts =  _.extend({}, params); // as params is a weird arrayobj right here
        delete opts.x;
        delete opts.y;
        delete opts.z;
        delete opts.callback;
        if ( ! opts.table ) {
          opts.table = opts.token;
          // interactivity is encoded in token
          delete opts.interactivity;
        }
        _.defaults(opts, {
          dbname:'', dbuser:'', table:'',
          format:'', geom_type:'', sql:'',
          interactivity:'', layer:'', style:'', style_version:''
        });
        return _.template(base ? baseKey : renderKey, opts);
    };

    // If renderer cache entry exists at req-derived key, return it,
    // else generate a new one and save at key.
    //
    // Caches lifetime is driven by the timeout passed at RendererCache
    // construction time.
    //
    //
    // @param callback will be called with (err, renderer)
    //        If `err` is null the renderer should be
    //        ready for you to use (calling getTile or getGrid).
    //        Note that the object is a proxy to the actual TileStore
    //        so you won't get the whole TileLive interface available.
    //        If you need that, use the .get() function.
    //        In order to reduce memory usage call renderer.release()
    //        when you're sure you won't need it anymore.
    //
    //
    me.getRenderer = function(req, callback) {

        var params = req.params;
        var cache_buster = this.getCacheBusterValue(params.cache_buster);

        // setup
        var key  = this.createKey(params);

        var cache_entry = this.renderers[key];

        if (this.shouldRecreateRenderer(cache_entry, cache_buster)) {

            cache_entry = this.renderers[key] = new CacheEntry();
            cache_entry.cache_buster = cache_buster;
            cache_entry.key = key; // TODO: drop, is unused
            cache_entry._addRef(); // we add another ref for this.renderers[key]

            var that = this;

            cache_entry.on('error', function(err) {
              console.log("Removing RendererCache " + key + " on error " + err);
              that.del(key);
            });

            Step(
              function invokeHook () {
                if ( ! params.processRendererCache ) return true;
                params.processRendererCache(cache_entry, req, this);
              },
              function makeRenderer (err) {
                if ( err ) { callback(err); return; }
                that.makeRenderer(req, function(err, renderer) {
                  if ( req.profiler ) req.profiler.done('makeRenderer-'+req.params.format.replace('.','_'));
                  cache_entry.setReady(err, renderer);
                });
              }
            );
        }

        cache_entry.pushCallback(callback);
    };

    me.getCacheBusterValue = function(cache_buster) {
        if (_.isUndefined(cache_buster)) {
            return 0;
        }
        if (_.isNumber(cache_buster)) {
            return Math.min(this._getMaxCacheBusterValue(), cache_buster);
        }
        return cache_buster;
    };

    me._getMaxCacheBusterValue = function() {
        return Date.now();
    };

    me.shouldRecreateRenderer = function(cacheEntry, cacheBuster) {
        if (cacheEntry) {
            var entryCacheBuster = parseFloat(cacheEntry.cache_buster),
                requestCacheBuster = parseFloat(cacheBuster);

            if (isNaN(entryCacheBuster) || isNaN(requestCacheBuster)) {
                return cacheEntry.cache_buster != cacheBuster;
            }
            return requestCacheBuster > entryCacheBuster;
        }
        return true;
    };

    me.dbParamsFromReqParams = function(params) {
        var dbParams = {};
        if ( params.dbuser ) dbParams.user = params.dbuser;
        if ( params.dbpassword ) dbParams.pass = params.dbpassword;
        if ( params.dbhost ) dbParams.host = params.dbhost;
        if ( params.dbport ) dbParams.port = params.dbport;
        if ( params.dbname ) dbParams.dbname = params.dbname;
        return dbParams;
    };

    me.mapConfigToMMLBuilderConfig = function(mapConfig) {
        var cfg = mapConfig.obj();
        var sql = [];
        var style = [];
        var interactivity = [];
        var style_version = cfg.hasOwnProperty('global_cartocss_version') ? cfg.global_cartocss_version : [];
        for ( var i=0; i<cfg.layers.length; ++i ) {
          var lyr = cfg.layers[i];
          if ( mapConfig.layerType(i) != 'mapnik' ) continue;
          if ( ! lyr.hasOwnProperty('options') )
            throw new Error("Missing options from layer " + i + " of layergroup config");
          var lyropt = lyr.options;
          if ( ! lyropt.hasOwnProperty('sql') )
            throw new Error("Missing sql for layer " + i + " options");
          // Wrap SQL requests in mapnik format if sent
          sql.push( "(" + lyropt.sql.replace(/;\s*$/, '') + ") as cdbq");
          if ( ! lyropt.hasOwnProperty('cartocss') )
            throw new Error("Missing cartocss for layer " + i + " options");
          style.push(lyropt.cartocss);
          if ( _.isArray(style_version) ) {
            if ( ! lyropt.hasOwnProperty('cartocss_version') ) {
              throw new Error("Missing cartocss_version for layer " + i + " options");
            }
            style_version.push(lyropt.cartocss_version);
          }
          // NOTE: interactivity used to be a string as of version 1.0.0
          if ( _.isArray(lyropt.interactivity) ) {
            lyropt.interactivity = lyropt.interactivity.join(',');
          }
          interactivity.push(lyropt.interactivity);
        }
        if ( ! sql.length ) throw new Error("No 'mapnik' layers in MapConfig");
        return { sql: sql, style: style, style_version: style_version, interactivity: interactivity, ttl: 0 }
    };

    function TileliveAdaptor(renderer, format) {
      this.renderer = renderer;
      this.close = this.renderer.close.bind(this.renderer);
      this.get = function() { return renderer; };
      if ( format == 'png' ) {
        this.getTile = this.renderer.getTile.bind(this.renderer);
      }
      else if ( format == 'grid.json' ) {
        this.getTile = this.renderer.getGrid.bind(this.renderer);
      }
      else throw new Error("Unsupported format " + format);
    }

    // controls the instantiation of mapnik renderer objects from mapnik XML
    me.makeRendererMapnik = function(req, callback){
        var that = this;
        var params = _.extend({}, req.params);
        var mml_builder;

        // returns a tilelive renderer by:
        // 1. generating or retrieving mapnik XML
        // 2. configuring a full mml document with mapnik XML, interactivity and other elements set
        Step(
            function getConfig(){
              if ( ! params.token ) return null;
              map_store.load(params.token, this);
            },
            function initBuilder(err, mapConfig) {
              if (err) throw err;
              if ( params.token ) {
                delete params.interactivity; // will be rewritten from MapConfig
                params = _.defaults(params, that.mapConfigToMMLBuilderConfig(mapConfig));
                delete params.token;
              }
              // create an mapnik mml builder object
              mml_builder = mml_store.mml_builder(params, this);
            },
            function generateXML(err){
                if (err) throw err;
                mml_builder.toXML(this);
            },
            function loadMapnik(err, data){
                if (err) throw err;

                var query = {
                    // TODO: document what `base` is
                    base: that.createKey(params) + '/xia',
                    metatile: that.mapnik_opts.metatile,
                    bufferSize: that.mapnik_opts.bufferSize,
                    internal_cache: false
                };

                // build full document to pass to tilelive
                var uri = {
                    query: query,
                    protocol: 'mapnik:',
                    slashes: true,
                    xml: data,
                    strict: !!params.strict, // force boolean
                    mml: {
                        format: params.format
                    }
                };

                // hand off to tilelive to create a renderer
                tilelive.load(uri, this);
            },
            function makeAdaptor(err, source){
                if ( err ) throw err;
                return new TileliveAdaptor(source, params.format);
            },
            function returnCallback(err, source){
                callback(err, source);
            }
        );
    };

    function TorqueAdaptor(renderer) {
      this.renderer = renderer;
      this.close = function() { /* nothing to do */ };
      this.get = function() { return renderer; };
      this.getTile = this.renderer.getTile.bind(this.renderer);
    }

    // controls the instantiation of mapnik renderer objects from mapnik XML
    me.makeRendererTorque = function(req, callback){
        var that = this;
        var params = _.extend({}, req.params);
        if ( ! params.token || ! params.hasOwnProperty('layer') ) {
          callback(new Error("Torque renderer can only be initialized with map token and layer id"));
          return;
        }

        Step(
            function getConfig(){
              map_store.load(params.token, this);
            },
            function initRenderer(err, mapConfig) {
              if (err) throw err;
              var dbParams = that.dbParamsFromReqParams(params);
              var format = params.format;
              var layer = params.layer;
              torqueFactory.getRenderer(mapConfig.obj(), dbParams, format, layer, this);
            },
            function makeAdaptor(err, renderer){
                if ( err ) throw err;
                return new TorqueAdaptor(renderer);
            },
            function returnCallback(err, renderer){
                callback(err, renderer);
            }
        );
    };

    me.makeRenderer = function(req, callback){
      var params = req.params;
      if ( params.format.match(/^(png|grid\.json)$/) ) {
        me.makeRendererMapnik(req, callback);
      }
      else if ( params.format.match(/torque/) ) {
        me.makeRendererTorque(req, callback);
      }
      else {
        callback(new Error("Unsupported format " + params.format));
      }
    };

    // delete all renderers in cache
    me.purge = function(){
        var that = this;
        _.each(_.keys(that.renderers), function(key){
            that.del(key);
        });
    };

    // Clears out all renderers related to a given table, regardless of other arguments
    me.reset = function(req){
        var base_key = this.createKey(req.params, true);
        var regex = new RegExp('^' + base_key + '.*');
        var that = this;

        _.each(_.keys(this.renderers), function(key){
            if(key.match(regex)){
                that.del(key);
            }
        });
    };

    // drain render pools, remove renderer and associated timeout calls
    me.del = function(id){
        var cache_entry = this.renderers[id];
        delete this.renderers[id];
        cache_entry.release();
    };

    return me;
};
<|endoftext|>


<commit_before>const configParser = require('../../config');
const pkgWorker = require('../package');
const { extractDependencies } = require('../../manager');

module.exports = {
  renovateDepType,
  getDepConfig,
};

async function renovateDepType(packageContent, config) {
  logger.setMeta({
    repository: config.repository,
    packageFile: config.packageFile,
    depType: config.depType,
  });
  logger.debug('renovateDepType()');
  logger.trace({ config });
  if (config.enabled === false) {
    logger.debug('depType is disabled');
    return [];
  }
  let deps = await extractDependencies(packageContent, config);
  if (config.lerna || config.workspaces || config.workspaceDir) {
    deps = deps.filter(
      dependency => config.monorepoPackages.indexOf(dependency.depName) === -1
    );
  }
  deps = deps.filter(
    dependency => config.ignoreDeps.indexOf(dependency.depName) === -1
  );
  logger.debug(`filtered deps length is ${deps.length}`);
  logger.debug({ deps }, `filtered deps`);
  // Obtain full config for each dependency
  const depConfigs = deps.map(dep => module.exports.getDepConfig(config, dep));
  logger.trace({ config: depConfigs }, `depConfigs`);
  // renovateDepType can return more than one upgrade each
  const pkgWorkers = depConfigs.map(depConfig =>
    pkgWorker.renovatePackage(depConfig)
  );
  // Use Promise.all to execute npm queries in parallel
  const allUpgrades = await Promise.all(pkgWorkers);
  logger.trace({ config: allUpgrades }, `allUpgrades`);
  // Squash arrays into one
  const combinedUpgrades = [].concat(...allUpgrades);
  logger.trace({ config: combinedUpgrades }, `combinedUpgrades`);
  return combinedUpgrades;
}

function getDepConfig(depTypeConfig, dep) {
  const { depName: dependency } = dep;
  let depConfig = configParser.mergeChildConfig(depTypeConfig, dep);
  // Apply any matching package rules
  if (depConfig.packageRules) {
    logger.trace(
      { dependency, packageRules: depConfig.packageRules },
      `Checking against ${depConfig.packageRules.length} packageRules`
    );
    depConfig.packageRules.forEach(packageRule => {
      let applyRule = false;
      if (!(packageRule.packageNames || packageRule.packagePatterns)) {
        logger.debug(
          { packageRule, dependency },
          'packageRule is missing packageNames and packagePatterns so will match anything'
        );
        applyRule = true;
      } else if (
        packageRule.packageNames &&
        packageRule.packageNames.includes(depConfig.depName)
      ) {
        logger.debug(
          { dependency, packageNames: packageRule.packageNames },
          'Matched packageNames'
        );
        applyRule = true;
      } else if (packageRule.packagePatterns) {
        logger.debug({ dependency }, 'Checking against packagePatterns');
        for (const packagePattern of packageRule.packagePatterns) {
          const packageRegex = new RegExp(
            packagePattern === '^*$' || packagePattern === '*'
              ? '.*'
              : packagePattern
          );
          logger.trace(
            { dependency, packagePattern },
            'Checking against packagePattern'
          );
          if (depConfig.depName.match(packageRegex)) {
            logger.debug(
              `${depConfig.depName} matches against ${packageRegex}`
            );
            applyRule = true;
          }
        }
      }
      if (
        packageRule.excludePackageNames &&
        packageRule.excludePackageNames.includes(depConfig.depName)
      ) {
        applyRule = false;
      } else if (packageRule.excludePackagePatterns) {
        for (const packagePattern of packageRule.excludePackagePatterns) {
          const packageRegex = new RegExp(packagePattern);
          if (depConfig.depName.match(packageRegex)) {
            applyRule = false;
          }
        }
      }
      if (applyRule) {
        // Package rule config overrides any existing config
        depConfig = configParser.mergeChildConfig(depConfig, packageRule);
        delete depConfig.packageNames;
        delete depConfig.packagePatterns;
        delete depConfig.excludePackageNames;
        delete depConfig.excludePackagePatterns;
      }
    });
  }
  return configParser.filterConfig(depConfig, 'package');
}
<commit_msg>chore: lower debug to trace for packagePatterns<commit_after>const configParser = require('../../config');
const pkgWorker = require('../package');
const { extractDependencies } = require('../../manager');

module.exports = {
  renovateDepType,
  getDepConfig,
};

async function renovateDepType(packageContent, config) {
  logger.setMeta({
    repository: config.repository,
    packageFile: config.packageFile,
    depType: config.depType,
  });
  logger.debug('renovateDepType()');
  logger.trace({ config });
  if (config.enabled === false) {
    logger.debug('depType is disabled');
    return [];
  }
  let deps = await extractDependencies(packageContent, config);
  if (config.lerna || config.workspaces || config.workspaceDir) {
    deps = deps.filter(
      dependency => config.monorepoPackages.indexOf(dependency.depName) === -1
    );
  }
  deps = deps.filter(
    dependency => config.ignoreDeps.indexOf(dependency.depName) === -1
  );
  logger.debug(`filtered deps length is ${deps.length}`);
  logger.debug({ deps }, `filtered deps`);
  // Obtain full config for each dependency
  const depConfigs = deps.map(dep => module.exports.getDepConfig(config, dep));
  logger.trace({ config: depConfigs }, `depConfigs`);
  // renovateDepType can return more than one upgrade each
  const pkgWorkers = depConfigs.map(depConfig =>
    pkgWorker.renovatePackage(depConfig)
  );
  // Use Promise.all to execute npm queries in parallel
  const allUpgrades = await Promise.all(pkgWorkers);
  logger.trace({ config: allUpgrades }, `allUpgrades`);
  // Squash arrays into one
  const combinedUpgrades = [].concat(...allUpgrades);
  logger.trace({ config: combinedUpgrades }, `combinedUpgrades`);
  return combinedUpgrades;
}

function getDepConfig(depTypeConfig, dep) {
  const { depName: dependency } = dep;
  let depConfig = configParser.mergeChildConfig(depTypeConfig, dep);
  // Apply any matching package rules
  if (depConfig.packageRules) {
    logger.trace(
      { dependency, packageRules: depConfig.packageRules },
      `Checking against ${depConfig.packageRules.length} packageRules`
    );
    depConfig.packageRules.forEach(packageRule => {
      let applyRule = false;
      if (!(packageRule.packageNames || packageRule.packagePatterns)) {
        logger.debug(
          { packageRule, dependency },
          'packageRule is missing packageNames and packagePatterns so will match anything'
        );
        applyRule = true;
      } else if (
        packageRule.packageNames &&
        packageRule.packageNames.includes(depConfig.depName)
      ) {
        logger.debug(
          { dependency, packageNames: packageRule.packageNames },
          'Matched packageNames'
        );
        applyRule = true;
      } else if (packageRule.packagePatterns) {
        logger.trace({ dependency }, 'Checking against packagePatterns');
        for (const packagePattern of packageRule.packagePatterns) {
          const packageRegex = new RegExp(
            packagePattern === '^*$' || packagePattern === '*'
              ? '.*'
              : packagePattern
          );
          logger.trace(
            { dependency, packagePattern },
            'Checking against packagePattern'
          );
          if (depConfig.depName.match(packageRegex)) {
            logger.debug(
              `${depConfig.depName} matches against ${packageRegex}`
            );
            applyRule = true;
          }
        }
      }
      if (
        packageRule.excludePackageNames &&
        packageRule.excludePackageNames.includes(depConfig.depName)
      ) {
        applyRule = false;
      } else if (packageRule.excludePackagePatterns) {
        for (const packagePattern of packageRule.excludePackagePatterns) {
          const packageRegex = new RegExp(packagePattern);
          if (depConfig.depName.match(packageRegex)) {
            applyRule = false;
          }
        }
      }
      if (applyRule) {
        // Package rule config overrides any existing config
        depConfig = configParser.mergeChildConfig(depConfig, packageRule);
        delete depConfig.packageNames;
        delete depConfig.packagePatterns;
        delete depConfig.excludePackageNames;
        delete depConfig.excludePackagePatterns;
      }
    });
  }
  return configParser.filterConfig(depConfig, 'package');
}
<|endoftext|>


<commit_before>/**
 * The MIT License (MIT)
 *
 * Copyright (c) 2016-2017 Mickael Jeanroy
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the 'Software'), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

'use strict';

const fs = require('fs');
const path = require('path');
const tmp = require('tmp');
const rollup = require('rollup');
const commonjs = require('rollup-plugin-commonjs');
const bowerResolve = require('../dist/rollup-plugin-bower-resolve');

describe('bowerResolve', () => {
  let cwd;
  let tmpDir;

  beforeEach(() => {
    cwd = process.cwd();
  });

  afterEach(() => {
    process.chdir(cwd);
  });

  beforeEach(() => {
    tmpDir = tmp.dirSync({
      unsafeCleanup: true,
    });
  });

  afterEach(() => {
    tmpDir.removeCallback();
  });

  it('should resolve dependency path of direct dependency', (done) => {
    const test1 = path.join(__dirname, 'it', 'test1');

    process.chdir(test1);

    const plugin = bowerResolve();
    const promise = plugin.resolveId('underscore', './index.js');

    promise.then((dependencyPath) => {
      expect(dependencyPath).toBeDefined();
      expect(dependencyPath).toBe(path.join(test1, 'vendors', 'underscore', 'underscore.js'));
      done();
    });

    promise.catch((err) => {
      done.fail(err);
    });
  });

  it('should resolve dependency path of transitive dependency', (done) => {
    const test2 = path.join(__dirname, 'it', 'test2');

    process.chdir(test2);

    const plugin = bowerResolve();
    const promise = plugin.resolveId('underscore', './index.js');

    promise.then((dependencyPath) => {
      expect(dependencyPath).toBeDefined();
      expect(dependencyPath).toBe(path.join(test2, 'vendors', 'underscore', 'underscore.js'));
      done();
    });

    promise.catch((err) => {
      done.fail(err);
    });
  });

  it('should bundle file', (done) => {
    const test1Dir = path.join(__dirname, 'it', 'test1');
    process.chdir(test1Dir);

    const bundleOutput = path.join(tmpDir.name, 'bundle.js');
    const bundleInput = path.join(test1Dir, 'bundle.js');

    const rollupConfig = {
      entry: bundleInput,
      dest: bundleOutput,
      format: 'es',
      plugins: [
        bowerResolve(),
        commonjs(),
      ],
    };

    rollup.rollup(rollupConfig)
      .then((bundle) => bundle.write(rollupConfig))
      .then(() => {
        fs.readFile(bundleOutput, 'utf8', (err, data) => {
          if (err) {
            done.fail(err);
          }

          const content = data.toString();
          expect(content).toBeDefined();
          expect(content).toContain('_.VERSION');
          done();
        });
      });
  });
});
<commit_msg>test: migrate test for rollup ~0.48.0 compatibility<commit_after>/**
 * The MIT License (MIT)
 *
 * Copyright (c) 2016-2017 Mickael Jeanroy
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the 'Software'), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

'use strict';

const fs = require('fs');
const path = require('path');
const tmp = require('tmp');
const rollup = require('rollup');
const commonjs = require('rollup-plugin-commonjs');
const bowerResolve = require('../dist/rollup-plugin-bower-resolve');

describe('bowerResolve', () => {
  let cwd;
  let tmpDir;

  beforeEach(() => {
    cwd = process.cwd();
  });

  afterEach(() => {
    process.chdir(cwd);
  });

  beforeEach(() => {
    tmpDir = tmp.dirSync({
      unsafeCleanup: true,
    });
  });

  afterEach(() => {
    tmpDir.removeCallback();
  });

  it('should resolve dependency path of direct dependency', (done) => {
    const test1 = path.join(__dirname, 'it', 'test1');

    process.chdir(test1);

    const plugin = bowerResolve();
    const promise = plugin.resolveId('underscore', './index.js');

    promise.then((dependencyPath) => {
      expect(dependencyPath).toBeDefined();
      expect(dependencyPath).toBe(path.join(test1, 'vendors', 'underscore', 'underscore.js'));
      done();
    });

    promise.catch((err) => {
      done.fail(err);
    });
  });

  it('should resolve dependency path of transitive dependency', (done) => {
    const test2 = path.join(__dirname, 'it', 'test2');

    process.chdir(test2);

    const plugin = bowerResolve();
    const promise = plugin.resolveId('underscore', './index.js');

    promise.then((dependencyPath) => {
      expect(dependencyPath).toBeDefined();
      expect(dependencyPath).toBe(path.join(test2, 'vendors', 'underscore', 'underscore.js'));
      done();
    });

    promise.catch((err) => {
      done.fail(err);
    });
  });

  it('should bundle file', (done) => {
    const test1Dir = path.join(__dirname, 'it', 'test1');
    process.chdir(test1Dir);

    const bundleOutput = path.join(tmpDir.name, 'bundle.js');
    const bundleInput = path.join(test1Dir, 'bundle.js');

    const rollupConfig = {
      // Keep the `entry` option because of `rollup-plugin-commonjs`.
      entry: bundleInput,

      input: bundleInput,

      output: {
        file: bundleOutput,
        format: 'es',
      },

      plugins: [
        bowerResolve(),
        commonjs(),
      ],
    };

    rollup.rollup(rollupConfig)
      .then((bundle) => bundle.write(rollupConfig.output))
      .then(() => {
        fs.readFile(bundleOutput, 'utf8', (err, data) => {
          if (err) {
            done.fail(err);
          }

          const content = data.toString();
          expect(content).toBeDefined();
          expect(content).toContain('_.VERSION');
          done();
        });
      });
  });
});
<|endoftext|>


<commit_before>var jsonstream = require("JSONstream");
var through = require("through2");
var fs= require('fs');

fs.createReadStream('t.json')
  .pipe(jsonstream.parse('features'))
  .pipe(through(function(chunk, enc, cb){
    console.log("Got chunk: %s", chunk.toString());
  }))
<commit_msg>Testing shows deficiencies in the JSONStream module. Depends on through style streams.. Should reimplement on readable-stream-based stream2 style<commit_after>var jsonstream = require("JSONstream");
var through = require("through2");
var fs= require('fs');

var jStream = jsonstream.parse('features..type');
jStream.on('root',function(){console.log(arguments)});
jStream.on('error',function(e){console.log(e)});

fs.createReadStream('t.json')
  .pipe(jStream)
  .pipe(through(function(chunk, enc, cb){
    console.log(typeof chunk);
    console.log(chunk);
    this.push(chunk);
  })).pipe(process.stdout);
<|endoftext|>