<commit_before>var path = require('path'),
    fs = require('fs'),
    External = require('./external'),
    Step = require('step'),
    _ = require('underscore')._,
    sys = require('sys'),
    srs = require('srs'),
    carto = require('carto'),
    tree = require('carto/tree');

require.paths.unshift(path.join(__dirname, '..', 'lib'));

// Rendering circuitry for JSON map manifests.
// This is node-only for the time being.
carto.Renderer = function Renderer(env) {
    env = _.extend({}, env);
    if (!env.data_dir) env.data_dir = '/tmp/';
    if (!env.local_data_dir) env.local_data_dir = '';
    if (!env.validation_data) env.validation_data = false;

    return {
        // Keep a copy of passed-in environment variables
        env: env,

        // Ensure that map layers have a populated SRS value and attempt to
        // autodetect SRS if missing. Requires that node-srs is available and
        // that any remote datasources have been localized.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        ensureSRS: function(m, callback) {
            Step(
                function autodetectSRS() {
                    var group = this.group();
                    var autodetect = _.filter(m.Layer, function(l) { return !l.srs; });
                    _.each(autodetect, function(l) {
                        var next = group();
                        Step(
                            function() {
                                fs.readdir(path.dirname(l.Datasource.file), this);
                            },
                            function(err, files) {
                                // Confirm that layers have .prj files
                                // in line with the .shp and .dbf, etc files.
                                var prj = _.detect(files, function(f) {
                                    return path.extname(f).toLowerCase() == '.prj';
                                });
                                if (prj) {
                                    prj = path.join(path.dirname(l.Datasource.file), prj);
                                    fs.readFile(prj, 'utf-8', this);
                                } else {
                                    this(new Error('No projection found'));
                                }
                            },
                            function(err, srs_txt) {
                                if (!err) {
                                    try {
                                        l.srs = srs.parse(srs_txt).proj4;
                                    } catch (err) {}
                                    if (!l.srs) {
                                        try {
                                            l.srs = srs.parse('ESRI::' + srs_txt).proj4;
                                        } catch(err) {
                                            // TODO: handle parse problems
                                        }
                                    }
                                }
                                next(err);
                            }
                        );
                    });
                    // If no layers missing SRS information, next.
                    autodetect.length === 0 && group()();
                },
                function waitForDetection(err) {
                    m.Layer = _.filter(m.Layer, function(l) { return l.srs; });
                    callback(err, m);
                }
            );
        },

        // Download any file-based remote datsources.
        //
        // Usable as an entry point: does not expect any modification to
        // the map object beyond JSON parsing.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        localizeExternals: function(m, callback) {
            var that = this;
            if (env.only_validate === true) {
                callback(null, m);
                return;
            }
            Step(
                function downloadExternals() {
                    var group = this.group();
                    m.Layer.forEach(function(l) {
                        if (l.Datasource.file) {
                            var next = group();
                            new External(that.env, l.Datasource.file)
                                .on('err', function() {
                                    next('Datasource could not be downloaded.');
                                })
                                .on('complete', function(external) {
                                    external.findDataFile(function(err, file) {
                                        if (err) {
                                            next(err);
                                        }
                                        else if (file) {
                                            l.Datasource.file = file;
                                            next(null);
                                        }
                                        else {
                                            next("No .shp file found in zipfile.");
                                        }
                                    });
                                });
                        } else {
                            throw 'Layer does not have a datasource';
                        }
                    });
                    // Continue even if we don't have any layers.
                    group()();
                },
                function waitForDownloads(err) {
                    callback(err, m);
                }
            );
        },

        // Download any remote stylesheets
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        localizeStyle: function(m, callback) {
            var that = this;
            Step(
                function downloadStyles() {
                    var group = this.group();
                    m.Stylesheet.forEach(function(s, k) {
                        if (!s.id) {
                            var next = group();
                            // TODO: handle stylesheet externals
                            // that fail.
                            new External(that.env, s)
                                .on('complete', function(external) {
                                    m.Stylesheet[k] = external.path();
                                    next();
                                });
                        }
                    });
                    group()();
                },
                function waitForDownloads(err, results) {
                    callback(err, m);
                }
            );
        },

        // Compile (already downloaded) styles with carto,
        // calling callback with an array of [map object, [stylesheet objects]]
        //
        // Called with the results of localizeStyle or localizeExternals:
        // expects not to handle downloading.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        style: function(m, callback) {
            var that = this;
            Step(
                function loadStyles() {
                    var group = this.group();
                    m.Stylesheet.forEach(function(s) {
                        var next = group();
                        // If a stylesheet is an object with an id
                        // property, it will have a .data property
                        // as well which can be parsed directly
                        if (s.id) {
                            next(null, [s.id, s.data]);
                        // Otherwise the value is assumed to be a
                        // string and is read from the filesystem
                        } else {
                            fs.readFile(s, 'utf-8', function(err, data) {
                                next(err, [s, data]);
                            });
                        }
                    });
                },
                function compileStyles(e, results) {
                    var options = {},
                        group = this.group();
                    results.forEach(function(result) {
                        var next = group();
                        var parsingTime = +new Date;
                        // Maintain a parsing environment from
                        // stylesheet to stylesheet, so that frames
                        // and effects are maintained.
                        var parse_env = _.extend(
                            _.extend(
                                that.env,
                                this.env
                            ), { filename: result[0] });
                        new carto.Parser(parse_env).parse(result[1],
                            function(err, tree) {
                            if (env.benchmark) console.warn('Parsing time: '
                                + ((new Date - parsingTime))
                                + 'ms');
                            try {
                                next(err, [
                                    result[0],
                                    tree]);
                                return;
                            } catch (e) {
                                throw e;
                                return;
                            }
                        });
                    });
                },
                function waitForCompilation(err, res) {
                    callback(err, m, res);
                }
            );
        },

        addRules: function(current, definition, existing) {
            var newFilters = definition.filters;
            var newRules = definition.rules;
            var updatedFilters, clone, previous;

            // The current definition might have been split up into
            // multiple definitions already.
            for (var k = 0; k < current.length; k++) {
                updatedFilters = current[k].filters.cloneWith(newFilters);
                if (updatedFilters) {
                    previous = existing[updatedFilters];
                    if (previous) {
                        // There's already a definition with those exact
                        // filters. Add the current definitions' rules
                        // and stop processing it as the existing rule
                        // has already gone down the inheritance chain.
                        previous.addRules(newRules);
                    } else {
                        clone = current[k].clone(updatedFilters);
                        // Make sure that we're only maintaining the clone
                        // when we did actually add rules. If not, there's
                        // no need to keep the clone around.
                        if (clone.addRules(newRules)) {
                            // We inserted an element before this one, so we need
                            // to make sure that in the next loop iteration, we're
                            // not performing the same task for this element again,
                            // hence the k++.
                            current.splice(k, 0, clone);
                            k++;
                        }
                    }
                } else if (updatedFilters === null) {
                    // Filters can be added, but they don't change the
                    // filters. This means we don't have to split the
                    // definition.
                    current[k].addRules(newRules);
                }
            }
            return current;
        },

        /**
         * Apply inherited styles from their
         * ancestors to them.
         */
        inheritRules: function(definitions) {
            var inheritTime = +new Date();
            // definitions are ordered by specificity,
            // high (index 0) to low
            var byAttachment = {}, byFilter = {};
            var result = [];
            var current, previous, attachment;

            for (var i = 0; i < definitions.length; i++) {
                attachment = definitions[i].attachment;
                if (!byAttachment[attachment]) {
                    byAttachment[attachment] = [];
                    byAttachment[attachment].attachment = attachment;
                    byFilter[attachment] = {};
                    result.push(byAttachment[attachment]);
                }

                current = [ definitions[i] ];
                // Iterate over all subsequent rules.
                for (var j = i + 1; j < definitions.length; j++) {
                    if (definitions[j].attachment === attachment) {
                        // Only inherit rules from the same attachment.
                        current = this.addRules(current, definitions[j], byFilter);
                    }
                }

                for (var j = 0; j < current.length; j++) {
                    byFilter[attachment][current[j].filters] = current[j];
                    byAttachment[attachment].push(current[j]);
                }
            }

            if (env.benchmark) console.warn('Inheritance time: ' + ((new Date - inheritTime)) + 'ms');

            return result;
        },

        // Find a rule like Map { background-color: #fff; },
        // if any, and return a list of properties to be inserted
        // into the <Map element of the resulting XML.
        //
        // - @param {Array} rulesets the output of toList.
        // - @param {Object} env.
        // - @return {String} rendered properties.
        getMapProperties: function(rulesets, env) {
            var rules = {};
            rulesets.filter(function(r) {
                return r.elements.join('') === 'Map';
            }).forEach(function(r) {
                for (var i = 0; i < r.rules.length; i++) {
                    var key = r.rules[i].name;
                    if (!(key in tree.Reference.data.symbolizers.map)) {
                        throw {
                            message: 'Rule ' + key + ' not allowed for Map.',
                            index: r.rules[i].index
                        };
                    }
                    rules[key] = r.rules[i].eval(env).toXML(env);
                }
            });
            return rules;
        },

        // Prepare full XML map output. Called with the results
        // of this.style
        //
        // - @param {Array} res array of [map object, stylesheets].
        // - @param {Function} callback
        template: function(err, m, stylesheets, callback) {
            // frames is a container for variables and other less.js
            // constructs.
            //
            // effects is a container for side-effects, which currently
            // are limited to FontSets.
            //
            // deferred_externals is a list of externals, like
            // pattern images, etc (by string URL)
            // referred to by url() constructs in styles
            var that = this,
                env = _.extend({
                    effects: [],
                    deferred_externals: []
                }, that.env),
                output = [];

            if (err) {
                callback(err);
                return;
            }

            var rulesets = _.flatten(stylesheets.map(function(rulesets) {
                return rulesets[1].toList(env);
            }));

            // Iterate through layers and create styles custom-built
            // for each of them, and apply those styles to the layers.
            m.Layer.forEach(function(l) {
                l.styles = [];
                // Classes are given as space-separated alphanumeric
                // strings.
                var classes = (l['class'] || '').split(/\s+/g);

                var matching = rulesets.filter(function(definition) {
                    return definition.appliesTo(l.name, classes);
                });

                var definitions = that.inheritRules(matching);

                for (var i = 0; i < definitions.length; i++) {
                    var style = new tree.Style(l.name, definitions[i].attachment, definitions[i]);
                    if (style) {
                        l.styles.push(style.name);

                        // env.effects can be modified by this call
                        output.push(style.toXML(env));
                    }
                }

                var nl = new carto.tree.Layer(l);
                output.push(nl.toXML());
            });
            output.unshift(env.effects.map(function(e) {
                return e.toXML(env);
            }).join('\n'));

            Step(
                function() {
                    var group = this.group();
                    try {
                        var map_properties = that.getMapProperties(rulesets, env);
                    } catch(err) {
                        env.error(err);
                        callback(env.errors);
                        return;
                    }

                    if (!map_properties.srs) {
                        map_properties.srs = 'srs="+proj=merc +a=6378137 ' +
                            '+b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 ' +
                            '+k=1.0 +units=m +nadgrids=@null +no_defs"';
                    }

                    // The only_validate flag can be set to prevent
                    // any externals from being downloaded - for instance,
                    // when only validation is needed.
                    if (env.only_validate !== true &&
                        env.deferred_externals.length) {
                        env.deferred_externals.forEach(function(def) {
                            var next = group();
                            new External(that.env, def)
                                .on('complete', function(external) {
                                    next(null, map_properties);
                                });
                        });
                    } else {
                        var next = group();
                        next(null, map_properties);
                    }
                },
                function(err, map_properties) {
                    var properties = [];
                    for (var key in map_properties[0]) {
                        properties.push(' ' + map_properties[0][key]);
                    }

                    output.unshift(
                        '<?xml version="1.0" '
                        + 'encoding="utf-8"?>\n'
                        + '<!DOCTYPE Map[]>\n'
                        + '<Map' + properties.join('') + '>\n');

                    output.push('</Map>');

                    if (env.errors.length) {
                        callback(env.errors);
                    } else {
                        callback(null, output.join('\n'))
                    }
                }
            );
        },

        // Prepare a MML document (given as a string) into a
        // fully-localized XML file ready for Mapnik2 consumption
        //
        // - @param {String} str the JSON file as a string.
        // - @param {Function} callback to be called with err,
        //   XML representation.
        render: function(str, callback) {
            var m = (typeof str == "string") ? JSON.parse(str) : str,
                that = this;

            var localizingTime = +new Date;
            this.localizeExternals(m, function(err, res) {
                that.ensureSRS(res, function(err, res) {
                    that.localizeStyle(res, function(err, res) {
                        if (env.benchmark) console.warn('Localizing time: '
                            + ((new Date - localizingTime)) + 'ms');
                        that.style(res, function(err, m, res) {
                            var compilingTime = +new Date;
                            that.template(err, m, res, function(err, res) {
                                if (env.benchmark) console.warn('Compiling time: '
                                    + ((new Date - compilingTime)) + 'ms');
                                callback(err, res);
                            });
                        });
                    });
                });
            });
        }
    };
};

module.exports = carto;
<commit_msg>Don't call ensureSRS contents when only_validate is true. Fixes #30.<commit_after>var path = require('path'),
    fs = require('fs'),
    External = require('./external'),
    Step = require('step'),
    _ = require('underscore')._,
    sys = require('sys'),
    srs = require('srs'),
    carto = require('carto'),
    tree = require('carto/tree');

require.paths.unshift(path.join(__dirname, '..', 'lib'));

// Rendering circuitry for JSON map manifests.
// This is node-only for the time being.
carto.Renderer = function Renderer(env) {
    env = _.extend({}, env);
    if (!env.data_dir) env.data_dir = '/tmp/';
    if (!env.local_data_dir) env.local_data_dir = '';
    if (!env.validation_data) env.validation_data = false;

    return {
        // Keep a copy of passed-in environment variables
        env: env,

        // Ensure that map layers have a populated SRS value and attempt to
        // autodetect SRS if missing. Requires that node-srs is available and
        // that any remote datasources have been localized.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        ensureSRS: function(m, callback) {
            if (env.only_validate === true) {
                callback(null, m);
                return;
            }
            Step(
                function autodetectSRS() {
                    var group = this.group();
                    var autodetect = _.filter(m.Layer, function(l) { return !l.srs; });
                    _.each(autodetect, function(l) {
                        var next = group();
                        Step(
                            function() {
                                fs.readdir(path.dirname(l.Datasource.file), this);
                            },
                            function(err, files) {
                                // Confirm that layers have .prj files
                                // in line with the .shp and .dbf, etc files.
                                var prj = _.detect(files, function(f) {
                                    return path.extname(f).toLowerCase() == '.prj';
                                });
                                if (prj) {
                                    prj = path.join(path.dirname(l.Datasource.file), prj);
                                    fs.readFile(prj, 'utf-8', this);
                                } else {
                                    this(new Error('No projection found'));
                                }
                            },
                            function(err, srs_txt) {
                                if (!err) {
                                    try {
                                        l.srs = srs.parse(srs_txt).proj4;
                                    } catch (err) {}
                                    if (!l.srs) {
                                        try {
                                            l.srs = srs.parse('ESRI::' + srs_txt).proj4;
                                        } catch(err) {
                                            // TODO: handle parse problems
                                        }
                                    }
                                }
                                next(err);
                            }
                        );
                    });
                    // If no layers missing SRS information, next.
                    autodetect.length === 0 && group()();
                },
                function waitForDetection(err) {
                    m.Layer = _.filter(m.Layer, function(l) { return l.srs; });
                    callback(err, m);
                }
            );
        },

        // Download any file-based remote datsources.
        //
        // Usable as an entry point: does not expect any modification to
        // the map object beyond JSON parsing.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        localizeExternals: function(m, callback) {
            var that = this;
            if (env.only_validate === true) {
                callback(null, m);
                return;
            }
            Step(
                function downloadExternals() {
                    var group = this.group();
                    m.Layer.forEach(function(l) {
                        if (l.Datasource.file) {
                            var next = group();
                            new External(that.env, l.Datasource.file)
                                .on('err', function() {
                                    next('Datasource could not be downloaded.');
                                })
                                .on('complete', function(external) {
                                    external.findDataFile(function(err, file) {
                                        if (err) {
                                            next(err);
                                        }
                                        else if (file) {
                                            l.Datasource.file = file;
                                            next(null);
                                        }
                                        else {
                                            next("No .shp file found in zipfile.");
                                        }
                                    });
                                });
                        } else {
                            throw 'Layer does not have a datasource';
                        }
                    });
                    // Continue even if we don't have any layers.
                    group()();
                },
                function waitForDownloads(err) {
                    callback(err, m);
                }
            );
        },

        // Download any remote stylesheets
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        localizeStyle: function(m, callback) {
            var that = this;
            Step(
                function downloadStyles() {
                    var group = this.group();
                    m.Stylesheet.forEach(function(s, k) {
                        if (!s.id) {
                            var next = group();
                            // TODO: handle stylesheet externals
                            // that fail.
                            new External(that.env, s)
                                .on('complete', function(external) {
                                    m.Stylesheet[k] = external.path();
                                    next();
                                });
                        }
                    });
                    group()();
                },
                function waitForDownloads(err, results) {
                    callback(err, m);
                }
            );
        },

        // Compile (already downloaded) styles with carto,
        // calling callback with an array of [map object, [stylesheet objects]]
        //
        // Called with the results of localizeStyle or localizeExternals:
        // expects not to handle downloading.
        //
        // - @param {Object} m map object.
        // - @param {Function} callback
        style: function(m, callback) {
            var that = this;
            Step(
                function loadStyles() {
                    var group = this.group();
                    m.Stylesheet.forEach(function(s) {
                        var next = group();
                        // If a stylesheet is an object with an id
                        // property, it will have a .data property
                        // as well which can be parsed directly
                        if (s.id) {
                            next(null, [s.id, s.data]);
                        // Otherwise the value is assumed to be a
                        // string and is read from the filesystem
                        } else {
                            fs.readFile(s, 'utf-8', function(err, data) {
                                next(err, [s, data]);
                            });
                        }
                    });
                },
                function compileStyles(e, results) {
                    var options = {},
                        group = this.group();
                    results.forEach(function(result) {
                        var next = group();
                        var parsingTime = +new Date;
                        // Maintain a parsing environment from
                        // stylesheet to stylesheet, so that frames
                        // and effects are maintained.
                        var parse_env = _.extend(
                            _.extend(
                                that.env,
                                this.env
                            ), { filename: result[0] });
                        (new carto.Parser(parse_env)).parse(result[1],
                            function(err, tree) {
                            if (env.benchmark) console.warn('Parsing time: '
                                + ((new Date - parsingTime))
                                + 'ms');
                            try {
                                next(err, [
                                    result[0],
                                    tree]);
                                return;
                            } catch (e) {
                                throw e;
                                return;
                            }
                        });
                    });
                },
                function waitForCompilation(err, res) {
                    callback(err, m, res);
                }
            );
        },

        addRules: function(current, definition, existing) {
            var newFilters = definition.filters;
            var newRules = definition.rules;
            var updatedFilters, clone, previous;

            // The current definition might have been split up into
            // multiple definitions already.
            for (var k = 0; k < current.length; k++) {
                updatedFilters = current[k].filters.cloneWith(newFilters);
                if (updatedFilters) {
                    previous = existing[updatedFilters];
                    if (previous) {
                        // There's already a definition with those exact
                        // filters. Add the current definitions' rules
                        // and stop processing it as the existing rule
                        // has already gone down the inheritance chain.
                        previous.addRules(newRules);
                    } else {
                        clone = current[k].clone(updatedFilters);
                        // Make sure that we're only maintaining the clone
                        // when we did actually add rules. If not, there's
                        // no need to keep the clone around.
                        if (clone.addRules(newRules)) {
                            // We inserted an element before this one, so we need
                            // to make sure that in the next loop iteration, we're
                            // not performing the same task for this element again,
                            // hence the k++.
                            current.splice(k, 0, clone);
                            k++;
                        }
                    }
                } else if (updatedFilters === null) {
                    // Filters can be added, but they don't change the
                    // filters. This means we don't have to split the
                    // definition.
                    current[k].addRules(newRules);
                }
            }
            return current;
        },

        /**
         * Apply inherited styles from their
         * ancestors to them.
         */
        inheritRules: function(definitions) {
            var inheritTime = +new Date();
            // definitions are ordered by specificity,
            // high (index 0) to low
            var byAttachment = {}, byFilter = {};
            var result = [];
            var current, previous, attachment;

            for (var i = 0; i < definitions.length; i++) {
                attachment = definitions[i].attachment;
                if (!byAttachment[attachment]) {
                    byAttachment[attachment] = [];
                    byAttachment[attachment].attachment = attachment;
                    byFilter[attachment] = {};
                    result.push(byAttachment[attachment]);
                }

                current = [ definitions[i] ];
                // Iterate over all subsequent rules.
                for (var j = i + 1; j < definitions.length; j++) {
                    if (definitions[j].attachment === attachment) {
                        // Only inherit rules from the same attachment.
                        current = this.addRules(current, definitions[j], byFilter);
                    }
                }

                for (var j = 0; j < current.length; j++) {
                    byFilter[attachment][current[j].filters] = current[j];
                    byAttachment[attachment].push(current[j]);
                }
            }

            if (env.benchmark) console.warn('Inheritance time: ' + ((new Date - inheritTime)) + 'ms');

            return result;
        },

        // Find a rule like Map { background-color: #fff; },
        // if any, and return a list of properties to be inserted
        // into the <Map element of the resulting XML.
        //
        // - @param {Array} rulesets the output of toList.
        // - @param {Object} env.
        // - @return {String} rendered properties.
        getMapProperties: function(rulesets, env) {
            var rules = {};
            rulesets.filter(function(r) {
                return r.elements.join('') === 'Map';
            }).forEach(function(r) {
                for (var i = 0; i < r.rules.length; i++) {
                    var key = r.rules[i].name;
                    if (!(key in tree.Reference.data.symbolizers.map)) {
                        throw {
                            message: 'Rule ' + key + ' not allowed for Map.',
                            index: r.rules[i].index
                        };
                    }
                    rules[key] = r.rules[i].eval(env).toXML(env);
                }
            });
            return rules;
        },

        // Prepare full XML map output. Called with the results
        // of this.style
        //
        // - @param {Array} res array of [map object, stylesheets].
        // - @param {Function} callback
        template: function(err, m, stylesheets, callback) {
            // frames is a container for variables and other less.js
            // constructs.
            //
            // effects is a container for side-effects, which currently
            // are limited to FontSets.
            //
            // deferred_externals is a list of externals, like
            // pattern images, etc (by string URL)
            // referred to by url() constructs in styles
            var that = this,
                env = _.extend({
                    effects: [],
                    deferred_externals: []
                }, that.env),
                output = [];

            if (err) {
                callback(err);
                return;
            }

            var rulesets = _.flatten(stylesheets.map(function(rulesets) {
                return rulesets[1].toList(env);
            }));

            // Iterate through layers and create styles custom-built
            // for each of them, and apply those styles to the layers.
            m.Layer.forEach(function(l) {
                l.styles = [];
                // Classes are given as space-separated alphanumeric
                // strings.
                var classes = (l['class'] || '').split(/\s+/g);

                var matching = rulesets.filter(function(definition) {
                    return definition.appliesTo(l.name, classes);
                });

                var definitions = that.inheritRules(matching);

                for (var i = 0; i < definitions.length; i++) {
                    var style = new tree.Style(l.name, definitions[i].attachment, definitions[i]);
                    if (style) {
                        l.styles.push(style.name);

                        // env.effects can be modified by this call
                        output.push(style.toXML(env));
                    }
                }

                var nl = new carto.tree.Layer(l);
                output.push(nl.toXML());
            });
            output.unshift(env.effects.map(function(e) {
                return e.toXML(env);
            }).join('\n'));

            Step(
                function() {
                    var group = this.group();
                    try {
                        var map_properties = that.getMapProperties(rulesets, env);
                    } catch(err) {
                        env.error(err);
                        callback(env.errors);
                        return;
                    }

                    if (!map_properties.srs) {
                        map_properties.srs = 'srs="+proj=merc +a=6378137 ' +
                            '+b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 ' +
                            '+k=1.0 +units=m +nadgrids=@null +no_defs"';
                    }

                    // The only_validate flag can be set to prevent
                    // any externals from being downloaded - for instance,
                    // when only validation is needed.
                    if (env.only_validate !== true &&
                        env.deferred_externals.length) {
                        env.deferred_externals.forEach(function(def) {
                            var next = group();
                            new External(that.env, def)
                                .on('complete', function(external) {
                                    next(null, map_properties);
                                });
                        });
                    } else {
                        var next = group();
                        next(null, map_properties);
                    }
                },
                function(err, map_properties) {
                    var properties = [];
                    for (var key in map_properties[0]) {
                        properties.push(' ' + map_properties[0][key]);
                    }

                    output.unshift(
                        '<?xml version="1.0" '
                        + 'encoding="utf-8"?>\n'
                        + '<!DOCTYPE Map[]>\n'
                        + '<Map' + properties.join('') + '>\n');

                    output.push('</Map>');

                    if (env.errors.length) {
                        callback(env.errors);
                    } else {
                        callback(null, output.join('\n'))
                    }
                }
            );
        },

        // Prepare a MML document (given as a string) into a
        // fully-localized XML file ready for Mapnik2 consumption
        //
        // - @param {String} str the JSON file as a string.
        // - @param {Function} callback to be called with err,
        //   XML representation.
        render: function(str, callback) {
            var m = (typeof str == "string") ? JSON.parse(str) : str,
                that = this;

            var localizingTime = +new Date;
            this.localizeExternals(m, function(err, res) {
                that.ensureSRS(res, function(err, res) {
                    that.localizeStyle(res, function(err, res) {
                        if (env.benchmark) console.warn('Localizing time: '
                            + ((new Date - localizingTime)) + 'ms');
                        that.style(res, function(err, m, res) {
                            var compilingTime = +new Date;
                            that.template(err, m, res, function(err, res) {
                                if (env.benchmark) console.warn('Compiling time: '
                                    + ((new Date - compilingTime)) + 'ms');
                                callback(err, res);
                            });
                        });
                    });
                });
            });
        }
    };
};

module.exports = carto;
<|endoftext|>


<commit_before>'use strict';

const _ = require('underscore');
const express = require('express');
const windshaft = require('windshaft');
const { mapnik } = windshaft;

const jsonReplacer = require('./utils/json-replacer');

const ApiRouter = require('./api/api-router');
const ServerInfoController = require('./server-info-controller');

const StatsClient = require('./stats/client');

module.exports = function createServer (serverOptions) {
    validateOptions(serverOptions);

    // Make stats client globally accessible
    global.statsClient = StatsClient.getInstance(serverOptions.statsd);

    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);

    bootstrapFonts(serverOptions);

    const app = express();

    app.enable('jsonp callback');
    app.disable('x-powered-by');
    app.disable('etag');
    app.set('json replacer', jsonReplacer());

    const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
    apiRouter.register(app);

    const versions = getAndValidateVersions(serverOptions);

    const serverInfoController = new ServerInfoController(versions);
    serverInfoController.register(app);

    return app;
};

function validateOptions(opts) {
    if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
        throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
    }
}

function bootstrapFonts(opts) {
    // Set carto renderer configuration for MMLStore
    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
    var cenv = opts.grainstore.carto_env;
    cenv.validation_data = cenv.validation_data || {};
    if ( ! cenv.validation_data.fonts ) {
        mapnik.register_system_fonts();
        mapnik.register_default_fonts();
        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
    }
}

function mapnikVersion(opts) {
    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}

function getAndValidateVersions(options) {
    // jshint undef:false
    var warn = console.warn.bind(console);
    // jshint undef:true

    var packageDefinition = require('../../package.json');

    var declaredDependencies = packageDefinition.dependencies || {};
    var installedDependenciesVersions = {
        camshaft: require('camshaft').version,
        grainstore: windshaft.grainstore.version(),
        mapnik: windshaft.mapnik.versions.mapnik,
        node_mapnik: windshaft.mapnik.version,
        'turbo-carto': require('turbo-carto').version,
        windshaft: windshaft.version,
        windshaft_cartodb: packageDefinition.version
    };

    var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
    dependenciesToValidate.forEach(function(depName) {
        var declaredDependencyVersion = declaredDependencies[depName];
        var installedDependencyVersion = installedDependenciesVersions[depName];
        if (declaredDependencyVersion !== installedDependencyVersion) {
            warn(
                'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
                depName, installedDependencyVersion, declaredDependencyVersion
            );
        }
    });

    // Be nice and warn if configured mapnik version is != installed mapnik version
    if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
        warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' +
            ' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
    }

    return installedDependenciesVersions;
}
<commit_msg>Ignore line<commit_after>'use strict';

const _ = require('underscore');
const express = require('express');
const windshaft = require('windshaft');
const { mapnik } = windshaft;

const jsonReplacer = require('./utils/json-replacer');

const ApiRouter = require('./api/api-router');
const ServerInfoController = require('./server-info-controller');

const StatsClient = require('./stats/client');

module.exports = function createServer (serverOptions) {
    validateOptions(serverOptions);

    // Make stats client globally accessible
    global.statsClient = StatsClient.getInstance(serverOptions.statsd);

    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);

    bootstrapFonts(serverOptions);

    const app = express();

    app.enable('jsonp callback');
    app.disable('x-powered-by');
    app.disable('etag');
    app.set('json replacer', jsonReplacer());

    const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
    apiRouter.register(app);

    const versions = getAndValidateVersions(serverOptions);

    const serverInfoController = new ServerInfoController(versions);
    serverInfoController.register(app);

    return app;
};

function validateOptions(opts) {
    if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
        throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
    }
}

function bootstrapFonts(opts) {
    // Set carto renderer configuration for MMLStore
    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
    var cenv = opts.grainstore.carto_env;
    cenv.validation_data = cenv.validation_data || {};
    if ( ! cenv.validation_data.fonts ) {
        mapnik.register_system_fonts();
        mapnik.register_default_fonts();
        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
    }
}

function mapnikVersion(opts) {
    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}

function getAndValidateVersions(options) {
    var warn = console.warn.bind(console); // jshint ignore:line

    var packageDefinition = require('../../package.json');

    var declaredDependencies = packageDefinition.dependencies || {};
    var installedDependenciesVersions = {
        camshaft: require('camshaft').version,
        grainstore: windshaft.grainstore.version(),
        mapnik: windshaft.mapnik.versions.mapnik,
        node_mapnik: windshaft.mapnik.version,
        'turbo-carto': require('turbo-carto').version,
        windshaft: windshaft.version,
        windshaft_cartodb: packageDefinition.version
    };

    var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
    dependenciesToValidate.forEach(function(depName) {
        var declaredDependencyVersion = declaredDependencies[depName];
        var installedDependencyVersion = installedDependenciesVersions[depName];
        if (declaredDependencyVersion !== installedDependencyVersion) {
            warn(
                'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
                depName, installedDependencyVersion, declaredDependencyVersion
            );
        }
    });

    // Be nice and warn if configured mapnik version is != installed mapnik version
    if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
        warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' +
            ' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
    }

    return installedDependenciesVersions;
}
<|endoftext|>


<commit_before>var assert = require('assert');

var express = require('express');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var _ = require('underscore');
var step = require('step');

var controller = require('./controllers');

var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
var FastlyCacheBackend = require('./cache/backend/fastly');

var StatsClient = require('./stats/client');
var Profiler = require('./stats/profiler_proxy');
var RendererStatsReporter = require('./stats/reporter/renderer');

var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;

var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection');

var CdbRequest = require('./models/cdb_request');
var cdbRequest = new CdbRequest();

var LZMA = require('lzma').LZMA;
// Whitelist query parameters and attach format
var REQUEST_QUERY_PARAMS_WHITELIST = [
    'config',
    'map_key',
    'api_key',
    'auth_token',
    'callback'
];

var lzmaWorker = new LZMA();

var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});

module.exports = function(serverOptions) {
    // Make stats client globally accessible
    global.statsClient = StatsClient.getInstance(serverOptions.statsd);

    var redisPool = new RedisPool(_.defaults(global.environment.redis, {
        name: 'windshaft:server',
        unwatchOnRelease: false,
        noReadyCheck: true
    }));

    redisPool.on('status', function(status) {
        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
        global.statsClient.gauge(keyPrefix + 'count', status.count);
        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
    });

    var metadataBackend = cartodbRedis({pool: redisPool});
    var pgConnection = new PgConnection(metadataBackend);
    var pgQueryRunner = new PgQueryRunner(pgConnection);
    var queryTablesApi = new QueryTablesApi(pgQueryRunner);
    var userLimitsApi = new UserLimitsApi(metadataBackend, {
        limits: {
            cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
            render: serverOptions.renderer.mapnik.limits.render || 0
        }
    });

    var templateMaps = new TemplateMaps(redisPool, {
        max_user_templates: global.environment.maxUserTemplates
    });

    var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));

    function invalidateNamedMap (owner, templateName) {
        var startTime = Date.now();
        surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
            var logMessage = JSON.stringify({
                username: owner,
                type: 'named_map_invalidation',
                elapsed: Date.now() - startTime,
                error: !!err ? JSON.stringify(err.message) : undefined
            });
            if (err) {
                console.warn(logMessage);
            } else {
                console.info(logMessage);
            }
        });
    }
    ['update', 'delete'].forEach(function(eventType) {
        templateMaps.on(eventType, invalidateNamedMap);
    });

    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);

    validateOptions(serverOptions);

    bootstrapFonts(serverOptions);

    // initialize express server
    var app = bootstrap(serverOptions);
    // Extend windshaft with all the elements of the options object
    _.extend(app, serverOptions);

    var mapStore  = new windshaft.storage.MapStore({
        pool: redisPool,
        expire_time: serverOptions.grainstore.default_layergroup_ttl
    });

    var onTileErrorStrategy;
    if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
        onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
            if (err && err.message === 'Render timed out' && format === 'png') {
                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
            } else {
                return callback(err, tile, headers, stats);
            }
        };
    }

    var rendererFactory = new windshaft.renderer.Factory({
        onTileErrorStrategy: onTileErrorStrategy,
        mapnik: {
            redisPool: redisPool,
            grainstore: serverOptions.grainstore,
            mapnik: serverOptions.renderer.mapnik
        },
        http: serverOptions.renderer.http
    });

    // initialize render cache
    var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, {
        ttl: 60000, // 60 seconds TTL by default
        statsInterval: 60000 // reports stats every milliseconds defined here
    });
    var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
    var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
    rendererStatsReporter.start();

    var attributesBackend = new windshaft.backend.Attributes(mapStore);
    var previewBackend = new windshaft.backend.Preview(rendererCache);
    var tileBackend = new windshaft.backend.Tile(rendererCache);
    var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
    var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);

    var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
    app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
    ['update', 'delete'].forEach(function(eventType) {
        templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
    });

    var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);

    var TablesExtentApi = require('./api/tables_extent_api');
    var tablesExtentApi = new TablesExtentApi(pgQueryRunner);

    /*******************************************************************************************************************
     * Routing
     ******************************************************************************************************************/

    app.all('*', function(req, res, next) {
        req.context.user = cdbRequest.userByReq(req);
        next();
    });

    new controller.Layergroup(
        app,
        authApi,
        pgConnection,
        mapStore,
        tileBackend,
        previewBackend,
        attributesBackend,
        surrogateKeysCache,
        userLimitsApi,
        queryTablesApi,
        layergroupAffectedTablesCache
    ).register(app);

    new controller.Map(
        app,
        authApi,
        pgConnection,
        templateMaps,
        mapBackend,
        metadataBackend,
        queryTablesApi,
        surrogateKeysCache,
        userLimitsApi,
        layergroupAffectedTablesCache
    ).register(app);

    new controller.NamedMaps(
        app,
        authApi,
        pgConnection,
        namedMapProviderCache,
        tileBackend,
        previewBackend,
        surrogateKeysCache,
        tablesExtentApi
    ).register(app);

    new controller.NamedMapsAdmin(templateMaps, authApi, pgConnection).register(app);

    new controller.ServerInfo().register(app);

    /*******************************************************************************************************************
     * END Routing
     ******************************************************************************************************************/

    return app;
};

function validateOptions(opts) {
    if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
        throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
    }

    // Be nice and warn if configured mapnik version is != instaled mapnik version
    if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) {
        console.warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
            ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')');
    }
}

function bootstrapFonts(opts) {
    // Set carto renderer configuration for MMLStore
    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
    var cenv = opts.grainstore.carto_env;
    cenv.validation_data = cenv.validation_data || {};
    if ( ! cenv.validation_data.fonts ) {
        mapnik.register_system_fonts();
        mapnik.register_default_fonts();
        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
    }
}

function bootstrap(opts) {
    var app;
    if (_.isObject(opts.https)) {
        // use https if possible
        app = express.createServer(opts.https);
    } else {
        // fall back to http by default
        app = express.createServer();
    }
    app.enable('jsonp callback');
    app.use(express.bodyParser());

    app.use(function bootstrap$prepareRequestResponse(req, res, next) {
        req.context = req.context || {};
        req.profiler = new Profiler({
            statsd_client: global.statsClient,
            profile: opts.useProfiler
        });

        res.removeHeader('x-powered-by');

        res.sendResponse = function(args) {
            if (global.environment && global.environment.api_hostname) {
                res.header('X-Served-By-Host', global.environment.api_hostname);
            }

            if (req.params && req.params.dbhost) {
                res.header('X-Served-By-DB-Host', req.params.dbhost);
            }

            if (req.profiler) {
                res.header('X-Tiler-Profiler', req.profiler.toJSONString());
            }

            res.send.apply(res, args);

            if (req.profiler ) {
                try {
                    // May throw due to dns, see
                    // See http://github.com/CartoDB/Windshaft/issues/166
                    req.profiler.sendStats();
                } catch (err) {
                    console.error("error sending profiling stats: " + err);
                }
            }
        };
        res.sendError = function(err, label) {

            label = label || 'UNKNOWN';

            var statusCode = findStatusCode(err);

            // use console.log for statusCode != 500 ?
            if (statusCode >= 500) {
                console.error('[%s ERROR] -- %d: %s', label, statusCode, err);
            } else {
                console.warn('[%s WARN] -- %d: %s', label, statusCode, err);
            }

            // If a callback was requested, force status to 200
            if (req.query.callback) {
                statusCode = 200;
            }

            // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
            var message = (_.isString(err) ? err : err.message) || 'Unknown error';
            // Strip connection info, if any
            message = message
                // See https://github.com/CartoDB/Windshaft/issues/173
                .replace(/Connection string: '[^']*'\\n/, '')
                // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
                .replace(/is the server.*encountered/im, 'encountered');

            var errorResponseBody = { errors: [message] };

            res.sendResponse([errorResponseBody, statusCode]);
        };
        next();
    });

    // temporary measure until we upgrade to newer version expressjs so we can check err.status
    app.use(function(err, req, res, next) {
        if (err) {
            if (err.name === 'SyntaxError') {
                res.send({ errors: [err.name + ': ' + err.message] }, 400);
            } else {
                next(err);
            }
        } else {
            next();
        }
    });

    setupLogger(app, opts);

    return app;
}

function setupLogger(app, opts) {
    if (opts.log_format) {
        var loggerOpts = {
            // Allowing for unbuffered logging is mainly
            // used to avoid hanging during unit testing.
            // TODO: provide an explicit teardown function instead,
            //       releasing any event handler or timer set by
            //       this component.
            buffer: !opts.unbuffered_logging,
            // optional log format
            format: opts.log_format
        };
        if (global.log4js) {
            app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'})));
        } else {
            app.use(express.logger(loggerOpts));
        }
    }
}

function surrogateKeysCacheBackends(serverOptions) {
    var cacheBackends = [];

    if (serverOptions.varnish_purge_enabled) {
        cacheBackends.push(
            new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
        );
    }

    if (serverOptions.fastly &&
        !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
        cacheBackends.push(
            new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
        );
    }

    return cacheBackends;
}

function findStatusCode(err) {
    var statusCode;
    if ( err.http_status ) {
        statusCode = err.http_status;
    } else {
        statusCode = statusFromErrorMessage('' + err);
    }
    return statusCode;
}
module.exports.findStatusCode = findStatusCode;

function statusFromErrorMessage(errMsg) {
    // Find an appropriate statusCode based on message
    var statusCode = 400;
    if ( -1 !== errMsg.indexOf('permission denied') ) {
        statusCode = 403;
    }
    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
        statusCode = 403;
    }
    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
        statusCode = 400;
    }
    else if ( -1 !== errMsg.indexOf('does not exist') ) {
        if ( -1 !== errMsg.indexOf(' role ') ) {
            statusCode = 403; // role 'xxx' does not exist
        } else {
            statusCode = 404;
        }
    }
    return statusCode;
}

function mapnikVersion(opts) {
    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}
<commit_msg>Remove unused variables<commit_after>var express = require('express');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var _ = require('underscore');

var controller = require('./controllers');

var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
var FastlyCacheBackend = require('./cache/backend/fastly');

var StatsClient = require('./stats/client');
var Profiler = require('./stats/profiler_proxy');
var RendererStatsReporter = require('./stats/reporter/renderer');

var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;

var TemplateMaps = require('./backends/template_maps.js');
var QueryTablesApi = require('./api/query_tables_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection');

var CdbRequest = require('./models/cdb_request');
var cdbRequest = new CdbRequest();

var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});

module.exports = function(serverOptions) {
    // Make stats client globally accessible
    global.statsClient = StatsClient.getInstance(serverOptions.statsd);

    var redisPool = new RedisPool(_.defaults(global.environment.redis, {
        name: 'windshaft:server',
        unwatchOnRelease: false,
        noReadyCheck: true
    }));

    redisPool.on('status', function(status) {
        var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
        global.statsClient.gauge(keyPrefix + 'count', status.count);
        global.statsClient.gauge(keyPrefix + 'unused', status.unused);
        global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
    });

    var metadataBackend = cartodbRedis({pool: redisPool});
    var pgConnection = new PgConnection(metadataBackend);
    var pgQueryRunner = new PgQueryRunner(pgConnection);
    var queryTablesApi = new QueryTablesApi(pgQueryRunner);
    var userLimitsApi = new UserLimitsApi(metadataBackend, {
        limits: {
            cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
            render: serverOptions.renderer.mapnik.limits.render || 0
        }
    });

    var templateMaps = new TemplateMaps(redisPool, {
        max_user_templates: global.environment.maxUserTemplates
    });

    var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));

    function invalidateNamedMap (owner, templateName) {
        var startTime = Date.now();
        surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
            var logMessage = JSON.stringify({
                username: owner,
                type: 'named_map_invalidation',
                elapsed: Date.now() - startTime,
                error: !!err ? JSON.stringify(err.message) : undefined
            });
            if (err) {
                console.warn(logMessage);
            } else {
                console.info(logMessage);
            }
        });
    }
    ['update', 'delete'].forEach(function(eventType) {
        templateMaps.on(eventType, invalidateNamedMap);
    });

    serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);

    validateOptions(serverOptions);

    bootstrapFonts(serverOptions);

    // initialize express server
    var app = bootstrap(serverOptions);
    // Extend windshaft with all the elements of the options object
    _.extend(app, serverOptions);

    var mapStore  = new windshaft.storage.MapStore({
        pool: redisPool,
        expire_time: serverOptions.grainstore.default_layergroup_ttl
    });

    var onTileErrorStrategy;
    if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
        onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
            if (err && err.message === 'Render timed out' && format === 'png') {
                return callback(null, timeoutErrorTile, { 'Content-Type': 'image/png' }, {});
            } else {
                return callback(err, tile, headers, stats);
            }
        };
    }

    var rendererFactory = new windshaft.renderer.Factory({
        onTileErrorStrategy: onTileErrorStrategy,
        mapnik: {
            redisPool: redisPool,
            grainstore: serverOptions.grainstore,
            mapnik: serverOptions.renderer.mapnik
        },
        http: serverOptions.renderer.http
    });

    // initialize render cache
    var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, {
        ttl: 60000, // 60 seconds TTL by default
        statsInterval: 60000 // reports stats every milliseconds defined here
    });
    var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
    var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
    rendererStatsReporter.start();

    var attributesBackend = new windshaft.backend.Attributes(mapStore);
    var previewBackend = new windshaft.backend.Preview(rendererCache);
    var tileBackend = new windshaft.backend.Tile(rendererCache);
    var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
    var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);

    var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
    app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;

    var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
    ['update', 'delete'].forEach(function(eventType) {
        templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
    });

    var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);

    var TablesExtentApi = require('./api/tables_extent_api');
    var tablesExtentApi = new TablesExtentApi(pgQueryRunner);

    /*******************************************************************************************************************
     * Routing
     ******************************************************************************************************************/

    app.all('*', function(req, res, next) {
        req.context.user = cdbRequest.userByReq(req);
        next();
    });

    new controller.Layergroup(
        app,
        authApi,
        pgConnection,
        mapStore,
        tileBackend,
        previewBackend,
        attributesBackend,
        surrogateKeysCache,
        userLimitsApi,
        queryTablesApi,
        layergroupAffectedTablesCache
    ).register(app);

    new controller.Map(
        app,
        authApi,
        pgConnection,
        templateMaps,
        mapBackend,
        metadataBackend,
        queryTablesApi,
        surrogateKeysCache,
        userLimitsApi,
        layergroupAffectedTablesCache
    ).register(app);

    new controller.NamedMaps(
        app,
        authApi,
        pgConnection,
        namedMapProviderCache,
        tileBackend,
        previewBackend,
        surrogateKeysCache,
        tablesExtentApi
    ).register(app);

    new controller.NamedMapsAdmin(templateMaps, authApi, pgConnection).register(app);

    new controller.ServerInfo().register(app);

    /*******************************************************************************************************************
     * END Routing
     ******************************************************************************************************************/

    return app;
};

function validateOptions(opts) {
    if (!_.isString(opts.base_url) || !_.isString(opts.base_url_mapconfig) || !_.isString(opts.base_url_templated)) {
        throw new Error("Must initialise server with: 'base_url'/'base_url_mapconfig'/'base_url_templated' URLs");
    }

    // Be nice and warn if configured mapnik version is != instaled mapnik version
    if (mapnik.versions.mapnik !== opts.grainstore.mapnik_version) {
        console.warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
            ' != configured mapnik version (' + opts.grainstore.mapnik_version + ')');
    }
}

function bootstrapFonts(opts) {
    // Set carto renderer configuration for MMLStore
    opts.grainstore.carto_env = opts.grainstore.carto_env || {};
    var cenv = opts.grainstore.carto_env;
    cenv.validation_data = cenv.validation_data || {};
    if ( ! cenv.validation_data.fonts ) {
        mapnik.register_system_fonts();
        mapnik.register_default_fonts();
        cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
    }
}

function bootstrap(opts) {
    var app;
    if (_.isObject(opts.https)) {
        // use https if possible
        app = express.createServer(opts.https);
    } else {
        // fall back to http by default
        app = express.createServer();
    }
    app.enable('jsonp callback');
    app.use(express.bodyParser());

    app.use(function bootstrap$prepareRequestResponse(req, res, next) {
        req.context = req.context || {};
        req.profiler = new Profiler({
            statsd_client: global.statsClient,
            profile: opts.useProfiler
        });

        res.removeHeader('x-powered-by');

        res.sendResponse = function(args) {
            if (global.environment && global.environment.api_hostname) {
                res.header('X-Served-By-Host', global.environment.api_hostname);
            }

            if (req.params && req.params.dbhost) {
                res.header('X-Served-By-DB-Host', req.params.dbhost);
            }

            if (req.profiler) {
                res.header('X-Tiler-Profiler', req.profiler.toJSONString());
            }

            res.send.apply(res, args);

            if (req.profiler ) {
                try {
                    // May throw due to dns, see
                    // See http://github.com/CartoDB/Windshaft/issues/166
                    req.profiler.sendStats();
                } catch (err) {
                    console.error("error sending profiling stats: " + err);
                }
            }
        };
        res.sendError = function(err, label) {

            label = label || 'UNKNOWN';

            var statusCode = findStatusCode(err);

            // use console.log for statusCode != 500 ?
            if (statusCode >= 500) {
                console.error('[%s ERROR] -- %d: %s', label, statusCode, err);
            } else {
                console.warn('[%s WARN] -- %d: %s', label, statusCode, err);
            }

            // If a callback was requested, force status to 200
            if (req.query.callback) {
                statusCode = 200;
            }

            // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
            var message = (_.isString(err) ? err : err.message) || 'Unknown error';
            // Strip connection info, if any
            message = message
                // See https://github.com/CartoDB/Windshaft/issues/173
                .replace(/Connection string: '[^']*'\\n/, '')
                // See https://travis-ci.org/CartoDB/Windshaft/jobs/20703062#L1644
                .replace(/is the server.*encountered/im, 'encountered');

            var errorResponseBody = { errors: [message] };

            res.sendResponse([errorResponseBody, statusCode]);
        };
        next();
    });

    // temporary measure until we upgrade to newer version expressjs so we can check err.status
    app.use(function(err, req, res, next) {
        if (err) {
            if (err.name === 'SyntaxError') {
                res.send({ errors: [err.name + ': ' + err.message] }, 400);
            } else {
                next(err);
            }
        } else {
            next();
        }
    });

    setupLogger(app, opts);

    return app;
}

function setupLogger(app, opts) {
    if (opts.log_format) {
        var loggerOpts = {
            // Allowing for unbuffered logging is mainly
            // used to avoid hanging during unit testing.
            // TODO: provide an explicit teardown function instead,
            //       releasing any event handler or timer set by
            //       this component.
            buffer: !opts.unbuffered_logging,
            // optional log format
            format: opts.log_format
        };
        if (global.log4js) {
            app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'})));
        } else {
            app.use(express.logger(loggerOpts));
        }
    }
}

function surrogateKeysCacheBackends(serverOptions) {
    var cacheBackends = [];

    if (serverOptions.varnish_purge_enabled) {
        cacheBackends.push(
            new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
        );
    }

    if (serverOptions.fastly &&
        !!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
        cacheBackends.push(
            new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
        );
    }

    return cacheBackends;
}

function findStatusCode(err) {
    var statusCode;
    if ( err.http_status ) {
        statusCode = err.http_status;
    } else {
        statusCode = statusFromErrorMessage('' + err);
    }
    return statusCode;
}
module.exports.findStatusCode = findStatusCode;

function statusFromErrorMessage(errMsg) {
    // Find an appropriate statusCode based on message
    var statusCode = 400;
    if ( -1 !== errMsg.indexOf('permission denied') ) {
        statusCode = 403;
    }
    else if ( -1 !== errMsg.indexOf('authentication failed') ) {
        statusCode = 403;
    }
    else if (errMsg.match(/Postgis Plugin.*[\s|\n].*column.*does not exist/)) {
        statusCode = 400;
    }
    else if ( -1 !== errMsg.indexOf('does not exist') ) {
        if ( -1 !== errMsg.indexOf(' role ') ) {
            statusCode = 403; // role 'xxx' does not exist
        } else {
            statusCode = 404;
        }
    }
    return statusCode;
}

function mapnikVersion(opts) {
    return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}
<|endoftext|>


<commit_before>
var shs = require('../')

var tape = require('tape')
var pull = require('pull-stream')

var cl = require('chloride')
var deepEqual = require('deep-equal')
var bitflipper = require('pull-bitflipper')
var Hang = require('pull-hang')

function hash (str) {
  return cl.crypto_hash_sha256(new Buffer(str))
}

var alice = cl.crypto_sign_seed_keypair(hash('alice'))
var bob   = cl.crypto_sign_seed_keypair(hash('bob'))
var wally = cl.crypto_sign_seed_keypair(hash('wally'))

//var secure = require('../secure')

var app_key = hash('app_key')

function unauthorized (_, cb) {
  cb(new Error('unauthorized'))
}

function authorized (_, cb) {
  cb()
}

tape('test handshake', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err, stream) {

      if(err) throw err

      pull(
        pull.values([new Buffer('hello there')]),
        stream,
        pull.collect(function (err, hello_there) {
          t.equal(hello_there.toString(), 'hello there')
          console.log('output:', hello_there.join(''))
          t.end()
        })
      )

    })

  var r = Math.random()
  var bobHS = shs.server(bob, function (public, cb) {
      t.deepEqual(public, alice.publicKey)

      if(deepEqual(public, alice.publicKey)) cb(null, {okay: true, random: r})
      else
        cb(new Error('unauthorized'))

    }, app_key, 100)
      (function (err, stream) {

        if(err) throw err

        t.deepEqual(stream.auth, {okay: true, random: r})
        pull(stream, pull.through(function (data) {
          console.log('echo:', data.toString())
        }), stream) //ECHO
      })

  pull(
    aliceHS,
    pull.through(console.log.bind(console, 'A->B')),
    bobHS,
    pull.through(console.log.bind(console, 'A<-B')),
    aliceHS
  )

})

function bitflipTest(t, test) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err) {
      t.ok(err, 'Alice errored')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, function (public, cb) {
      t.deepEqual(public, alice.publicKey)

      if(deepEqual(public, alice.publicKey)) cb(null)
      else
        cb(new Error('unauthorized'))

    }, app_key, 100)
      (function (err) {
        t.ok(err, 'Bob errored')
        if(++errs === 2) t.end()
      })

  test(aliceHS, bobHS)

}

tape('test auth fails when first packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bitflipper(1),
      bobHS,
      aliceHS
    )
  })
})

tape('test auth fails when 2nd packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bobHS,
      bitflipper(1),
      aliceHS
    )
  })
})

tape('test auth fails when 3rd packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bitflipper(2),
      bobHS,
      aliceHS
    )
  })
})

tape('test auth fails when 4th packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bobHS,
      bitflipper(2),
      aliceHS
    )
  })
})

tape('test error cb when client is not authorized', function (t) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err) {
      t.ok(err, 'Bob hungup')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, unauthorized, app_key, 100)
      (function (err) {
        t.ok(err, 'client unauthorized')
        if(++errs === 2) t.end()
      })

  pull(aliceHS, bobHS, aliceHS)
})

tape('test error cb when client get wrong number', function (t) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (wally.publicKey, function (err) {
      t.ok(err, 'Bob hungup')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, unauthorized, app_key, 100)
      (function (err) {
        t.ok(err, 'client unauthorized')
        if(++errs === 2) t.end()
      })

  pull(aliceHS, bobHS, aliceHS)
})

tape('error if created without public key', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
  t.throws(function () {
    aliceHS()
  })
  t.end()
})

tape('unauthorized connection must cb once', function (t) {
  t.plan(2)
  var n = 2
  var aliceHS = shs.client(alice, app_key, 100)
  var bobHS = shs.server(bob, authorized, app_key, 100)

  var as = aliceHS(bob.publicKey, function (err, stream) {
    console.log('Alice')
    t.ok(err, 'client connect should fail')
    next()
  })

  pull(as, bobHS(function (err, stream) {
    console.log('Bob')
    t.ok(err, 'server connect should fail')
    next()
  }), as)

  function next () {
    if(--n) return
    t.end()
  }

})

tape('client timeout error if there is no response', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err, stream) {
      t.ok(err)
      t.end()
    })

  pull(
    Hang(),
    aliceHS
  )
  //do nothing, so aliceHS should timeout
})

tape('server timeout error if there is no response', function (t) {

  var bobHS = shs.server(alice, authorized, app_key, 100)
    (function (err, stream) {
      t.ok(err)
      t.end()
    })

  pull(
    Hang(),
    bobHS
  )
  //do nothing, so aliceHS should timeout
})

tape('test handshake', function (t) {

  var aliceHS = shs.client(hash('alice'), app_key, 100)
    (bob.publicKey, function (err, stream) {

      if(err) throw err

    })

  var r = Math.random()
  var bobHS = shs.server(hash('bob'), function (public, cb) {
      t.deepEqual(public, alice.publicKey)
      cb(null, {okay: true, random: r})
    }, app_key, 100)
      (function (err, stream) {
        if(err) throw err

        t.deepEqual(stream.auth, {okay: true, random: r})
        t.end()
      })

  pull(
    aliceHS,
    bobHS,
    aliceHS
  )

})

<commit_msg>test: initialize handshake with null local keypair, and use a seed instead<commit_after>
var shs = require('../')

var tape = require('tape')
var pull = require('pull-stream')

var cl = require('chloride')
var deepEqual = require('deep-equal')
var bitflipper = require('pull-bitflipper')
var Hang = require('pull-hang')

function hash (str) {
  return cl.crypto_hash_sha256(new Buffer(str))
}

var alice = cl.crypto_sign_seed_keypair(hash('alice'))
var bob   = cl.crypto_sign_seed_keypair(hash('bob'))
var wally = cl.crypto_sign_seed_keypair(hash('wally'))

//var secure = require('../secure')

var app_key = hash('app_key')

function unauthorized (_, cb) {
  cb(new Error('unauthorized'))
}

function authorized (_, cb) {
  cb()
}

tape('test handshake', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err, stream) {

      if(err) throw err

      pull(
        pull.values([new Buffer('hello there')]),
        stream,
        pull.collect(function (err, hello_there) {
          t.equal(hello_there.toString(), 'hello there')
          console.log('output:', hello_there.join(''))
          t.end()
        })
      )

    })

  var r = Math.random()
  var bobHS = shs.server(bob, function (public, cb) {
      t.deepEqual(public, alice.publicKey)

      if(deepEqual(public, alice.publicKey)) cb(null, {okay: true, random: r})
      else
        cb(new Error('unauthorized'))

    }, app_key, 100)
      (function (err, stream) {

        if(err) throw err

        t.deepEqual(stream.auth, {okay: true, random: r})
        pull(stream, pull.through(function (data) {
          console.log('echo:', data.toString())
        }), stream) //ECHO
      })

  pull(
    aliceHS,
    pull.through(console.log.bind(console, 'A->B')),
    bobHS,
    pull.through(console.log.bind(console, 'A<-B')),
    aliceHS
  )

})

function bitflipTest(t, test) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err) {
      t.ok(err, 'Alice errored')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, function (public, cb) {
      t.deepEqual(public, alice.publicKey)

      if(deepEqual(public, alice.publicKey)) cb(null)
      else
        cb(new Error('unauthorized'))

    }, app_key, 100)
      (function (err) {
        t.ok(err, 'Bob errored')
        if(++errs === 2) t.end()
      })

  test(aliceHS, bobHS)

}

tape('test auth fails when first packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bitflipper(1),
      bobHS,
      aliceHS
    )
  })
})

tape('test auth fails when 2nd packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bobHS,
      bitflipper(1),
      aliceHS
    )
  })
})

tape('test auth fails when 3rd packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bitflipper(2),
      bobHS,
      aliceHS
    )
  })
})

tape('test auth fails when 4th packet is flipped', function (t) {
  bitflipTest(t, function (aliceHS, bobHS) {
    pull(
      aliceHS,
      bobHS,
      bitflipper(2),
      aliceHS
    )
  })
})

tape('test error cb when client is not authorized', function (t) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err) {
      t.ok(err, 'Bob hungup')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, unauthorized, app_key, 100)
      (function (err) {
        t.ok(err, 'client unauthorized')
        if(++errs === 2) t.end()
      })

  pull(aliceHS, bobHS, aliceHS)
})

tape('test error cb when client get wrong number', function (t) {
  var errs = 0
  var aliceHS = shs.client(alice, app_key, 100)
    (wally.publicKey, function (err) {
      t.ok(err, 'Bob hungup')
      if(++errs === 2) t.end()
    })

  var bobHS = shs.server(bob, unauthorized, app_key, 100)
      (function (err) {
        t.ok(err, 'client unauthorized')
        if(++errs === 2) t.end()
      })

  pull(aliceHS, bobHS, aliceHS)
})

tape('error if created without public key', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
  t.throws(function () {
    aliceHS()
  })
  t.end()
})

tape('unauthorized connection must cb once', function (t) {
  t.plan(2)
  var n = 2
  var aliceHS = shs.client(alice, app_key, 100)
  var bobHS = shs.server(bob, authorized, app_key, 100)

  var as = aliceHS(bob.publicKey, function (err, stream) {
    console.log('Alice')
    t.ok(err, 'client connect should fail')
    next()
  })

  pull(as, bobHS(function (err, stream) {
    console.log('Bob')
    t.ok(err, 'server connect should fail')
    next()
  }), as)

  function next () {
    if(--n) return
    t.end()
  }

})

tape('client timeout error if there is no response', function (t) {

  var aliceHS = shs.client(alice, app_key, 100)
    (bob.publicKey, function (err, stream) {
      t.ok(err)
      t.end()
    })

  pull(
    Hang(),
    aliceHS
  )
  //do nothing, so aliceHS should timeout
})

tape('server timeout error if there is no response', function (t) {

  var bobHS = shs.server(alice, authorized, app_key, 100)
    (function (err, stream) {
      t.ok(err)
      t.end()
    })

  pull(
    Hang(),
    bobHS
  )
  //do nothing, so aliceHS should timeout
})

tape('test handshake', function (t) {

  var aliceHS = shs.client(null, app_key, 100)
    (bob.publicKey, hash('alice'), function (err, stream) {

      if(err) throw err

    })

  var r = Math.random()
  var bobHS = shs.server(bob, function (public, cb) {
      t.deepEqual(public, alice.publicKey)
      cb(null, {okay: true, random: r})
    }, app_key, 100)
      (function (err, stream) {
        if(err) throw err

        t.deepEqual(stream.auth, {okay: true, random: r})
        t.end()
      })

  pull(
    aliceHS,
    bobHS,
    aliceHS
  )

})

<|endoftext|>