<commit_before>var _ = require("lodash");
var Q = require("q");
var path = require("path");
var semver = require("semver");

var pkg = require("../package.json");
var i18n = require("./utils/i18n");

// Default plugins added to each books
var defaultsPlugins = ["highlight"];

// Normalize a list of plugins to use
function normalizePluginsList(plugins, addDefaults) {
    // Normalize list to an array
    plugins = _.isString(plugins) ? plugins.split(",") : (plugins || []);

    // Remove empty parts
    plugins = _.compact(plugins);

    // Divide as {name, version} to handle format like "myplugin@1.0.0"
    plugins = _.map(plugins, function(plugin) {
        if (plugin.name) return plugin;

        var parts = plugin.split("@");
        return {
            "name": parts[0],
            "version": parts[1] // optional
        };
    });

    // List plugins to remove
    var toremove = _.chain(plugins)
    .filter(function(plugin) {
        return plugin.name.length > 0 && plugin.name[0] == "-";
    })
    .map(function(plugin) {
        return plugin.name.slice(1);
    })
    .value();

    // Merge with defaults
    if (addDefaults !== false) {
        plugins = _.chain(plugins)
        .concat(_.map(defaultsPlugins, function(plugin) {
            return { "name": plugin };
        }))
        .uniq()
        .value();
    }

    // Build final list
    plugins = _.filter(plugins, function(plugin) {
        return !_.contains(toremove, plugin.name) && !(plugin.name.length > 0 && plugin.name[0] == "-");
    });

    return plugins;
}

var Configuration = function(book, options) {
    var that = this;

    this.book = book;

    this.options = _.cloneDeep(Configuration.DEFAULT);
    this.options = _.merge(this.options, options || {});

    // options.input == book.root
    Object.defineProperty(this.options, "input", {
        get: function () {
            return that.book.root;
        }
    });

    // options.originalInput == book.parent.root
    Object.defineProperty(this.options, "originalInput", {
        get: function () {
            return that.book.parent? that.book.parent.root : undefined;
        }
    });

    // options.originalOutput == book.parent.options.output
    Object.defineProperty(this.options, "originalOutput", {
        get: function () {
            return that.book.parent? that.book.parent.options.output : undefined;
        }
    });
};

// Read and parse the configuration
Configuration.prototype.load = function() {
    var that = this;

    return Q()
    .then(function() {
        var configPath, _config;

        try {
            configPath = require.resolve(
                that.book.resolve(that.options.configFile)
            );

            // Invalidate node.js cache for livreloading
            delete require.cache[configPath];

            _config = require(configPath);
            that.options = _.merge(
                that.options,
                _.omit(_config, "configFile", "defaultsPlugins", "generator", "extension")
            );
        }
        catch(err) {
            if (err instanceof SyntaxError) return Q.reject(err);
            return Q();
        }
    })
    .then(function() {
        if (!semver.satisfies(pkg.version, that.options.gitbook)) {
            throw "GitBook version doesn't satisfy version required by the book: "+that.options.gitbook;
        }
        if (that.options.gitbook != "*" && !semver.satisfies(semver.inc(pkg.version, "patch"), that.options.gitbook)) {
            that.book.log.warn.ln("gitbook version specified in your book.json might be too strict for future patches, \""+(_.first(pkg.version.split("."))+".x.x")+"\" is more adequate");
        }

        that.options.output = path.resolve(that.options.output || that.book.resolve("_book"));
        that.options.plugins = normalizePluginsList(that.options.plugins);
        that.options.defaultsPlugins = normalizePluginsList(that.options.defaultsPlugins || "", false);
        that.options.plugins = _.union(that.options.plugins, that.options.defaultsPlugins);
        that.options.plugins = _.uniq(that.options.plugins, "name");

        // Default value for text direction (from language)
        if (!that.options.direction) {
            var lang = i18n.getCatalog(that.options.language);
            if (lang) that.options.direction = lang.direction;
        }

        that.options.gitbook = pkg.version;
    });
};

// Extend the configuration
Configuration.prototype.extend = function(options) {
    _.extend(this.options, options);
};

// Get structure file
Configuration.prototype.getStructure = function(name) {
    return this.options.structure[name].split(".").slice(0, -1).join(".");
};

// Return normalized language
Configuration.prototype.normalizeLanguage = function() {
    return i18n.normalizeLanguage(this.options.language);
};

// Return a configuration
Configuration.prototype.get = function(key, def) {
    return _.get(this.options, key, def);
};

// Default configuration
Configuration.DEFAULT = {
    // Options that can"t be extend
    "configFile": "book",
    "generator": "website",
    "extension": null,

    // Book metadats (somes are extracted from the README by default)
    "title": null,
    "description": null,
    "isbn": null,
    "language": "en",
    "direction": null,
    "author": null,

    // version of gitbook to use
    "gitbook": "*",

    // Structure
    "structure": {
        "langs": "LANGS.md",
        "readme": "README.md",
        "glossary": "GLOSSARY.md",
        "summary": "SUMMARY.md"
    },

    // CSS Styles
    "styles": {
        "website": "styles/website.css",
        "print": "styles/print.css",
        "ebook": "styles/ebook.css",
        "pdf": "styles/pdf.css",
        "mobi": "styles/mobi.css",
        "epub": "styles/epub.css"
    },

    // Plugins list, can contain "-name" for removing default plugins
    "plugins": [],

    // Global configuration for plugins
    "pluginsConfig": {
        "fontSettings": {
            "theme": null, //"sepia", "night" or "white",
            "family": "sans",// "serif" or "sans",
            "size": 2 // 1 - 4
        }
    },

    // Variables for templating
    "variables": {},

    // Set another theme with your own layout
    // It"s recommended to use plugins or add more options for default theme, though
    // See https://github.com/GitbookIO/gitbook/issues/209
    "theme": path.resolve(__dirname, "../theme"),

    // Links in template (null: default, false: remove, string: new value)
    "links": {
        // Custom links at top of sidebar
        "sidebar": {
            //"Custom link name": "https://customlink.com"
        },

        // Sharing links
        "sharing": {
            "google": null,
            "facebook": null,
            "twitter": null,
            "weibo": null,
            "all": null
        }
    },

    // Options for PDF generation
    "pdf": {
        // Add toc at the end of the file
        "toc": true,

        // Add page numbers to the bottom of every page
        "pageNumbers": false,

        // Font size for the file content
        "fontSize": 12,

        // Paper size for the pdf
        // Choices are [u’a0’, u’a1’, u’a2’, u’a3’, u’a4’, u’a5’, u’a6’, u’b0’, u’b1’, u’b2’, u’b3’, u’b4’, u’b5’, u’b6’, u’legal’, u’letter’]
        "paperSize": "a4",

        // How to mark detected chapters.
        // Choices are “pagebreak”, “rule”, "both" or “none”.
        "chapterMark" : "pagebreak",

        // An XPath expression. Page breaks are inserted before the specified elements.
        // To disable use the expression: "/"
        "pageBreaksBefore": "/",

        // Margin (in pts)
        // Note: 72 pts equals 1 inch
        "margin": {
            "right": 62,
            "left": 62,
            "top": 56,
            "bottom": 56
        },

        //Header HTML template. Available variables: _PAGENUM_, _TITLE_, _AUTHOR_ and _SECTION_.
        "headerTemplate": "",

        //Footer HTML template. Available variables: _PAGENUM_, _TITLE_, _AUTHOR_ and _SECTION_.
        "footerTemplate": ""
    }
};

module.exports= Configuration;
<commit_msg>Throw correct error in configuration<commit_after>var _ = require("lodash");
var Q = require("q");
var path = require("path");
var semver = require("semver");

var pkg = require("../package.json");
var i18n = require("./utils/i18n");

// Default plugins added to each books
var defaultsPlugins = ["highlight"];

// Normalize a list of plugins to use
function normalizePluginsList(plugins, addDefaults) {
    // Normalize list to an array
    plugins = _.isString(plugins) ? plugins.split(",") : (plugins || []);

    // Remove empty parts
    plugins = _.compact(plugins);

    // Divide as {name, version} to handle format like "myplugin@1.0.0"
    plugins = _.map(plugins, function(plugin) {
        if (plugin.name) return plugin;

        var parts = plugin.split("@");
        return {
            "name": parts[0],
            "version": parts[1] // optional
        };
    });

    // List plugins to remove
    var toremove = _.chain(plugins)
    .filter(function(plugin) {
        return plugin.name.length > 0 && plugin.name[0] == "-";
    })
    .map(function(plugin) {
        return plugin.name.slice(1);
    })
    .value();

    // Merge with defaults
    if (addDefaults !== false) {
        plugins = _.chain(plugins)
        .concat(_.map(defaultsPlugins, function(plugin) {
            return { "name": plugin };
        }))
        .uniq()
        .value();
    }

    // Build final list
    plugins = _.filter(plugins, function(plugin) {
        return !_.contains(toremove, plugin.name) && !(plugin.name.length > 0 && plugin.name[0] == "-");
    });

    return plugins;
}

var Configuration = function(book, options) {
    var that = this;

    this.book = book;

    this.options = _.cloneDeep(Configuration.DEFAULT);
    this.options = _.merge(this.options, options || {});

    // options.input == book.root
    Object.defineProperty(this.options, "input", {
        get: function () {
            return that.book.root;
        }
    });

    // options.originalInput == book.parent.root
    Object.defineProperty(this.options, "originalInput", {
        get: function () {
            return that.book.parent? that.book.parent.root : undefined;
        }
    });

    // options.originalOutput == book.parent.options.output
    Object.defineProperty(this.options, "originalOutput", {
        get: function () {
            return that.book.parent? that.book.parent.options.output : undefined;
        }
    });
};

// Read and parse the configuration
Configuration.prototype.load = function() {
    var that = this;

    return Q()
    .then(function() {
        var configPath, _config;

        try {
            configPath = require.resolve(
                that.book.resolve(that.options.configFile)
            );

            // Invalidate node.js cache for livreloading
            delete require.cache[configPath];

            _config = require(configPath);
            that.options = _.merge(
                that.options,
                _.omit(_config, "configFile", "defaultsPlugins", "generator", "extension")
            );
        }
        catch(err) {
            if (err instanceof SyntaxError) return Q.reject(err);
            return Q();
        }
    })
    .then(function() {
        if (!semver.satisfies(pkg.version, that.options.gitbook)) {
            throw new Error("GitBook version doesn't satisfy version required by the book: "+that.options.gitbook);
        }
        if (that.options.gitbook != "*" && !semver.satisfies(semver.inc(pkg.version, "patch"), that.options.gitbook)) {
            that.book.log.warn.ln("gitbook version specified in your book.json might be too strict for future patches, \""+(_.first(pkg.version.split("."))+".x.x")+"\" is more adequate");
        }

        that.options.output = path.resolve(that.options.output || that.book.resolve("_book"));
        that.options.plugins = normalizePluginsList(that.options.plugins);
        that.options.defaultsPlugins = normalizePluginsList(that.options.defaultsPlugins || "", false);
        that.options.plugins = _.union(that.options.plugins, that.options.defaultsPlugins);
        that.options.plugins = _.uniq(that.options.plugins, "name");

        // Default value for text direction (from language)
        if (!that.options.direction) {
            var lang = i18n.getCatalog(that.options.language);
            if (lang) that.options.direction = lang.direction;
        }

        that.options.gitbook = pkg.version;
    });
};

// Extend the configuration
Configuration.prototype.extend = function(options) {
    _.extend(this.options, options);
};

// Get structure file
Configuration.prototype.getStructure = function(name) {
    return this.options.structure[name].split(".").slice(0, -1).join(".");
};

// Return normalized language
Configuration.prototype.normalizeLanguage = function() {
    return i18n.normalizeLanguage(this.options.language);
};

// Return a configuration
Configuration.prototype.get = function(key, def) {
    return _.get(this.options, key, def);
};

// Default configuration
Configuration.DEFAULT = {
    // Options that can"t be extend
    "configFile": "book",
    "generator": "website",
    "extension": null,

    // Book metadats (somes are extracted from the README by default)
    "title": null,
    "description": null,
    "isbn": null,
    "language": "en",
    "direction": null,
    "author": null,

    // version of gitbook to use
    "gitbook": "*",

    // Structure
    "structure": {
        "langs": "LANGS.md",
        "readme": "README.md",
        "glossary": "GLOSSARY.md",
        "summary": "SUMMARY.md"
    },

    // CSS Styles
    "styles": {
        "website": "styles/website.css",
        "print": "styles/print.css",
        "ebook": "styles/ebook.css",
        "pdf": "styles/pdf.css",
        "mobi": "styles/mobi.css",
        "epub": "styles/epub.css"
    },

    // Plugins list, can contain "-name" for removing default plugins
    "plugins": [],

    // Global configuration for plugins
    "pluginsConfig": {
        "fontSettings": {
            "theme": null, //"sepia", "night" or "white",
            "family": "sans",// "serif" or "sans",
            "size": 2 // 1 - 4
        }
    },

    // Variables for templating
    "variables": {},

    // Set another theme with your own layout
    // It"s recommended to use plugins or add more options for default theme, though
    // See https://github.com/GitbookIO/gitbook/issues/209
    "theme": path.resolve(__dirname, "../theme"),

    // Links in template (null: default, false: remove, string: new value)
    "links": {
        // Custom links at top of sidebar
        "sidebar": {
            //"Custom link name": "https://customlink.com"
        },

        // Sharing links
        "sharing": {
            "google": null,
            "facebook": null,
            "twitter": null,
            "weibo": null,
            "all": null
        }
    },

    // Options for PDF generation
    "pdf": {
        // Add toc at the end of the file
        "toc": true,

        // Add page numbers to the bottom of every page
        "pageNumbers": false,

        // Font size for the file content
        "fontSize": 12,

        // Paper size for the pdf
        // Choices are [u’a0’, u’a1’, u’a2’, u’a3’, u’a4’, u’a5’, u’a6’, u’b0’, u’b1’, u’b2’, u’b3’, u’b4’, u’b5’, u’b6’, u’legal’, u’letter’]
        "paperSize": "a4",

        // How to mark detected chapters.
        // Choices are “pagebreak”, “rule”, "both" or “none”.
        "chapterMark" : "pagebreak",

        // An XPath expression. Page breaks are inserted before the specified elements.
        // To disable use the expression: "/"
        "pageBreaksBefore": "/",

        // Margin (in pts)
        // Note: 72 pts equals 1 inch
        "margin": {
            "right": 62,
            "left": 62,
            "top": 56,
            "bottom": 56
        },

        //Header HTML template. Available variables: _PAGENUM_, _TITLE_, _AUTHOR_ and _SECTION_.
        "headerTemplate": "",

        //Footer HTML template. Available variables: _PAGENUM_, _TITLE_, _AUTHOR_ and _SECTION_.
        "footerTemplate": ""
    }
};

module.exports= Configuration;
<|endoftext|>


<commit_before>;(function(window, undefined) {
	"use strict";

	var CC={},
	// this is typically window
	root = this || window,
	con = root.console,
	aps = Array.prototype.slice,

    // make sure we can save history, or don't save it at all
    persistHistory = (root.localStorage && root.JSON);

    // inherit the console prototype
    if(con) {
    	CC = Object.create(con);
    }

	if (typeof module !== 'undefined') {
        module.exports = CC;
    } else {
        root.cconsole = CC;
    }

    // Default settings
    CC.settings = {
		standBy: false,
		debug: false,
		environment : "development", // production || development
		history: persistHistory ? JSON.parse(root.localStorage.getItem('CC.history')) || [] : [],
		// limit the size of the array to 100
		maxHistory: 100,
		filterLog: false,
		notify: function(error, identifier) { return; } // un-implemented, meant to be replaced by developer code.
    };

    // Add the ability to override the settings
    CC.set = function(settings) {
    	if(typeof settings === "undefined" ||
    		typeof settings !== "object" ||
    		settings instanceof String ||
    		settings instanceof Array) {
    		throw Error("You must pass an plain object to set settings. EG: { }");
    	}
    	// allow the setting of settings in a much easier way
    	// hasOwnProperty is to avoid times where you may have passed an odd function with inherited props
    	for(var setting in settings) if(settings.hasOwnProperty(setting)) {
    		CC.settings[setting] = settings[setting];
    	}
    };
    CC.clearFilter = function() {
    	CC.settings.filterLog = false;
    };
    CC.setFilter = function(string) {
    	// if not string, don't do it
    	if(typeof string !== "string" && !(string instanceof String)) {
    		throw new Error("Filters must be of type string");
    	}

    	CC.settings.filterLog = string;
    };

	// we intend to overwrite these methods
	var methods = [
		'error',
		'warn',
		'info',
		'debug',
		'log'
	],
	// these methods will just 'pass through' to the original console or use console.log if it doesn't support them
	pass_methods = [
		'assert',
		'clear',
		'count',
		'dir',
		'dirxml',
		'exception',
		'group',
		'groupCollapsed',
		'groupEnd',
		'markTimeline',
		'profile',
		'profileEnd',
		'table',
		'time',
		'timeEnd',
		'timeStamp',
		'trace',
		'memory',
		'profiles'
	],
	idx = pass_methods.length,

	loadMethods = function(method){
		CC[method]=function() {
			var args = aps.call(arguments);

			if(CC.settings.standBy || ( CC.settings.debug && con && con[method] )){
				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);
			}
		};
	},

	loadTraceMethods = function(idx,method){
		CC[method] = function() {
			var args = aps.call(arguments);

			if(CC.settings.standBy) {
				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);
				return;
			}

			if(method !== 'error' && CC.settings.environment === "production" ) {
				return;
			}

			// everything past this point is not in production code, unless it is an error

			try {
				var log_arr,
					error = new Error(),
					stackError = error && error.stack,
					trace, line_trace,identifier="", re = /\//g,
					alwaysShow = ['error','warn'];

				// Sometimes when we try to get the stack trace,
				// things go poorly. We still want to continue on
				// our merry way however. We've noticed this
				// behavior particularly when highcharts is involved
				if (stackError && typeof stackError === 'string') {
					try {
						trace = stackError.split(/\r\n|\r|\n/);
						trace = trace[trace.length-1].split(/\//);
					}catch (ignore){
						trace = ['Failed to retrieve stack trace.'];
					}
				}

				line_trace = trace[trace.length-1].split(/:/);

				if(method!=='debug') {
					args.push(line_trace[0] + " Line "+line_trace[1]);
					identifier = line_trace[0].replace(re,"~")+":"+line_trace[1];
				} else {
					args.push(((new Error() && new Error().stack)?new Error().stack:'').split(/\r\n|\r|\n/));
				}

				// errors and warning (but the if above only allows errors in prod) should always go to the console in dev.
				if ( !con || ( alwaysShow.indexOf(method) < 0 && (!CC.settings.debug || args[0].indexOf(CC.settings.filterLog) < 0   ))) {
					return;
				}

				log_arr = [method].concat( args );
				log_arr.push(new Date().getTime());
				// push to the beginning, not the end
				CC.settings.history.unshift( log_arr );
				// make sure we aren't running out of memory here
				if(CC.settings.history.length > CC.settings.maxHistory) {
					// drop the most recent off the top
					CC.settings.history.pop();
				}
				if( persistHistory ) {
					root.localStorage.setItem('CC.history',root.JSON.stringify(CC.settings.history));
				}

				// the method must be an error becaue of previous if statement
				if(CC.settings.environment === 'production') {
					CC.settings.notify( log_arr,  identifier);
					return;
				}
				// push to the actual console

				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);

			} catch(ignore){}
		};
	};

	// Add CC as the console
	root.console = CC;

	/* passthrough any console methods */
	while(--idx >= 0){
		loadMethods(pass_methods[idx]);
	}

	// reuse the vars
	idx = methods.length;

	while(--idx >= 0){
		loadTraceMethods(idx,methods[idx]);
	}

})(window);<commit_msg>Better history and timestamps<commit_after>;(function(window, undefined) {
	"use strict";

	var CC={},
	// this is typically window
	root = this || window,
	con = root.console,
	aps = Array.prototype.slice,

    // make sure we can save history, or don't save it at all
    persistHistory = (root.localStorage && root.JSON);

    // inherit the console prototype
    if(con) {
    	CC = Object.create(con);
    }

	if (typeof module !== 'undefined') {
        module.exports = CC;
    } else {
        root.cconsole = CC;
    }

    // Default settings
    CC.settings = {
		standBy: false,
		timestamp: true,
		debug: false,
		environment : "development", // production || development
		history: persistHistory ? JSON.parse(root.localStorage.getItem('CC.history')) || [] : [],
		// limit the size of the array to 100
		maxHistory: 100,
		filterLog: false,
		notify: function(error, identifier) { return; } // un-implemented, meant to be replaced by developer code.
    };

    // Add the ability to override the settings
    CC.set = function(settings) {
    	if(typeof settings === "undefined" ||
    		typeof settings !== "object" ||
    		settings instanceof String ||
    		settings instanceof Array) {
    		throw Error("You must pass an plain object to set settings. EG: { }");
    	}
    	// allow the setting of settings in a much easier way
    	// hasOwnProperty is to avoid times where you may have passed an odd function with inherited props
    	for(var setting in settings) if(settings.hasOwnProperty(setting)) {
    		CC.settings[setting] = settings[setting];
    	}
    };
    CC.clearFilter = function() {
    	CC.settings.filterLog = false;
    };
    CC.setFilter = function(string) {
    	// if not string, don't do it
    	if(typeof string !== "string" && !(string instanceof String)) {
    		throw new Error("Filters must be of type string");
    	}

    	CC.settings.filterLog = string;
    };

	// we intend to overwrite these methods
	var methods = [
		'error',
		'warn',
		'info',
		'debug',
		'log'
	],
	// these methods will just 'pass through' to the original console or use console.log if it doesn't support them
	pass_methods = [
		'assert',
		'clear',
		'count',
		'dir',
		'dirxml',
		'exception',
		'group',
		'groupCollapsed',
		'groupEnd',
		'markTimeline',
		'profile',
		'profileEnd',
		'table',
		'time',
		'timeEnd',
		'timeStamp',
		'trace',
		'memory',
		'profiles'
	],
	idx = pass_methods.length,
	updateHistory = function(msg) {
		// push to the beginning, not the end
		CC.settings.history.unshift( msg );
		// make sure we aren't running out of memory here
		if(CC.settings.history.length > CC.settings.maxHistory) {
			// drop the most recent off the top
			CC.settings.history.pop();
		}
		if( persistHistory ) {
			root.localStorage.setItem('CC.history',root.JSON.stringify(CC.settings.history));
		}
	},
	loadMethods = function(method){
		CC[method]=function() {
			var args = aps.call(arguments);

			if(CC.settings.standBy || ( CC.settings.debug && con && con[method] )){
				if(CC.settings.timestamp) {
					args.unshift(new Date().toISOString());
				}
				updateHistory([method].concat( args ));
				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);
			}
		};
	},
	loadTraceMethods = function(idx,method){
		CC[method] = function() {
			var args = aps.call(arguments);

			if(CC.settings.standBy) {
				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);
				return;
			}

			if(method !== 'error' && CC.settings.environment === "production" ) {
				return;
			}

			// everything past this point is not in production code, unless it is an error

			try {
				var log_arr,
					error = new Error(),
					stackError = error && error.stack,
					trace, line_trace,identifier="", re = /\//g,
					alwaysShow = ['error','warn'];

				// Sometimes when we try to get the stack trace,
				// things go poorly. We still want to continue on
				// our merry way however. We've noticed this
				// behavior particularly when highcharts is involved
				if (stackError && typeof stackError === 'string') {
					try {
						trace = stackError.split(/\r\n|\r|\n/);
						trace = trace[trace.length-1].split(/\//);
					}catch (ignore){
						trace = ['Failed to retrieve stack trace.'];
					}
				}

				line_trace = trace[trace.length-1].split(/:/);

				if(method!=='debug') {
					args.push(line_trace[0] + " Line "+line_trace[1]);
					identifier = line_trace[0].replace(re,"~")+":"+line_trace[1];
				} else {
					args.push(((new Error() && new Error().stack)?new Error().stack:'').split(/\r\n|\r|\n/));
				}

				// errors and warning (but the if above only allows errors in prod) should always go to the console in dev.
				if ( !con || ( alwaysShow.indexOf(method) < 0 && (!CC.settings.debug || args[0].indexOf(CC.settings.filterLog) < 0   ))) {
					return;
				}

				if(CC.settings.timestamp) {
					args.unshift(new Date().toISOString());
				}

				log_arr = [method].concat( args );

				updateHistory(log_arr);

				// the method must be an error becaue of previous if statement
				if(CC.settings.environment === 'production') {
					CC.settings.notify( log_arr,  identifier);
					return;
				}
				// push to the actual console

				con.firebug ? con[method].apply(root, args) :
					con[method] ? con[method].apply(con,args) :
					con.log.apply(con,args);

			} catch(ignore){}
		};
	};

	// Add CC as the console
	root.console = CC;

	/* passthrough any console methods */
	while(--idx >= 0){
		loadMethods(pass_methods[idx]);
	}

	// reuse the vars
	idx = methods.length;

	while(--idx >= 0){
		loadTraceMethods(idx,methods[idx]);
	}

})(window);<|endoftext|>


<commit_before>var events = require("events");
var util = require("util");

var DataCollector = function(expected, recordEmpty) {
	this.expected = expected;
	this.recordEmpty = recordEmpty;
	this.named = (typeof(this.expected) != 'number');
	this.items = {};
	//this.on('ready', function(data){console.log('Emitted ready with data: ' + JSON.stringify(data));});
};
util.inherits(DataCollector, events.EventEmitter);

DataCollector.prototype.isFinished = function() {
	if (this.named) {
		var evLen = this.expected.length;
		for (var i=0;i<evLen;i++) {
			if (typeof(this.items[this.expected[i]]) == 'undefined') return false;
		}
		return true;
	} else return (this.items.length >= this.expected);
};

DataCollector.prototype.collect = function(name, value) {
	var k = (this.named) ? name : this.items.length;
	var val = (this.named) ? value : name;

	if (val !== undefined || this.recordEmpty) {
		this.items[k] == value;
	} else if (!this.named) this.expected--;

	this.emit(name, value);
	if (this.isFinished()) this.emit('ready', this.items);

};

DataCollector.prototype.get = function(name, callback) {
	if (!this.named) {
		callback();
	} else if (!this.items.hasOwnProperty(name)) {
		this.once(name, callback);
	}  else callback(this.items[name]);
};

module.exports = DataCollector;<commit_msg>Added extra handling for collect(undefined)<commit_after>var events = require("events");
var util = require("util");

var DataCollector = function(expected, recordEmpty) {
	this.expected = expected;
	this.recordEmpty = recordEmpty;
	this.named = (typeof(this.expected) != 'number');
	this.items = {};
	//this.on('ready', function(data){console.log('Emitted ready with data: ' + JSON.stringify(data));});
};
util.inherits(DataCollector, events.EventEmitter);

DataCollector.prototype.isFinished = function() {
	if (this.named) {
		var evLen = this.expected.length;
		for (var i=0;i<evLen;i++) {
			if (typeof(this.items[this.expected[i]]) == 'undefined') return false;
		}
		return true;
	} else return (this.items.length >= this.expected);
};

DataCollector.prototype.collect = function(name, value) {
	var k = (this.named) ? name : this.items.length;
	var val = (this.named) ? value : name;

	if (this.named && !this.expected.hasOwnProperty(k)) {
		// do nothing;
	} else if (val !== undefined || this.recordEmpty) {
		this.items[k] == (val !== undefined) ? val : null;
	} else if (this.named) {
		var idx = this.expected.indexOf(k);
		if (idx !== -1) delete this.expected[idx];
	} else this.expected--;

	this.emit(name, value);
	if (this.isFinished()) this.emit('ready', this.items);

};

DataCollector.prototype.get = function(name, callback) {
	if (!this.named) {
		callback();
	} else if (!this.items.hasOwnProperty(name)) {
		this.once(name, callback);
	}  else callback(this.items[name]);
};

module.exports = DataCollector;<|endoftext|>


<commit_before>module.exports = {
  "GIVE_X11_TIME_TIMEOUT": 80,
  "POLL_ALL_APPS_STARTED_INTERVAL": 2000,
  "POLL_ALL_MAX_TIMEOUT": 120000,
  "SAVE_SESSION_IN_PRETTY_FORMAT": true,
  "WM_CLASS_AND_EXECUTABLE_FILE_MAP": {
    "gnome-terminal-server.Gnome-terminal": "gnome-terminal",
    "google-chrome.Google-chrome": "google-chrome.desktop",
    "Mail.Thunderbird": "thunderbird.desktop",
    "nautilus.Nautilus": "nautilus",
    "Navigator.Firefox": "firefox.desktop",
    "Navigator.Pale": "palemoon.desktop",
    "skype.Skype": "skypeforlinux.desktop",
    "sun-awt-X11-XFramePeer.jetbrains-idea": "jetbrains-idea.desktop",
    "VirtualBox.VirtualBox": "virtualbox.desktop",
    "Telegram.TelegramDesktop": "telegram-desktop_telegramdesktop.desktop",
    "keepassxc.keepassxc": "keepassxc_keepassxc.desktop"
  },
  "WM_CLASS_EXCLUSIONS": [
    "N/A",
    "tilda.Tilda",
    "Popup.desktop",
    "update-manager.Update-manager",
    "desktop_window.Nautilus",
    "electron.Electron",
    "guake.Main.py",
    'gnome-software.Gnome-software'
  ],
  "WM_META_MAP": {
    "WM_CLASS(STRING)": "wmClassName",
    "_NET_WM_STATE(ATOM)": "states",
    "_NET_WM_DESKTOP(CARDINAL)": "wmCurrentDesktopNr",
    "WM_NAME(UTF8_STRING)": "wmTitle",
    "_NET_WM_PID(CARDINAL)": "wmPid",
    "_NET_WM_WINDOW_TYPE(ATOM)": "wmType",
    "_BAMF_DESKTOP_FILE(STRING)": "executableFile"
  },
  "WM_META_MAP_NUMBER_TYPES": [
    "_NET_WM_PID(CARDINAL)",
    "_NET_WM_DESKTOP(CARDINAL)"
  ],
  "DESKTOP_FILE_LOCATIONS": [
    "{home}/.local/share/applications",
    "{home}/.gnome/apps/",
    "/usr/share/applications",
    "/usr/local/share/applications",
    "/usr/share/app-install",
    "{home}/.config/autostart/",
    "/var/lib/snapd/desktop/applications"
  ],
  "CMD": {
    "GET_DISPLAY_ID": "xrandr --query | grep '[^s]connected '",
    "XPROP_ID": "xprop -id"
  }
};
<commit_msg>add slack app name and new folder for desktopfiles<commit_after>module.exports = {
  "GIVE_X11_TIME_TIMEOUT": 80,
  "POLL_ALL_APPS_STARTED_INTERVAL": 2000,
  "POLL_ALL_MAX_TIMEOUT": 120000,
  "SAVE_SESSION_IN_PRETTY_FORMAT": true,
  "WM_CLASS_AND_EXECUTABLE_FILE_MAP": {
    "gnome-terminal-server.Gnome-terminal": "gnome-terminal",
    "google-chrome.Google-chrome": "google-chrome.desktop",
    "Mail.Thunderbird": "thunderbird.desktop",
    "nautilus.Nautilus": "nautilus",
    "Navigator.Firefox": "firefox.desktop",
    "Navigator.Pale": "palemoon.desktop",
    "skype.Skype": "skypeforlinux.desktop",
    "sun-awt-X11-XFramePeer.jetbrains-idea": "jetbrains-idea.desktop",
    "VirtualBox.VirtualBox": "virtualbox.desktop",
    "Telegram.TelegramDesktop": "telegram-desktop_telegramdesktop.desktop",
    "keepassxc.keepassxc": "keepassxc_keepassxc.desktop",
    "slack.Slack": "com.slack.Slack.desktop"
  },
  "WM_CLASS_EXCLUSIONS": [
    "N/A",
    "tilda.Tilda",
    "Popup.desktop",
    "update-manager.Update-manager",
    "desktop_window.Nautilus",
    "electron.Electron",
    "guake.Main.py",
    'gnome-software.Gnome-software'
  ],
  "WM_META_MAP": {
    "WM_CLASS(STRING)": "wmClassName",
    "_NET_WM_STATE(ATOM)": "states",
    "_NET_WM_DESKTOP(CARDINAL)": "wmCurrentDesktopNr",
    "WM_NAME(UTF8_STRING)": "wmTitle",
    "_NET_WM_PID(CARDINAL)": "wmPid",
    "_NET_WM_WINDOW_TYPE(ATOM)": "wmType",
    "_BAMF_DESKTOP_FILE(STRING)": "executableFile"
  },
  "WM_META_MAP_NUMBER_TYPES": [
    "_NET_WM_PID(CARDINAL)",
    "_NET_WM_DESKTOP(CARDINAL)"
  ],
  "DESKTOP_FILE_LOCATIONS": [
    "{home}/.local/share/applications",
    "{home}/.gnome/apps/",
    "/usr/share/applications",
    "/usr/local/share/applications",
    "/usr/share/app-install",
    "{home}/.config/autostart/",
    "/var/lib/snapd/desktop/applications",
    "/var/lib/flatpak/app"
  ],
  "CMD": {
    "GET_DISPLAY_ID": "xrandr --query | grep '[^s]connected '",
    "XPROP_ID": "xprop -id"
  }
};
<|endoftext|>


<commit_before>/*
 * @copyright Copyright (c) Sematext Group, Inc. - All Rights Reserved
 *
 * @licence SPM for Docker is free-to-use, proprietary software.
 * THIS IS PROPRIETARY SOURCE CODE OF Sematext Group, Inc. (Sematext)
 * This source code may not be copied, reverse engineered, or altered for any purpose.
 * This source code is to be used exclusively by users and customers of Sematext.
 * Please see the full license (found in LICENSE in this distribution) for details on its license and the licenses of its dependencies.
 */
var stats = require('docker-stats')
var flatten = require('flat')
var through = require('through2')
var Aggregator = require('./aggregator.js')
var SpmAgent = require('spm-agent')
var config = SpmAgent.Config

var Docker = require('dockerode')
var docker = new Docker()
var containerCount = 0
var GOOGLE_PAUSE_REGEX = /google_containers\/pause/
var opts = {
  docker: null, //  here goes options for Dockerode
  events: null, //  an instance of docker-allcontainers,
  statsinterval: 19,
  matchByName: process.env.SPM_MATCH_BY_NAME,
  matchByImage: process.env.SPM_MATCH_BY_IMAGE,
  skipByName: process.env.SPM_SKIP_BY_NAME,
  skipByImage: process.env.SPM_SKIP_BY_IMAGE
}
// holds aggregation object for each container
var cMetrics = {}
var pausedContainers = -1
var allContainers = require('docker-allcontainers')
var events = allContainers(opts)
var DockerEvents = require('docker-events')
var de = new DockerEvents({docker: new Docker()})
de.start()
events.on('error', console.error)
// remove inactive containers
events.on('stop', function (container) {
  if (!container || !container.id) {
    return
  }
  var id = container.id.substring(0, 12)
  delete cMetrics[id]
})

de.on('connect', function () {
  de.on('_message', function (message) {
    if (message && message.status === 'pause' && message.Actor && message.Actor.ID) {
      pausedContainers = pausedContainers + 1
    }
    if (message && message.status === 'unpause' && message.Actor && message.Actor.ID) {
      if (pausedContainers > 0) {
        pausedContainers = pausedContainers - 1
      }
    }
  })
})

// remove inactive containers
setInterval(function cleanUpMetrics () {
  var now = Date.now()
  var containers = Object.keys(cMetrics)
  containers.forEach(function removeEntry (entry) {
    if (cMetrics[entry] && cMetrics[entry].lastUpdate !== undefined && ((now - cMetrics[entry].lastUpdate) > 30000)) {
      delete cMetrics[entry]
    }
  })
}, 30000)

function countPauseContainers (cb) {
  docker.listContainers(function (err, info) {
    if (err) {
      return cb(err)
    } else {
      if (info) {
        var k8sPause = info.filter(function (c) {
          return GOOGLE_PAUSE_REGEX.test(c.Image) && c.State === 'running'
        })
        if (k8sPause) {
          return cb(null, k8sPause.length)
        } else {
          return cb(null, 0)
        }
      } else {
        return cb(new Error('countPauseContainers: no container info in docker.listContainers call'))
      }
    }
  })
}

function transformStatsApi1_21 (stats) {
  if (!stats.networks) {
    return stats
  }
  var result = {
    rx_bytes: 0,
    rx_packets: 0,
    rx_errors: 0,
    rx_dropped: 0,
    tx_bytes: 0,
    tx_packets: 0,
    tx_errors: 0,
    tx_dropped: 0
  }
  Object.keys(stats.networks).forEach(function aggNetworkMetrics (key) {
    Object.keys(result).forEach(function (metric) {
      result[metric] = result[metric] + getValueAsNum(stats.networks[key][metric])
    })
  })
  stats.network = result
  return stats
}

// handling different structures of docker stats
function transformStats (stats) {
  if (stats.network && stats.network.rx_bytes) {
    // API < 1.21
    return stats
  } else {
    return transformStatsApi1_21(stats)
  }
}

function processStats (chunk, enc, cb) {
  var stats = flatten(transformStats(chunk.stats))
  var metric = {}
  SpmAgent.Logger.debug(stats)
  if (getValueAsNum(stats['memory_stats.usage']) === 0 &&
      getValueAsNum(stats['cpu_stats.cpu_usage.cpu_percent']) === 0) {
      // no mem & cpu used, container stopped - workaround for bug in docker-stats
      // reporting stats for stopped containers
    return cb()
  }
  if (cMetrics[chunk.id] === undefined) {
    cMetrics[chunk.id] = new Aggregator()
  }
  metric.name = chunk.name
  metric.dockerId = chunk.id
  metric.image = chunk.image
  var agg = cMetrics[chunk.id]
  agg.info = metric

  metrics.forEach(function (key) {
    var value = getValueAsNum(stats[key] || 0)
    if (/service_time_/i.test(key) || /wait_time_/i.test(key)) {
      // micro seconds to ms
      value = value / (1000 * 1000)
    }
    agg.update(Date.now(), key, value, useDivForAggregation[key] || false)
  })
  // setTimeout (cb, 10)
  cb()
}

function getDockerStats (dockerStatsCallback) {
  var dockerStatsStream = stats(opts)
  dockerStatsStream.pipe(through.obj(processStats))
  dockerStatsStream.on('closed', function () {
    SpmAgent.Logger.error('docker stats stream closed')
  })
  dockerStatsStream.once('error', function (err) {
    SpmAgent.Logger.error('docker stats stream error:' + err)
    // try to reconnect
    setTimeout(function () {
      getDockerStats(dockerStatsCallback)
    }, 15000)
    try {
      dockerStatsStream.close()
    } catch (ex) {
      SpmAgent.Logger.error(ex)
    }
  })
}

function getStatsForContainer (container, aggStats) {
  var agg = cMetrics[container]
  if (GOOGLE_PAUSE_REGEX.test(cMetrics[container].info.image)) {
    return null
  }
  if (!agg) {
    return null
  }
  var values = []
  aggStats.forEach(function (key) {
    var metricStats = agg.get(key)
    metricStats.id = container
    values.push(metricStats)
  })
  agg.reset()
  return values
}

var cpuMetrics = [
  // CPU
  // 'cpu_stats.cpu_usage.total_usage',
  // 'cpu_stats.cpu_usage.percpu_usage.0',
  // 'cpu_stats.cpu_usage.usage_in_kernelmode',
  // 'cpu_stats.cpu_usage.usage_in_usermode',
  'cpu_stats.cpu_usage.cpu_percent',
  // 'cpu_stats.system_cpu_usage',
  // 'cpu_stats.throttling_data.periods',
  // 'cpu_stats.throttling_data.throttled_periods',
  'cpu_stats.throttling_data.throttled_time'
]

var networkMetrics = [
  // NETWORK
  'network.rx_bytes',
  'network.rx_packets',
  'network.rx_errors',
  'network.rx_dropped',
  'network.tx_bytes',
  'network.tx_packets',
  'network.tx_errors',
  'network.tx_dropped'
]

var memoryMetrics = [
  // MEMORY
  // 'memory_stats.total_rss',
  'memory_stats.usage',
  'memory_stats.limit',
  'memory_stats.failcnt',
  // 'memory_stats.max_usage',
  // 'memory_stats.active_anon',
  // 'memory_stats.active_file',
  // 'memory_stats.cache',
  // 'memory_stats.hierarchical_memory_limit',
  // 'memory_stats.inactive_anon',
  // 'memory_stats.inactive_file',
  // 'memory_stats.mapped_file',
  'memory_stats.stats.pgfault',
  // 'memory_stats.pgmajfault',
  'memory_stats.stats.pgpgin',
  'memory_stats.stats.pgpgout'
  // 'memory_stats.rss',
  // 'memory_stats.rss_huge',
  // MEMORY FAIL COUNTER

  // MEMORY LIMIT

// MEMORY TOTAL
// 'memory_stats.total_active_anon',
// 'memory_stats.total_active_file',
// 'memory_stats.total_cache',
// 'memory_stats.total_inactive_anon',
// 'memory_total_inactive_file',
// 'memory_stats.total_mapped_file',
// 'memory_stats.total_pgfault',
// 'memory_stats.total_pgmajfault',
// 'memory_stats.total_pgpgin',
// 'memory_stats.total_pgpgout',
// 'memory_stats.total_rss',
// 'memory_stats.total_rss_huge',
// 'memory_stats.total_unevictable',
// 'memory_stats.stats.total_writeback',
// 'memory_stats.unevictable',
// 'memory_stats.writeback'
]

var ioMetrics = [
  // IO STATS
  // READ
  'blkio_stats.io_service_bytes_recursive.0.value',
  // READ time
  'blkio_stats.io_service_time_recursive.0.value',
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value',
  // WRITE
  'blkio_stats.io_service_bytes_recursive.1.value',
  // Write time
  'blkio_stats.io_service_time_recursive.1.value',
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value'
// 'blkio_io_service_bytes_recursive',
// 'blkio_io_serviced_recursive',
// 'blkio_io_queue_recursive',
// 'blkio_io_service_time_recursive',
// 'blkio_io_wait_time_recursive',
// 'blkio_io_merged_recursive',
// 'blkio_io_time_recursive',
// 'blkio_sectors_recursive'
]
var metrics = []
metrics = metrics.concat(cpuMetrics).concat(networkMetrics).concat(memoryMetrics).concat(ioMetrics)
var avgAggregation = {
  'cpu_stats.cpu_usage.cpu_percent': true,
  'cpu_stats.throttling_data.throttled_time': true,
  'memory_stats.usage': true,
  'memory_stats.limit': true,
  // READ time
  'blkio_stats.io_service_time_recursive.0.value': true,
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value': true,
  // Write time
  'blkio_stats.io_service_time_recursive.1.value': true,
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value': true
}
var useDivForAggregation = {
  'network.rx_bytes': true,
  'network.rx_packets': true,
  'network.rx_errors': true,
  'network.rx_dropped': true,
  'network.tx_bytes': true,
  'network.tx_packets': true,
  'network.tx_errors': true,
  'network.tx_dropped': true,
  'memory_stats.failcnt': true,
  'memory_stats.stats.pgfault': true,
  'memory_stats.stats.pgpgin': true,
  'memory_stats.stats.pgpgout': true,
  'cpu_stats.throttling_data.throttled_time': true,
  'blkio_stats.io_service_bytes_recursive.0.value': true,
  // READ time
  'blkio_stats.io_service_time_recursive.0.value': true,
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value': true,
  // WRITE
  'blkio_stats.io_service_bytes_recursive.1.value': true,
  // Write time
  'blkio_stats.io_service_time_recursive.1.value': true,
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value': true
}

function getValueAsNum (value) {
  var rv = 0
  var nuVal = Number(value)
  if (!isNaN(nuVal)) {
    rv = nuVal
  }
  return rv
}

function getStatsValue (metric) {
  if (avgAggregation[metric.name]) {
    return getValueAsNum(metric.mean)
  } else {
    return getValueAsNum(metric.sum)
  }
}
function dockerStatsCollector (statsCallback) {
  // start streaming docker stats
  getDockerStats()
  setInterval(function () {
    statsCallback({type: 'container', name: 'count', value: containerCount})
    var cpuPercentTotal = 0
    for (var container in cMetrics) {
      var values = getStatsForContainer(container, metrics)
      if (!values) {
        statsCallback(null)
      } else {
        values = values.map(function (v) {
          var rv = getStatsValue(v)
          return rv
        })
        var stats = {dockerId: container, image: cMetrics[container].info.image, name: cMetrics[container].info.name, value: values, fieldNames: metrics}
        statsCallback(stats)
        if (values && values.length > 0) {
          cpuPercentTotal = cpuPercentTotal + values[0]
        }
      }
    }
    statsCallback({type: 'container', name: 'totalcpu', value: cpuPercentTotal})
  }, config.collectionInterval)
}

function containerCountQuery () {
  countPauseContainers(function (err, pauseCount) {
    var k8sPauseCount = pauseCount || 0
    if (err) {
      k8sPauseCount = 0
      SpmAgent.Logger.debug('Error counting k8s pause containers: ' + err)
    }
    SpmAgent.Logger.debug('k8s google pause containers: ' + k8sPauseCount)
    docker.info(function dockerInfoHandler (err, data) {
      if (err) {
        SpmAgent.Logger.error(err)
      }
      if (data && data.ContainersPaused && pausedContainers < 0) {
        // set inital pausedContainer value
        pausedContainers = Number(data.ContainersPaused)
      }
      if (data && data.ContainersRunning) {
        containerCount = Number(data.ContainersRunning) - k8sPauseCount
      } else {
        // no info from Docker API
        // fall-back counters based on status information
        containerCount = Object.keys(cMetrics).length - k8sPauseCount - pausedContainers
        SpmAgent.Logger.error('error', 'error dockerInfo.ContainersRunning not reported ')
      }
      SpmAgent.Logger.debug('containerCount ' + containerCount)
    })
  })
}

setInterval(containerCountQuery, 10000)
containerCountQuery()

module.exports = dockerStatsCollector
<commit_msg>fix for '0 values in metrics'<commit_after>/*
 * @copyright Copyright (c) Sematext Group, Inc. - All Rights Reserved
 *
 * @licence SPM for Docker is free-to-use, proprietary software.
 * THIS IS PROPRIETARY SOURCE CODE OF Sematext Group, Inc. (Sematext)
 * This source code may not be copied, reverse engineered, or altered for any purpose.
 * This source code is to be used exclusively by users and customers of Sematext.
 * Please see the full license (found in LICENSE in this distribution) for details on its license and the licenses of its dependencies.
 */
var stats = require('docker-stats')
var flatten = require('flat')
var through = require('through2')
var Aggregator = require('./aggregator.js')
var SpmAgent = require('spm-agent')
var config = SpmAgent.Config

var Docker = require('dockerode')
var docker = new Docker()
var containerCount = 0
var GOOGLE_PAUSE_REGEX = /google_containers\/pause/
var opts = {
  docker: null, //  here goes options for Dockerode
  events: null, //  an instance of docker-allcontainers,
  statsinterval: 19,
  matchByName: process.env.SPM_MATCH_BY_NAME,
  matchByImage: process.env.SPM_MATCH_BY_IMAGE,
  skipByName: process.env.SPM_SKIP_BY_NAME,
  skipByImage: process.env.SPM_SKIP_BY_IMAGE
}
// holds aggregation object for each container
var cMetrics = {}
var pausedContainers = -1
var allContainers = require('docker-allcontainers')
var events = allContainers(opts)
var DockerEvents = require('docker-events')
var de = new DockerEvents({docker: new Docker()})
de.start()
events.on('error', console.error)
// remove inactive containers
events.on('stop', function (container) {
  if (!container || !container.id) {
    return
  }
  var id = container.id.substring(0, 12)
  delete cMetrics[id]
})

de.on('connect', function () {
  de.on('_message', function (message) {
    if (message && message.status === 'pause' && message.Actor && message.Actor.ID) {
      pausedContainers = pausedContainers + 1
    }
    if (message && message.status === 'unpause' && message.Actor && message.Actor.ID) {
      if (pausedContainers > 0) {
        pausedContainers = pausedContainers - 1
      }
    }
  })
})

// remove inactive containers
setInterval(function cleanUpMetrics () {
  var now = Date.now()
  var containers = Object.keys(cMetrics)
  containers.forEach(function removeEntry (entry) {
    if (cMetrics[entry] && cMetrics[entry].lastUpdate !== undefined && ((now - cMetrics[entry].lastUpdate) > 30000)) {
      delete cMetrics[entry]
    }
  })
}, 30000)

function countPauseContainers (cb) {
  docker.listContainers(function (err, info) {
    if (err) {
      return cb(err)
    } else {
      if (info) {
        var k8sPause = info.filter(function (c) {
          return GOOGLE_PAUSE_REGEX.test(c.Image) && c.State === 'running'
        })
        if (k8sPause) {
          return cb(null, k8sPause.length)
        } else {
          return cb(null, 0)
        }
      } else {
        return cb(new Error('countPauseContainers: no container info in docker.listContainers call'))
      }
    }
  })
}

function transformStatsApi1_21 (stats) {
  if (!stats.networks) {
    return stats
  }
  var result = {
    rx_bytes: 0,
    rx_packets: 0,
    rx_errors: 0,
    rx_dropped: 0,
    tx_bytes: 0,
    tx_packets: 0,
    tx_errors: 0,
    tx_dropped: 0
  }
  Object.keys(stats.networks).forEach(function aggNetworkMetrics (key) {
    Object.keys(result).forEach(function (metric) {
      result[metric] = result[metric] + getValueAsNum(stats.networks[key][metric])
    })
  })
  stats.network = result
  return stats
}

// handling different structures of docker stats
function transformStats (stats) {
  if (stats.network && stats.network.rx_bytes) {
    // API < 1.21
    return stats
  } else {
    return transformStatsApi1_21(stats)
  }
}

function processStats (chunk, enc, cb) {
  var stats = flatten(transformStats(chunk.stats))
  var metric = {}
  SpmAgent.Logger.debug(stats)
  if (getValueAsNum(stats['memory_stats.usage']) === 0 &&
      getValueAsNum(stats['cpu_stats.cpu_usage.cpu_percent']) === 0) {
      // no mem & cpu used, container stopped - workaround for bug in docker-stats
      // reporting stats for stopped containers
    return cb()
  }
  if (cMetrics[chunk.id] === undefined) {
    cMetrics[chunk.id] = new Aggregator()
  }
  metric.name = chunk.name
  metric.dockerId = chunk.id
  metric.image = chunk.image
  var agg = cMetrics[chunk.id]
  agg.info = metric

  metrics.forEach(function (key) {
    var value = getValueAsNum(stats[key] || 0)
    if (/service_time_/i.test(key) || /wait_time_/i.test(key)) {
      // micro seconds to ms
      value = value / (1000 * 1000)
    }
    agg.update(Date.now(), key, value, useDivForAggregation[key] || false)
  })
  // setTimeout (cb, 10)
  cb()
}

function getDockerStats (dockerStatsCallback) {
  var dockerStatsStream = stats(opts)
  dockerStatsStream.pipe(through.obj(processStats))
  dockerStatsStream.on('closed', function () {
    SpmAgent.Logger.error('docker stats stream closed')
  })
  dockerStatsStream.once('error', function (err) {
    SpmAgent.Logger.error('docker stats stream error:' + err)
    // try to reconnect
    setTimeout(function () {
      getDockerStats(dockerStatsCallback)
    }, 15000)
    try {
      dockerStatsStream.close()
    } catch (ex) {
      SpmAgent.Logger.error(ex)
    }
  })
}

function getStatsForContainer (container, aggStats) {
  var agg = cMetrics[container]
  if (GOOGLE_PAUSE_REGEX.test(cMetrics[container].info.image)) {
    return null
  }
  if (!agg) {
    return null
  }
  var values = []
  aggStats.forEach(function (key) {
    var metricStats = agg.get(key)
    if (!metricStats.err) {
      metricStats.id = container
      values.push(metricStats)
    }
  })
  agg.reset()
  return values
}

var cpuMetrics = [
  // CPU
  // 'cpu_stats.cpu_usage.total_usage',
  // 'cpu_stats.cpu_usage.percpu_usage.0',
  // 'cpu_stats.cpu_usage.usage_in_kernelmode',
  // 'cpu_stats.cpu_usage.usage_in_usermode',
  'cpu_stats.cpu_usage.cpu_percent',
  // 'cpu_stats.system_cpu_usage',
  // 'cpu_stats.throttling_data.periods',
  // 'cpu_stats.throttling_data.throttled_periods',
  'cpu_stats.throttling_data.throttled_time'
]

var networkMetrics = [
  // NETWORK
  'network.rx_bytes',
  'network.rx_packets',
  'network.rx_errors',
  'network.rx_dropped',
  'network.tx_bytes',
  'network.tx_packets',
  'network.tx_errors',
  'network.tx_dropped'
]

var memoryMetrics = [
  // MEMORY
  // 'memory_stats.total_rss',
  'memory_stats.usage',
  'memory_stats.limit',
  'memory_stats.failcnt',
  // 'memory_stats.max_usage',
  // 'memory_stats.active_anon',
  // 'memory_stats.active_file',
  // 'memory_stats.cache',
  // 'memory_stats.hierarchical_memory_limit',
  // 'memory_stats.inactive_anon',
  // 'memory_stats.inactive_file',
  // 'memory_stats.mapped_file',
  'memory_stats.stats.pgfault',
  // 'memory_stats.pgmajfault',
  'memory_stats.stats.pgpgin',
  'memory_stats.stats.pgpgout'
  // 'memory_stats.rss',
  // 'memory_stats.rss_huge',
  // MEMORY FAIL COUNTER

  // MEMORY LIMIT

// MEMORY TOTAL
// 'memory_stats.total_active_anon',
// 'memory_stats.total_active_file',
// 'memory_stats.total_cache',
// 'memory_stats.total_inactive_anon',
// 'memory_total_inactive_file',
// 'memory_stats.total_mapped_file',
// 'memory_stats.total_pgfault',
// 'memory_stats.total_pgmajfault',
// 'memory_stats.total_pgpgin',
// 'memory_stats.total_pgpgout',
// 'memory_stats.total_rss',
// 'memory_stats.total_rss_huge',
// 'memory_stats.total_unevictable',
// 'memory_stats.stats.total_writeback',
// 'memory_stats.unevictable',
// 'memory_stats.writeback'
]

var ioMetrics = [
  // IO STATS
  // READ
  'blkio_stats.io_service_bytes_recursive.0.value',
  // READ time
  'blkio_stats.io_service_time_recursive.0.value',
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value',
  // WRITE
  'blkio_stats.io_service_bytes_recursive.1.value',
  // Write time
  'blkio_stats.io_service_time_recursive.1.value',
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value'
// 'blkio_io_service_bytes_recursive',
// 'blkio_io_serviced_recursive',
// 'blkio_io_queue_recursive',
// 'blkio_io_service_time_recursive',
// 'blkio_io_wait_time_recursive',
// 'blkio_io_merged_recursive',
// 'blkio_io_time_recursive',
// 'blkio_sectors_recursive'
]
var metrics = []
metrics = metrics.concat(cpuMetrics).concat(networkMetrics).concat(memoryMetrics).concat(ioMetrics)
var avgAggregation = {
  'cpu_stats.cpu_usage.cpu_percent': true,
  'cpu_stats.throttling_data.throttled_time': true,
  'memory_stats.usage': true,
  'memory_stats.limit': true,
  // READ time
  'blkio_stats.io_service_time_recursive.0.value': true,
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value': true,
  // Write time
  'blkio_stats.io_service_time_recursive.1.value': true,
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value': true
}
var useDivForAggregation = {
  'network.rx_bytes': true,
  'network.rx_packets': true,
  'network.rx_errors': true,
  'network.rx_dropped': true,
  'network.tx_bytes': true,
  'network.tx_packets': true,
  'network.tx_errors': true,
  'network.tx_dropped': true,
  'memory_stats.failcnt': true,
  'memory_stats.stats.pgfault': true,
  'memory_stats.stats.pgpgin': true,
  'memory_stats.stats.pgpgout': true,
  'cpu_stats.throttling_data.throttled_time': true,
  'blkio_stats.io_service_bytes_recursive.0.value': true,
  // READ time
  'blkio_stats.io_service_time_recursive.0.value': true,
  // READ wait time
  'blkio_stats.io_wait_time_recursive.0.value': true,
  // WRITE
  'blkio_stats.io_service_bytes_recursive.1.value': true,
  // Write time
  'blkio_stats.io_service_time_recursive.1.value': true,
  // Write wait time
  'blkio_stats.io_wait_time_recursive.1.value': true
}

function getValueAsNum (value) {
  var rv = 0
  var nuVal = Number(value)
  if (!isNaN(nuVal)) {
    rv = nuVal
  }
  return rv
}

function getStatsValue (metric) {
  if (avgAggregation[metric.name]) {
    return getValueAsNum(metric.mean)
  } else {
    return getValueAsNum(metric.sum)
  }
}
function dockerStatsCollector (statsCallback) {
  // start streaming docker stats
  getDockerStats()
  setInterval(function () {
    statsCallback({type: 'container', name: 'count', value: containerCount})
    var cpuPercentTotal = 0
    for (var container in cMetrics) {
      var values = getStatsForContainer(container, metrics)
      if (!values) {
        statsCallback(null)
      } else {
        values = values.map(function (v) {
          var rv = getStatsValue(v)
          return rv
        })
        var stats = {dockerId: container, image: cMetrics[container].info.image, name: cMetrics[container].info.name, value: values, fieldNames: metrics}
        statsCallback(stats)
        if (values && values.length > 0) {
          cpuPercentTotal = cpuPercentTotal + values[0]
        }
      }
    }
    statsCallback({type: 'container', name: 'totalcpu', value: cpuPercentTotal})
  }, config.collectionInterval)
}

function containerCountQuery () {
  countPauseContainers(function (err, pauseCount) {
    var k8sPauseCount = pauseCount || 0
    if (err) {
      k8sPauseCount = 0
      SpmAgent.Logger.debug('Error counting k8s pause containers: ' + err)
    }
    SpmAgent.Logger.debug('k8s google pause containers: ' + k8sPauseCount)
    docker.info(function dockerInfoHandler (err, data) {
      if (err) {
        SpmAgent.Logger.error(err)
      }
      if (data && data.ContainersPaused && pausedContainers < 0) {
        // set inital pausedContainer value
        pausedContainers = Number(data.ContainersPaused)
      }
      if (data && data.ContainersRunning) {
        containerCount = Number(data.ContainersRunning) - k8sPauseCount
      } else {
        // no info from Docker API
        // fall-back counters based on status information
        containerCount = Object.keys(cMetrics).length - k8sPauseCount - pausedContainers
        SpmAgent.Logger.error('error', 'error dockerInfo.ContainersRunning not reported ')
      }
      SpmAgent.Logger.debug('containerCount ' + containerCount)
    })
  })
}

setInterval(containerCountQuery, 10000)
containerCountQuery()

module.exports = dockerStatsCollector
<|endoftext|>


<commit_before>'use strict';

const debug = require('debug')('migro:drivers:mysql');

const Promise = require('bluebird');
const mysql = require('mysql');

const queries = require('./queries');

class MySQLDriver {
  constructor (config) {
    this.config = config;
    this._mysql = Promise.promisifyAll(mysql.createConnection(config));

    this._initialized = false;
  }

  async init () {
    try {
      await this._mysql.connectAsync();

      const res = await this._mysql.queryAsync(queries.mysql.CHECK_MIGRATIONS_TABLE_EXISTENCE);

      if (res[0].exists) {
        this._initialized = true;
        return;
      }

      await this._mysql.queryAsync(queries.mysql.CREATE_MIGRATIONS_TABLE);
      this._initialized = true;
    } catch (err) {
      this._lastError = err;
      debug(err);
      this._initialized = false;
    }
  }

  async getLastMigration () {
    debug('getLastMigration');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_VERSION_AND_MIGRATION);
    debug('last version & migration:', res[0]);

    return {
      version: (res[0] || {}).version,
      name: (res[0] || {}).name
    };
  }

  async getLastVersionMigrations () {
    debug('getLastVersionMigrations');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_VERSION_AND_ALL_ITS_MIGRATIONS);
    debug('last version & all its migration:', res);

    return res.map(row => ({
      version: (row || {}).version,
      name: (row || {}).name
    }));
  }

  async getLastNMigrations (count) {
    debug('getLastNMigrations');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_N_MIGRATIONS, [count]);
    debug(`last ${count} migrations`, res);

    return res.map(row => ({
      version: (row || {}).version,
      name: (row || {}).name
    }));
  }

  async execute (version, name, fn, isUp = true) {
    if (!this._initialized) {
      await this.init();
    }

    await this._mysql.queryAsync('START TRANSACTION;');

    try {
      await fn(this._mysql);
      await this._mysql.queryAsync(isUp ? queries.mysql.INSERT_MIGRATION : queries.mysql.DELETE_MIGRATION, [version, name]);
      await this._mysql.queryAsync('COMMIT;');
    } catch (err) {
      debug(`Error during migration ${version}:${name}`, err);
      await this._mysql.queryAsync('ROLLBACK;');
    }
  }

  async exit () {
    await this._mysql.endAsync();
  }
}

module.exports = MySQLDriver;
<commit_msg>Final implmentation of MySQL driver.<commit_after>'use strict';

const debug = require('debug')('migro:drivers:mysql');

const Promise = require('bluebird');
const mysql = require('mysql');

const queries = require('./queries');

class MySQLDriver {
  constructor (config) {
    this.config = config;
    this._mysql = Promise.promisifyAll(mysql.createConnection(config));

    this._initialized = false;
  }

  async init () {
    try {
      await this._mysql.connectAsync();

      const res = await this._mysql.queryAsync(queries.mysql.CHECK_MIGRATIONS_TABLE_EXISTENCE, [this.config.database]);

      if (res[0].exists) {
        this._initialized = true;
        return;
      }

      await this._mysql.queryAsync(queries.mysql.CREATE_MIGRATIONS_TABLE);
      this._initialized = true;
    } catch (err) {
      this._lastError = err;
      debug(err);
      this._initialized = false;
    }
  }

  async getLastMigration () {
    debug('getLastMigration');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_VERSION_AND_MIGRATION);
    debug('last version & migration:', res[0]);

    return {
      version: (res[0] || {}).version,
      name: (res[0] || {}).name
    };
  }

  async getLastVersionMigrations () {
    debug('getLastVersionMigrations');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_VERSION_AND_ALL_ITS_MIGRATIONS);
    debug('last version & all its migration:', res);

    return res.map(row => ({
      version: (row || {}).version,
      name: (row || {}).name
    }));
  }

  async getLastNMigrations (count) {
    debug('getLastNMigrations');

    if (!this._initialized) {
      await this.init();
    }

    const res = await this._mysql.queryAsync(queries.mysql.GET_LAST_N_MIGRATIONS, [parseInt(count)]);
    debug(`last ${count} migrations`, res);

    return res.map(row => ({
      version: (row || {}).version,
      name: (row || {}).name
    }));
  }

  async execute (version, name, fn, isUp = true) {
    if (!this._initialized) {
      await this.init();
    }

    await this._mysql.queryAsync('START TRANSACTION;');

    try {
      await fn(this._mysql);
      await this._mysql.queryAsync(isUp ? queries.mysql.INSERT_MIGRATION : queries.mysql.DELETE_MIGRATION, [version, name]);
      await this._mysql.queryAsync('COMMIT;');
    } catch (err) {
      debug(`Error during migration ${version}:${name}`, err);
      await this._mysql.queryAsync('ROLLBACK;');
    }
  }

  async exit () {
    await this._mysql.endAsync();
  }
}

module.exports = MySQLDriver;
<|endoftext|>


<commit_before>var _ = require('lodash');
var AWS = require('aws-sdk');
var events = require('events');
var moment = require('moment');
var path = require('path');
var async = require('async');
var util = require('util');

var Uploader = require('s3-streaming-upload').Uploader;

var ReadableStream = require('./readable-stream');

function DynamoBackup(options) {
    options = options || {};
    this.excludedTables = options.excludedTables || [];
    this.includedTables = options.includedTables;
    this.readPercentage = options.readPercentage || .25;
    this.backupPath = options.backupPath;
    this.bucket = options.bucket;
    this.stopOnFailure = options.stopOnFailure || false;
    this.base64Binary = options.base64Binary || false;
    this.awsAccessKey = options.awsAccessKey || process.env.AWS_ACCESS_KEY_ID;
    this.awsSecretKey = options.awsSecretKey || process.env.AWS_SECRET_ACCESS_KEY;
    this.awsRegion = options.awsRegion || process.env.AWS_DEFAULT_REGION || 'us-east-1';
    this.debug = Boolean(options.debug);

    AWS.config.update({
        accessKeyId: this.awsAccessKey,
        secretAccessKey: this.secretAccessKey,
        region: this.awsRegion
    });
}

util.inherits(DynamoBackup, events.EventEmitter);

DynamoBackup.prototype.listTables = function (callback) {
    var self = this;
    self._fetchTables(null, [], callback);
}

DynamoBackup.prototype.backupTable = function (tableName, backupPath, callback) {
    var self = this;
    var stream = new ReadableStream();

    if (callback === undefined) {
        callback = backupPath;
        backupPath = self._getBackupPath();
    }

    var upload = new Uploader({
        accessKey:  self.awsAccessKey,
        secretKey:  self.awsSecretKey,
        region:     self.awsRegion,
        bucket:     self.bucket,
        objectName: path.join(backupPath, tableName + '.json'),
        stream:     stream,
        debug:      self.debug
    });

    upload.send(function(err) {
        if (err) {
            self.emit('error', {
                table: tableName,
                err:   err
            });
        }
        return callback(err);
    });

    self._copyTable(
        tableName,
        function(items) {
            items.forEach(function(item) {
                if (self.base64Binary) {
                    _.each(item, function (value, key) {
                        if (value && value.B) {
                            value.B = new Buffer(value.B).toString('base64');
                        }
                    });
                }
                stream.append(JSON.stringify(item));
                stream.append('\n');
            });
        },
        function(err) {
            stream.end();
            if(err) {
                self.emit('error', {
                    table: tableName,
                    err:   err
                });
            }
        }
    );
}

DynamoBackup.prototype.backupAllTables = function (callback) {
    var self = this;
    var backupPath = self._getBackupPath();

    self.listTables(function(err, tables) {
        if (err) {
            return callback(err);
        }
        var includedTables = self.includedTables || tables;
        tables = _.difference(tables, self.excludedTables);
        tables = _.intersection(tables, includedTables);

        async.each(tables,
            function(tableName, done) {
                var startTime = moment();
                self.emit('start-backup', tableName);
                self.backupTable(tableName, backupPath, function(err) {
                    if (err) {
                        self.emit('error', {
                            tableName: tableName,
                            error: err
                        });
                        if (self.stopOnFailure) {
                            return done(err);
                        }
                    }
                    self.emit('end-backup', tableName);
                    done();
                })
            },
            callback
        );
    });
}

DynamoBackup.prototype._getBackupPath = function() {
    var self = this;
    var now = moment();
    return self.backupPath || ('DynamoDB-backup-' + now.format('YYYY-MM-DD-HH-mm-ss'));
}

DynamoBackup.prototype._copyTable = function (tableName, itemsReceived, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    ddb.describeTable({ TableName: tableName }, function(err, data) {
        if (err) {
            return callback(err);
        }

        var readPercentage = self.readPercentage;
        var limit = Math.max((data.Table.ProvisionedThroughput.ReadCapacityUnits * readPercentage)|0, 1);

        self._streamItems(tableName, null, limit, itemsReceived, callback);
    });
}

DynamoBackup.prototype._streamItems = function fetchItems(tableName, startKey, limit, itemsReceived, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    var params = {
        Limit: limit,
        ReturnConsumedCapacity: 'NONE',
        TableName: tableName
    };
    if (startKey) {
        params.ExclusiveStartKey = startKey;
    }
    ddb.scan(params, function(err, data) {
        if (err) {
            return callback(err);
        }

        if(data.Items.length > 0) {
            itemsReceived(data.Items);
        }

        if (!data.LastEvaluatedKey || _.keys(data.LastEvaluatedKey).length === 0) {
            return callback();
        }
        self._streamItems(tableName, data.LastEvaluatedKey, limit, itemsReceived, callback);
    });
}

DynamoBackup.prototype._fetchTables = function(lastTable, tables, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    var params = {};
    if (lastTable) {
        params.ExclusiveStartTableName = lastTable;
    }
    ddb.listTables(params, function(err, data) {
        if (err) {
            return callback(err, null);
        }
        tables = tables.concat(data.TableNames);
        if (data.LastEvaluatedTableName) {
            self._fetchTables(data.LastEvaluatedTableName, tables, callback);
        } else {
            callback(null, tables);
        }
    });
};

module.exports = DynamoBackup;
<commit_msg>Removed duplicate error event<commit_after>var _ = require('lodash');
var AWS = require('aws-sdk');
var events = require('events');
var moment = require('moment');
var path = require('path');
var async = require('async');
var util = require('util');

var Uploader = require('s3-streaming-upload').Uploader;

var ReadableStream = require('./readable-stream');

function DynamoBackup(options) {
    options = options || {};
    this.excludedTables = options.excludedTables || [];
    this.includedTables = options.includedTables;
    this.readPercentage = options.readPercentage || .25;
    this.backupPath = options.backupPath;
    this.bucket = options.bucket;
    this.stopOnFailure = options.stopOnFailure || false;
    this.base64Binary = options.base64Binary || false;
    this.awsAccessKey = options.awsAccessKey || process.env.AWS_ACCESS_KEY_ID;
    this.awsSecretKey = options.awsSecretKey || process.env.AWS_SECRET_ACCESS_KEY;
    this.awsRegion = options.awsRegion || process.env.AWS_DEFAULT_REGION || 'us-east-1';
    this.debug = Boolean(options.debug);

    AWS.config.update({
        accessKeyId: this.awsAccessKey,
        secretAccessKey: this.secretAccessKey,
        region: this.awsRegion
    });
}

util.inherits(DynamoBackup, events.EventEmitter);

DynamoBackup.prototype.listTables = function (callback) {
    var self = this;
    self._fetchTables(null, [], callback);
}

DynamoBackup.prototype.backupTable = function (tableName, backupPath, callback) {
    var self = this;
    var stream = new ReadableStream();

    if (callback === undefined) {
        callback = backupPath;
        backupPath = self._getBackupPath();
    }

    var upload = new Uploader({
        accessKey:  self.awsAccessKey,
        secretKey:  self.awsSecretKey,
        region:     self.awsRegion,
        bucket:     self.bucket,
        objectName: path.join(backupPath, tableName + '.json'),
        stream:     stream,
        debug:      self.debug
    });

    upload.send(function(err) {
        if (err) {
            self.emit('error', {
                table: tableName,
                err:   err
            });
        }
        return callback(err);
    });

    self._copyTable(
        tableName,
        function(items) {
            items.forEach(function(item) {
                if (self.base64Binary) {
                    _.each(item, function (value, key) {
                        if (value && value.B) {
                            value.B = new Buffer(value.B).toString('base64');
                        }
                    });
                }
                stream.append(JSON.stringify(item));
                stream.append('\n');
            });
        },
        function(err) {
            stream.end();
            if(err) {
                self.emit('error', {
                    table: tableName,
                    err:   err
                });
            }
        }
    );
}

DynamoBackup.prototype.backupAllTables = function (callback) {
    var self = this;
    var backupPath = self._getBackupPath();

    self.listTables(function(err, tables) {
        if (err) {
            return callback(err);
        }
        var includedTables = self.includedTables || tables;
        tables = _.difference(tables, self.excludedTables);
        tables = _.intersection(tables, includedTables);

        async.each(tables,
            function(tableName, done) {
                var startTime = moment();
                self.emit('start-backup', tableName);
                self.backupTable(tableName, backupPath, function(err) {
                    if (err) {
                        if (self.stopOnFailure) {
                            return done(err);
                        }
                    }
                    self.emit('end-backup', tableName);
                    done();
                })
            },
            callback
        );
    });
}

DynamoBackup.prototype._getBackupPath = function() {
    var self = this;
    var now = moment();
    return self.backupPath || ('DynamoDB-backup-' + now.format('YYYY-MM-DD-HH-mm-ss'));
}

DynamoBackup.prototype._copyTable = function (tableName, itemsReceived, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    ddb.describeTable({ TableName: tableName }, function(err, data) {
        if (err) {
            return callback(err);
        }

        var readPercentage = self.readPercentage;
        var limit = Math.max((data.Table.ProvisionedThroughput.ReadCapacityUnits * readPercentage)|0, 1);

        self._streamItems(tableName, null, limit, itemsReceived, callback);
    });
}

DynamoBackup.prototype._streamItems = function fetchItems(tableName, startKey, limit, itemsReceived, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    var params = {
        Limit: limit,
        ReturnConsumedCapacity: 'NONE',
        TableName: tableName
    };
    if (startKey) {
        params.ExclusiveStartKey = startKey;
    }
    ddb.scan(params, function(err, data) {
        if (err) {
            return callback(err);
        }

        if(data.Items.length > 0) {
            itemsReceived(data.Items);
        }

        if (!data.LastEvaluatedKey || _.keys(data.LastEvaluatedKey).length === 0) {
            return callback();
        }
        self._streamItems(tableName, data.LastEvaluatedKey, limit, itemsReceived, callback);
    });
}

DynamoBackup.prototype._fetchTables = function(lastTable, tables, callback) {
    var self = this;
    var ddb = new AWS.DynamoDB();
    var params = {};
    if (lastTable) {
        params.ExclusiveStartTableName = lastTable;
    }
    ddb.listTables(params, function(err, data) {
        if (err) {
            return callback(err, null);
        }
        tables = tables.concat(data.TableNames);
        if (data.LastEvaluatedTableName) {
            self._fetchTables(data.LastEvaluatedTableName, tables, callback);
        } else {
            callback(null, tables);
        }
    });
};

module.exports = DynamoBackup;
<|endoftext|>


<commit_before>module.exports = function (sails) {

	/**
	 * Module dependencies.
	 */

	var util		= require( '../util' ),
	async			= require('async'),
	startServer		= require('./start')(sails),
	bodyParserRetry	= require('./bodyParserRetry')(sails);

	/**
	 * Configure the encapsulated Express server
	 */

	return function loadExpress (cb) {

		// Required to be here due to dynmaic NODE_ENV settings via command line args
		var express = require('express'),
			expresstUtils = require('../../node_modules/express/lib/utils.js');

		var bodyParserEnabled = util.isFunction(sails.config.express.bodyParser),
			bodyParserRetryEnabled = sails.config.express.retryBodyParserWithJSON,
			bodyParser = sails.config.express.bodyParser();

		// Define housing object for Express in the Sails namespace
		sails.express = {};
		sails.http = sails.express;

		// Create express server
		var app = sails.express.app = express();

		// (required by Express 3.x)
		var usingSSL = ( ( sails.config.serverOptions && sails.config.serverOptions.key && sails.config.serverOptions.cert ) ||
					( sails.config.express && sails.config.express.serverOptions && sails.config.express.serverOptions.key && sails.config.express.serverOptions.cert ));

		// Get the appropriate server creation method for the protocol
		var createServer = usingSSL ? require('https').createServer : require('http').createServer;

		// Use serverOptions if they were specified
		// Manually create http server using Express app instance
		if (sails.config.express.serverOptions) {
			sails.express.server = createServer(sails.config.express.serverOptions, sails.express.app);
		}
		else sails.express.server = createServer(sails.express.app);

		// Set up location of server-side views and their engine
		sails.express.app.set('views', sails.config.paths.views);

		var engine = sails.config.views.engine;
		var depPath = sails.config.appPath + '/node_modules/' + engine;
		var engineModule;
		try {
			engineModule = require(depPath);
		}
		catch (e) {
			if (e) {
				sails.log.error('Your configured server-side view engine (' + engine + ') could not be found.');
				sails.log.error('You\'re probably just missing it as a dependency though.');
				sails.log.error('To install ' + engine + ', run:  `npm install ' + engine + ' --save`');
				throw e;
			}
		}

		app.engine(engine, engineModule.__express);
		sails.express.app.set('view engine', engine);

		// Install Express middleware
		/////////////////////////////////////////////////////////////////

		// When Sails binds routes, bind them to the internal Express router
		sails.on('router:bind', function (route) {

			route = util.clone(route);

			// TODO: Add support for error domains

			app[route.verb || 'all'](route.path, route.target);
		});

		// When Sails unbinds routes, remove them from the internal Express router
		sails.on('router:unbind', function (path, method) {

			var newRoutes = [];
			util.each(app.routes[method], function(expressRoute) {
				if (expressRoute.path != path) {
					newRoutes.push(expressRoute);
				}
			});
			app.routes[method] = newRoutes;

		});

		// When Sails is ready, start the express server
		sails.on('ready', startServer);

		// Use the specified cookieParser
		var cookieParser = sails.config.express.cookieParser;
		var sessionSecret = sails.config.session.secret;
		if (util.isFunction(cookieParser)) {
			app.use(cookieParser(sessionSecret));
			if (sails.config.environment === 'development') {
				sails.log.verbose('Using secret: '+sessionSecret+' in cookie parser');
			}
		}

		// Connect session to express
		app.use(express.session(sails.config.session));

		// Limit file uploads per route
		// pre-calculate regexp for performance
		var limitedRoutes = [];
		for (var key in sails.config.routes) {
			var route = sails.config.routes[key];
			if (typeof route === 'object' && route.uploadLimit) {
				limitedRoutes.push({matcher: expresstUtils.pathRegexp(key), limit: route.uploadLimit});
			}
		}

		app.use(function(req, res, next) {

			// Limit upload based on route
			for (var key in limitedRoutes){
				var match = limitedRoutes[key].matcher.exec(req.url);
				if(match) {
					return express.limit(limitedRoutes[key].limit)(req, res, next);
				}

			}

			// default to a limit of 10mb
			return express.limit('10mb')(req, res, next);
		});

		// Use body parser, if enabled
		if (bodyParserEnabled) {
			app.use(bodyParser);

			app.use(function handleBodyParserError (err, req, res, next) {

				// Add key middleware
				if (sails.config.hooks.request) {
					sails._mixinLocals(req,res);
					sails._mixinResError(req,res);
					sails._mixinServerMetadata(req,res);
					sails._mixinReqQualifiers(req, res);
				}
				if (sails.config.hooks.views) {
					sails._mixinResView(req,res, function unused () {
						next('Unable to parse HTTP body :: ' + util.inspect(err));
					});
				}

				// Since an error occurred with the body parser,
				// we need to pick up the middleware necessary
				// to serve the error route which we would have
				// gotten in the router and keep going
				next('Unable to parse HTTP body :: ' + util.inspect(err));
			});

			// Retry body parser to try and force JSON, if enabled
			if (bodyParserRetryEnabled) {
				app.use(bodyParserRetry);
			}

		}

		// Use CSRF middleware if enabled
		if(sails.config.csrf){

			// Require CSRF token for non-GET requests
			app.use(express.csrf());

			// Pass csrf token to templates via locals
			app.use(function(req, res, next){
				res.locals._csrf = req.session._csrf;
				next();
			});
		}

		// Use CORES if set per route
		var coresRoutes = [];
		for (var key in sails.config.routes) {
			var route = sails.config.routes[key];
			if (typeof route === 'object' && route.cores) {
			  	if (typeof route.cores === 'boolean') {
      				route.cores = '*';
				}
				coresRoutes.push({matcher: expresstUtils.pathRegexp(key), origin: route.cores});
			}
		}

		app.use(function(req, res, next) {

			// Set CORES header based on route
			for (var key in coresRoutes){
				var match = coresRoutes[key].matcher.exec(req.url);
				if(match) {
				  res.setHeader('Access-Control-Allow-Origin', coresRoutes[key].origin);
				  return next();
				}

			}

			next();
		});

		// Allow full REST simulation for clients which don't support it natively
		// (by using _method parameter)
		if (sails.config.express.methodOverride) {
			app.use(sails.config.express.methodOverride());
		}

		// Allow usage of custom express middleware
		// Must be before the router
		if (sails.config.express.customMiddleware) {
			sails.config.express.customMiddleware(app);
		}

		// Add powered-by Sails header
		app.use(function(req, res, next) {
			res.header('X-Powered-By', 'Sails <sailsjs.org>');
			next();
		});

		// Set up express router
		// Route last so all the middleware gets called first
		sails.log.verbose('Using Express router...');

		// Bind static routes and blueprints
		app.use(app.router);

		// Configure flat file server
		// Bind static files AFTER routing
		sails.log.verbose('Configuring express.static flat-file middleware...');
		var flatFileServer = express['static'](sails.config.paths['public'], {
			maxAge: sails.config.cache.maxAge
		});
		sails.express.app.use(flatFileServer);
		app.use(express.favicon());

		// Make some MIME type exceptions
		express['static'].mime.define({
			'application/font-woff': ['woff']
		});

		// Wait until everything else is done,
		// then bind configured 404 behavior
		app.use(sails.config[404]);

		// When Sails is finished routing ALL routes (including implicit routes)
		// add our default error handler
		app.use(sails.config[500]);

		return cb();

	};

};
<commit_msg>fixed upload limit routing<commit_after>module.exports = function (sails) {

	/**
	 * Module dependencies.
	 */

	var util		= require( '../util' ),
	async			= require('async'),
	startServer		= require('./start')(sails),
	bodyParserRetry	= require('./bodyParserRetry')(sails);

	/**
	 * Configure the encapsulated Express server
	 */

	return function loadExpress (cb) {

		// Required to be here due to dynmaic NODE_ENV settings via command line args
		var express = require('express'),
			expresstUtils = require('../../node_modules/express/lib/utils.js');

		var bodyParserEnabled = util.isFunction(sails.config.express.bodyParser),
			bodyParserRetryEnabled = sails.config.express.retryBodyParserWithJSON,
			bodyParser = sails.config.express.bodyParser();

		// Define housing object for Express in the Sails namespace
		sails.express = {};
		sails.http = sails.express;

		// Create express server
		var app = sails.express.app = express();

		// (required by Express 3.x)
		var usingSSL = ( ( sails.config.serverOptions && sails.config.serverOptions.key && sails.config.serverOptions.cert ) ||
					( sails.config.express && sails.config.express.serverOptions && sails.config.express.serverOptions.key && sails.config.express.serverOptions.cert ));

		// Get the appropriate server creation method for the protocol
		var createServer = usingSSL ? require('https').createServer : require('http').createServer;

		// Use serverOptions if they were specified
		// Manually create http server using Express app instance
		if (sails.config.express.serverOptions) {
			sails.express.server = createServer(sails.config.express.serverOptions, sails.express.app);
		}
		else sails.express.server = createServer(sails.express.app);

		// Set up location of server-side views and their engine
		sails.express.app.set('views', sails.config.paths.views);

		var engine = sails.config.views.engine;
		var depPath = sails.config.appPath + '/node_modules/' + engine;
		var engineModule;
		try {
			engineModule = require(depPath);
		}
		catch (e) {
			if (e) {
				sails.log.error('Your configured server-side view engine (' + engine + ') could not be found.');
				sails.log.error('You\'re probably just missing it as a dependency though.');
				sails.log.error('To install ' + engine + ', run:  `npm install ' + engine + ' --save`');
				throw e;
			}
		}

		app.engine(engine, engineModule.__express);
		sails.express.app.set('view engine', engine);

		// Install Express middleware
		/////////////////////////////////////////////////////////////////

		// When Sails binds routes, bind them to the internal Express router
		sails.on('router:bind', function (route) {

			route = util.clone(route);

			// TODO: Add support for error domains

			app[route.verb || 'all'](route.path, route.target);
		});

		// When Sails unbinds routes, remove them from the internal Express router
		sails.on('router:unbind', function (path, method) {

			var newRoutes = [];
			util.each(app.routes[method], function(expressRoute) {
				if (expressRoute.path != path) {
					newRoutes.push(expressRoute);
				}
			});
			app.routes[method] = newRoutes;

		});

		// When Sails is ready, start the express server
		sails.on('ready', startServer);

		// Use the specified cookieParser
		var cookieParser = sails.config.express.cookieParser;
		var sessionSecret = sails.config.session.secret;
		if (util.isFunction(cookieParser)) {
			app.use(cookieParser(sessionSecret));
			if (sails.config.environment === 'development') {
				sails.log.verbose('Using secret: '+sessionSecret+' in cookie parser');
			}
		}

		// Connect session to express
		app.use(express.session(sails.config.session));

		// Limit file uploads per route
		app.use(function(req, res, next) {

			// Match the url to a config route
			for (var key in sails.config.routes) {
				var route = sails.config.routes[key];
				if (typeof route === 'object' && route.uploadLimit) {
					var match = expresstUtils.pathRegexp(key).exec(req.url);
					if(match) {
						return express.limit(route.uploadLimit)(req, res, next);
					}
				}
			}

			// default to a limit of 10mb
			return express.limit('10mb')(req, res, next);
		});

		// Use body parser, if enabled
		if (bodyParserEnabled) {
			app.use(bodyParser);

			app.use(function handleBodyParserError (err, req, res, next) {

				// Add key middleware
				if (sails.config.hooks.request) {
					sails._mixinLocals(req,res);
					sails._mixinResError(req,res);
					sails._mixinServerMetadata(req,res);
					sails._mixinReqQualifiers(req, res);
				}
				if (sails.config.hooks.views) {
					sails._mixinResView(req,res, function unused () {
						next('Unable to parse HTTP body :: ' + util.inspect(err));
					});
				}

				// Since an error occurred with the body parser,
				// we need to pick up the middleware necessary
				// to serve the error route which we would have
				// gotten in the router and keep going
				next('Unable to parse HTTP body :: ' + util.inspect(err));
			});

			// Retry body parser to try and force JSON, if enabled
			if (bodyParserRetryEnabled) {
				app.use(bodyParserRetry);
			}

		}

		// Use CSRF middleware if enabled
		if(sails.config.csrf){

			// Require CSRF token for non-GET requests
			app.use(express.csrf());

			// Pass csrf token to templates via locals
			app.use(function(req, res, next){
				res.locals._csrf = req.session._csrf;
				next();
			});
		}

		// Use CORES if set per route
		var coresRoutes = [];
		for (var key in sails.config.routes) {
			var route = sails.config.routes[key];
			if (typeof route === 'object' && route.cores) {
			  	if (typeof route.cores === 'boolean') {
      				route.cores = '*';
				}
				coresRoutes.push({matcher: expresstUtils.pathRegexp(key), origin: route.cores});
			}
		}

		app.use(function(req, res, next) {

			// Set CORES header based on route
			for (var key in coresRoutes){
				var match = coresRoutes[key].matcher.exec(req.url);
				if(match) {
				  res.setHeader('Access-Control-Allow-Origin', coresRoutes[key].origin);
				  return next();
				}

			}

			next();
		});

		// Allow full REST simulation for clients which don't support it natively
		// (by using _method parameter)
		if (sails.config.express.methodOverride) {
			app.use(sails.config.express.methodOverride());
		}

		// Allow usage of custom express middleware
		// Must be before the router
		if (sails.config.express.customMiddleware) {
			sails.config.express.customMiddleware(app);
		}

		// Add powered-by Sails header
		app.use(function(req, res, next) {
			res.header('X-Powered-By', 'Sails <sailsjs.org>');
			next();
		});

		// Set up express router
		// Route last so all the middleware gets called first
		sails.log.verbose('Using Express router...');

		// Bind static routes and blueprints
		app.use(app.router);

		// Configure flat file server
		// Bind static files AFTER routing
		sails.log.verbose('Configuring express.static flat-file middleware...');
		var flatFileServer = express['static'](sails.config.paths['public'], {
			maxAge: sails.config.cache.maxAge
		});
		sails.express.app.use(flatFileServer);
		app.use(express.favicon());

		// Make some MIME type exceptions
		express['static'].mime.define({
			'application/font-woff': ['woff']
		});

		// Wait until everything else is done,
		// then bind configured 404 behavior
		app.use(sails.config[404]);

		// When Sails is finished routing ALL routes (including implicit routes)
		// add our default error handler
		app.use(sails.config[500]);

		return cb();

	};

};
<|endoftext|>