<commit_before>var async = require('async')
    , assert = require('assert')
    , host = process.env.COREOS_HOST || '127.0.0.1'
    , etcd = new (require('node-etcd'))(host)
    , winston = require('winston')
    , bunyan = require('bunyan')
    , Stream = require('stream').Stream
    , util = require('util')
    , kafka = require('kafka-node')
    , EventEmitter = require('events').EventEmitter;

var last_resort_logger = bunyan.createLogger({ name: 'auth0-coreos-tools-last-resort-logging' });

var KafkaStream = exports.KafkaStream = function (options) {
    assert.ok(options, 'options must be specified');
    assert.ok(typeof options === 'object', 'options must be an object');
    assert.ok(typeof options.zookeeper === 'string', 'options.zookeeper must be specfied');
    assert.ok(Array.isArray(options.topics), 'options.topic must be an array of string topics');
    Stream.call(this);
    this.name = options.name || 'kafka';
    this.writable = true;
    this.options = options;
    this._pending = [];
    this._client = new kafka.Client(this.options.zookeeper, 'kafka-transport');
    this._producer = new kafka.Producer(this._client);
    var self = this;
    this._producer
        .once('ready', function () {
            self._ready = true;
            KafkaStream.prototype._send_pending.apply(self);
        })
        .on('error', function (error) {
            last_resort_logger.error(error, 'kafka logging error');
            self._error = error;
            self.emit('error', error);
        });
};

util.inherits(KafkaStream, Stream);

KafkaStream.prototype.close = function close() {
    this.writable = false;
    this._client.close();
};

KafkaStream.prototype.destroy = function destroy() {
    this.writable = false;
    this.close();
};

KafkaStream.prototype.end = function end() {
    if (arguments.length > 0)
            this.write.apply(this, Array.prototype.slice.call(arguments));

    this.writable = false;
    this.close();
};

KafkaStream.prototype.write = function write(r) {
    if (this._error)
        this.emit('error', new Error('Attempt to write to Kafka after previous error.'));
    else if (!this.writable)
        this.emit('error', new Error('Attempt to write to Kafka after stream was closed.'));
    else {
        this._pending.push(r);
        this._send_pending();
    }
};

KafkaStream.prototype._send_pending = function () {
    if (this._ready && this._pending.length > 0) {
        var current = this._pending;
        this._pending = [];
        var messages = [];
        current.forEach(function (entry) {
            messages.push(typeof entry === 'string' ? entry : JSON.stringify(entry));
        });
        var payload = [];
        this.options.topics.forEach(function (topic) {
            payload.push({
                topic: topic,
                messages: messages
            });
        });
        this._producer.send(payload, function (error) {
            if (error)
                last_resort_logger.error(error, 'kafka logging error');
        });
    }
};

function create_logger(name, topics) {
    var logger_options = {
        name: name,
        streams: [{ level: 'info', stream: process.stdout }]
    };

    var kafka_stream;
    if (Array.isArray(topics) && process.env.KAFKA_ZOOKEEPER) {
        // Enable Kafka stream
        kafka_stream = new KafkaStream({
            zookeeper: process.env.KAFKA_ZOOKEEPER,
            topics: topics
        });
        logger_options.streams.push({ level: 'info', stream: kafka_stream });
    }

    var result = bunyan.createLogger(logger_options);
    result.close = function () {
        if (kafka_stream) {
            kafka_stream.close();
            kafka_stream = undefined;
        }
    };
    result.on('error', function () {
        // ignore logging errors
    });

    return result;
};

exports.create_logger = create_logger;

var logger = create_logger('auth0-coreos-tools', ['system']);

exports.set_logger = function (new_logger) {
    logger = new_logger.child({ 'auth0-coreos-tools' : 1 });
};

// Accepts an array of strings as configuration keys.
// Each {key1} corresponds to /config/{key1} entry in etcd.
// Returns an object with properties corresponding to input keys.
// Each property is an EventEmitter. It emits the `modified` event
// when the value of the configuration key. In addition, each EventEmitter
// has a `current` property which returns the current string value stored in etcd.
exports.etcd_config = function (keys, path, callback) {
    if (typeof path === 'function') {
        callback = path;
        path = '/config/';
    }
    var config = {};
    async.each(
        keys,
        function (key, callback) {
            etcd.get(path + key, function (error, result) {
                if (error) {
                    error.key = path + key;
                    logger.error(error, 'unable to read etcd key');
                    return callback(error);
                }

                var watch = etcd.watcher(path + key, result.node.modifiedIndex + 1);
                watch.current = result.node.value;
                watch.on('change', function (result) {
                    if (watch.current !== result.node.value)
                        this.emit('modified', result.node.value);
                    watch.current = result.node.value;
                });
                watch.on('error', function (error) {
                    error.key = path + key;
                    logger.error(error, 'error watching etcd key');
                    throw error; // TODO (tjanczuk): consider surfacing etcd errors differently?
                });

                config[key] = watch;
                callback();
            });
        },
        function (error) {
            callback && (error ? callback(error) : callback(null, config));
        }
    );

    return config; // for convenience of REPL
};

// Converts response from etcd into a JS object representing only
// the data and its hierarchy. Etcd directories are represented as JS objects.

function etcd2js(node) {
    if (node.value !== undefined)
        return node.value;
    var result = {};
    if (Array.isArray(node.nodes)) {
        node.nodes.forEach(function (child) {
            result[child.key.substring(node.key.length === 1 ? 1 : node.key.length + 1)]
                = etcd2js(child);
        });
    }
    return result;
}

exports.etcd2js = etcd2js;

// Gets the key from etcd (recursively) and returns a JS representation of
// the hierarchy.
exports.etcd_get = function (key, callback) {
    etcd.get(key, { recursive: true }, function (error, result) {
        error ? callback(error): callback(null, etcd2js(result.node));
    });
};

// Convert object property names to uppercase
exports.upper = function (o) {
    var r = {};
    for (var k in o)
        r[k.toUpperCase()] = typeof o[k] === 'object' ? upper(o[k]) : o[k];
    return r;
};

// The .lowr async function completes in either of two situations:
//
// 1. If the sync_root is not locked, it obtains an exclusive, distributed lock of the sync_root
// via etcd and returns a function that can be used to release the lock as a callback parameter.
//
// 2. If the sync_root is locked, it waits for that lock to be released, and returns without any results.
//
// This logic allows several homogenic, distributed processes to ensure that a certain workload is
// completed by only one of them.
//
// The lock TTL in etcd is configured with the `/config/lock.ttl` confguration entry.
// While the lock is held, it is reset to that TTL every `/config/lock.renew` period.
var lock_id = process.pid + '.' + Date.now();
exports.lowr = function (sync_root, etcdconfig, callback) {
    logger.info({ sync_root: sync_root }, 'lowr enter');
    // Attempt to take the lock
    etcd.set(
        '/lock/' + sync_root,
        lock_id,
        { prevExist: false, ttl: +etcdconfig.lock_ttl.current },
        function (error, result) {
            if (error) {
                if (error.errorCode === 105) {
                    logger.info({ sync_root: sync_root }, 'lowr lock already taken');
                    // Lock is currently held, wait for its release
                    var watch = etcd.watcher('/lock/' + sync_root, error.error.index + 1);
                    watch.on('change', function (result) {
                        if (result.node.value === undefined)
                            done();
                    });
                    watch.on('error', done);

                    function done(error) {
                        if (error)
                            logger.error(error, 'lowr wait error');
                        else
                            logger.info({ sync_root: sync_root }, 'lowr finished wait');
                        watch.stop();
                        callback(error);
                    }

                    return;
                }

                logger.error({ sync_root: sync_root, error: error }, 'lowr etcd error');

                return callback(error);
            }

            var current_lock = Math.floor(Math.random() * 999999);
            logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr obtained lock');

            // Lock was taken, renew it periodically until released

            var renew = setInterval(function () {
                logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr renew lock');
                etcd.set('/lock/' + sync_root, lock_id, { ttl: +etcdconfig.lock_ttl.current });
            }, +etcdconfig.lock_renew.current * 1000);

            // Return a function that must be used to release the lock

            callback(null, function (callback) {
                logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr delete lock');
                clearInterval(renew);
                etcd.compareAndDelete('/lock/' + sync_root, lock_id, function (error) {
                    return callback && callback(error);
                });
            });
        });
};

var KafkaTransport = exports.KafkaTransport = function (options) {
    assert.ok(options, 'options must be specified');
    assert.ok(typeof options === 'object', 'options must be an object');
    assert.ok(typeof options.zookeeper === 'string', 'options.zookeeper must be specfied');
    assert.ok(Array.isArray(options.topics), 'options.topic must be an array of string topics');
    this.name = options.name || 'kafka';
    this.level = options.level || 'info';
    this.options = options;
    this._pending = [];
    this._client = new kafka.Client(this.options.zookeeper, 'kafka-transport');
    this._producer = new kafka.Producer(this._client);
    var self = this;
    this._producer
        .once('ready', function () {
            self._ready = true;
            self._send_pending();
        })
        .on('error', function (error) {
            self._error = error;
            winston.error('winston-kafka error', error);
        });
};

util.inherits(KafkaTransport, winston.Transport);

KafkaTransport.prototype.log = function (level, msg, meta, callback) {
    if (!this._error) {
        this._pending.push({ time: new Date(), level: level, msg: msg, meta: meta, callback: callback });
        this._send_pending();
    }
    else
        callback(this._error);
};

KafkaTransport.prototype._send_pending = function () {
    if (this._ready && this._pending.length > 0) {
        var current = this._pending;
        this._pending = [];
        var messages = [];
        current.forEach(function (entry) {
            var meta;
            if (entry.meta && Object.getOwnPropertyNames(entry.meta).length > 0)
                meta = ' ' + util.inspect(entry.meta);
            messages.push(
                entry.time.toISOString() + ' '
                + entry.level + ': '
                + entry.msg
                + (meta || '')
            );
        });
        var payload = [];
        this.options.topics.forEach(function (topic) {
            payload.push({
                topic: topic,
                messages: messages
            });
        });
        this._producer.send(payload, function (error) {
            current.forEach(function (entry) {
                entry.callback && entry.callback(error);
            });
        });
    }
};

exports.setup_kafka = function (options, callback) {
    var ev = new EventEmitter();
    var config;
    var zookeeper;

    async.series([
        function (callback) {
            // Get config for enable_kafka_logs
            last_resort_logger.info('checking if kafka logging is enabled');
            exports.etcd_config([
                'enable_kafka_logs'
            ], function (error, data) {
                config = data;
                callback(error);
            });
        },
        function (callback) {
            // Get zookeeper config
            last_resort_logger.info({ enable_kafka_logs: +config.enable_kafka_logs.current }, 'kafka logs enabled');
            if (+config.enable_kafka_logs.current)
                setup_zookeeper(callback);
            else
                callback();
        },
        function (callback) {
            // Set up first logger
            setup_logger(+config.enable_kafka_logs.current, zookeeper ? zookeeper.route.current : undefined);

            // Keep track of changes to logging settings
            config.enable_kafka_logs.on('modified', function (kafka_enabled) {
                last_resort_logger.info({ enable_kafka_logs: +kafka_enabled, zookeeper: zookeeper ? zookeeper.route.current : undefined },
                 'enable_kafka_logs changed');
                if (+kafka_enabled && !zookeeper)
                    setup_zookeeper(function () {
                        setup_logger(+kafka_enabled, zookeeper ? zookeeper.route.current : undefined);
                    });
                else
                    setup_logger(+kafka_enabled, zookeeper ? zookeeper.route.current : undefined);
            });

            callback();
        }
    ], function (error) {
        if (!ev.logger)
            setup_logger(false);
        callback(error, ev.logger, zookeeper ? zookeeper.route.current : undefined);
    });

    function setup_zookeeper(callback) {
        last_resort_logger.info('getting zookeeper configuration');
        exports.etcd_config(['route'], '/services/zookeeper/', function (error, data) {
            if (error) {
                last_resort_logger.error(error, 'error getting zookeeper configuration');
                return callback(error);
            }
            zookeeper = data;
            zookeeper.route.on('modified', function (new_zookeeper) {
                last_resort_logger.info({ zookeeper: new_zookeeper }, 'zookeeper address has changed');
                setup_logger(+config.enable_kafka_logs.current, new_zookeeper);
            });
            callback();
        });
    }

    function setup_logger(kafka_enabled, zookeeper_route) {
        process.env.KAFKA_ZOOKEEPER = (kafka_enabled && zookeeper_route) ? zookeeper_route : '';
        last_resort_logger.info({
            kafka_enabled: kafka_enabled,
            zookeeper: zookeeper_route,
            previous_logger: ev.logger !== undefined,
            kafka_zookeper: process.env.KAFKA_ZOOKEEPER,
            options: options
        }, 'setting up new logger');
        if (ev.logger)
            try { ev.logger.close(); ev.logger = undefined; } catch (e) {}
        ev.logger = exports.create_logger(options.name, options.topics);
        ev.logger.info({ kafka_logging: kafka_enabled, zookeeper: zookeeper_route
            , logging_streams: (ev.logger && Array.isArray(ev.logger.streams)) ? ev.logger.streams.length : 0 }
            ,'updated logger');
        exports.set_logger(ev.logger);
        process.nextTick(function () {
            last_resort_logger.info({ logger: typeof ev.logger, zookeeper: zookeeper_route },
                'emitting kafka modified event');
            ev.emit('modified', ev.logger, zookeeper_route);
        });
    }

    return ev;
};
<commit_msg>delayed kafkastream close<commit_after>var async = require('async')
    , assert = require('assert')
    , host = process.env.COREOS_HOST || '127.0.0.1'
    , etcd = new (require('node-etcd'))(host)
    , winston = require('winston')
    , bunyan = require('bunyan')
    , Stream = require('stream').Stream
    , util = require('util')
    , kafka = require('kafka-node')
    , EventEmitter = require('events').EventEmitter;

var last_resort_logger = bunyan.createLogger({ name: 'auth0-coreos-tools-last-resort-logging' });

var KafkaStream = exports.KafkaStream = function (options) {
    assert.ok(options, 'options must be specified');
    assert.ok(typeof options === 'object', 'options must be an object');
    assert.ok(typeof options.zookeeper === 'string', 'options.zookeeper must be specfied');
    assert.ok(Array.isArray(options.topics), 'options.topic must be an array of string topics');
    Stream.call(this);
    this.name = options.name || 'kafka';
    this.writable = true;
    this.options = options;
    this._pending = [];
    this._client = new kafka.Client(this.options.zookeeper, 'kafka-transport');
    this._producer = new kafka.Producer(this._client);
    var self = this;
    this._producer
        .once('ready', function () {
            self._ready = true;
            KafkaStream.prototype._send_pending.apply(self);
        })
        .on('error', function (error) {
            last_resort_logger.error(error, 'kafka logging error');
            self._error = error;
            self.emit('error', error);
        });
};

util.inherits(KafkaStream, Stream);

KafkaStream.prototype.close = function close() {
    if (!this._closed) {
        this._closed = true;
        this.writable = false;
        if (this._pending.length > 0)
            this._close_pending = true;
        else
            this._client.close();
    }
};

KafkaStream.prototype.destroy = function destroy() {
    this.writable = false;
    this.close();
};

KafkaStream.prototype.end = function end() {
    if (arguments.length > 0)
        this.write.apply(this, Array.prototype.slice.call(arguments));

    this.writable = false;
    this.close();
};

KafkaStream.prototype.write = function write(r) {
    if (this._error)
        this.emit('error', new Error('Attempt to write to Kafka after previous error.'));
    else if (!this.writable)
        this.emit('error', new Error('Attempt to write to Kafka after stream was closed.'));
    else {
        this._pending.push(r);
        this._send_pending();
    }
};

KafkaStream.prototype._send_pending = function () {
    if (this._ready && this._pending.length > 0) {
        var current = this._pending;
        this._pending = [];
        var messages = [];
        current.forEach(function (entry) {
            messages.push(typeof entry === 'string' ? entry : JSON.stringify(entry));
        });
        var payload = [];
        this.options.topics.forEach(function (topic) {
            payload.push({
                topic: topic,
                messages: messages
            });
        });
        var self = this;
        this._producer.send(payload, function (error) {
            if (error)
                last_resort_logger.error(error, 'kafka logging error');
            if (self._close_pending) {
                self.writable = false;
                self._client.close();
            }
        });
    }
};

function create_logger(name, topics) {
    var logger_options = {
        name: name,
        streams: [{ level: 'info', stream: process.stdout }]
    };

    var kafka_stream;
    if (Array.isArray(topics) && process.env.KAFKA_ZOOKEEPER) {
        // Enable Kafka stream
        kafka_stream = new KafkaStream({
            zookeeper: process.env.KAFKA_ZOOKEEPER,
            topics: topics
        });
        logger_options.streams.push({ level: 'info', stream: kafka_stream });
    }

    var result = bunyan.createLogger(logger_options);
    result.close = function () {
        if (kafka_stream) {
            kafka_stream.close();
            kafka_stream = undefined;
        }
    };
    result.on('error', function () {
        // ignore logging errors
    });

    return result;
};

exports.create_logger = create_logger;

var logger = create_logger('auth0-coreos-tools', ['system']);

exports.set_logger = function (new_logger) {
    logger = new_logger.child({ 'auth0-coreos-tools' : 1 });
};

// Accepts an array of strings as configuration keys.
// Each {key1} corresponds to /config/{key1} entry in etcd.
// Returns an object with properties corresponding to input keys.
// Each property is an EventEmitter. It emits the `modified` event
// when the value of the configuration key. In addition, each EventEmitter
// has a `current` property which returns the current string value stored in etcd.
exports.etcd_config = function (keys, path, callback) {
    if (typeof path === 'function') {
        callback = path;
        path = '/config/';
    }
    var config = {};
    async.each(
        keys,
        function (key, callback) {
            etcd.get(path + key, function (error, result) {
                if (error) {
                    error.key = path + key;
                    logger.error(error, 'unable to read etcd key');
                    return callback(error);
                }

                var watch = etcd.watcher(path + key, result.node.modifiedIndex + 1);
                watch.current = result.node.value;
                watch.on('change', function (result) {
                    if (watch.current !== result.node.value)
                        this.emit('modified', result.node.value);
                    watch.current = result.node.value;
                });
                watch.on('error', function (error) {
                    error.key = path + key;
                    logger.error(error, 'error watching etcd key');
                    throw error; // TODO (tjanczuk): consider surfacing etcd errors differently?
                });

                config[key] = watch;
                callback();
            });
        },
        function (error) {
            callback && (error ? callback(error) : callback(null, config));
        }
    );

    return config; // for convenience of REPL
};

// Converts response from etcd into a JS object representing only
// the data and its hierarchy. Etcd directories are represented as JS objects.

function etcd2js(node) {
    if (node.value !== undefined)
        return node.value;
    var result = {};
    if (Array.isArray(node.nodes)) {
        node.nodes.forEach(function (child) {
            result[child.key.substring(node.key.length === 1 ? 1 : node.key.length + 1)]
                = etcd2js(child);
        });
    }
    return result;
}

exports.etcd2js = etcd2js;

// Gets the key from etcd (recursively) and returns a JS representation of
// the hierarchy.
exports.etcd_get = function (key, callback) {
    etcd.get(key, { recursive: true }, function (error, result) {
        error ? callback(error): callback(null, etcd2js(result.node));
    });
};

// Convert object property names to uppercase
exports.upper = function (o) {
    var r = {};
    for (var k in o)
        r[k.toUpperCase()] = typeof o[k] === 'object' ? upper(o[k]) : o[k];
    return r;
};

// The .lowr async function completes in either of two situations:
//
// 1. If the sync_root is not locked, it obtains an exclusive, distributed lock of the sync_root
// via etcd and returns a function that can be used to release the lock as a callback parameter.
//
// 2. If the sync_root is locked, it waits for that lock to be released, and returns without any results.
//
// This logic allows several homogenic, distributed processes to ensure that a certain workload is
// completed by only one of them.
//
// The lock TTL in etcd is configured with the `/config/lock.ttl` confguration entry.
// While the lock is held, it is reset to that TTL every `/config/lock.renew` period.
var lock_id = process.pid + '.' + Date.now();
exports.lowr = function (sync_root, etcdconfig, callback) {
    logger.info({ sync_root: sync_root }, 'lowr enter');
    // Attempt to take the lock
    etcd.set(
        '/lock/' + sync_root,
        lock_id,
        { prevExist: false, ttl: +etcdconfig.lock_ttl.current },
        function (error, result) {
            if (error) {
                if (error.errorCode === 105) {
                    logger.info({ sync_root: sync_root }, 'lowr lock already taken');
                    // Lock is currently held, wait for its release
                    var watch = etcd.watcher('/lock/' + sync_root, error.error.index + 1);
                    watch.on('change', function (result) {
                        if (result.node.value === undefined)
                            done();
                    });
                    watch.on('error', done);

                    function done(error) {
                        if (error)
                            logger.error(error, 'lowr wait error');
                        else
                            logger.info({ sync_root: sync_root }, 'lowr finished wait');
                        watch.stop();
                        callback(error);
                    }

                    return;
                }

                logger.error({ sync_root: sync_root, error: error }, 'lowr etcd error');

                return callback(error);
            }

            var current_lock = Math.floor(Math.random() * 999999);
            logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr obtained lock');

            // Lock was taken, renew it periodically until released

            var renew = setInterval(function () {
                logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr renew lock');
                etcd.set('/lock/' + sync_root, lock_id, { ttl: +etcdconfig.lock_ttl.current });
            }, +etcdconfig.lock_renew.current * 1000);

            // Return a function that must be used to release the lock

            callback(null, function (callback) {
                logger.info({ sync_root: sync_root, lock_num: current_lock }, 'lowr delete lock');
                clearInterval(renew);
                etcd.compareAndDelete('/lock/' + sync_root, lock_id, function (error) {
                    return callback && callback(error);
                });
            });
        });
};

var KafkaTransport = exports.KafkaTransport = function (options) {
    assert.ok(options, 'options must be specified');
    assert.ok(typeof options === 'object', 'options must be an object');
    assert.ok(typeof options.zookeeper === 'string', 'options.zookeeper must be specfied');
    assert.ok(Array.isArray(options.topics), 'options.topic must be an array of string topics');
    this.name = options.name || 'kafka';
    this.level = options.level || 'info';
    this.options = options;
    this._pending = [];
    this._client = new kafka.Client(this.options.zookeeper, 'kafka-transport');
    this._producer = new kafka.Producer(this._client);
    var self = this;
    this._producer
        .once('ready', function () {
            self._ready = true;
            self._send_pending();
        })
        .on('error', function (error) {
            self._error = error;
            winston.error('winston-kafka error', error);
        });
};

util.inherits(KafkaTransport, winston.Transport);

KafkaTransport.prototype.log = function (level, msg, meta, callback) {
    if (!this._error) {
        this._pending.push({ time: new Date(), level: level, msg: msg, meta: meta, callback: callback });
        this._send_pending();
    }
    else
        callback(this._error);
};

KafkaTransport.prototype._send_pending = function () {
    if (this._ready && this._pending.length > 0) {
        var current = this._pending;
        this._pending = [];
        var messages = [];
        current.forEach(function (entry) {
            var meta;
            if (entry.meta && Object.getOwnPropertyNames(entry.meta).length > 0)
                meta = ' ' + util.inspect(entry.meta);
            messages.push(
                entry.time.toISOString() + ' '
                + entry.level + ': '
                + entry.msg
                + (meta || '')
            );
        });
        var payload = [];
        this.options.topics.forEach(function (topic) {
            payload.push({
                topic: topic,
                messages: messages
            });
        });
        this._producer.send(payload, function (error) {
            current.forEach(function (entry) {
                entry.callback && entry.callback(error);
            });
        });
    }
};

exports.setup_kafka = function (options, callback) {
    var ev = new EventEmitter();
    var config;
    var zookeeper;

    async.series([
        function (callback) {
            // Get config for enable_kafka_logs
            last_resort_logger.info('checking if kafka logging is enabled');
            exports.etcd_config([
                'enable_kafka_logs'
            ], function (error, data) {
                config = data;
                callback(error);
            });
        },
        function (callback) {
            // Get zookeeper config
            last_resort_logger.info({ enable_kafka_logs: +config.enable_kafka_logs.current }, 'kafka logs enabled');
            if (+config.enable_kafka_logs.current)
                setup_zookeeper(callback);
            else
                callback();
        },
        function (callback) {
            // Set up first logger
            setup_logger(+config.enable_kafka_logs.current, zookeeper ? zookeeper.route.current : undefined);

            // Keep track of changes to logging settings
            config.enable_kafka_logs.on('modified', function (kafka_enabled) {
                last_resort_logger.info({ enable_kafka_logs: +kafka_enabled, zookeeper: zookeeper ? zookeeper.route.current : undefined },
                 'enable_kafka_logs changed');
                if (+kafka_enabled && !zookeeper)
                    setup_zookeeper(function () {
                        setup_logger(+kafka_enabled, zookeeper ? zookeeper.route.current : undefined);
                    });
                else
                    setup_logger(+kafka_enabled, zookeeper ? zookeeper.route.current : undefined);
            });

            callback();
        }
    ], function (error) {
        if (!ev.logger)
            setup_logger(false);
        callback(error, ev.logger, zookeeper ? zookeeper.route.current : undefined);
    });

    function setup_zookeeper(callback) {
        last_resort_logger.info('getting zookeeper configuration');
        exports.etcd_config(['route'], '/services/zookeeper/', function (error, data) {
            if (error) {
                last_resort_logger.error(error, 'error getting zookeeper configuration');
                return callback(error);
            }
            zookeeper = data;
            zookeeper.route.on('modified', function (new_zookeeper) {
                last_resort_logger.info({ zookeeper: new_zookeeper }, 'zookeeper address has changed');
                setup_logger(+config.enable_kafka_logs.current, new_zookeeper);
            });
            callback();
        });
    }

    function setup_logger(kafka_enabled, zookeeper_route) {
        process.env.KAFKA_ZOOKEEPER = (kafka_enabled && zookeeper_route) ? zookeeper_route : '';
        last_resort_logger.info({
            kafka_enabled: kafka_enabled,
            zookeeper: zookeeper_route,
            previous_logger: ev.logger !== undefined,
            kafka_zookeper: process.env.KAFKA_ZOOKEEPER,
            options: options
        }, 'setting up new logger');
        if (ev.logger)
            try { ev.logger.close(); ev.logger = undefined; } catch (e) {}
        ev.logger = exports.create_logger(options.name, options.topics);
        ev.logger.info({ kafka_logging: kafka_enabled, zookeeper: zookeeper_route
            , logging_streams: (ev.logger && Array.isArray(ev.logger.streams)) ? ev.logger.streams.length : 0 }
            ,'updated logger');
        exports.set_logger(ev.logger);
        process.nextTick(function () {
            last_resort_logger.info({ logger: typeof ev.logger, zookeeper: zookeeper_route },
                'emitting kafka modified event');
            ev.emit('modified', ev.logger, zookeeper_route);
        });
    }

    return ev;
};
<|endoftext|>


<commit_before>"use strict";
var EventEmitter = require("events");
var util = require("util");
var instance = null;

function MockAppService() {
    EventEmitter.call(this);
}
util.inherits(MockAppService, EventEmitter);

MockAppService.prototype.listen = function(port) {
    // NOP
};

MockAppService.prototype._trigger = function(eventType, content) {
    var listeners = instance.listeners(eventType);
    var promises = listeners.map(function(l) {
        return l(content);
    });
    if (promises.length === 1) {
        return promises[0];
    }
    return Promise.all(promises);
};

MockAppService.prototype._queryAlias = function(alias) {
    if (!this.onAliasQuery) {
        throw new Error("IRC AS hasn't hooked into onAliasQuery yet.");
    }
    return this.onAliasQuery(alias);
};

MockAppService.prototype._queryUser = function(user) {
    if (!this.onUserQuery) {
        throw new Error("IRC AS hasn't hooked into onUserQuery yet.");
    }
    return this.onUserQuery(user);
};

function MockAppServiceProxy() {
    if (!instance) {
        instance = new MockAppService();
    }
    return instance;
}
MockAppServiceProxy.instance = function() {
    instance = new MockAppService();
    return instance;
};
MockAppServiceProxy.resetInstance = function() {
    if (instance) {
        instance.removeAllListeners();
    }
    instance = null;
};

module.exports = MockAppServiceProxy;
<commit_msg>Make instance() not create new singletons<commit_after>"use strict";
var EventEmitter = require("events");
var util = require("util");
var instance = null;

function MockAppService() {
    EventEmitter.call(this);
}
util.inherits(MockAppService, EventEmitter);

MockAppService.prototype.listen = function(port) {
    // NOP
};

MockAppService.prototype._trigger = function(eventType, content) {
    var listeners = instance.listeners(eventType);
    var promises = listeners.map(function(l) {
        return l(content);
    });
    if (promises.length === 1) {
        return promises[0];
    }
    return Promise.all(promises);
};

MockAppService.prototype._queryAlias = function(alias) {
    if (!this.onAliasQuery) {
        throw new Error("IRC AS hasn't hooked into onAliasQuery yet.");
    }
    return this.onAliasQuery(alias);
};

MockAppService.prototype._queryUser = function(user) {
    if (!this.onUserQuery) {
        throw new Error("IRC AS hasn't hooked into onUserQuery yet.");
    }
    return this.onUserQuery(user);
};

function MockAppServiceProxy() {
    if (!instance) {
        instance = new MockAppService();
    }
    return instance;
}

MockAppServiceProxy.instance = function() {
    if (!instance) {
        instance = new MockAppService();
    }
    return instance;
};

MockAppServiceProxy.resetInstance = function() {
    if (instance) {
        instance.removeAllListeners();
    }
    instance = null;
};

module.exports = MockAppServiceProxy;
<|endoftext|>


<commit_before>var net = require('net');
var clc = require('cli-color');
var Jetty = require("jetty");
var jetty = new Jetty(process.stdout);

var level = {
  info: clc.blue,
  error: clc.red,
  ok: clc.green,
  warning: clc.yellow
}

function repeat(interval, fn) {
  fn();
  setTimeout(repeat.bind(undefined, interval, fn), interval);
}

function slice(string, length) {
  var s = string.substring(0, length);
  return new String(new Array(length).join(" ") + s).slice(s.length)
}

function fill(string, length) {
  var diff = length - string.length;
  return string + new Array(diff).join("-");
}

function compareState(a, b) {
  if (a.name > b.name) {
    return 1;
  }
  if (a.name < b.name) {
    return -1;
  }
  return 0;
}

function printStatus(connectionState) {
  jetty.clear();
  jetty.moveTo([0,0]);
  var groups = {};
  for (var value of connectionState.values()) {
    if (!groups[value.group]) {
      groups[value.group] = [];
    }
    groups[value.group].push(value);
  }
  Object.getOwnPropertyNames(groups).forEach(function (group) {
    jetty.text(fill("----- " + group + " -", 63) + "\n");
    groups[group].sort(compareState).forEach(function (state) {
      jetty.text(slice(state.name, 30) + " - " + level[state.level](slice(state.status, 30)));
      jetty.text('\n');
    });
  });
}

function createStateObject(msg, c) {
  var state = JSON.parse(msg);
  if (!state.name) {
    state.name = c.remoteAddress;
  }
  if (!state.level) {
    state.level = "info";
  }
  if (!state.status) {
    state.status = "unknown";
    state.level = "warning";
  }
  if (!state.group) {
    state.group = "no group";
  }
  return state;
}

function handleBuffer(buffer, c, connectionState) {
  var index = buffer.indexOf("}");
  if (index !== -1) {
    var msg = buffer.substring(0, index + 1);
    buffer = buffer.substring(index + 1);
    try {
      connectionState.set(c, createStateObject(msg, c));
    } catch (e) {
      // ignore invalid messages
    }
    return handleBuffer(buffer, c);
  } else {
    return buffer;
  }
}

function start(port) {
  var connectionState = new Map();

  var server = net.createServer(function(c) {
    connectionState.set(c, {
      name: "Client connected",
      status: "Waiting...",
      level: "warning",
      group: "no group"
    });
    var buffer = "";
    c.on('end', function() {
      setTimeout(function() {
        connectionState.delete(c);
      }, 10000);
    });
    c.on('data', function(data) {
      buffer = handleBuffer(buffer + data.toString(), c, connectionState);
    });
  });
  server.listen(port, function() {
    console.log('Server started...');
  });
  repeat(2000, printStatus.bind(undefined, connectionState));
}

module.exports = function() {
  return {
    startServer: start
  };
}<commit_msg>Sort groups<commit_after>var net = require('net');
var clc = require('cli-color');
var Jetty = require("jetty");
var jetty = new Jetty(process.stdout);

var level = {
  info: clc.blue,
  error: clc.red,
  ok: clc.green,
  warning: clc.yellow
}

function repeat(interval, fn) {
  fn();
  setTimeout(repeat.bind(undefined, interval, fn), interval);
}

function slice(string, length) {
  var s = string.substring(0, length);
  return new String(new Array(length).join(" ") + s).slice(s.length)
}

function fill(string, length) {
  var diff = length - string.length;
  return string + new Array(diff).join("-");
}

function compareState(a, b) {
  if (a.name > b.name) {
    return 1;
  }
  if (a.name < b.name) {
    return -1;
  }
  return 0;
}

function printStatus(connectionState) {
  jetty.clear();
  jetty.moveTo([0,0]);
  var groups = {};
  for (var value of connectionState.values()) {
    if (!groups[value.group]) {
      groups[value.group] = [];
    }
    groups[value.group].push(value);
  }
  Object.getOwnPropertyNames(groups).sort().forEach(function (group) {
    jetty.text(fill("----- " + group + " -", 63) + "\n");
    groups[group].sort(compareState).forEach(function (state) {
      jetty.text(slice(state.name, 30) + " - " + level[state.level](slice(state.status, 30)));
      jetty.text('\n');
    });
  });
}

function createStateObject(msg, c) {
  var state = JSON.parse(msg);
  if (!state.name) {
    state.name = c.remoteAddress;
  }
  if (!state.level) {
    state.level = "info";
  }
  if (!state.status) {
    state.status = "unknown";
    state.level = "warning";
  }
  if (!state.group) {
    state.group = "no group";
  }
  return state;
}

function handleBuffer(buffer, c, connectionState) {
  var index = buffer.indexOf("}");
  if (index !== -1) {
    var msg = buffer.substring(0, index + 1);
    buffer = buffer.substring(index + 1);
    try {
      connectionState.set(c, createStateObject(msg, c));
    } catch (e) {
      // ignore invalid messages
    }
    return handleBuffer(buffer, c);
  } else {
    return buffer;
  }
}

function start(port) {
  var connectionState = new Map();

  var server = net.createServer(function(c) {
    connectionState.set(c, {
      name: "Client connected",
      status: "Waiting...",
      level: "warning",
      group: "no group"
    });
    var buffer = "";
    c.on('end', function() {
      setTimeout(function() {
        connectionState.delete(c);
      }, 10000);
    });
    c.on('data', function(data) {
      buffer = handleBuffer(buffer + data.toString(), c, connectionState);
    });
  });
  server.listen(port, function() {
    console.log('Server started...');
  });
  repeat(2000, printStatus.bind(undefined, connectionState));
}

module.exports = function() {
  return {
    startServer: start
  };
}<|endoftext|>


<commit_before>/**
 *  @author  jwyuan and lewuathe
 *  @license Apache-2.0
 *  @version 0.2.3
 *  @requires request
 *  @requires querystring
 *  @see http://docs.treasuredata.com/articles/rest-api-node-client
 *
 *  @example
 *  var TD = require('td');
 *  var client = new TD('TREASURE_DATA_API_KEY');
 *
 *  var fnPrint = function(err, results) {
 *    console.log(results);
 *  };
 *  client.listDatabase(function(err, results) {
 *    for (var i = 0; i < results.databases.length; i++) {
 *      client.listTables(results.databases[i].name, fnPrint);
 *    }
 *  });
 */

var request = require('request').defaults({encoding: 'utf8'});
var qs = require('querystring');

/**
 *  TreasureData REST API Client
 *  @constructor
 *  @param {string} apikey - The API key available from user account.
 *  @param {object} options - Specify the endpoint of TreasureData api.
 *                            options.host = 'api.treasuredata.com'
 *                            options.protocol = 'http'
 */
function TDClient(apikey, options) {
    this.options = options || {};
    this.options.apikey = apikey;
    this.options.host = this.options.host || 'api.treasuredata.com';
    this.options.protocol = this.options.protocol || 'http';
}

TDClient.prototype = {
    // PUBLIC METHOD
    get baseUrl() {
        return this.options.protocol + '://' + this.options.host;
    },

    /**
     *  Return the list of all databases belongs to given account
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *  @example
     *  // Results object
     *  {name: 'db1', count: 1, created_at: 'XXX',
     *   updated_at: 'YYY', organization: null,
     *   permission: 'administrator'}
     */
    listDatabases: function(callback) {
        this._request("/v3/database/list", {
            method: 'GET',
            json: true
        }, callback);
    },

    /**
     *  Delete the given named database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    deleteDatabase: function(db, callback) {
        this._request("/v3/database/delete/" + qs.escape(db), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Create the given named database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createDatabase: function(db, callback) {
        this._request("/v3/database/create/" + qs.escape(db), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Return the list of all tables belongs to given database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    listTables: function(db, callback) {
        this._request("/v3/table/list/" + qs.escape(db), {
            json: true
        }, callback);
    },

    /**
     *  Create log type table in the given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createLogTable: function(db, table, callback) {
        this.createTable(db, table, 'log', callback);
    },

    /**
     *  Create item type table in the given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createItemTable: function(db, table, callback) {
        this.createTable(db, table, 'item', callback);
    },

    /**
     *  Create table in given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {string} type - The type of table ('log' or 'item')
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createTable: function(db, table, type, callback) {
        this._request("/v3/table/create/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(type), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Swap the content of two tables
     *  @param {string} db - The name of database
     *  @param {string} table1 - The first table
     *  @param {string} table2 - The second table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    swapTable: function(db, table1, table2, callback) {
        this._request("/v3/table/swap/" + qs.escape(db) + "/" + qs.escape(table1) + "/" + qs.escape(table2), {
            method: 'POST',
            json: true
        }, callback);
    },

    updateSchema: function(db, table, schema_json, callback) {
        this._request("/v3/table/update-schema/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            body: schema_json,
            json: true
        }, callback);
    },

    deleteTable: function(db, table, callback) {
        this._request("/v3/table/update-schema/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            json: true
        }, callback);
    },

    tail: function(db, table, count, to, from, callback) {
        if (typeof count === 'function') {
            callback = count;
            count = null;
        } else if (typeof to === 'function') {
            callback = to;
            to = null;
        } else if (typeof from === 'function') {
            callback = from;
            from = null;
        }

        var params = {
            // format: 'msgpack'
        };
        if (count) {
            params.count = count;
        }
        if (to) {
            params.to = to;
        }
        if (from) {
            params.from = from;
        }

        this._request("/v3/table/tail/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'GET',
            qs: params
        }, callback);
    },

    /**
     *  Return the list of all jobs run by your account
     *  @param {string} from - The start of the range of list
     *  @param {string} to - The end of the range of list
     *  @param {string} status - The status of returned jobs
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *
     */
    listJobs: function(from, to, status, conditions, callback) {
        if (typeof from === 'function') {
            callback = from;
            from = 0;
        } else if (typeof to === 'function') {
            callback = to;
            to = null;
        } else if (typeof status === 'function') {
            callback = status;
            status = null;
        } else if (typeof conditions === 'function') {
            callback = conditions;
            conditions = null;
        }

        var params = {
            from: from
        };
        if (to) {
            params.to = to;
        }
        if (status) {
            params.status = status;
        }
        if (conditions) {
            params.conditions = conditions;
        }

        this._request("/v3/job/list/", {
            qs: params,
            json: true
        }, callback);
    },

    /**
     *  Returns the status and logs of a given job
     *  @param {string} job_id - The job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *
     */
    showJob: function(job_id, callback) {
        this._request("/v3/job/show/" + qs.escape(job_id), {
            json: true
        }, callback);
    },

    /**
     *  Returns the result of a specific job.
     *  @param {string} job_id - The job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    jobResult: function(job_id, callback) {
        this._request("/v3/job/result/" + qs.escape(job_id), {
            method: 'GET',
            qs: { format: 'tsv' }
        }, callback);
    },

    /**
     *  Kill the currently running job
     *  @param {string} job_id - the job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    kill: function(job_id, callback) {
        this._request("/v3/job/kill/" + qs.escape(job_id), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Submit Hive type job
     *  @param {string} db - The name of database
     *  @param {string} query - The Hive query which run on given database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    hiveQuery: function(db, query, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        this._query(db, 'hive', query, opts, callback)
    },

    prestoQuery: function(db, query, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        this._query(db, 'presto', query, opts, callback)
    },

    // Export API
    export: function(db, table, storage_type, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        opts.storage_type = storage_type;

        this._request("/v3/export/run/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    // Schedule API
    createSchedule: function(name, opts, callback) {
        if (!opts.cron || !opts.query) {
            return callback(new Error('opts.cron and opts.query is required!'), {});
        }
        opts.type = 'hive';
        this._request("/v3/schedule/create/" + qs.escape(name), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    deleteSchedule: function(name, callback) {
        this._request("/v3/schedule/delete/" + qs.escape(name), {
            method: 'POST',
            json: true
        }, callback);
    },

    listSchedules: function(callback) {
        this._request("/v3/schedule/list", {
            method: 'GET',
            json: true
        }, callback);
    },

    updateSchedule: function(name, params, callback) {
        this._request("/v3/schedule/update/" + qs.escape(name), {
            body: params,
            method: 'POST',
            json: true
        }, callback);
    },

    history: function(name, from, to, callback) {
        if (typeof from === 'function') {
            callback = from;
            from = 0;
        } else if (typeof from === 'function') {
            callback = to;
            to = null;
        }
        var params = {};
        if (from) {
            params.from = from;
        }
        if (to) {
            params.to = to;
        }

        this._request("/v3/schedule/history/" + qs.escape(name), {
            method: 'GET',
            qs: params,
            json: true
        }, callback);
    },

    runSchedule: function(name, time, num, callback) {
        if (typeof num === 'function') {
            callback = num;
            num = null;
        }

        var params = {};
        if (num) {
            params.num = num;
        }

        this._request("/v3/schedule/run/" + qs.escape(name) + "/" + qs.escape(time), {
            method: 'POST',
            qs: params,
            json: true
        }, callback);
    },

    // Import API
    import: function(db, table, format, stream, size, unique_id, callback) {
        if (typeof unique_id === 'function') {
            callback = unique_id;
            unique_id = null;
        }

        var path;
        if (unique_id) {
            path = "/v3/table/import_with_id/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(unique_id) + "/" + qs.escape(format);
        } else {
            path = "/v3/table/import/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(format);
        }

        this._put(path, {
            method: 'PUT',
            headers: {
                'Content-Type': 'application/octet-stream',
                'Content-Length': size
            }
        }, stream, callback);
    },

    // Result API
    listResult: function(callback) {
        this._request("/v3/result/list", {
            json: true
        }, callback);
    },

    createResult: function(name, url, callback) {
        this._request("/v3/result/create/" + qs.escape(name), {
            method: 'POST',
            body: { 'url': url },
            json: true
        }, callback);
    },

    deleteResult: function(name, url, callback) {
        this._request("/v3/result/delete/" + qs.escape(name), {
            method: 'POST',
            body: { 'url': url },
            json: true
        }, callback);
    },

    // Server Status API
    serverStatus: function(callback) {
        this._request("/v3/system/server_status", {
            json: true
        }, callback);
    },

    /**
     * _query: Protected method
     * @protected
     */
    _query: function(db, query_type, q, opts, callback) {
        opts.query = q;

        this._request("/v3/job/issue/" + query_type + "/" + qs.escape(db), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    /**
     * _request: Protected method
     * @protected
     */
    _request: function(path, options, callback) {
        if (typeof options === 'function') {
            callback = options;
            options  = {};
        }
        options.uri = this.baseUrl + path;
        options.headers = { 'Authorization': 'TD1 ' + this.options.apikey };
        callback = callback || noop;

        request(options, function (err, res, body) {
            if (err) { return callback(err, undefined); }
            if (res.statusCode < 200 || res.statusCode > 299) {
                return callback(new Error((body && body.error) || 'HTTP ' + res.statusCode),
                    body || {});
            }

            callback(null, body || {});
        });
    },

    /**
     *  _put: Protected method
     *  @protected
     */
    _put: function(path, options, stream, callback) {
        options.uri = this.baseUrl + path;
        options.headers = { 'Authorization': 'TD1 ' + this.options.apikey };

        stream.pipe(request(options, function (err, res, body) {
            if (err) { return callback(err, undefined); }
            if (res.statusCode < 200 || res.statusCode > 299) {
                return callback(new Error((body && body.error) || 'HTTP ' + res.statusCode),
                    body || {});
            }

            callback(null, body || {});
        }));
    }
};

module.exports = TDClient;

// Private and Utility Methods
function noop() {}
<commit_msg>Add query method document<commit_after>/**
 *  @author  jwyuan and lewuathe
 *  @license Apache-2.0
 *  @version 0.2.3
 *  @requires request
 *  @requires querystring
 *  @see http://docs.treasuredata.com/articles/rest-api-node-client
 *
 *  @example
 *  var TD = require('td');
 *  var client = new TD('TREASURE_DATA_API_KEY');
 *
 *  var fnPrint = function(err, results) {
 *    console.log(results);
 *  };
 *  client.listDatabase(function(err, results) {
 *    for (var i = 0; i < results.databases.length; i++) {
 *      client.listTables(results.databases[i].name, fnPrint);
 *    }
 *  });
 */

var request = require('request').defaults({encoding: 'utf8'});
var qs = require('querystring');

/**
 *  TreasureData REST API Client
 *  @constructor
 *  @param {string} apikey - The API key available from user account.
 *  @param {object} options - Specify the endpoint of TreasureData api.
 *                            options.host = 'api.treasuredata.com'
 *                            options.protocol = 'http'
 */
function TDClient(apikey, options) {
    this.options = options || {};
    this.options.apikey = apikey;
    this.options.host = this.options.host || 'api.treasuredata.com';
    this.options.protocol = this.options.protocol || 'http';
}

TDClient.prototype = {
    // PUBLIC METHOD
    get baseUrl() {
        return this.options.protocol + '://' + this.options.host;
    },

    /**
     *  Return the list of all databases belongs to given account
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *  @example
     *  // Results object
     *  {name: 'db1', count: 1, created_at: 'XXX',
     *   updated_at: 'YYY', organization: null,
     *   permission: 'administrator'}
     */
    listDatabases: function(callback) {
        this._request("/v3/database/list", {
            method: 'GET',
            json: true
        }, callback);
    },

    /**
     *  Delete the given named database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    deleteDatabase: function(db, callback) {
        this._request("/v3/database/delete/" + qs.escape(db), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Create the given named database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createDatabase: function(db, callback) {
        this._request("/v3/database/create/" + qs.escape(db), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Return the list of all tables belongs to given database
     *  @param {string} db - The name of database
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    listTables: function(db, callback) {
        this._request("/v3/table/list/" + qs.escape(db), {
            json: true
        }, callback);
    },

    /**
     *  Create log type table in the given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createLogTable: function(db, table, callback) {
        this.createTable(db, table, 'log', callback);
    },

    /**
     *  Create item type table in the given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createItemTable: function(db, table, callback) {
        this.createTable(db, table, 'item', callback);
    },

    /**
     *  Create table in given database
     *  @param {string} db - The name of database
     *  @param {string} table - The name of table
     *  @param {string} type - The type of table ('log' or 'item')
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    createTable: function(db, table, type, callback) {
        this._request("/v3/table/create/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(type), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Swap the content of two tables
     *  @param {string} db - The name of database
     *  @param {string} table1 - The first table
     *  @param {string} table2 - The second table
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    swapTable: function(db, table1, table2, callback) {
        this._request("/v3/table/swap/" + qs.escape(db) + "/" + qs.escape(table1) + "/" + qs.escape(table2), {
            method: 'POST',
            json: true
        }, callback);
    },

    updateSchema: function(db, table, schema_json, callback) {
        this._request("/v3/table/update-schema/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            body: schema_json,
            json: true
        }, callback);
    },

    deleteTable: function(db, table, callback) {
        this._request("/v3/table/update-schema/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            json: true
        }, callback);
    },

    tail: function(db, table, count, to, from, callback) {
        if (typeof count === 'function') {
            callback = count;
            count = null;
        } else if (typeof to === 'function') {
            callback = to;
            to = null;
        } else if (typeof from === 'function') {
            callback = from;
            from = null;
        }

        var params = {
            // format: 'msgpack'
        };
        if (count) {
            params.count = count;
        }
        if (to) {
            params.to = to;
        }
        if (from) {
            params.from = from;
        }

        this._request("/v3/table/tail/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'GET',
            qs: params
        }, callback);
    },

    /**
     *  Return the list of all jobs run by your account
     *  @param {string} from - The start of the range of list
     *  @param {string} to - The end of the range of list
     *  @param {string} status - The status of returned jobs
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *
     */
    listJobs: function(from, to, status, conditions, callback) {
        if (typeof from === 'function') {
            callback = from;
            from = 0;
        } else if (typeof to === 'function') {
            callback = to;
            to = null;
        } else if (typeof status === 'function') {
            callback = status;
            status = null;
        } else if (typeof conditions === 'function') {
            callback = conditions;
            conditions = null;
        }

        var params = {
            from: from
        };
        if (to) {
            params.to = to;
        }
        if (status) {
            params.status = status;
        }
        if (conditions) {
            params.conditions = conditions;
        }

        this._request("/v3/job/list/", {
            qs: params,
            json: true
        }, callback);
    },

    /**
     *  Returns the status and logs of a given job
     *  @param {string} job_id - The job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     *
     */
    showJob: function(job_id, callback) {
        this._request("/v3/job/show/" + qs.escape(job_id), {
            json: true
        }, callback);
    },

    /**
     *  Returns the result of a specific job.
     *  @param {string} job_id - The job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    jobResult: function(job_id, callback) {
        this._request("/v3/job/result/" + qs.escape(job_id), {
            method: 'GET',
            qs: { format: 'tsv' }
        }, callback);
    },

    /**
     *  Kill the currently running job
     *  @param {string} job_id - the job id
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    kill: function(job_id, callback) {
        this._request("/v3/job/kill/" + qs.escape(job_id), {
            method: 'POST',
            json: true
        }, callback);
    },

    /**
     *  Submit Hive type job
     *  @param {string} db - The name of database
     *  @param {string} query - The Hive query which run on given database
     *  @param {object} opts - Optoins supported member are `result`,
     *                         `priority` and `retry_limit`
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    hiveQuery: function(db, query, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        this._query(db, 'hive', query, opts, callback)
    },

    /**
     *  Submit Presto type job
     *  @param {string} db - The name of database
     *  @param {string} query - The Presto query which run on given database
     *  @param {object} opts - Optoins supported member are `result`,
     *                         `priority` and `retry_limit`
     *  @param {function} callback - Callback function which receives
     *                               error object and results object
     */
    prestoQuery: function(db, query, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        this._query(db, 'presto', query, opts, callback)
    },

    // Export API
    export: function(db, table, storage_type, opts, callback) {
        if (typeof opts === 'function') {
            callback = opts;
            opts = {};
        }
        opts.storage_type = storage_type;

        this._request("/v3/export/run/" + qs.escape(db) + "/" + qs.escape(table), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    // Schedule API
    createSchedule: function(name, opts, callback) {
        if (!opts.cron || !opts.query) {
            return callback(new Error('opts.cron and opts.query is required!'), {});
        }
        opts.type = 'hive';
        this._request("/v3/schedule/create/" + qs.escape(name), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    deleteSchedule: function(name, callback) {
        this._request("/v3/schedule/delete/" + qs.escape(name), {
            method: 'POST',
            json: true
        }, callback);
    },

    listSchedules: function(callback) {
        this._request("/v3/schedule/list", {
            method: 'GET',
            json: true
        }, callback);
    },

    updateSchedule: function(name, params, callback) {
        this._request("/v3/schedule/update/" + qs.escape(name), {
            body: params,
            method: 'POST',
            json: true
        }, callback);
    },

    history: function(name, from, to, callback) {
        if (typeof from === 'function') {
            callback = from;
            from = 0;
        } else if (typeof from === 'function') {
            callback = to;
            to = null;
        }
        var params = {};
        if (from) {
            params.from = from;
        }
        if (to) {
            params.to = to;
        }

        this._request("/v3/schedule/history/" + qs.escape(name), {
            method: 'GET',
            qs: params,
            json: true
        }, callback);
    },

    runSchedule: function(name, time, num, callback) {
        if (typeof num === 'function') {
            callback = num;
            num = null;
        }

        var params = {};
        if (num) {
            params.num = num;
        }

        this._request("/v3/schedule/run/" + qs.escape(name) + "/" + qs.escape(time), {
            method: 'POST',
            qs: params,
            json: true
        }, callback);
    },

    // Import API
    import: function(db, table, format, stream, size, unique_id, callback) {
        if (typeof unique_id === 'function') {
            callback = unique_id;
            unique_id = null;
        }

        var path;
        if (unique_id) {
            path = "/v3/table/import_with_id/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(unique_id) + "/" + qs.escape(format);
        } else {
            path = "/v3/table/import/" + qs.escape(db) + "/" + qs.escape(table) + "/" + qs.escape(format);
        }

        this._put(path, {
            method: 'PUT',
            headers: {
                'Content-Type': 'application/octet-stream',
                'Content-Length': size
            }
        }, stream, callback);
    },

    // Result API
    listResult: function(callback) {
        this._request("/v3/result/list", {
            json: true
        }, callback);
    },

    createResult: function(name, url, callback) {
        this._request("/v3/result/create/" + qs.escape(name), {
            method: 'POST',
            body: { 'url': url },
            json: true
        }, callback);
    },

    deleteResult: function(name, url, callback) {
        this._request("/v3/result/delete/" + qs.escape(name), {
            method: 'POST',
            body: { 'url': url },
            json: true
        }, callback);
    },

    // Server Status API
    serverStatus: function(callback) {
        this._request("/v3/system/server_status", {
            json: true
        }, callback);
    },

    /**
     * _query: Protected method
     * @protected
     */
    _query: function(db, query_type, q, opts, callback) {
        opts.query = q;

        this._request("/v3/job/issue/" + query_type + "/" + qs.escape(db), {
            method: 'POST',
            body: opts,
            json: true
        }, callback);
    },

    /**
     * _request: Protected method
     * @protected
     */
    _request: function(path, options, callback) {
        if (typeof options === 'function') {
            callback = options;
            options  = {};
        }
        options.uri = this.baseUrl + path;
        options.headers = { 'Authorization': 'TD1 ' + this.options.apikey };
        callback = callback || noop;

        request(options, function (err, res, body) {
            if (err) { return callback(err, undefined); }
            if (res.statusCode < 200 || res.statusCode > 299) {
                return callback(new Error((body && body.error) || 'HTTP ' + res.statusCode),
                    body || {});
            }

            callback(null, body || {});
        });
    },

    /**
     *  _put: Protected method
     *  @protected
     */
    _put: function(path, options, stream, callback) {
        options.uri = this.baseUrl + path;
        options.headers = { 'Authorization': 'TD1 ' + this.options.apikey };

        stream.pipe(request(options, function (err, res, body) {
            if (err) { return callback(err, undefined); }
            if (res.statusCode < 200 || res.statusCode > 299) {
                return callback(new Error((body && body.error) || 'HTTP ' + res.statusCode),
                    body || {});
            }

            callback(null, body || {});
        }));
    }
};

module.exports = TDClient;

// Private and Utility Methods
function noop() {}
<|endoftext|>


<commit_before>'use babel';
/* @flow */

import { CompositeDisposable } from 'atom';
import path from 'path';

import handleData from './message.js';

import type { LinterMessage } from './message';

type BufferType = {
  cachedText: string;
  getText(): string;
}

type TextEditorType = {
  getPath(): string;
  buffer: BufferType;
  isModified(): boolean;
}

type Linter = {
  grammarScopes: Array<string>;
  scope: 'file' | 'project';
  name?: string;
  lintOnFly: boolean;
  lint(TextEditor: TextEditorType): Array<LinterMessage> | Promise<Array<LinterMessage>>;
};

export default {
  config: {
    executablePath: {
      type: 'string',
      default: 'flow',
      description: 'Absolute path to the Flow executable on your system.',
    },
  },

  activate(): void {
    require('atom-package-deps').install('linter-flow');

    this.subscriptions = new CompositeDisposable();
    this.subscriptions.add(atom.config.observe('linter-flow.executablePath', (pathToFlow) => {
      this.pathToFlow = pathToFlow;
    }));
    this.subscriptions.add(atom.config.observe('linter-flow.enableAll', (enableAll) => {
      this.enableAll = enableAll;
    }));
  },

  deactivate(): void {
    const helpers = require('atom-linter');
    console.log('deactivating... lintet-flow');
    helpers.exec(this.pathToFlow, ['stop'], {});
    this.subscriptions.dispose();
  },

  provideLinter(): Linter {
    const helpers = require('atom-linter');

    return {
      grammarScopes: [
        'source.js',
        'source.js.jsx',
        'source.babel',
        'source.js-semantic',
        'source.es6',
      ],
      scope: 'project',
      name: 'Flow',
      lintOnFly: true,
      lint: (TextEditor) => {
        const filePath = TextEditor.getPath();
        const fileText = TextEditor.buffer.cachedText;

        // Is flow enabled for current file ?
        if (fileText.indexOf('@flow') === -1) {
          return Promise.resolve([]);
        }

        // Check if .flowconfig file is present
        const flowConfig = helpers.find(filePath, '.flowconfig');
        if (!flowConfig) {
          atom.notifications.addWarning('[Linter-Flow] Missing .flowconfig file.', {
            detail: 'To get started with Flow, run `flow init`.',
            dismissable: true,
          });
          return Promise.resolve([]);
        }

        let args;
        let options;

        // Use `check-contents` for unsaved files. and `status` for saved files.
        if (TextEditor.isModified()) {
          args = ['check-contents', '--json', filePath];
          options = { cwd: path.dirname(filePath), stdin: fileText };
        } else {
          args = ['status', '--json', filePath];
          options = { cwd: path.dirname(filePath) };
        }

        return helpers
          .exec(this.pathToFlow, args, options)
          .catch((error: String) => {
            if (error.indexOf('Launching Flow server') !== -1) {
              return helpers.exec(this.pathToFlow, args, options);
            }
            return '{}';
          })
          .then((str: string) => str.slice(str.indexOf('{')))
          .then(JSON.parse)
          .then(handleData);
      },
    };
  },
};
<commit_msg>feat(flow-comment-detection): slightly stricter check for flow comment<commit_after>'use babel';
/* @flow */

import { CompositeDisposable } from 'atom';
import path from 'path';

import handleData from './message.js';

import type { LinterMessage } from './message';

type BufferType = {
  cachedText: string;
  getText(): string;
}

type TextEditorType = {
  getPath(): string;
  buffer: BufferType;
  isModified(): boolean;
}

type Linter = {
  grammarScopes: Array<string>;
  scope: 'file' | 'project';
  name?: string;
  lintOnFly: boolean;
  lint(TextEditor: TextEditorType): Array<LinterMessage> | Promise<Array<LinterMessage>>;
};

export default {
  config: {
    executablePath: {
      type: 'string',
      default: 'flow',
      description: 'Absolute path to the Flow executable on your system.',
    },
  },

  activate(): void {
    require('atom-package-deps').install('linter-flow');

    this.subscriptions = new CompositeDisposable();
    this.subscriptions.add(atom.config.observe('linter-flow.executablePath', (pathToFlow) => {
      this.pathToFlow = pathToFlow;
    }));
    this.subscriptions.add(atom.config.observe('linter-flow.enableAll', (enableAll) => {
      this.enableAll = enableAll;
    }));
  },

  deactivate(): void {
    const helpers = require('atom-linter');
    console.log('deactivating... lintet-flow');
    helpers.exec(this.pathToFlow, ['stop'], {});
    this.subscriptions.dispose();
  },

  provideLinter(): Linter {
    const helpers = require('atom-linter');

    return {
      grammarScopes: [
        'source.js',
        'source.js.jsx',
        'source.babel',
        'source.js-semantic',
        'source.es6',
      ],
      scope: 'project',
      name: 'Flow',
      lintOnFly: true,
      lint: (TextEditor) => {
        const filePath = TextEditor.getPath();
        const fileText = TextEditor.buffer.cachedText;

        // Is flow enabled for current file ?
        const firstComStart = fileText.indexOf('\/*');
        const firstComEnd = fileText.indexOf('*\/');
        if (firstComStart !== -1 &&
          firstComEnd !== -1 &&
          fileText.slice(firstComStart + 2, firstComEnd).indexOf('@flow') === -1) {
          return Promise.resolve([]);
        }

        // Check if .flowconfig file is present
        const flowConfig = helpers.find(filePath, '.flowconfig');
        if (!flowConfig) {
          atom.notifications.addWarning('[Linter-Flow] Missing .flowconfig file.', {
            detail: 'To get started with Flow, run `flow init`.',
            dismissable: true,
          });
          return Promise.resolve([]);
        }

        let args;
        let options;

        // Use `check-contents` for unsaved files. and `status` for saved files.
        if (TextEditor.isModified()) {
          args = ['check-contents', '--json', filePath];
          options = { cwd: path.dirname(filePath), stdin: fileText };
        } else {
          args = ['status', '--json', filePath];
          options = { cwd: path.dirname(filePath) };
        }

        return helpers
          .exec(this.pathToFlow, args, options)
          .catch((error: String) => {
            if (error.indexOf('Launching Flow server') !== -1) {
              return helpers.exec(this.pathToFlow, args, options);
            }
            console.log(error);
            return '{}';
          })
          .then((str: string) => str.slice(str.indexOf('{')))
          .then(JSON.parse)
          .then(handleData);
      },
    };
  },
};
<|endoftext|>


<commit_before>'use strict';

var url = require('url');
var logger;

exports = module.exports = function historyApiFallback (options) {
  options = options || {};

  logger = defaultLogger(options);

  return function(req, res, next) {
    var headers = req.headers;
    if (req.method !== 'GET') {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the method is not GET.'
      );
      return next();
    } else if (!headers || typeof headers.accept !== 'string') {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client did not send an HTTP accept header.'
      );
      return next();
    } else if (headers.accept.indexOf('application/json') === 0) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client prefers JSON.'
      );
      return next();
    } else if (!acceptsHtml(headers.accept)) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client does not accept HTML.'
      );
      return next();
    }

    var parsedUrl = url.parse(req.url);
    if (parsedUrl.pathname.indexOf('.') !== -1) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the path includes a dot (.) character.'
      );
      return next();
    }

    var rewriteTarget = options.index || '/index.html';
    options.rewrites = options.rewrites || [];
    for (var i in options.rewrites) {
      var rewrite = options.rewrites[i],
        pattern = new RegExp(rewrite.pattern);
      if (parsedUrl.pathname.match(pattern) !== null) {
        rewriteTarget = rewrite.target;
      }
    }

    logger('Rewriting', req.method, req.url, 'to', rewriteTarget);
    req.url = rewriteTarget;
    next();
  };
};

module.exports.setLogger = function (newLogger) {
  logger = newLogger || defaultLogger();
};

function acceptsHtml (header) {
  return header.indexOf('text/html') !== -1 || header.indexOf('*/*') !== -1;
}

function defaultLogger (options) {
  return function () {
    var verbose = options.verbose || false;
    if (verbose) {
      console.log(Array.prototype.join.call(arguments, ' '));
    }
  };
}
<commit_msg>Revert to previous logger API<commit_after>'use strict';

var url = require('url');
var logger;

exports = module.exports = function historyApiFallback (options) {
  options = options || {};

  logger = defaultLogger(options);

  return function(req, res, next) {
    var headers = req.headers;
    if (req.method !== 'GET') {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the method is not GET.'
      );
      return next();
    } else if (!headers || typeof headers.accept !== 'string') {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client did not send an HTTP accept header.'
      );
      return next();
    } else if (headers.accept.indexOf('application/json') === 0) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client prefers JSON.'
      );
      return next();
    } else if (!acceptsHtml(headers.accept)) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the client does not accept HTML.'
      );
      return next();
    }

    var parsedUrl = url.parse(req.url);
    if (parsedUrl.pathname.indexOf('.') !== -1) {
      logger(
        'Not rewriting',
        req.method,
        req.url,
        'because the path includes a dot (.) character.'
      );
      return next();
    }

    var rewriteTarget = options.index || '/index.html';
    options.rewrites = options.rewrites || [];
    for (var i in options.rewrites) {
      var rewrite = options.rewrites[i],
        pattern = new RegExp(rewrite.pattern);
      if (parsedUrl.pathname.match(pattern) !== null) {
        rewriteTarget = rewrite.target;
      }
    }

    logger('Rewriting', req.method, req.url, 'to', rewriteTarget);
    req.url = rewriteTarget;
    next();
  };
};

module.exports.setLogger = function (newLogger) {
  logger = newLogger || defaultLogger();
};

function acceptsHtml (header) {
  return header.indexOf('text/html') !== -1 || header.indexOf('*/*') !== -1;
}

function defaultLogger (options) {
  return function () {
    var verbose = options.verbose || false;
    if (verbose) {
      console.log.apply(console, arguments);
    }
  };
}
<|endoftext|>


<commit_before>/**
 * 按入口页面输出项目
 */
var path = require('path');
var fs = require('fs');
var shell = require('shelljs');
var utils = require('./utils');
require("consoleplusplus");
console.disableTimestamp();
var plugins = require('./plugins');
var puz = require('./puzzle');
var include = require('./include');
var cwd = process.cwd();

var config = {};

if (fs.existsSync(path.resolve(cwd, './configs.js'))) {
    config = require(path.resolve(cwd, './configs.js'));
} else if (fs.existsSync(path.resolve(cwd, './kConfig/configs.js'))) {
    config = require(path.resolve(cwd, './kConfig/configs.js'));
}

var concatServerBaseUrl = '.';
var projectPublishRoot = '';

var getPageDir = function (subPage) {
    var projectPath = path.resolve(cwd, './__publish__');
    return /\/pages\/|\\pages\\/.test(subPage) ? path.join(projectPath, path.relative(path.resolve(cwd, './pages'), path.dirname(subPage)))//处于正宗的pages文件夹中，播出pages这一层，建立文件夹
        : path.join(projectPath, path.relative(cwd, path.dirname(subPage)));
};

/**
 * 创建项目输出目录
 * @param pageUrl
 */
var mkProjectDir = function () {
    var projectPath = path.resolve(cwd, './__publish__');
    projectPublishRoot = projectPath;
    if (!fs.existsSync(path.resolve(cwd, './__publish__'))) {
        fs.mkdirSync(path.resolve(cwd, './__publish__'));
    }
    fs.mkdirSync(projectPath + '/styles');
    fs.mkdirSync(projectPath + '/scripts');
    fs.mkdirSync(projectPath + '/statics');

    pageToPublish.forEach(function (subPage) {
        var dirPath = getPageDir(subPage);
        if (!fs.existsSync(dirPath)) {
            shell.mkdir('-p', dirPath);
        }
    });
};

/**
 * 发布一个项目
 * @param pageUrl 项目入口文件路径
 * @param toServer 是否上传到服务器
 */
var publishOnePage = function (pageUrl, toServer) {
    console.info('#yellow{<-----pack ' + pageUrl + '----->}:');
    var projectRoot = getPageDir(pageUrl);
    var pageName = path.basename(path.basename(pageUrl, '.html'), '.htm');
    var htmlCode = analyzeHtml(pageUrl, path.resolve(cwd, './__publish__'), pageName, toServer);
    var minify = require('html-minifier').minify;
    if (!/^<\?xml.*\?>/.test(htmlCode)) {
        htmlCode = minify(htmlCode, config.htmlmin || {});
    }
    console.info('#yellow{minify ' + pageUrl + ' done}\n\n');
    fs.writeFileSync(projectRoot + '/' + pageName + '.htm', htmlCode);
    console.info('#yellow{<-----pack ' + pageUrl + ' done----->}\n\n');
};

/**
 * 处理匹配的类型
 * @param type
 * @param args
 * @param htmlpath
 * @param projectRoot
 * @param pageName
 * @returns {*}
 */
var handleMatch = function (type, args, htmlpath, projectRoot, pageName) {
    var plugin = plugins.use(args);
    if (plugin[type] !== undefined) {
        return plugin[type](args[0], htmlpath, projectRoot, pageName, concatServerBaseUrl, projectPublishRoot);
    } else {
        return args[0];
    }
};

/**
 * 获取<script ... src="path"></script>
 * 获取<img ... src="path"></script>
 * 获取<link ... rel="stylesheet" href="path" />
 * @param htmlpath html文件的路径
 * @param projectRoot 项目根目录
 * @param pageName 页面文件名
 * @param toServer  是否上传到服务器，当不上传到服务器，只在本地测试时，需要把include的页面也直接静态的合并到页面代码中，否则不支持include语法
 *
 */
var analyzeHtml = function (htmlpath, projectRoot, pageName, toServer) {
    var content = puz.puz(path.resolve(cwd, htmlpath));
    if (!toServer) {
        content = include.include(path.resolve(cwd, htmlpath), content);
    }
    var reg = new RegExp(plugins.reg, 'ig');
    plugins.reset();
    content.replace(reg, function (m) {
        handleMatch('preReplace', arguments, htmlpath, projectRoot, pageName);//replace前做的准备
    });
    return content.replace(reg, function (m) {
        return handleMatch('replace', arguments, htmlpath, projectRoot, pageName);
    });
};

var isLegalFile = function (url) {
    var pass = false;
    var filters = config.htmlPublishFilter || [/\.htm$|\.html$/];
    for (var idx = 0; idx < filters.length; idx++) {
        if (filters[idx].test(url)) {
            pass = true;
            break;
        }
    }
    return pass;
};

/**
 * 发布所有的页面
 * @param dir
 */
var pageToPublish = [];
var collectAllPages = function (dir) {
    var stat = fs.statSync(dir);
    if (stat.isDirectory()) {
        var files = fs.readdirSync(dir);
        files.forEach(function (value) {
            var realPath = path.join(dir, value);
            var stat = fs.statSync(realPath);
            if (stat.isDirectory() && value !== 'src' && value !== 'kConfig' && value !== 'lib' && value !== 'output') {
                collectAllPages(realPath);
            } else if (isLegalFile(realPath)) {
                pageToPublish.push(realPath);
            }
        });
    } else {
        pageToPublish.push(dir);
    }
};

/**
 * 输出项目
 * @param pageUrl 项目入口页面
 */
exports.publish = function (pageUrl) {
    pageToPublish = [];//重置
    shell.rm('-rf', path.resolve(cwd, './__publish__'));//清除原生成文件
    var toServer = /.*_toserver$/g.test(pageUrl); //是否上传到服务器
    if (toServer) { //pageUrl 以 _toserver结尾，表示要发布到服务端，包括cdn，加上配置文件中的serverUrl
        concatServerBaseUrl = config.serverUrl;
        if (concatServerBaseUrl === undefined) {
            console.error('Please check out your config file, and set the attribute "serverUrl"!');
            return;
        }
        pageUrl = pageUrl.replace(/_toserver$/g, '');//去掉 _toserver 后缀
    }

    collectAllPages(path.resolve(cwd, pageUrl || ""));

    mkProjectDir(pageUrl);

    pageToPublish.forEach(function (value) {
        publishOnePage(value, toServer);
    });

    var pageName = path.basename(path.basename(pageUrl, '.html'), '.htm');
    //拷贝需要的静态资源文件
    if (Array.isArray(config.staticResource)) {
        config.staticResource.forEach(function (val) {
            shell.cp('-Rf',
                path.resolve(cwd, val.source),
                path.resolve(cwd + '/__publish__/' + pageName, val.target));  //styles
        });
    }
};

exports.getConfig = function () {
    return config;
};<commit_msg>MOD:  pageName 可能会是undefined<commit_after>/**
 * 按入口页面输出项目
 */
var path = require('path');
var fs = require('fs');
var shell = require('shelljs');
var utils = require('./utils');
require("consoleplusplus");
console.disableTimestamp();
var plugins = require('./plugins');
var puz = require('./puzzle');
var include = require('./include');
var cwd = process.cwd();

var config = {};

if (fs.existsSync(path.resolve(cwd, './configs.js'))) {
    config = require(path.resolve(cwd, './configs.js'));
} else if (fs.existsSync(path.resolve(cwd, './kConfig/configs.js'))) {
    config = require(path.resolve(cwd, './kConfig/configs.js'));
}

var concatServerBaseUrl = '.';
var projectPublishRoot = '';

var getPageDir = function (subPage) {
    var projectPath = path.resolve(cwd, './__publish__');
    return /\/pages\/|\\pages\\/.test(subPage) ? path.join(projectPath, path.relative(path.resolve(cwd, './pages'), path.dirname(subPage)))//处于正宗的pages文件夹中，播出pages这一层，建立文件夹
        : path.join(projectPath, path.relative(cwd, path.dirname(subPage)));
};

/**
 * 创建项目输出目录
 * @param pageUrl
 */
var mkProjectDir = function () {
    var projectPath = path.resolve(cwd, './__publish__');
    projectPublishRoot = projectPath;
    if (!fs.existsSync(path.resolve(cwd, './__publish__'))) {
        fs.mkdirSync(path.resolve(cwd, './__publish__'));
    }
    fs.mkdirSync(projectPath + '/styles');
    fs.mkdirSync(projectPath + '/scripts');
    fs.mkdirSync(projectPath + '/statics');

    pageToPublish.forEach(function (subPage) {
        var dirPath = getPageDir(subPage);
        if (!fs.existsSync(dirPath)) {
            shell.mkdir('-p', dirPath);
        }
    });
};

/**
 * 发布一个项目
 * @param pageUrl 项目入口文件路径
 * @param toServer 是否上传到服务器
 */
var publishOnePage = function (pageUrl, toServer) {
    console.info('#yellow{<-----pack ' + pageUrl + '----->}:');
    var projectRoot = getPageDir(pageUrl);
    var pageName = path.basename(path.basename(pageUrl, '.html'), '.htm');
    var htmlCode = analyzeHtml(pageUrl, path.resolve(cwd, './__publish__'), pageName, toServer);
    var minify = require('html-minifier').minify;
    if (!/^<\?xml.*\?>/.test(htmlCode)) {
        htmlCode = minify(htmlCode, config.htmlmin || {});
    }
    console.info('#yellow{minify ' + pageUrl + ' done}\n\n');
    fs.writeFileSync(projectRoot + '/' + pageName + '.htm', htmlCode);
    console.info('#yellow{<-----pack ' + pageUrl + ' done----->}\n\n');
};

/**
 * 处理匹配的类型
 * @param type
 * @param args
 * @param htmlpath
 * @param projectRoot
 * @param pageName
 * @returns {*}
 */
var handleMatch = function (type, args, htmlpath, projectRoot, pageName) {
    var plugin = plugins.use(args);
    if (plugin[type] !== undefined) {
        return plugin[type](args[0], htmlpath, projectRoot, pageName, concatServerBaseUrl, projectPublishRoot);
    } else {
        return args[0];
    }
};

/**
 * 获取<script ... src="path"></script>
 * 获取<img ... src="path"></script>
 * 获取<link ... rel="stylesheet" href="path" />
 * @param htmlpath html文件的路径
 * @param projectRoot 项目根目录
 * @param pageName 页面文件名
 * @param toServer  是否上传到服务器，当不上传到服务器，只在本地测试时，需要把include的页面也直接静态的合并到页面代码中，否则不支持include语法
 *
 */
var analyzeHtml = function (htmlpath, projectRoot, pageName, toServer) {
    var content = puz.puz(path.resolve(cwd, htmlpath));
    if (!toServer) {
        content = include.include(path.resolve(cwd, htmlpath), content);
    }
    var reg = new RegExp(plugins.reg, 'ig');
    plugins.reset();
    content.replace(reg, function (m) {
        handleMatch('preReplace', arguments, htmlpath, projectRoot, pageName);//replace前做的准备
    });
    return content.replace(reg, function (m) {
        return handleMatch('replace', arguments, htmlpath, projectRoot, pageName);
    });
};

var isLegalFile = function (url) {
    var pass = false;
    var filters = config.htmlPublishFilter || [/\.htm$|\.html$/];
    for (var idx = 0; idx < filters.length; idx++) {
        if (filters[idx].test(url)) {
            pass = true;
            break;
        }
    }
    return pass;
};

/**
 * 发布所有的页面
 * @param dir
 */
var pageToPublish = [];
var collectAllPages = function (dir) {
    var stat = fs.statSync(dir);
    if (stat.isDirectory()) {
        var files = fs.readdirSync(dir);
        files.forEach(function (value) {
            var realPath = path.join(dir, value);
            var stat = fs.statSync(realPath);
            if (stat.isDirectory() && value !== 'src' && value !== 'kConfig' && value !== 'lib' && value !== 'output') {
                collectAllPages(realPath);
            } else if (isLegalFile(realPath)) {
                pageToPublish.push(realPath);
            }
        });
    } else {
        pageToPublish.push(dir);
    }
};

/**
 * 输出项目
 * @param pageUrl 项目入口页面
 */
exports.publish = function (pageUrl) {
    pageToPublish = [];//重置
    shell.rm('-rf', path.resolve(cwd, './__publish__'));//清除原生成文件
    var toServer = /.*_toserver$/g.test(pageUrl); //是否上传到服务器
    if (toServer) { //pageUrl 以 _toserver结尾，表示要发布到服务端，包括cdn，加上配置文件中的serverUrl
        concatServerBaseUrl = config.serverUrl;
        if (concatServerBaseUrl === undefined) {
            console.error('Please check out your config file, and set the attribute "serverUrl"!');
            return;
        }
        pageUrl = pageUrl.replace(/_toserver$/g, '');//去掉 _toserver 后缀
    }

    collectAllPages(path.resolve(cwd, pageUrl || ""));

    mkProjectDir(pageUrl);

    pageToPublish.forEach(function (value) {
        publishOnePage(value, toServer);
    });

    var pageName = path.basename(path.basename(pageUrl, '.html'), '.htm');
    //拷贝需要的静态资源文件
    if (Array.isArray(config.staticResource)) {
        config.staticResource.forEach(function (val) {
            shell.cp('-Rf',
                path.resolve(cwd, val.source),
                path.resolve(cwd + '/__publish__/' + (pageName || ''), val.target));  //styles
        });
    }
};

exports.getConfig = function () {
    return config;
};<|endoftext|>