<commit_before>var http = require("http"),
    https = require("https"),
    urllib = require("url"),
    utillib = require("util"),
    zlib = require('zlib'),
    dns = require('dns'),
    Stream = require("stream").Readable,
    CookieJar = require("./cookiejar").CookieJar,
    encodinglib = require("encoding"),
    net = require("net");

exports.FetchStream = FetchStream;
exports.CookieJar = CookieJar;

exports.fetchUrl = fetchUrl;

function FetchStream(url, options){
    Stream.call(this);

    options = options || {};

    this.url = url;
    if(!this.url){
        return this.emit("error", new Error("url not defined"));
    }

    this.userAgent = options.userAgent || "FetchStream";

    this._redirect_count = 0;

    this.options = options || {};
    this.normalizeOptions();

    // prevent errors before "error" handler is set by defferring actions
    if(typeof setImmediate != "undefined"){
        setImmediate(this.runStream.bind(this, url));
    }else{
        process.nextTick(this.runStream.bind(this, url));
    }
	this.responseBuffer = new Buffer(0,'binary');
	this.ended = false;
	this.readyToRead = 0;
}
utillib.inherits(FetchStream, Stream);

FetchStream.prototype._read = function(size) {
	if (this.ended && this.responseBuffer.length === 0) {
		this.push(null);
		return;
	}
	this.readyToRead += size;
	this.drainBuffer();
};

FetchStream.prototype.drainBuffer = function() {
	if (this.readyToRead === 0) return;
	if (this.responseBuffer.length === 0) return;
	var push;
	var rest;
	if (this.responseBuffer.length > this.readyToRead) {
		push = new Buffer( this.readyToRead, 'binary' );
		this.responseBuffer.copy( push, 0, 0, this.readyToRead );
		rest = new Buffer( this.responseBuffer.length - this.readyToRead, 'binary' );
		this.responseBuffer.copy( rest, 0, this.readyToRead );
	} else {
		push = this.responseBuffer;
		rest = new Buffer( 0, 'binary' );
	}
	this.responseBuffer = rest;
	this.readyToRead = 0;
	if(this.options.encoding) {
		this.push( push, this.options.encoding );
	} else {
		this.push( push );
	}
};

FetchStream.prototype.destroy = function(ex) {
	this.emit("destroy",ex);
};

FetchStream.prototype.normalizeOptions = function(){

    // cookiejar
    this.cookieJar = this.options.cookieJar || new CookieJar();

    // default redirects - 10
    // if disableRedirect is set, then 0
    if(!this.options.disableRedirect && typeof this.options.maxRedirects != "number" &&
      !(this.options.maxRedirects instanceof Number)){
        this.options.maxRedirects = 10;
    }else if(this.options.disableRedirects){
        this.options.maxRedirects = 0;
    }

    // normalize header keys
    // HTTP and HTTPS takes in key names in case insensitive but to find
    // an exact value from an object key name needs to be case sensitive
    // so we're just lowercasing all input keys
    this.options.headers = this.options.headers || {};

    var keys = Object.keys(this.options.headers),
        newheaders = {},
        i;

    for(i=keys.length-1; i>=0; i--){
        newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
    }

    this.options.headers = newheaders;

    if(!this.options.headers["user-agent"]){
        this.options.headers["user-agent"] = this.userAgent;
    }

    if(!this.options.headers["pragma"]){
        this.options.headers["pragma"] = "no-cache";
    }

    if(!this.options.headers["cache-control"]){
        this.options.headers["cache-control"] = "no-cache";
    }

    if(!this.options.disableGzip){
        this.options.headers['accept-encoding'] = 'gzip, deflate';
    }else{
        delete this.options.headers['accept-encoding'];
    }

    // max length for the response,
    // if not set, default is Infinity
    if(!this.options.maxResponseLength){
        this.options.maxResponseLength = Infinity;
    }

    // method:
    // defaults to GET, or when payload present to POST
    if(!this.options.method){
        this.options.method = this.options.payload || this.options.payloadSize?"POST":"GET";
    }

    // set cookies
    // takes full cookie definition strings as params
    if(this.options.cookies){
        for(var i=0; i<this.options.cookies.length; i++){
            this.cookieJar.setCookie(this.options.cookies[i], this.url);
        }
    }

    // rejectUnauthorized
    if (typeof this.options.rejectUnauthorized === 'undefined') {
      this.options.rejectUnauthorized = true;
    }
}

FetchStream.prototype.parseUrl = function(url){
    var urlparts = urllib.parse(url, false, true),
        transport,
        urloptions = {
            host: urlparts.hostname || urlparts.host,
            port: urlparts.port,
            path: urlparts.pathname + (urlparts.search || "") || "/",
            method: this.options.method,
            rejectUnauthorized: this.options.rejectUnauthorized
        };

    if("agent" in this.options){
        urloptions.agent = this.options.agent;
    }

    switch(urlparts.protocol){
        case "https:":
            transport = https;
            break;
        case "http:":
        default:
            transport = http;
            break;
    }

    if(!urloptions.port){
        switch(urlparts.protocol){
            case "https:":
                urloptions.port = 443;
                break;
            case "http:":
            default:
                urloptions.port = 80;
                break;
        }
    }

    urloptions.headers = this.options.headers;

    return {
        urloptions: urloptions,
        transport: transport
    }
}

FetchStream.prototype.setEncoding = function(encoding){
    this.options.encoding = encoding;
}

FetchStream.prototype.absoluteUrl = function(url, base){

    var target_url = urllib.parse(url, false, true),
        base_url = urllib.parse(base || "", false, true),
        base_path, target_path, final_path;

    // if protocol is set, then it's good to go
    if(target_url.protocol){
        return url;
    }

    // the url might be int the form of "//www.example.com" with leading slashes -
    // the protocol from the base url must be used, defaults to http
    if(target_url.hostname){
        return (base_url.protocol || "http:") + (url.substr(0,2)!="//"?"//":"") + url;
    }

    // this is absolute path for relative domain
    if((target_url.pathname || "/").substr(0,1)=="/"){
        return (base_url.protocol || "http:") + "//" + (base_url.hostname || "") + url;
    }

    // relative path
    // remove also .. and . directory references
    base_path = (base_url.pathname || "/").split("/");
    base_path.pop(); // ditch the last element, empty for dir or a file name

    target_path = (target_url.pathname || "/").split("/");

    target_path = base_path.concat(target_path);
    final_path = [];

    target_path.forEach(function(dir){
        if(dir=="."){
            return;
        }

        if(dir==".."){
            final_path.pop();
            return;
        }

        if(dir){
            final_path.push(dir);
        }
    });

    return (base_url.protocol || "http:") + "//" + (base_url.hostname || "") + "/" +
        final_path.join("/") + (target_url.search || "");
}

FetchStream.prototype.runStream = function(url){
    var url_data = this.parseUrl(url),
        cookies = this.cookieJar.getCookies(url);

    if(cookies){
        url_data.urloptions.headers.cookie = cookies;
    }else{
        delete url_data.urloptions.headers.cookie;
    }

    if(this.options.payload){
        url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload || "","utf-8");
    }

    if(this.options.payloadSize){
        url_data.urloptions.headers['content-length'] = this.options.payloadSize;
    }

    if (this.options.asyncDnsLoookup) {
        var dnsCallback = (function (err, addresses){
            if (err) {
                this.emit("error", err);
                return;
            }

            url_data.urloptions.headers['host'] = url_data.urloptions.hostname || url_data.urloptions.host;
            url_data.urloptions.hostname = addresses[0];
            url_data.urloptions.host = url_data.urloptions.headers['host'] + (url_data.urloptions.port? ':' + url_data.urloptions.port: '');

            this._runStream(url_data, url);
        }).bind(this);

        if(net.isIP(url_data.urloptions.host)){
            dnsCallback(null, [url_data.urloptions.host]);
        }else{
            dns.resolve4(url_data.urloptions.host, dnsCallback);
        }
    } else {
        this._runStream(url_data, url);
    }
}

FetchStream.prototype._runStream = function(url_data, url){

    var req = url_data.transport.request(url_data.urloptions, (function(res) {

        // catch new cookies before potential redirect
        if(Array.isArray(res.headers['set-cookie'])){
            for(var i=0; i<res.headers['set-cookie'].length; i++){
                this.cookieJar.setCookie(res.headers['set-cookie'][i], url)
            }
        }

        if([301, 302, 303, 307, 308].indexOf(res.statusCode)>=0){
            if(!this.options.disableRedirects && this.options.maxRedirects>this._redirect_count && res.headers.location){
                this._redirect_count++;
                this.runStream(this.absoluteUrl(res.headers.location, url));
                return;
            }
        }

        this.meta = {
            status: res.statusCode,
            responseHeaders: res.headers,
            finalUrl: url,
            redirectCount: this._redirect_count,
            cookieJar: this.cookieJar
        }

        var curlen = 0,
            maxlen,

            receive = (function(chunk){
                if(curlen + chunk.length > this.options.maxResponseLength){
                    maxlen = this.options.maxResponseLength - curlen;
                }else{
                    maxlen = chunk.length;
                }
                if(maxlen<=0)return;

                curlen += Math.min(maxlen, chunk.length);
                if (maxlen>=chunk.length) {
					if (this.responseBuffer.length === 0) {
						this.responseBuffer = chunk;
					} else {
						this.responseBuffer = Buffer.concat([ this.responseBuffer, chunk ]);
					}
                } else {
					this.responseBuffer = Buffer.concat([ this.responseBuffer, chunk ], this.responseBuffer.length + maxlen);
                }
				this.drainBuffer();
            }).bind(this),

            error = (function(e){
				this.ended = true;
                this.emit("error", e);
				this.drainBuffer();
            }).bind(this),

            end = (function(){
				this.ended = true;
				if (this.responseBuffer.length === 0) {
					this.push(null);
				}
            }).bind(this),

            unpack = (function(type, res){
                var z = zlib["create"+type]();
                z.on("data", receive);
                z.on("error", error);
                z.on("end", end);
                res.pipe(z);
            }).bind(this);

        this.emit("meta", this.meta);

        if(res.headers['content-encoding']){
            switch(res.headers['content-encoding'].toLowerCase().trim()){
                case "gzip":
                    return unpack("Gunzip", res);
                case "deflate":
                    return unpack("InflateRaw", res);
            }
        }

        res.on('data', receive);
        res.on('end', end);

    }).bind(this));

    req.on('error', (function(e){
        this.emit("error", e);
    }).bind(this));

    if (this.options.timeout) {
        req.setTimeout(this.options.timeout, req.abort.bind(req));
    }
	this.on("destroy",req.abort.bind(req));

    if(this.options.payload){
        req.end(this.options.payload);
    }else if(this.options.payloadStream){
        this.options.payloadStream.pipe(req);
        this.options.payloadStream.resume();
    }else{
        req.end();
    }
}

function fetchUrl(url, options, callback){
    if(!callback && typeof options=="function"){
        callback = options;
        options = undefined;
    }
    options = options || {};

    var fetchstream = new FetchStream(url, options),
        response_data, chunks = [], length=0, curpos=0, buffer,
        content_type,
        callbackFired = false;

    fetchstream.on("meta", function(meta){
        response_data = meta;
        content_type = _parseContentType(meta.responseHeaders['content-type']);
    });

    fetchstream.on("data", function(chunk){
        if(chunk){
            chunks.push(chunk);
            length += chunk.length;
        }
    });

    fetchstream.on("error", function(error){
        if(error && error.code == 'HPE_INVALID_CONSTANT'){
            // skip invalid formatting errors
            return;
        }
        if(callbackFired){
            return;
        }
        callbackFired = true;
        callback(error);
    });

    fetchstream.on("end", function(error){
        if(callbackFired){
            return;
        }
        callbackFired = true;

        buffer = new Buffer(length);
        for(var i=0, len = chunks.length; i<len; i++){
            chunks[i].copy(buffer, curpos);
            curpos += chunks[i].length;
        }

        if(content_type.mimeType == "text/html"){
            content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
        }

        content_type.charset = (options.overrideCharset || content_type.charset || "utf-8").trim().toLowerCase();

        if(!this.options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)){
            buffer = encodinglib.convert(buffer, "UTF-8", content_type.charset);
        }

        if(this.options.outputEncoding){
            callback(null, response_data, buffer.toString(this.options.outputEncoding));
        }else{
            callback(null, response_data, buffer);
        }

    });
}

function _parseContentType(str){
    if(!str){
        return {};
    }
    var parts = str.split(";"),
        mimeType = parts.shift(),
        charset, chparts;

    for(var i=0, len = parts.length; i<len; i++){
        chparts = parts[i].split("=");
        if(chparts.length>1){
            if(chparts[0].trim().toLowerCase() == "charset"){
                charset = chparts[1];
            }
        }
    }

    return {
        mimeType: (mimeType || "").trim().toLowerCase(),
        charset: (charset || "UTF-8").trim().toLowerCase() // defaults to UTF-8
    }
}

function _findHTMLCharset(htmlbuffer){

    var body = htmlbuffer.toString("ascii"),
        input, meta, charset;

    if(meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i)){
        input = meta[0];
    }

    if(input){
        charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
        if(charset){
            charset = (charset[1] || "").trim().toLowerCase();
        }
    }

    if(!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))){
        charset = (meta[1] || "").trim().toLowerCase();
    }

    return charset;
}

<commit_msg>fixed tabs to spaces<commit_after>var http = require("http"),
    https = require("https"),
    urllib = require("url"),
    utillib = require("util"),
    zlib = require('zlib'),
    dns = require('dns'),
    Stream = require("stream").Readable,
    CookieJar = require("./cookiejar").CookieJar,
    encodinglib = require("encoding"),
    net = require("net");

exports.FetchStream = FetchStream;
exports.CookieJar = CookieJar;

exports.fetchUrl = fetchUrl;

function FetchStream(url, options){
    Stream.call(this);

    options = options || {};

    this.url = url;
    if(!this.url){
        return this.emit("error", new Error("url not defined"));
    }

    this.userAgent = options.userAgent || "FetchStream";

    this._redirect_count = 0;

    this.options = options || {};
    this.normalizeOptions();

    // prevent errors before "error" handler is set by defferring actions
    if(typeof setImmediate != "undefined"){
        setImmediate(this.runStream.bind(this, url));
    }else{
        process.nextTick(this.runStream.bind(this, url));
    }
    this.responseBuffer = new Buffer(0,'binary');
    this.ended = false;
    this.readyToRead = 0;
}
utillib.inherits(FetchStream, Stream);

FetchStream.prototype._read = function(size) {
    if (this.ended && this.responseBuffer.length === 0) {
        this.push(null);
        return;
    }
    this.readyToRead += size;
    this.drainBuffer();
};

FetchStream.prototype.drainBuffer = function() {
    if (this.readyToRead === 0) return;
    if (this.responseBuffer.length === 0) return;
    var push;
    var rest;
    if (this.responseBuffer.length > this.readyToRead) {
        push = new Buffer( this.readyToRead, 'binary' );
        this.responseBuffer.copy( push, 0, 0, this.readyToRead );
        rest = new Buffer( this.responseBuffer.length - this.readyToRead, 'binary' );
        this.responseBuffer.copy( rest, 0, this.readyToRead );
    } else {
        push = this.responseBuffer;
        rest = new Buffer( 0, 'binary' );
    }
    this.responseBuffer = rest;
    this.readyToRead = 0;
    if(this.options.encoding) {
        this.push( push, this.options.encoding );
    } else {
        this.push( push );
    }
};

FetchStream.prototype.destroy = function(ex) {
    this.emit("destroy",ex);
};

FetchStream.prototype.normalizeOptions = function(){

    // cookiejar
    this.cookieJar = this.options.cookieJar || new CookieJar();

    // default redirects - 10
    // if disableRedirect is set, then 0
    if(!this.options.disableRedirect && typeof this.options.maxRedirects != "number" &&
      !(this.options.maxRedirects instanceof Number)){
        this.options.maxRedirects = 10;
    }else if(this.options.disableRedirects){
        this.options.maxRedirects = 0;
    }

    // normalize header keys
    // HTTP and HTTPS takes in key names in case insensitive but to find
    // an exact value from an object key name needs to be case sensitive
    // so we're just lowercasing all input keys
    this.options.headers = this.options.headers || {};

    var keys = Object.keys(this.options.headers),
        newheaders = {},
        i;

    for(i=keys.length-1; i>=0; i--){
        newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
    }

    this.options.headers = newheaders;

    if(!this.options.headers["user-agent"]){
        this.options.headers["user-agent"] = this.userAgent;
    }

    if(!this.options.headers["pragma"]){
        this.options.headers["pragma"] = "no-cache";
    }

    if(!this.options.headers["cache-control"]){
        this.options.headers["cache-control"] = "no-cache";
    }

    if(!this.options.disableGzip){
        this.options.headers['accept-encoding'] = 'gzip, deflate';
    }else{
        delete this.options.headers['accept-encoding'];
    }

    // max length for the response,
    // if not set, default is Infinity
    if(!this.options.maxResponseLength){
        this.options.maxResponseLength = Infinity;
    }

    // method:
    // defaults to GET, or when payload present to POST
    if(!this.options.method){
        this.options.method = this.options.payload || this.options.payloadSize?"POST":"GET";
    }

    // set cookies
    // takes full cookie definition strings as params
    if(this.options.cookies){
        for(var i=0; i<this.options.cookies.length; i++){
            this.cookieJar.setCookie(this.options.cookies[i], this.url);
        }
    }

    // rejectUnauthorized
    if (typeof this.options.rejectUnauthorized === 'undefined') {
      this.options.rejectUnauthorized = true;
    }
}

FetchStream.prototype.parseUrl = function(url){
    var urlparts = urllib.parse(url, false, true),
        transport,
        urloptions = {
            host: urlparts.hostname || urlparts.host,
            port: urlparts.port,
            path: urlparts.pathname + (urlparts.search || "") || "/",
            method: this.options.method,
            rejectUnauthorized: this.options.rejectUnauthorized
        };

    if("agent" in this.options){
        urloptions.agent = this.options.agent;
    }

    switch(urlparts.protocol){
        case "https:":
            transport = https;
            break;
        case "http:":
        default:
            transport = http;
            break;
    }

    if(!urloptions.port){
        switch(urlparts.protocol){
            case "https:":
                urloptions.port = 443;
                break;
            case "http:":
            default:
                urloptions.port = 80;
                break;
        }
    }

    urloptions.headers = this.options.headers;

    return {
        urloptions: urloptions,
        transport: transport
    }
}

FetchStream.prototype.setEncoding = function(encoding){
    this.options.encoding = encoding;
}

FetchStream.prototype.absoluteUrl = function(url, base){

    var target_url = urllib.parse(url, false, true),
        base_url = urllib.parse(base || "", false, true),
        base_path, target_path, final_path;

    // if protocol is set, then it's good to go
    if(target_url.protocol){
        return url;
    }

    // the url might be int the form of "//www.example.com" with leading slashes -
    // the protocol from the base url must be used, defaults to http
    if(target_url.hostname){
        return (base_url.protocol || "http:") + (url.substr(0,2)!="//"?"//":"") + url;
    }

    // this is absolute path for relative domain
    if((target_url.pathname || "/").substr(0,1)=="/"){
        return (base_url.protocol || "http:") + "//" + (base_url.hostname || "") + url;
    }

    // relative path
    // remove also .. and . directory references
    base_path = (base_url.pathname || "/").split("/");
    base_path.pop(); // ditch the last element, empty for dir or a file name

    target_path = (target_url.pathname || "/").split("/");

    target_path = base_path.concat(target_path);
    final_path = [];

    target_path.forEach(function(dir){
        if(dir=="."){
            return;
        }

        if(dir==".."){
            final_path.pop();
            return;
        }

        if(dir){
            final_path.push(dir);
        }
    });

    return (base_url.protocol || "http:") + "//" + (base_url.hostname || "") + "/" +
        final_path.join("/") + (target_url.search || "");
}

FetchStream.prototype.runStream = function(url){
    var url_data = this.parseUrl(url),
        cookies = this.cookieJar.getCookies(url);

    if(cookies){
        url_data.urloptions.headers.cookie = cookies;
    }else{
        delete url_data.urloptions.headers.cookie;
    }

    if(this.options.payload){
        url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload || "","utf-8");
    }

    if(this.options.payloadSize){
        url_data.urloptions.headers['content-length'] = this.options.payloadSize;
    }

    if (this.options.asyncDnsLoookup) {
        var dnsCallback = (function (err, addresses){
            if (err) {
                this.emit("error", err);
                return;
            }

            url_data.urloptions.headers['host'] = url_data.urloptions.hostname || url_data.urloptions.host;
            url_data.urloptions.hostname = addresses[0];
            url_data.urloptions.host = url_data.urloptions.headers['host'] + (url_data.urloptions.port? ':' + url_data.urloptions.port: '');

            this._runStream(url_data, url);
        }).bind(this);

        if(net.isIP(url_data.urloptions.host)){
            dnsCallback(null, [url_data.urloptions.host]);
        }else{
            dns.resolve4(url_data.urloptions.host, dnsCallback);
        }
    } else {
        this._runStream(url_data, url);
    }
}

FetchStream.prototype._runStream = function(url_data, url){

    var req = url_data.transport.request(url_data.urloptions, (function(res) {

        // catch new cookies before potential redirect
        if(Array.isArray(res.headers['set-cookie'])){
            for(var i=0; i<res.headers['set-cookie'].length; i++){
                this.cookieJar.setCookie(res.headers['set-cookie'][i], url)
            }
        }

        if([301, 302, 303, 307, 308].indexOf(res.statusCode)>=0){
            if(!this.options.disableRedirects && this.options.maxRedirects>this._redirect_count && res.headers.location){
                this._redirect_count++;
                this.runStream(this.absoluteUrl(res.headers.location, url));
                return;
            }
        }

        this.meta = {
            status: res.statusCode,
            responseHeaders: res.headers,
            finalUrl: url,
            redirectCount: this._redirect_count,
            cookieJar: this.cookieJar
        }

        var curlen = 0,
            maxlen,

            receive = (function(chunk){
                if(curlen + chunk.length > this.options.maxResponseLength){
                    maxlen = this.options.maxResponseLength - curlen;
                }else{
                    maxlen = chunk.length;
                }
                if(maxlen<=0)return;

                curlen += Math.min(maxlen, chunk.length);
                if (maxlen>=chunk.length) {
                    if (this.responseBuffer.length === 0) {
                        this.responseBuffer = chunk;
                    } else {
                        this.responseBuffer = Buffer.concat([ this.responseBuffer, chunk ]);
                    }
                } else {
                    this.responseBuffer = Buffer.concat([ this.responseBuffer, chunk ], this.responseBuffer.length + maxlen);
                }
                this.drainBuffer();
            }).bind(this),

            error = (function(e){
                this.ended = true;
                this.emit("error", e);
                this.drainBuffer();
            }).bind(this),

            end = (function(){
                this.ended = true;
                if (this.responseBuffer.length === 0) {
                    this.push(null);
                }
            }).bind(this),

            unpack = (function(type, res){
                var z = zlib["create"+type]();
                z.on("data", receive);
                z.on("error", error);
                z.on("end", end);
                res.pipe(z);
            }).bind(this);

        this.emit("meta", this.meta);

        if(res.headers['content-encoding']){
            switch(res.headers['content-encoding'].toLowerCase().trim()){
                case "gzip":
                    return unpack("Gunzip", res);
                case "deflate":
                    return unpack("InflateRaw", res);
            }
        }

        res.on('data', receive);
        res.on('end', end);

    }).bind(this));

    req.on('error', (function(e){
        this.emit("error", e);
    }).bind(this));

    if (this.options.timeout) {
        req.setTimeout(this.options.timeout, req.abort.bind(req));
    }
    this.on("destroy",req.abort.bind(req));

    if(this.options.payload){
        req.end(this.options.payload);
    }else if(this.options.payloadStream){
        this.options.payloadStream.pipe(req);
        this.options.payloadStream.resume();
    }else{
        req.end();
    }
}

function fetchUrl(url, options, callback){
    if(!callback && typeof options=="function"){
        callback = options;
        options = undefined;
    }
    options = options || {};

    var fetchstream = new FetchStream(url, options),
        response_data, chunks = [], length=0, curpos=0, buffer,
        content_type,
        callbackFired = false;

    fetchstream.on("meta", function(meta){
        response_data = meta;
        content_type = _parseContentType(meta.responseHeaders['content-type']);
    });

    fetchstream.on("data", function(chunk){
        if(chunk){
            chunks.push(chunk);
            length += chunk.length;
        }
    });

    fetchstream.on("error", function(error){
        if(error && error.code == 'HPE_INVALID_CONSTANT'){
            // skip invalid formatting errors
            return;
        }
        if(callbackFired){
            return;
        }
        callbackFired = true;
        callback(error);
    });

    fetchstream.on("end", function(error){
        if(callbackFired){
            return;
        }
        callbackFired = true;

        buffer = new Buffer(length);
        for(var i=0, len = chunks.length; i<len; i++){
            chunks[i].copy(buffer, curpos);
            curpos += chunks[i].length;
        }

        if(content_type.mimeType == "text/html"){
            content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
        }

        content_type.charset = (options.overrideCharset || content_type.charset || "utf-8").trim().toLowerCase();

        if(!this.options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)){
            buffer = encodinglib.convert(buffer, "UTF-8", content_type.charset);
        }

        if(this.options.outputEncoding){
            callback(null, response_data, buffer.toString(this.options.outputEncoding));
        }else{
            callback(null, response_data, buffer);
        }

    });
}

function _parseContentType(str){
    if(!str){
        return {};
    }
    var parts = str.split(";"),
        mimeType = parts.shift(),
        charset, chparts;

    for(var i=0, len = parts.length; i<len; i++){
        chparts = parts[i].split("=");
        if(chparts.length>1){
            if(chparts[0].trim().toLowerCase() == "charset"){
                charset = chparts[1];
            }
        }
    }

    return {
        mimeType: (mimeType || "").trim().toLowerCase(),
        charset: (charset || "UTF-8").trim().toLowerCase() // defaults to UTF-8
    }
}

function _findHTMLCharset(htmlbuffer){

    var body = htmlbuffer.toString("ascii"),
        input, meta, charset;

    if(meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i)){
        input = meta[0];
    }

    if(input){
        charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
        if(charset){
            charset = (charset[1] || "").trim().toLowerCase();
        }
    }

    if(!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))){
        charset = (meta[1] || "").trim().toLowerCase();
    }

    return charset;
}

<|endoftext|>


<commit_before>import { combineReducers } from 'redux'
import { routeReducer } from 'redux-simple-router'
import { some, get, includes, omit, partition, merge, transform } from 'lodash'
import comments from './comments'
import commentsByPost from './commentsByPost'
import communities from './communities'
import communitiesByQuery from './communitiesByQuery'
import networks from './networks'
import networkEdits from './networkEdits'
import people from './people'
import peopleByQuery from './peopleByQuery'
import postEdits from './postEdits'
import postsByQuery from './postsByQuery'
import posts from './posts'
import projectsByQuery from './projectsByQuery'
import projects from './projects'
import projectEdits from './projectEdits'
import { appendUniq, mergeList } from './util'

import {
  CANCEL_TYPEAHEAD,
  CHECK_FRESHNESS_POSTS,
  CLEAR_CACHE,
  CREATE_COMMUNITY,
  CREATE_POST,
  CREATE_NETWORK,
  FETCH_ACTIVITY,
  FETCH_LIVE_STATUS,
  FETCH_COMMUNITIES,
  FETCH_INVITATIONS,
  FETCH_ONBOARDING,
  FETCH_PEOPLE,
  FETCH_POSTS,
  FETCH_PROJECTS,
  FETCH_TAG,
  FETCH_LEFT_NAV_TAGS,
  FOLLOW_TAG_PENDING,
  LOGIN,
  LOGOUT,
  MARK_ACTIVITY_READ,
  MARK_ALL_ACTIVITIES_READ_PENDING,
  NAVIGATE,
  NOTIFY,
  REMOVE_NOTIFICATION,
  RESET_ERROR,
  RESET_COMMUNITY_VALIDATION,
  RESET_NETWORK_VALIDATION,
  SEARCH,
  SEND_COMMUNITY_INVITATION,
  SEND_PROJECT_INVITE,
  SET_CURRENT_COMMUNITY_ID,
  SET_LOGIN_ERROR,
  SET_META_TAGS,
  SET_SIGNUP_ERROR,
  SIGNUP,
  TOGGLE_MAIN_MENU,
  TOGGLE_USER_SETTINGS_SECTION,
  TYPEAHEAD,
  UPDATE_COMMUNITY_EDITOR,
  UPDATE_NETWORK_EDITOR,
  UPDATE_INVITATION_EDITOR,
  UPDATE_POST,
  UPDATE_PROJECT_INVITE,
  UPLOAD_IMAGE,
  VALIDATE_COMMUNITY_ATTRIBUTE,
  VALIDATE_COMMUNITY_ATTRIBUTE_PENDING,
  VALIDATE_NETWORK_ATTRIBUTE,
  VALIDATE_NETWORK_ATTRIBUTE_PENDING
} from '../actions'

const keyedCounter = (actionType, payloadKey, statePath = 'meta.cache.id') =>
  (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    if (type === actionType) {
      return {...state, [get(action, statePath)]: Number(payload[payloadKey])}
    }
    return state
  }

const keyedHasFreshItems = (actionType, bucket) =>
  (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state
    if (type === actionType) {
      return {...state, [meta.cacheId]: payload}
    }
    if (type === CLEAR_CACHE && payload.bucket === bucket) {
      return {...state, [payload.id]: false}
    }
    return state
  }

export default combineReducers({
  currentCommunityId: (state = null, action) => {
    let { error, type, payload } = action
    if (error) return state

    switch (type) {
      case SET_CURRENT_COMMUNITY_ID:
        return payload
    }

    return state
  },

  leftNavIsOpen: (state = true, action) => {
    let { error, type } = action
    if (error) return state

    switch (type) {
      case TOGGLE_MAIN_MENU:
        return !state
    }

    return state
  },

  errors: (state = {}, action) => {
    let { error, type, payload, meta } = action

    switch (type) {
      case RESET_ERROR:
        return {...state, [meta.type]: null}
    }

    if (!error) return state

    return {...state, [type]: {error, payload, meta}}
  },

  routing: (state = {path: ''}, action) => {
    if (action.error) return state

    switch (action.type) {
      case LOGOUT:
        return {path: '/login'}
      case NAVIGATE:
        return {path: action.payload}
      default:
        return routeReducer(state, action)
    }
  },

  login: (state = {}, action) => {
    let { type, payload, error } = action
    switch (type) {
      case LOGIN:
        if (error) return {error: payload.message}
        break
      case SET_LOGIN_ERROR:
        if (payload) return {error: payload}
    }
    return state
  },

  signup: (state = {}, action) => {
    let { type, payload, error } = action
    switch (type) {
      case SIGNUP:
        if (error) return {error: payload.message}
        break
      case SET_SIGNUP_ERROR:
        if (payload) return {error: payload}
    }
    return state
  },

  comments,
  commentsByPost,
  communities,
  communitiesByQuery,
  networks,
  networkEdits,
  people,
  peopleByQuery,
  posts,
  postsByQuery,
  postEdits,
  projects,
  projectsByQuery,
  projectEdits,

  pending: (state = {}, action) => {
    let { type, meta } = action

    let toggle = (targetType, useMeta) => {
      if (type === targetType) return {...state, [targetType]: false}
      if (type === targetType + '_PENDING') {
        return {...state, [targetType]: (useMeta && meta ? meta : true)}
      }
    }

    return toggle(FETCH_POSTS) ||
      toggle(FETCH_PEOPLE) ||
      toggle(UPLOAD_IMAGE, true) ||
      toggle(CREATE_POST) ||
      toggle(UPDATE_POST) ||
      toggle(FETCH_PROJECTS) ||
      toggle(CREATE_COMMUNITY) ||
      toggle(CREATE_NETWORK) ||
      toggle(FETCH_ACTIVITY) ||
      toggle(SEND_COMMUNITY_INVITATION) ||
      toggle(FETCH_INVITATIONS) ||
      toggle(FETCH_COMMUNITIES) ||
      toggle(SEND_PROJECT_INVITE) ||
      toggle(SEARCH) ||
      state
  },

  typeaheadMatches: (state = {}, action) => {
    let { error, type } = action
    if (error || !includes([TYPEAHEAD, CANCEL_TYPEAHEAD], type)) return state

    let { payload, meta: { id } } = action
    switch (type) {
      case TYPEAHEAD:
        return {...state, [id]: payload}
      case CANCEL_TYPEAHEAD:
        return {...state, [id]: []}
    }

    return state
  },

  hasFreshPostsByQuery: keyedHasFreshItems(CHECK_FRESHNESS_POSTS, 'postsByQuery'),

  totalPostsByQuery: keyedCounter(FETCH_POSTS, 'posts_total'),
  totalPeopleByQuery: keyedCounter(FETCH_PEOPLE, 'people_total'),
  totalProjectsByQuery: keyedCounter(FETCH_PROJECTS, 'projects_total'),
  totalCommunitiesByQuery: keyedCounter(FETCH_COMMUNITIES, 'communities_total'),

  communityValidation: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case VALIDATE_COMMUNITY_ATTRIBUTE_PENDING:
        return {
          ...state,
          pending: {...state.pending, [meta.key]: true}
        }
      case VALIDATE_COMMUNITY_ATTRIBUTE:
        return {
          ...state,
          [meta.key]: payload,
          pending: {...state.pending, [meta.key]: false}
        }
      case RESET_COMMUNITY_VALIDATION:
        return {...state, [meta.key]: null}
    }

    return state
  },

  communityEditor: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) {
      switch (type) {
        case CREATE_COMMUNITY:
          return {
            ...state,
            errors: {...state.errors, server: `Server error: ${payload.message}`}
          }
        default:
          return state
      }
    }

    switch (type) {
      case UPDATE_COMMUNITY_EDITOR:
        return {
          ...state,
          [meta.subtree]: {...state[meta.subtree], ...payload}
        }
      case UPLOAD_IMAGE:
        if (meta.id !== 'new') break
        if (meta.subject === 'community-avatar') {
          return {
            ...state,
            community: {...state.community, avatar_url: payload}
          }
        } else if (meta.subject === 'community-banner') {
          return {
            ...state,
            community: {...state.community, banner_url: payload}
          }
        }
    }

    return state
  },

  networkValidation: (state = {}, action) => {
    let { type, payload, error, meta } = action
    let { id, key } = meta || {}
    if (error) return state

    switch (type) {
      case VALIDATE_NETWORK_ATTRIBUTE_PENDING:
        return {
          ...state,
          [id]: {...state[id], pending: {...(state[id] || {}).pending, [key]: true}}
        }
      case VALIDATE_NETWORK_ATTRIBUTE:
        return {
          ...state,
          [id]: {
            ...state[id],
            [key]: payload,
            pending: {...state[id].pending, [key]: false}
          }
        }
      case RESET_NETWORK_VALIDATION:
        return {
          ...state,
          [id]: {...state[id], [key]: null}
        }
    }

    return state
  },

  networkEditor: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) {
      switch (type) {
        case CREATE_NETWORK:
          return {
            ...state,
            errors: {...state.errors, server: `Server error: ${payload.message}`}
          }
        default:
          return state
      }
    }

    switch (type) {
      case UPDATE_NETWORK_EDITOR:
        return {
          ...state,
          [meta.subtree]: {...state[meta.subtree], ...payload}
        }
      case UPLOAD_IMAGE:
        if (meta.id !== 'new') break
        if (meta.subject === 'network-avatar') {
          return {
            ...state,
            network: {...state.network, avatar_url: payload}
          }
        } else if (meta.subject === 'network-banner') {
          return {
            ...state,
            network: {...state.network, banner_url: payload}
          }
        }
    }

    return state
  },

  projectInvite: (state = {}, action) => {
    let { type, payload, error, meta } = action
    let { id } = meta || {}
    if (error) return state
    switch (type) {
      case UPDATE_PROJECT_INVITE:
        return {
          ...state,
          [id]: {...state[id], ...payload}
        }
    }

    return state
  },

  userSettingsEditor: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case TOGGLE_USER_SETTINGS_SECTION:
        return {
          ...state,
          expand: {...state.expand, [payload]: meta.forceOpen || !get(state.expand, payload)}
        }
    }
    return state
  },

  activities: (state = [], action) => {
    let { type, payload, error, meta } = action
    if (error) return state
    let normalize = activity => {
      let comment = activity.comment
      if (!comment) {
        return activity
      }
      return {...omit(activity, 'comment'), comment_id: comment.id}
    }
    var activityId
    switch (type) {
      case FETCH_ACTIVITY:
        return state.concat(payload.items.map(normalize))
      case MARK_ACTIVITY_READ:
        activityId = meta.activityId
        return state.map(a => a.id === activityId ? {...a, unread: false} : a)
      case MARK_ALL_ACTIVITIES_READ_PENDING:
        return state.map(a => ({...a, unread: false}))
    }
    return state
  },

  totalActivities: (state = 0, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case FETCH_ACTIVITY:
        return payload.total
    }
    return state
  },

  metaTags: (state = {}, action) => {
    let { type, payload } = action
    if (type === SET_META_TAGS) {
      // remove closing script tags to prevent js error
      // https://groups.google.com/a/chromium.org/forum/#!topic/chromium-extensions/6bRq60rgBWk
      return transform(payload, (acc, val, key) => acc[key] = val.replace('</script>', ''))
    }

    return state
  },

  invitations: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case FETCH_INVITATIONS:
        let { communityId, reset } = meta
        if (reset) return {...state, [communityId]: payload.items}
        return {
          ...state,
          [communityId]: [...(state[communityId] || []), ...payload.items]
        }
    }

    return state
  },

  totalInvitations: keyedCounter(FETCH_INVITATIONS, 'total', 'meta.communityId'),

  invitationEditor: (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case UPDATE_INVITATION_EDITOR:
        return {...state, [payload.field]: payload.value}
      case SEND_COMMUNITY_INVITATION:
        let { results } = payload
        let [ failures, successes ] = partition(results, r => r.error)
        let success, error
        let sl = successes.length
        if (sl > 0) {
          let pl = sl > 1
          success = `Sent invitation${pl ? 's' : ''} to ${sl} ${pl ? 'people' : 'person'}.`
        }
        if (some(failures)) {
          error = failures.map(r => `Couldn't send to ${r.email}: ${r.error}.`).join(' ')
        }
        return {...state, success, error}
    }
    return state
  },

  notifierMessages: (state = [], action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case NOTIFY:
        return [payload, ...state]
      case REMOVE_NOTIFICATION:
        return state.filter(n => n.id !== payload)
    }

    return state
  },

  onboarding: (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case FETCH_ONBOARDING:
        return payload
    }
    return state
  },

  searchResultsByQuery: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) return state
    switch (type) {
      case SEARCH:
        return appendUniq(state, meta.cache.id, payload.items)
    }
    return state
  },

  totalSearchResultsByQuery: keyedCounter(SEARCH, 'total'),

  pageTitle: (state = 'Hylo', action) => {
    let updateTitle = (title, count) => {
      let split = title.split(') ')
      let words = split.length > 1 ? split[1] : split[0]
      return (count > 0 ? `(${count}) ` : '') + words
    }

    let { type, payload, meta } = action
    switch (type) {
      case FETCH_ACTIVITY:
        if (meta.resetCount) {
          return updateTitle(state, 0)
        }
        break
      case FETCH_LIVE_STATUS:
        return updateTitle(state, payload.new_notification_count)
    }
    return state
  },

  tagsByCommunity: (state = {}, action) => {
    // meta.id here is whatever params.id is in CommunityProfile
    let { type, payload, meta } = action
    switch (type) {
      case FETCH_TAG:
        let oldCommunityTags = state[meta.id] || {}
        return {
          ...state,
          [meta.id]: {
            ...oldCommunityTags,
            [meta.tagName]: payload
          }
        }
      case FETCH_LEFT_NAV_TAGS:
        let labeledTags = payload.followed.map(f => merge(f, {followed: true}))
        .concat(payload.created.map(c => merge(c, {created: true})))
        return {
          ...state,
          [meta.id]: mergeList(state[meta.id] || {}, labeledTags, 'name')
        }
      case FOLLOW_TAG_PENDING:
        oldCommunityTags = state[meta.id] || {}
        var oldTag = oldCommunityTags[meta.tagName]
        return {
          ...state,
          [meta.id]: {
            ...oldCommunityTags,
            [meta.tagName]: {...oldTag, followed: !oldTag.followed}
          }
        }
    }
    return state
  }

})
<commit_msg>only transform metatags that are strings<commit_after>import { combineReducers } from 'redux'
import { routeReducer } from 'redux-simple-router'
import { some, get, includes, omit, partition, merge, transform } from 'lodash'
import comments from './comments'
import commentsByPost from './commentsByPost'
import communities from './communities'
import communitiesByQuery from './communitiesByQuery'
import networks from './networks'
import networkEdits from './networkEdits'
import people from './people'
import peopleByQuery from './peopleByQuery'
import postEdits from './postEdits'
import postsByQuery from './postsByQuery'
import posts from './posts'
import projectsByQuery from './projectsByQuery'
import projects from './projects'
import projectEdits from './projectEdits'
import { appendUniq, mergeList } from './util'

import {
  CANCEL_TYPEAHEAD,
  CHECK_FRESHNESS_POSTS,
  CLEAR_CACHE,
  CREATE_COMMUNITY,
  CREATE_POST,
  CREATE_NETWORK,
  FETCH_ACTIVITY,
  FETCH_LIVE_STATUS,
  FETCH_COMMUNITIES,
  FETCH_INVITATIONS,
  FETCH_ONBOARDING,
  FETCH_PEOPLE,
  FETCH_POSTS,
  FETCH_PROJECTS,
  FETCH_TAG,
  FETCH_LEFT_NAV_TAGS,
  FOLLOW_TAG_PENDING,
  LOGIN,
  LOGOUT,
  MARK_ACTIVITY_READ,
  MARK_ALL_ACTIVITIES_READ_PENDING,
  NAVIGATE,
  NOTIFY,
  REMOVE_NOTIFICATION,
  RESET_ERROR,
  RESET_COMMUNITY_VALIDATION,
  RESET_NETWORK_VALIDATION,
  SEARCH,
  SEND_COMMUNITY_INVITATION,
  SEND_PROJECT_INVITE,
  SET_CURRENT_COMMUNITY_ID,
  SET_LOGIN_ERROR,
  SET_META_TAGS,
  SET_SIGNUP_ERROR,
  SIGNUP,
  TOGGLE_MAIN_MENU,
  TOGGLE_USER_SETTINGS_SECTION,
  TYPEAHEAD,
  UPDATE_COMMUNITY_EDITOR,
  UPDATE_NETWORK_EDITOR,
  UPDATE_INVITATION_EDITOR,
  UPDATE_POST,
  UPDATE_PROJECT_INVITE,
  UPLOAD_IMAGE,
  VALIDATE_COMMUNITY_ATTRIBUTE,
  VALIDATE_COMMUNITY_ATTRIBUTE_PENDING,
  VALIDATE_NETWORK_ATTRIBUTE,
  VALIDATE_NETWORK_ATTRIBUTE_PENDING
} from '../actions'

const keyedCounter = (actionType, payloadKey, statePath = 'meta.cache.id') =>
  (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    if (type === actionType) {
      return {...state, [get(action, statePath)]: Number(payload[payloadKey])}
    }
    return state
  }

const keyedHasFreshItems = (actionType, bucket) =>
  (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state
    if (type === actionType) {
      return {...state, [meta.cacheId]: payload}
    }
    if (type === CLEAR_CACHE && payload.bucket === bucket) {
      return {...state, [payload.id]: false}
    }
    return state
  }

export default combineReducers({
  currentCommunityId: (state = null, action) => {
    let { error, type, payload } = action
    if (error) return state

    switch (type) {
      case SET_CURRENT_COMMUNITY_ID:
        return payload
    }

    return state
  },

  leftNavIsOpen: (state = true, action) => {
    let { error, type } = action
    if (error) return state

    switch (type) {
      case TOGGLE_MAIN_MENU:
        return !state
    }

    return state
  },

  errors: (state = {}, action) => {
    let { error, type, payload, meta } = action

    switch (type) {
      case RESET_ERROR:
        return {...state, [meta.type]: null}
    }

    if (!error) return state

    return {...state, [type]: {error, payload, meta}}
  },

  routing: (state = {path: ''}, action) => {
    if (action.error) return state

    switch (action.type) {
      case LOGOUT:
        return {path: '/login'}
      case NAVIGATE:
        return {path: action.payload}
      default:
        return routeReducer(state, action)
    }
  },

  login: (state = {}, action) => {
    let { type, payload, error } = action
    switch (type) {
      case LOGIN:
        if (error) return {error: payload.message}
        break
      case SET_LOGIN_ERROR:
        if (payload) return {error: payload}
    }
    return state
  },

  signup: (state = {}, action) => {
    let { type, payload, error } = action
    switch (type) {
      case SIGNUP:
        if (error) return {error: payload.message}
        break
      case SET_SIGNUP_ERROR:
        if (payload) return {error: payload}
    }
    return state
  },

  comments,
  commentsByPost,
  communities,
  communitiesByQuery,
  networks,
  networkEdits,
  people,
  peopleByQuery,
  posts,
  postsByQuery,
  postEdits,
  projects,
  projectsByQuery,
  projectEdits,

  pending: (state = {}, action) => {
    let { type, meta } = action

    let toggle = (targetType, useMeta) => {
      if (type === targetType) return {...state, [targetType]: false}
      if (type === targetType + '_PENDING') {
        return {...state, [targetType]: (useMeta && meta ? meta : true)}
      }
    }

    return toggle(FETCH_POSTS) ||
      toggle(FETCH_PEOPLE) ||
      toggle(UPLOAD_IMAGE, true) ||
      toggle(CREATE_POST) ||
      toggle(UPDATE_POST) ||
      toggle(FETCH_PROJECTS) ||
      toggle(CREATE_COMMUNITY) ||
      toggle(CREATE_NETWORK) ||
      toggle(FETCH_ACTIVITY) ||
      toggle(SEND_COMMUNITY_INVITATION) ||
      toggle(FETCH_INVITATIONS) ||
      toggle(FETCH_COMMUNITIES) ||
      toggle(SEND_PROJECT_INVITE) ||
      toggle(SEARCH) ||
      state
  },

  typeaheadMatches: (state = {}, action) => {
    let { error, type } = action
    if (error || !includes([TYPEAHEAD, CANCEL_TYPEAHEAD], type)) return state

    let { payload, meta: { id } } = action
    switch (type) {
      case TYPEAHEAD:
        return {...state, [id]: payload}
      case CANCEL_TYPEAHEAD:
        return {...state, [id]: []}
    }

    return state
  },

  hasFreshPostsByQuery: keyedHasFreshItems(CHECK_FRESHNESS_POSTS, 'postsByQuery'),

  totalPostsByQuery: keyedCounter(FETCH_POSTS, 'posts_total'),
  totalPeopleByQuery: keyedCounter(FETCH_PEOPLE, 'people_total'),
  totalProjectsByQuery: keyedCounter(FETCH_PROJECTS, 'projects_total'),
  totalCommunitiesByQuery: keyedCounter(FETCH_COMMUNITIES, 'communities_total'),

  communityValidation: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case VALIDATE_COMMUNITY_ATTRIBUTE_PENDING:
        return {
          ...state,
          pending: {...state.pending, [meta.key]: true}
        }
      case VALIDATE_COMMUNITY_ATTRIBUTE:
        return {
          ...state,
          [meta.key]: payload,
          pending: {...state.pending, [meta.key]: false}
        }
      case RESET_COMMUNITY_VALIDATION:
        return {...state, [meta.key]: null}
    }

    return state
  },

  communityEditor: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) {
      switch (type) {
        case CREATE_COMMUNITY:
          return {
            ...state,
            errors: {...state.errors, server: `Server error: ${payload.message}`}
          }
        default:
          return state
      }
    }

    switch (type) {
      case UPDATE_COMMUNITY_EDITOR:
        return {
          ...state,
          [meta.subtree]: {...state[meta.subtree], ...payload}
        }
      case UPLOAD_IMAGE:
        if (meta.id !== 'new') break
        if (meta.subject === 'community-avatar') {
          return {
            ...state,
            community: {...state.community, avatar_url: payload}
          }
        } else if (meta.subject === 'community-banner') {
          return {
            ...state,
            community: {...state.community, banner_url: payload}
          }
        }
    }

    return state
  },

  networkValidation: (state = {}, action) => {
    let { type, payload, error, meta } = action
    let { id, key } = meta || {}
    if (error) return state

    switch (type) {
      case VALIDATE_NETWORK_ATTRIBUTE_PENDING:
        return {
          ...state,
          [id]: {...state[id], pending: {...(state[id] || {}).pending, [key]: true}}
        }
      case VALIDATE_NETWORK_ATTRIBUTE:
        return {
          ...state,
          [id]: {
            ...state[id],
            [key]: payload,
            pending: {...state[id].pending, [key]: false}
          }
        }
      case RESET_NETWORK_VALIDATION:
        return {
          ...state,
          [id]: {...state[id], [key]: null}
        }
    }

    return state
  },

  networkEditor: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) {
      switch (type) {
        case CREATE_NETWORK:
          return {
            ...state,
            errors: {...state.errors, server: `Server error: ${payload.message}`}
          }
        default:
          return state
      }
    }

    switch (type) {
      case UPDATE_NETWORK_EDITOR:
        return {
          ...state,
          [meta.subtree]: {...state[meta.subtree], ...payload}
        }
      case UPLOAD_IMAGE:
        if (meta.id !== 'new') break
        if (meta.subject === 'network-avatar') {
          return {
            ...state,
            network: {...state.network, avatar_url: payload}
          }
        } else if (meta.subject === 'network-banner') {
          return {
            ...state,
            network: {...state.network, banner_url: payload}
          }
        }
    }

    return state
  },

  projectInvite: (state = {}, action) => {
    let { type, payload, error, meta } = action
    let { id } = meta || {}
    if (error) return state
    switch (type) {
      case UPDATE_PROJECT_INVITE:
        return {
          ...state,
          [id]: {...state[id], ...payload}
        }
    }

    return state
  },

  userSettingsEditor: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case TOGGLE_USER_SETTINGS_SECTION:
        return {
          ...state,
          expand: {...state.expand, [payload]: meta.forceOpen || !get(state.expand, payload)}
        }
    }
    return state
  },

  activities: (state = [], action) => {
    let { type, payload, error, meta } = action
    if (error) return state
    let normalize = activity => {
      let comment = activity.comment
      if (!comment) {
        return activity
      }
      return {...omit(activity, 'comment'), comment_id: comment.id}
    }
    var activityId
    switch (type) {
      case FETCH_ACTIVITY:
        return state.concat(payload.items.map(normalize))
      case MARK_ACTIVITY_READ:
        activityId = meta.activityId
        return state.map(a => a.id === activityId ? {...a, unread: false} : a)
      case MARK_ALL_ACTIVITIES_READ_PENDING:
        return state.map(a => ({...a, unread: false}))
    }
    return state
  },

  totalActivities: (state = 0, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case FETCH_ACTIVITY:
        return payload.total
    }
    return state
  },

  metaTags: (state = {}, action) => {
    let { type, payload } = action
    if (type === SET_META_TAGS) {
      // remove closing script tags to prevent js error
      // https://groups.google.com/a/chromium.org/forum/#!topic/chromium-extensions/6bRq60rgBWk
      return transform(payload, (acc, val, key) =>
        acc[key] = typeof val === 'string'
          ? val.replace('</script>', '')
          : val)
    }

    return state
  },

  invitations: (state = {}, action) => {
    let { type, payload, error, meta } = action
    if (error) return state

    switch (type) {
      case FETCH_INVITATIONS:
        let { communityId, reset } = meta
        if (reset) return {...state, [communityId]: payload.items}
        return {
          ...state,
          [communityId]: [...(state[communityId] || []), ...payload.items]
        }
    }

    return state
  },

  totalInvitations: keyedCounter(FETCH_INVITATIONS, 'total', 'meta.communityId'),

  invitationEditor: (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case UPDATE_INVITATION_EDITOR:
        return {...state, [payload.field]: payload.value}
      case SEND_COMMUNITY_INVITATION:
        let { results } = payload
        let [ failures, successes ] = partition(results, r => r.error)
        let success, error
        let sl = successes.length
        if (sl > 0) {
          let pl = sl > 1
          success = `Sent invitation${pl ? 's' : ''} to ${sl} ${pl ? 'people' : 'person'}.`
        }
        if (some(failures)) {
          error = failures.map(r => `Couldn't send to ${r.email}: ${r.error}.`).join(' ')
        }
        return {...state, success, error}
    }
    return state
  },

  notifierMessages: (state = [], action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case NOTIFY:
        return [payload, ...state]
      case REMOVE_NOTIFICATION:
        return state.filter(n => n.id !== payload)
    }

    return state
  },

  onboarding: (state = {}, action) => {
    let { type, payload, error } = action
    if (error) return state
    switch (type) {
      case FETCH_ONBOARDING:
        return payload
    }
    return state
  },

  searchResultsByQuery: (state = {}, action) => {
    let { type, payload, meta, error } = action
    if (error) return state
    switch (type) {
      case SEARCH:
        return appendUniq(state, meta.cache.id, payload.items)
    }
    return state
  },

  totalSearchResultsByQuery: keyedCounter(SEARCH, 'total'),

  pageTitle: (state = 'Hylo', action) => {
    let updateTitle = (title, count) => {
      let split = title.split(') ')
      let words = split.length > 1 ? split[1] : split[0]
      return (count > 0 ? `(${count}) ` : '') + words
    }

    let { type, payload, meta } = action
    switch (type) {
      case FETCH_ACTIVITY:
        if (meta.resetCount) {
          return updateTitle(state, 0)
        }
        break
      case FETCH_LIVE_STATUS:
        return updateTitle(state, payload.new_notification_count)
    }
    return state
  },

  tagsByCommunity: (state = {}, action) => {
    // meta.id here is whatever params.id is in CommunityProfile
    let { type, payload, meta } = action
    switch (type) {
      case FETCH_TAG:
        let oldCommunityTags = state[meta.id] || {}
        return {
          ...state,
          [meta.id]: {
            ...oldCommunityTags,
            [meta.tagName]: payload
          }
        }
      case FETCH_LEFT_NAV_TAGS:
        let labeledTags = payload.followed.map(f => merge(f, {followed: true}))
        .concat(payload.created.map(c => merge(c, {created: true})))
        return {
          ...state,
          [meta.id]: mergeList(state[meta.id] || {}, labeledTags, 'name')
        }
      case FOLLOW_TAG_PENDING:
        oldCommunityTags = state[meta.id] || {}
        var oldTag = oldCommunityTags[meta.tagName]
        return {
          ...state,
          [meta.id]: {
            ...oldCommunityTags,
            [meta.tagName]: {...oldTag, followed: !oldTag.followed}
          }
        }
    }
    return state
  }

})
<|endoftext|>


<commit_before>var semver = require("semver")
var depTypes = ["dependencies","devDependencies","optionalDependencies"]
var extractDescription = require("./extract_description")

module.exports = fixer = {
  fixRepositoryField: function(data) {
    if (data.repostories) {
      this.warn("'repositories' (plural) Not supported.\n" +
           "Please pick one as the 'repository' field");
      data.repository = data.repositories[0]
    }
    if (!data.repository) return;
    if (typeof data.repository === "string") {
      data.repository = {
        type: "git",
        url: data.repository
      }
    }
    var r = data.repository.url || ""
    // use the non-private urls
    r = r.replace(/^(https?|git):\/\/[^\@]+\@github.com/,
      '$1://github.com')
    r = r.replace(/^https?:\/\/github.com/,
      'git://github.com')
    if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
      this.warn("Probably broken git url: " + r)
    }
  }

, fixFilesField: function(data) {
    var files = data.files
    if (files && !Array.isArray(files)) {
      this.warn("Invalid 'files' member")
      delete data.files
    }
  }

, fixBinField: function(data) {
    if (!data.bin) return;
    if (typeof data.bin === "string") {
      var b = {}
      b[data.name] = data.bin
      data.bin = b
    }
  }

, fixManField: function(data) {
    if (!data.man) return;
    if (typeof data.man === "string") {
      data.man = [ data.man ]
    }
  }
, fixBundleDependenciesField: function(data) {
    var bdd = "bundledDependencies"
    var bd = "bundleDependencies"
    if (data[bdd] && !data[bd]) {
      data[bd] = data[bdd]
      delete data[bdd]
    }
  }

, fixDependencies: function(data) {
    objectifyDeps(data, this.warn)
    addOptionalDepsToDeps(data, this.warn)
    this.fixBundleDependenciesField(data)
  }

, fixKeywordsField: function (data, warn) {
    if (typeof data.keywords === "string") {
      data.keywords = data.keywords.split(/,\s+/)
    }
  }

, fixVersionField: function(data) {
    if (!data.version) {
      data.version = ""
      return true
    }
    if (!semver.valid(data.version)) {
      throw new Error("invalid version: "+ data.version)
    }
    data.version = semver.clean(data.version)
    return true
  }

, fixPeople: function(data) {
    modifyPeople(data, unParsePerson)
    modifyPeople(data, parsePerson)
  }

, fixNameField: function(data) {
    if (!data.name) {
      data.name = ""
      return true
    }
    data.name = data.name.trim()
    ensureValidName(data.name)
  }

, fixDescriptionField: function (data) {
    if (data.description && typeof data.description !== 'string') {
      this.warn("'description' field should be a string")
      delete data.description
    }
    if (data.readme && !data.description)
      data.description = extractDescription(data.readme)
  }

, fixReadmeField: function (data) {
    if (!data.readme) data.readme = "ERROR: No README data found!"
  }

, fixBugsField: function(data) {
    var url
    if (!data.bugs && data.repository && data.repository.url) {
      url = data.repository.url
      // inference below was taken from code for "npm bugs" command
      if (url.match(/^(https?:\/\/|git(:\/\/|@))github.com/)) {
        url = url.replace(/^git(@|:\/\/)/, "https://")
                 .replace(/^https?:\/\/github.com:/, "https://github.com/")
                 .replace(/\.git$/, '')+"/issues"
        data.bugs = {url: url}
      }
    }
    else if(data.bugs && typeof data.bugs == "string") {
      data.bugs = {url: url}

,  fixHomepageField: function(data) {
    if(typeof data.homepage !== "string") {
      this.warn("homepage field must be a string. Deleted.")
      delete data.homepage
    }
  }
}

function ensureValidName (name) {
  if (name.charAt(0) === "." ||
      name.match(/[\/@\s\+%:]/) ||
      name !== encodeURIComponent(name) ||
      name.toLowerCase() === "node_modules" ||
      name.toLowerCase() === "favicon.ico") {
        throw new Error("Invalid name: " + JSON.stringify(name))
  }
}

function modifyPeople (data, fn) {
  if (data.author) data.author = fn(data.author)
  ;["maintainers", "contributors"].forEach(function (set) {
    if (!Array.isArray(data[set])) return;
    data[set] = data[set].map(fn)
  })
  return data
}

function unParsePerson (person) {
  if (typeof person === "string") return person
  var name = person.name || ""
  var u = person.url || person.web
  var url = u ? (" ("+u+")") : ""
  var e = person.email || person.mail
  var email = e ? (" <"+e+">") : ""
  return name+email+url
}

function parsePerson (person) {
  if (typeof person !== "string") return person
  var name = person.match(/^([^\(<]+)/)
  var url = person.match(/\(([^\)]+)\)/)
  var email = person.match(/<([^>]+)>/)
  var obj = {}
  if (name && name[0].trim()) obj.name = name[0].trim()
  if (email) obj.email = email[1];
  if (url) obj.url = url[1];
  return obj
}

function addOptionalDepsToDeps (data, warn) {
  var o = data.optionalDependencies
  if (!o) return;
  var d = data.dependencies || {}
  Object.keys(o).forEach(function (k) {
    d[k] = o[k]
  })
  data.dependencies = d
}

function depObjectify (deps) {
  if (!deps) return {}
  if (typeof deps === "string") {
    deps = deps.trim().split(/[\n\r\s\t ,]+/)
  }
  if (!Array.isArray(deps)) return deps
  var o = {}
  deps.forEach(function (d) {
    d = d.trim().split(/(:?[@\s><=])/)
    var dn = d.shift()
    var dv = d.join("")
    dv = dv.trim()
    dv = dv.replace(/^@/, "")
    o[dn] = dv
  })
  return o
}

function objectifyDeps (data, warn) {
  depTypes.forEach(function (type) {
    if (!data[type]) return;
    data[type] = depObjectify(data[type])
  })
}<commit_msg>Ensure that if data.bugs exists, it is an object with (string) email, url fields, or both.<commit_after>var semver = require("semver")
var depTypes = ["dependencies","devDependencies","optionalDependencies"]
var extractDescription = require("./extract_description")

module.exports = fixer = {
  fixRepositoryField: function(data) {
    if (data.repostories) {
      this.warn("'repositories' (plural) Not supported.\n" +
           "Please pick one as the 'repository' field");
      data.repository = data.repositories[0]
    }
    if (!data.repository) return;
    if (typeof data.repository === "string") {
      data.repository = {
        type: "git",
        url: data.repository
      }
    }
    var r = data.repository.url || ""
    // use the non-private urls
    r = r.replace(/^(https?|git):\/\/[^\@]+\@github.com/,
      '$1://github.com')
    r = r.replace(/^https?:\/\/github.com/,
      'git://github.com')
    if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
      this.warn("Probably broken git url: " + r)
    }
  }

, fixFilesField: function(data) {
    var files = data.files
    if (files && !Array.isArray(files)) {
      this.warn("Invalid 'files' member")
      delete data.files
    }
  }

, fixBinField: function(data) {
    if (!data.bin) return;
    if (typeof data.bin === "string") {
      var b = {}
      b[data.name] = data.bin
      data.bin = b
    }
  }

, fixManField: function(data) {
    if (!data.man) return;
    if (typeof data.man === "string") {
      data.man = [ data.man ]
    }
  }
, fixBundleDependenciesField: function(data) {
    var bdd = "bundledDependencies"
    var bd = "bundleDependencies"
    if (data[bdd] && !data[bd]) {
      data[bd] = data[bdd]
      delete data[bdd]
    }
  }

, fixDependencies: function(data) {
    objectifyDeps(data, this.warn)
    addOptionalDepsToDeps(data, this.warn)
    this.fixBundleDependenciesField(data)
  }

, fixKeywordsField: function (data, warn) {
    if (typeof data.keywords === "string") {
      data.keywords = data.keywords.split(/,\s+/)
    }
  }

, fixVersionField: function(data) {
    if (!data.version) {
      data.version = ""
      return true
    }
    if (!semver.valid(data.version)) {
      throw new Error("invalid version: "+ data.version)
    }
    data.version = semver.clean(data.version)
    return true
  }

, fixPeople: function(data) {
    modifyPeople(data, unParsePerson)
    modifyPeople(data, parsePerson)
  }

, fixNameField: function(data) {
    if (!data.name) {
      data.name = ""
      return true
    }
    data.name = data.name.trim()
    ensureValidName(data.name)
  }

, fixDescriptionField: function (data) {
    if (data.description && typeof data.description !== 'string') {
      this.warn("'description' field should be a string")
      delete data.description
    }
    if (data.readme && !data.description)
      data.description = extractDescription(data.readme)
  }

, fixReadmeField: function (data) {
    if (!data.readme) data.readme = "ERROR: No README data found!"
  }

, fixBugsField: function(data) {
    var url
    if (!data.bugs && data.repository && data.repository.url) {
      url = data.repository.url
      // inference below was taken from code for "npm bugs" command
      if (url.match(/^(https?:\/\/|git(:\/\/|@))github.com/)) {
        url = url.replace(/^git(@|:\/\/)/, "https://")
                 .replace(/^https?:\/\/github.com:/, "https://github.com/")
                 .replace(/\.git$/, '')+"/issues"
        data.bugs = {url: url}
      }
    }
    else if(data.bugs)
      if(typeof data.bugs == "string") {
        data.bugs = {url: data.bugs}
      }
      else {
        oldBugs = data.bugs
        data.bugs = {}
        if(oldBugs.url) {
          if(typeof(oldBugs.url) == "string")
            data.bugs.url = oldBugs.url
          else
            this.warn("bugs.url field must be a string. Deleted.")
        }
        if(oldBugs.email) {
          if(typeof(oldBugs.email) == "string")
            data.bugs.email = oldBugs.email
          else
            this.warn("bugs.email field must be a string. Deleted.")
        }
      }
      if(!data.bugs.email && !data.bugs.url) {
        delete data.bugs
        this.warn("Normalized value of bugs field is an empty object. Deleted.")
      }
  }

,  fixHomepageField: function(data) {
    if(typeof data.homepage !== "string") {
      this.warn("homepage field must be a string. Deleted.")
      delete data.homepage
    }
  }
}

function ensureValidName (name) {
  if (name.charAt(0) === "." ||
      name.match(/[\/@\s\+%:]/) ||
      name !== encodeURIComponent(name) ||
      name.toLowerCase() === "node_modules" ||
      name.toLowerCase() === "favicon.ico") {
        throw new Error("Invalid name: " + JSON.stringify(name))
  }
}

function modifyPeople (data, fn) {
  if (data.author) data.author = fn(data.author)
  ;["maintainers", "contributors"].forEach(function (set) {
    if (!Array.isArray(data[set])) return;
    data[set] = data[set].map(fn)
  })
  return data
}

function unParsePerson (person) {
  if (typeof person === "string") return person
  var name = person.name || ""
  var u = person.url || person.web
  var url = u ? (" ("+u+")") : ""
  var e = person.email || person.mail
  var email = e ? (" <"+e+">") : ""
  return name+email+url
}

function parsePerson (person) {
  if (typeof person !== "string") return person
  var name = person.match(/^([^\(<]+)/)
  var url = person.match(/\(([^\)]+)\)/)
  var email = person.match(/<([^>]+)>/)
  var obj = {}
  if (name && name[0].trim()) obj.name = name[0].trim()
  if (email) obj.email = email[1];
  if (url) obj.url = url[1];
  return obj
}

function addOptionalDepsToDeps (data, warn) {
  var o = data.optionalDependencies
  if (!o) return;
  var d = data.dependencies || {}
  Object.keys(o).forEach(function (k) {
    d[k] = o[k]
  })
  data.dependencies = d
}

function depObjectify (deps) {
  if (!deps) return {}
  if (typeof deps === "string") {
    deps = deps.trim().split(/[\n\r\s\t ,]+/)
  }
  if (!Array.isArray(deps)) return deps
  var o = {}
  deps.forEach(function (d) {
    d = d.trim().split(/(:?[@\s><=])/)
    var dn = d.shift()
    var dv = d.join("")
    dv = dv.trim()
    dv = dv.replace(/^@/, "")
    o[dn] = dv
  })
  return o
}

function objectifyDeps (data, warn) {
  depTypes.forEach(function (type) {
    if (!data[type]) return;
    data[type] = depObjectify(data[type])
  })
}<|endoftext|>


<commit_before>'use strict';

let co         = require('co');
let Apps       = require('../lib/apps');
let Addons     = require('../lib/addons');
let Postgres   = require('../lib/postgres');
let h          = require('heroku-cli-util');

function wait(ms) {
  return function(done) {
    setTimeout(done, ms);
  };
}

function deleteApp(app, heroku) {
  co(function* () {
    console.error(`\nIn order to avoid being charged for any resources on ${app}, it is being destroyed...`);
    yield h.action(`Destroying app ${app}`, heroku.apps(app).delete());
    process.exit(1);
  });
}

module.exports = {
  topic: 'fork',
  needsAuth: true,
  needsApp: true,
  description: 'Fork an existing app into a new one',
  help: `Copy config vars and Heroku Postgres data, and re-provision add-ons to a new app.
New app name should not be an existing app. The new app will be created as part of the forking process.`,
  flags: [
    {name: 'stack', char: 's', description: 'specify a stack for the new app', hasValue: true},
    {name: 'region', description: 'specify a region', hasValue: true},
    {name: 'skip-pg', description: 'skip postgres databases', hasValue: false}
  ],
  args: [{name: 'NEWNAME', optional: true}],
  run: h.command(function* (context, heroku) {
    let stopping, newAppName;
    process.once('SIGINT', function () {
      stopping = true;
      if (newAppName) { deleteApp(newAppName, heroku); }
    });
    let apps = new Apps(heroku);
    let postgres = new Postgres(heroku);
    let addons = new Addons(heroku, postgres);

    let oldApp = yield apps.getApp(context.app);
    let slug   = yield apps.getLastSlug(oldApp);

    if (stopping) { return; }
    let newApp = yield apps.createNewApp(oldApp, context.args.NEWNAME, context.flags.stack, context.flags.region);
    newAppName = newApp.name;

    try {
      if (stopping) { return; }
      yield apps.copySlug(newApp, slug);

      yield wait(2000); // TODO remove this after api #4022
      if (stopping) { return; }
      yield addons.copyAddons(oldApp, newApp, context.flags['skip-pg']);

      if (stopping) { return; }
      yield addons.copyConfigVars(oldApp, newApp);

      console.log(`Fork complete. View it at ${newApp.web_url}`);
    } catch (err) {
      h.errorHandler({
        exit:    false,
        logPath: context.herokuDir + '/error.log',
      })(err);
      if (newAppName) {
        console.error(`\nThere was an error forking to ${newAppName}.`);
        deleteApp(newAppName, heroku);
      } else {
        process.exit(1);
      }
    }
  })
};
<commit_msg>switch to --from and --to for app names<commit_after>'use strict';

let co         = require('co');
let Apps       = require('../lib/apps');
let Addons     = require('../lib/addons');
let Postgres   = require('../lib/postgres');
let cli        = require('heroku-cli-util');

function wait(ms) {
  return function(done) {
    setTimeout(done, ms);
  };
}

function deleteApp(app, heroku) {
  co(function* () {
    console.error(`\nIn order to avoid being charged for any resources on ${app}, it is being destroyed...`);
    yield cli.action(`Destroying app ${app}`, heroku.apps(app).delete());
    process.exit(1);
  });
}

module.exports = {
  topic: 'fork',
  needsAuth: true,
  description: 'Fork an existing app into a new one',
  help: `Copy config vars and Heroku Postgres data, and re-provision add-ons to a new app.
New app name should not be an existing app. The new app will be created as part of the forking process.

Example:

  $ heroku fork --from my-production-app --to my-development-app`,
  flags: [
    {name: 'stack', char: 's', description: 'specify a stack for the new app', hasValue: true},
    {name: 'region', description: 'specify a region', hasValue: true},
    {name: 'skip-pg', description: 'skip postgres databases', hasValue: false},
    {name: 'from', description: 'app to fork from', hasValue: true},
    {name: 'to', description: 'app to create', hasValue: true},
    {name: 'app', hasValue: true, hidden: true}
  ],
  args: [{name: 'NEWNAME', optional: true, hidden: true}],
  run: cli.command(function* (context, heroku) {
    let stopping;
    let fromAppName = context.flags.from || context.flags.app;
    let toAppName   = context.flags.to || context.args.NEWNAME;
    if (!fromAppName) {
      cli.error('No source app specified.\nSpecify an app to fork from with --from APP');
    }
    if (context.flags.app) {
      cli.warn('Specifying the source app without --from APP is deprecated');
    }
    if (context.args.NEWNAME) {
      cli.warn('Specifying the new app without --to APP is deprecated');
    }
    process.once('SIGINT', function () {
      stopping = true;
      if (toAppName) { deleteApp(toAppName, heroku); }
    });
    let apps = new Apps(heroku);
    let postgres = new Postgres(heroku);
    let addons = new Addons(heroku, postgres);

    let oldApp = yield apps.getApp(fromAppName);
    let slug   = yield apps.getLastSlug(oldApp);

    if (stopping) { return; }
    let newApp = yield apps.createNewApp(oldApp, toAppName, context.flags.stack, context.flags.region);
    toAppName = newApp.name;

    try {
      if (stopping) { return; }
      yield apps.copySlug(newApp, slug);

      yield wait(2000); // TODO remove this after api #4022
      if (stopping) { return; }
      yield addons.copyAddons(oldApp, newApp, context.flags['skip-pg']);

      if (stopping) { return; }
      yield addons.copyConfigVars(oldApp, newApp);

      console.log(`Fork complete. View it at ${newApp.web_url}`);
    } catch (err) {
      cli.errorHandler({
        exit:    false,
        logPath: context.herokuDir + '/error.log',
      })(err);
      if (newAppName) {
        console.error(`\nThere was an error forking to ${newAppName}.`);
        deleteApp(newAppName, heroku);
      } else {
        process.exit(1);
      }
    }
  })
};
<|endoftext|>


<commit_before>define(['jquery',
        'underscore',
        'backbone',
        'base',
        'SQL',
        'text!views/statusbar.html',
        'bootstrap'
    ], function ($, _, Backbone, base, SQL, StatusBarTemplate) {

    var Status = {};

    Status.ClusterStatus = Backbone.Model.extend({

        defaults: {
            cluster_name: '',
            cluster_state: '',
            load: ['-.-', '-.-', '-.-'],
            replicated_data: 0,
            available_data: 0,
            records_total: 0,
            records_underreplicated: 0,
            records_unavailable: 0
        },

        _normalizeClusterLoad: function (nodes) {
            var nodes_count = 0;
            var load = [0.0, 0.0, 0.0];
            for (var node in nodes) {
                nodes_count++;
                for (var i=0; i<3; i++) {
                    load[i] = load[i]+nodes[node].os.load_average[i];
                }
            }
            for (var i=0; i<3; i++) {
                load[i] = load[i]/nodes_count;
                load[i] = load[i].toFixed(2);
            }
            return load;
        },

        _updateHealth: function () {
            var self = this,
                sq;

            sq = new SQL.Query("select sum(number_of_shards) from information_schema.tables where schema_name = 'doc'");
            sq.execute().done(function (res) {
                var configuredShards = 0;
                if (res.rowcount > 0) {
                    configuredShards = res.rows[0][0];
                }

                sq = new SQL.Query('select count(*), "primary", state from sys.shards group by "primary", state');
                sq.execute().done(function (res) {
                    var activePrimaryShards = 0;
                    var unassignedShards = 0;
                    for (var row in res.rows) {
                        if (res.rows[row][1] === true && res.rows[row][2] in {'STARTED':'', 'RELOCATING':''} ) {
                            activePrimaryShards = res.rows[row][0];
                        } else if (res.rows[row][2] == 'UNASSIGNED') {
                            unassignedShards = res.rows[row][0];
                        }
                    }
                    if (activePrimaryShards < configuredShards) {
                        self.set({
                            cluster_state: 'critical',
                        });
                    } else if (unassignedShards > 0) {
                        self.set({
                            cluster_state: 'warning',
                        });
                    } else {
                        self.set({
                            cluster_state: 'good',
                        });
                    }
                });
            }).fail(function (err) {});
        },

        _updateTableData: function (resTables, resRecords) {
            var self = this,
                records_total = 0,
                records_not_replicated = 0,
                records_unavailable = 0,
                table_state = {};

            // fill table state with response from 1st query
            for (var row in resTables.rows) {
                var current_row = resTables.rows[row];
                if (table_state[current_row[0]] === undefined) {
                    table_state[current_row[0]] = {'total':0, 'replicated': -1, 'avg_docs': 0, 'active_shards': 0};
                }
                if (current_row[2] === true) {
                    table_state[current_row[0]]['total'] = current_row[1];
                    table_state[current_row[0]]['avg_docs'] += current_row[4];
                    table_state[current_row[0]]['active_shards'] += current_row[5];
                } else if (current_row[6] != 'UNASSIGNED') {
                    table_state[current_row[0]]['replicated'] = current_row[1];
                }
            }
            // fill table state with response from 2st query
            if (resRecords !== undefined) {
                for (var row in resRecords.rows) {
                    var current_row = resRecords.rows[row];
                    if (table_state[current_row[0]] === undefined) {
                        table_state[current_row[0]] = {'total_shards':0};
                    }
                    table_state[current_row[0]]['total_shards'] = current_row[1];
                }
            }

            // calculated cluster numbers
            for (var table in table_state) {
                records_total += table_state[table]['total'];
                if (table_state[table]['replicated'] > -1) {
                    records_not_replicated += (table_state[table]['total'] - table_state[table]['replicated']);
                }
                var unavailable_shards = table_state[table]['total_shards'] - table_state[table]['active_shards'];
                    if (unavailable_shards > 0) {
                        records_unavailable = unavailable_shards * table_state[table]['avg_docs'];
                }
            }

            self.set({
                'records_total': records_total,
                'records_underreplicated': records_not_replicated.toFixed(0),
                'replicated_data': Math.floor(100-((records_not_replicated/records_total)*100)),
                'records_unavailable': records_unavailable.toFixed(0),
                'available_data': Math.floor(100-((records_unavailable/records_total)*100))
            });
        },

        _updateTableStatus: function () {
            var self = this,
                sqRecords,
                dRecords;

            sqRecords = new SQL.Query(
                'select table_name, sum(num_docs), "primary", relocating_node, avg(num_docs), count(*), state '+
                'from sys.shards group by table_name, "primary", relocating_node, state ' +
                'order by table_name, "primary"');
            sqTables = new SQL.Query(
                'select table_name, sum(number_of_shards) from information_schema.tables ' +
                'group by table_name');
            dRecords = sqRecords.execute();
            dTables = sqTables.execute();
            dRecords.done(function (resTables) {
                dTables.done(function (resRecords) {
                    self._updateTableData(resTables, resRecords);
                }).fail(function () {
                    self._updateTableData(resTables);
                });
            });

        },

        fetch: function () {
            var self = this,
                load;
            $.get('/_nodes/stats?all=true')
                .done(function (data) {
                    self.set({
                        cluster_name: data.cluster_name,
                        load: self._normalizeClusterLoad(data.nodes)
                    });
                })
                .error(function() {
                    delete self.data;
                });
            this._updateHealth();
            this._updateTableStatus();
            setTimeout(function () { self.fetch(); }, 5000);
        },

    });

    Status.StatusView = base.CrateView.extend({

        el: '#statusbar',
        template: _.template(StatusBarTemplate),
        model: Status.ClusterStatus,

        initialize: function () {
            this.listenTo(this.model, 'change', this.render);
        },

        render: function () {
            this.$el.html(this.template(this.model.toJSON()));
            return this;
        }
    });

    return Status;
});
<commit_msg>Fix overview to deal with system tables, style fixes<commit_after>define(['jquery',
        'underscore',
        'backbone',
        'base',
        'SQL',
        'text!views/statusbar.html',
        'bootstrap'
    ], function ($, _, Backbone, base, SQL, StatusBarTemplate) {

    var Status = {};

    Status.ClusterStatus = Backbone.Model.extend({

        defaults: {
            cluster_name: '',
            cluster_state: '',
            load: ['-.-', '-.-', '-.-'],
            replicated_data: 0,
            available_data: 0,
            records_total: 0,
            records_underreplicated: 0,
            records_unavailable: 0
        },

        _normalizeClusterLoad: function (nodes) {
            var i, node;
            var nodes_count = 0;
            var load = [0.0, 0.0, 0.0];
            for (node in nodes) {
                nodes_count++;
                for (i=0; i<3; i++) {
                    load[i] = load[i]+nodes[node].os.load_average[i];
                }
            }
            for (i=0; i<3; i++) {
                load[i] = load[i]/nodes_count;
                load[i] = load[i].toFixed(2);
            }
            return load;
        },

        _updateHealth: function () {
            var self = this,
                sq;

            sq = new SQL.Query("select sum(number_of_shards) from information_schema.tables where schema_name = 'doc'");
            sq.execute().done(function (res) {
                var configuredShards = 0;
                if (res.rowcount > 0) {
                    configuredShards = res.rows[0][0];
                }

                sq = new SQL.Query('select count(*), "primary", state from sys.shards group by "primary", state');
                sq.execute().done(function (res) {
                    var activePrimaryShards = 0;
                    var unassignedShards = 0;
                    for (var row in res.rows) {
                        if (res.rows[row][1] === true && res.rows[row][2] in {'STARTED':'', 'RELOCATING':''} ) {
                            activePrimaryShards = res.rows[row][0];
                        } else if (res.rows[row][2] == 'UNASSIGNED') {
                            unassignedShards = res.rows[row][0];
                        }
                    }
                    if (activePrimaryShards < configuredShards) {
                        self.set({
                            cluster_state: 'critical',
                        });
                    } else if (unassignedShards > 0) {
                        self.set({
                            cluster_state: 'warning',
                        });
                    } else {
                        self.set({
                            cluster_state: 'good',
                        });
                    }
                });
            }).fail(function (err) {});
        },

        _updateTableData: function (resTables, resRecords) {
            var self = this,
                records_total = 0,
                records_not_replicated = 0,
                records_unavailable = 0,
                table_state = {},
                row, current_row;

            // fill table state with response from 1st query
            for (row in resTables.rows) {
                current_row = resTables.rows[row];
                if (table_state[current_row[0]] === undefined) {
                    table_state[current_row[0]] = {'total':0, 'replicated': -1, 'avg_docs': 0, 'active_shards': 0};
                }
                if (current_row[2] === true) {
                    table_state[current_row[0]].total = current_row[1];
                    table_state[current_row[0]].avg_docs += current_row[4];
                    table_state[current_row[0]].active_shards += current_row[5];
                } else if (current_row[6] != 'UNASSIGNED') {
                    table_state[current_row[0]].replicated = current_row[1];
                }
            }
            // fill table state with response from 2st query
            if (resRecords !== undefined) {
                for (row in resRecords.rows) {
                    current_row = resRecords.rows[row];
                    if (table_state[current_row[0]] === undefined) {
                        table_state[current_row[0]] = {'total_shards':0};
                    }
                    table_state[current_row[0]].total_shards = current_row[1];
                }
            }

            // calculated cluster numbers
            for (var table in table_state) {
                if (table_state[table].total) {
                    records_total += table_state[table].total;
                }
                if (table_state[table].replicated > -1) {
                    records_not_replicated += (table_state[table].total - table_state[table].replicated);
                }
                var unavailable_shards = table_state[table].total_shards - table_state[table].active_shards;
                    if (unavailable_shards > 0) {
                        records_unavailable = unavailable_shards * table_state[table].avg_docs;
                }
            }

            self.set({
                'records_total': records_total,
                'records_underreplicated': records_not_replicated.toFixed(0),
                'replicated_data': Math.floor(100-((records_not_replicated/records_total)*100)),
                'records_unavailable': records_unavailable.toFixed(0),
                'available_data': Math.floor(100-((records_unavailable/records_total)*100))
            });
        },

        _updateTableStatus: function () {
            var self = this,
                sqRecords,
                dRecords;

            sqRecords = new SQL.Query(
                'select table_name, sum(num_docs), "primary", relocating_node, avg(num_docs), count(*), state '+
                'from sys.shards group by table_name, "primary", relocating_node, state ' +
                'order by table_name, "primary"');
            sqTables = new SQL.Query(
                'select table_name, sum(number_of_shards) from information_schema.tables ' +
                'group by table_name');
            dRecords = sqRecords.execute();
            dTables = sqTables.execute();
            dRecords.done(function (resTables) {
                dTables.done(function (resRecords) {
                    self._updateTableData(resTables, resRecords);
                }).fail(function () {
                    self._updateTableData(resTables);
                });
            });

        },

        fetch: function () {
            var self = this,
                load;
            $.get('/_nodes/stats?all=true')
                .done(function (data) {
                    self.set({
                        cluster_name: data.cluster_name,
                        load: self._normalizeClusterLoad(data.nodes)
                    });
                })
                .error(function() {
                    delete self.data;
                });
            this._updateHealth();
            this._updateTableStatus();
            setTimeout(function () { self.fetch(); }, 5000);
        },

    });

    Status.StatusView = base.CrateView.extend({

        el: '#statusbar',
        template: _.template(StatusBarTemplate),
        model: Status.ClusterStatus,

        initialize: function () {
            this.listenTo(this.model, 'change', this.render);
        },

        render: function () {
            this.$el.html(this.template(this.model.toJSON()));
            return this;
        }
    });

    return Status;
});
<|endoftext|>


<commit_before>import { combineReducers } from 'redux'
import { routerReducer } from 'react-router-redux'

const appReducer = combineReducers({
    routing: routerReducer
});

export default function resetStateReducer(state, action) {
    return appReducer(state, action);
}<commit_msg>Added redux-form integration<commit_after>import { combineReducers } from 'redux'
import { routerReducer } from 'react-router-redux'
import { reducer as formReducer } from 'redux-form'

const appReducer = combineReducers({
    routing: routerReducer,
    form: formReducer
});

export default function resetStateReducer(state, action) {
    return appReducer(state, action);
}<|endoftext|>