<commit_before>var ical = require("ical"),
    async = require("async"),
    request = require("request"),
    config = require("./config"),
    googleapis = require('googleapis'),
    crypto = require("crypto"),
    moment = require("moment");

var CLIENT_ID = config.CLIENTID;
var CLIENT_SECRET = config.CLIENTSECRET;

var SERVICE_ACCOUNT_EMAIL = '721976846481-1s5altpg8afuc4opnlr13nua86hg0ul9@developer.gserviceaccount.com';
var SERVICE_ACCOUNT_KEY_FILE = './key.pem';
var jwt = new googleapis.auth.JWT(
        SERVICE_ACCOUNT_EMAIL,
        SERVICE_ACCOUNT_KEY_FILE,
        null,
        ['https://www.googleapis.com/auth/calendar']);

var MEETUP_KEY = config.MEETUPKEY;
var MEETUP_URL = "https://api.meetup.com/find/groups?" +
                    "&sign=true" +
                    "&photo-host=public" +
                    "&category=34" + /* Technology */
                    "&lat=52.483056&lon=-1.893611" + /* Birmingham */
                    "&radius=5" + /* radius (in miles) */
                    "&page=4000" + /* results per page */
                    "&key=";

/* FetchIcalUrls functions have to return a list of {source, url} objects.
   A source must be a short word which identifies the source somehow
   (for example, "meetup" for meetup iCal files) -- it is used to make sure
   that IDs from different sources don't collide. For boring technical reasons
   (https://developers.google.com/google-apps/calendar/v3/reference/events#id)
   the source must also contain only characters 0-9 and a-v (not a-z, and not
   capital letters).

   NOTE: a source is expected and required to provide UIDs for each event, and
   those UIDs must be both unique in that source and unchanging over time; if
   we request data from that source again later and one of the events that comes
   back was in this fetch too, then it must have the same UID. Otherwise it will
   be duplicated in the b.io calendar. Any source which does not enforce this is
   stupid, but if it does, it is your responsibility to provide a URL which *does*
   enforce it.
*/

function fetchIcalUrlsFromLocalFile(cb) {
    cb(null, [
        //"http://lanyrd.com/topics/nodejs/nodejs.ics",
        //"http://lanyrd.com/topics/python/python.ics"
    ]);
}

function fetchIcalUrlsFromMeetup(cb) {
    var req = request(MEETUP_URL + MEETUP_KEY, function(err, response, body) {
        if (err) {
            console.log("Meetup: Error connecting:", err);
            return;
        }
        else if (response.statusCode != 200) {
            console.log("Meetup: HTTP error code:", response.statusCode);
            return;
        }
        else {
            try {
                results = JSON.parse(body);
                if (results.length === 0) {
                    console.log("Meetup: Warning: no results received:");
                }
                urls = [];
                for (var result in results) {
                    urls.push({source: "meetup", url: results[result].link + "events/ical/"});
                }
                cb(null, urls);
            } catch(e) {
                console.log("Meetup: Error parsing JSON:", e);
            }
        }
    });
}

// first, get a list of ics urls from various places
function mainJob() {
    async.parallel([
        fetchIcalUrlsFromLocalFile,
        fetchIcalUrlsFromMeetup
    ], function(err, results) {
        if (err) {
            console.log("We failed to get a list of ics URLs", err);
            return;
        }
        // flatten results list and fetch them all
        var icsurls = [];
        icsurls = icsurls.concat.apply(icsurls, results);
        async.map(icsurls, function(icsurlobj, cb) {
            request(icsurlobj.url, function(err, response, body) {
                if (err) {
                    console.log("Failed to fetch URL", icsurlobj.url, err);
                    body = null;
                }
                cb(null, {source: icsurlobj.source, body: body});
            });
        }, function(err, results) {
            if (err) {
                console.log("We failed to fetch any ics URLs", err);
                return;
            }
            // parse them all into ICS structures
            async.concat(results, function(icsbodyobj, cb) {
                var events = [];
                if (icsbodyobj.body) {
                    var parsedEvents = ical.parseICS(icsbodyobj.body);
                    for (var k in parsedEvents) {
                        if (parsedEvents.hasOwnProperty(k)) {
                            var ev = parsedEvents[k];
                            if (ev.type != "VEVENT") {
                                /* Some ical files, including those from meetup, contain VTIMEZONE entries.
                                   Skip them, since they are not actually events, and epic fail lies within. */
                                continue;
                            }
                            ev.icalLibraryId = k;
                            /* the birminghamIOCalendarID is the ID we eventually
                               use to store this event in Google Calendar. As for
                               sources, above, it must match /^[a-v0-9]+$/, and must
                               be unique in the calendar. So, we assume that event.uid
                               exists and is unique in the thing that we fetched, but
                               can take any form it likes, and we construct an actually
                               unique ID as "bio" + source + sha1(event.uid).hexdigest
                               (because the hex digest of anything is /^[0-9a-f]+$/).
                               That way, this ID is all of calendar-unique, probably
                               globally-unique (because of the "bio"), and suitable
                               for use as a gcal ID. */
                            var shasum = crypto.createHash('sha1');
                            shasum.update(ev.uid);
                            ev.birminghamIOCalendarID = "bio" + icsbodyobj.source + shasum.digest('hex');
                            events.push(ev);
                        }
                    }
                }
                cb(null, events);
            }, function(err, results) {
                if (err) {
                    console.log("We failed to create a list of events", err);
                    return;
                }

                // auth to the google calendar
                jwt.authorize(function(err, tokens) {
                    if (err) { console.log("Problem authorizing to Google", err); return; }
                    var gcal = googleapis.calendar('v3');

                    /* Get list of events */
                    gcal.events.list({auth: jwt, calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com'}, function(err, resp) {
                        if (err) { console.log("Problem getting existing events", err); return; }
                        // Make a list of existing events keyed by uid, which is the unique key we created
                        var existing = {};
                        resp.items.forEach(function(ev) { existing[ev.id] = ev; });
                        /* Now, go through each of our fetched events and either update them
                           if they exist, or create them if not. Note that we do not pass an
                           err in the update/insert to the callback, because that will terminate
                           the async.map; instead, we always say that there was no error, and
                           then if there was we pass it inside the results, so we can check later. */
                        async.map(results, function(ev, callback) {
                            if (existing[ev.birminghamIOCalendarID]) {
                                //console.log("Update event", ev.birminghamIOCalendarID);
                                gcal.events.patch({
                                    auth: jwt,
                                    calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                                    eventId: ev.birminghamIOCalendarID,
                                    resource: {
                                        start: { dateTime: moment(ev.start).format() },
                                        end: { dateTime: moment(ev.end).format() },
                                        description: ev.description || "",
                                        location: ev.location || "",
                                        summary: ev.summary
                                    }
                                }, function(err, resp) {
                                    if (err) {
                                        callback(null, {success: false, err: err, type: "update", event: ev});
                                        return;
                                    }
                                    callback(null, {success: true, type: "update", event: ev});
                                });
                            } else {
                                //console.log("Insert event", ev.birminghamIOCalendarID);
                                gcal.events.insert({
                                    auth: jwt,
                                    calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                                    resource: {
                                        start: { dateTime: moment(ev.start).format() },
                                        end: { dateTime: moment(ev.end).format() },
                                        id: ev.birminghamIOCalendarID,
                                        description: ev.description || "",
                                        location: ev.location || "",
                                        summary: ev.summary
                                    }
                                }, function(err, resp) {
                                    if (err) {
                                        callback(null, {success: false, err: err, type: "insert", event: ev});
                                        return;
                                    }
                                    callback(null, {success: true, type: "insert", event: ev});
                                });
                            }
                        }, function(err, results) {
                            if (err) { console.log("Update/insert got an error (this shouldn't happen!)", err); return; }
                            var successes = [], failures = [], inserts = 0, updates = 0;
                            results.forEach(function(r) {
                                if (r.success) {
                                    successes.push(r.event);
                                    if (r.type == "insert") { inserts += 1; }
                                    if (r.type == "update") { updates += 1; }
                                } else {
                                    failures.push({event: r.event, err: r.err});
                                }
                            });
                            console.log("Successfully dealt with", successes.length,
                                "events (" + inserts, "new events,", updates, "existing events)");
                            console.log("Failed to deal with", failures.length, "events");
                            if (failures.length > 0) {
                                console.log("== Failures ==");
                                failures.forEach(function(f) {
                                    console.log("Event", f.event.summary,
                                        "(" + f.event.uid + ", " + f.event.birminghamIOCalendarID + ")",
                                        JSON.stringify(f.err));
                                });
                            }
                        });
                    });

                    // temporarily create one event, just to see if inserting works. Remove this once we've made inserting work.
                    /*gcal.events.insert({
                        auth: jwt,
                        calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                        resource: {
                            start: { dateTime: "2014-11-16T14:00:01+00:00"},
                            end: { dateTime: "2014-11-16T15:00:01+00:00"},
                            id: "siltest00002",
                            description: "adescription",
                            location: "alocation",
                            summary: "ahack on calendar app"
                        }
                    }, function(err, resp) {
                        console.log("inserted, now delete.", err, resp);
                    });*/
                    // create a new calendar owned by the service account
                    //gcal.calendars.insert({auth:jwt, resource: { summary: "Calendar for Birmingham.IO"}}, function(err, resp) {
                    //    console.log("got response", err, resp);
                    //});
                    // delegate access to new calendar
                    //gcal.acl.insert({auth:jwt,  calendarId: "movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com", resource: {
                    //    role: "owner", scope: {type: "user", value: "info@birmingham.io"}
                    //}}, function(err, resp) {
                    //    console.log("got response", err, resp);
                    //})
                });
            });
        });
    });
}

if (require.main === module) {
    mainJob();
}<commit_msg>Events which have been deleted from the gcal but not from upstream are not actually deleted in gcal; they are set to status=cancelled. So, if that happens, set them back to status=confirmed. Also, log instances of events which are in the google calendar but are NOT upstream, to see whether we should delete them later or something.<commit_after>var ical = require("ical"),
    async = require("async"),
    request = require("request"),
    config = require("./config"),
    googleapis = require('googleapis'),
    crypto = require("crypto"),
    moment = require("moment");

var CLIENT_ID = config.CLIENTID;
var CLIENT_SECRET = config.CLIENTSECRET;

var SERVICE_ACCOUNT_EMAIL = '721976846481-1s5altpg8afuc4opnlr13nua86hg0ul9@developer.gserviceaccount.com';
var SERVICE_ACCOUNT_KEY_FILE = './key.pem';
var jwt = new googleapis.auth.JWT(
        SERVICE_ACCOUNT_EMAIL,
        SERVICE_ACCOUNT_KEY_FILE,
        null,
        ['https://www.googleapis.com/auth/calendar']);

var MEETUP_KEY = config.MEETUPKEY;
var MEETUP_URL = "https://api.meetup.com/find/groups?" +
                    "&sign=true" +
                    "&photo-host=public" +
                    "&category=34" + /* Technology */
                    "&lat=52.483056&lon=-1.893611" + /* Birmingham */
                    "&radius=5" + /* radius (in miles) */
                    "&page=4000" + /* results per page */
                    "&key=";

/* FetchIcalUrls functions have to return a list of {source, url} objects.
   A source must be a short word which identifies the source somehow
   (for example, "meetup" for meetup iCal files) -- it is used to make sure
   that IDs from different sources don't collide. For boring technical reasons
   (https://developers.google.com/google-apps/calendar/v3/reference/events#id)
   the source must also contain only characters 0-9 and a-v (not a-z, and not
   capital letters).

   NOTE: a source is expected and required to provide UIDs for each event, and
   those UIDs must be both unique in that source and unchanging over time; if
   we request data from that source again later and one of the events that comes
   back was in this fetch too, then it must have the same UID. Otherwise it will
   be duplicated in the b.io calendar. Any source which does not enforce this is
   stupid, but if it does, it is your responsibility to provide a URL which *does*
   enforce it.
*/

function fetchIcalUrlsFromLocalFile(cb) {
    cb(null, [
        //"http://lanyrd.com/topics/nodejs/nodejs.ics",
        //"http://lanyrd.com/topics/python/python.ics"
    ]);
}

function fetchIcalUrlsFromMeetup(cb) {
    var req = request(MEETUP_URL + MEETUP_KEY, function(err, response, body) {
        if (err) {
            console.log("Meetup: Error connecting:", err);
            return;
        }
        else if (response.statusCode != 200) {
            console.log("Meetup: HTTP error code:", response.statusCode);
            return;
        }
        else {
            try {
                results = JSON.parse(body);
                if (results.length === 0) {
                    console.log("Meetup: Warning: no results received:");
                }
                urls = [];
                for (var result in results) {
                    urls.push({source: "meetup", url: results[result].link + "events/ical/"});
                }
                cb(null, urls);
            } catch(e) {
                console.log("Meetup: Error parsing JSON:", e);
            }
        }
    });
}

// first, get a list of ics urls from various places
function mainJob() {
    async.parallel([
        fetchIcalUrlsFromLocalFile,
        fetchIcalUrlsFromMeetup
    ], function(err, results) {
        if (err) {
            console.log("We failed to get a list of ics URLs", err);
            return;
        }
        // flatten results list and fetch them all
        var icsurls = [];
        icsurls = icsurls.concat.apply(icsurls, results);
        async.map(icsurls, function(icsurlobj, cb) {
            request(icsurlobj.url, function(err, response, body) {
                if (err) {
                    console.log("Failed to fetch URL", icsurlobj.url, err);
                    body = null;
                }
                cb(null, {source: icsurlobj.source, body: body});
            });
        }, function(err, results) {
            if (err) {
                console.log("We failed to fetch any ics URLs", err);
                return;
            }
            // parse them all into ICS structures
            async.concat(results, function(icsbodyobj, cb) {
                var events = [];
                if (icsbodyobj.body) {
                    var parsedEvents = ical.parseICS(icsbodyobj.body);
                    for (var k in parsedEvents) {
                        if (parsedEvents.hasOwnProperty(k)) {
                            var ev = parsedEvents[k];
                            if (ev.type != "VEVENT") {
                                /* Some ical files, including those from meetup, contain VTIMEZONE entries.
                                   Skip them, since they are not actually events, and epic fail lies within. */
                                continue;
                            }
                            ev.icalLibraryId = k;
                            /* the birminghamIOCalendarID is the ID we eventually
                               use to store this event in Google Calendar. As for
                               sources, above, it must match /^[a-v0-9]+$/, and must
                               be unique in the calendar. So, we assume that event.uid
                               exists and is unique in the thing that we fetched, but
                               can take any form it likes, and we construct an actually
                               unique ID as "bio" + source + sha1(event.uid).hexdigest
                               (because the hex digest of anything is /^[0-9a-f]+$/).
                               That way, this ID is all of calendar-unique, probably
                               globally-unique (because of the "bio"), and suitable
                               for use as a gcal ID. */
                            var shasum = crypto.createHash('sha1');
                            shasum.update(ev.uid);
                            ev.birminghamIOCalendarID = "bio" + icsbodyobj.source + shasum.digest('hex');
                            events.push(ev);
                        }
                    }
                }
                cb(null, events);
            }, function(err, results) {
                if (err) {
                    console.log("We failed to create a list of events", err);
                    return;
                }

                // auth to the google calendar
                jwt.authorize(function(err, tokens) {
                    if (err) { console.log("Problem authorizing to Google", err); return; }
                    var gcal = googleapis.calendar('v3');

                    /* Get list of events */
                    gcal.events.list({auth: jwt, calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com', showDeleted: true}, function(err, resp) {
                        if (err) { console.log("Problem getting existing events", err); return; }
                        // Make a list of existing events keyed by uid, which is the unique key we created
                        var existing = {};
                        resp.items.forEach(function(ev) { existing[ev.id] = ev; });

                        // Make a list of events which are in the google calendar and are *not* upstream, and flag them
                        var deletedUpstream = [];
                        var presentUpstream = {};
                        results.forEach(function(upstr) {
                            presentUpstream[upstr.birminghamIOCalendarID] = "yes";
                        });
                        for (var bioid in existing) {
                            if (!presentUpstream[bioid]) {
                                deletedUpstream.push(existing[bioid]);
                            }
                        }

                        /* Now, go through each of our fetched events and either update them
                           if they exist, or create them if not. Note that we do not pass an
                           err in the update/insert to the callback, because that will terminate
                           the async.map; instead, we always say that there was no error, and
                           then if there was we pass it inside the results, so we can check later. */
                        async.map(results, function(ev, callback) {
                            if (existing[ev.birminghamIOCalendarID]) {
                                //console.log("Update event", ev.birminghamIOCalendarID);
                                gcal.events.patch({
                                    auth: jwt,
                                    calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                                    eventId: ev.birminghamIOCalendarID,
                                    resource: {
                                        start: { dateTime: moment(ev.start).format() },
                                        end: { dateTime: moment(ev.end).format() },
                                        description: ev.description || "",
                                        location: ev.location || "",
                                        summary: ev.summary,
                                        status: "confirmed"
                                    }
                                }, function(err, resp) {
                                    if (err) {
                                        callback(null, {success: false, err: err, type: "update", event: ev});
                                        return;
                                    }
                                    callback(null, {success: true, type: "update", event: ev});
                                });
                            } else {
                                //console.log("Insert event", ev.birminghamIOCalendarID);
                                gcal.events.insert({
                                    auth: jwt,
                                    calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                                    resource: {
                                        start: { dateTime: moment(ev.start).format() },
                                        end: { dateTime: moment(ev.end).format() },
                                        id: ev.birminghamIOCalendarID,
                                        description: ev.description || "",
                                        location: ev.location || "",
                                        summary: ev.summary,
                                        status: "confirmed"
                                    }
                                }, function(err, resp) {
                                    if (err) {
                                        callback(null, {success: false, err: err, type: "insert", event: ev});
                                        return;
                                    }
                                    callback(null, {success: true, type: "insert", event: ev});
                                });
                            }
                        }, function(err, results) {
                            if (err) { console.log("Update/insert got an error (this shouldn't happen!)", err); return; }
                            var successes = [], failures = [], inserts = 0, updates = 0;
                            results.forEach(function(r) {
                                if (r.success) {
                                    successes.push(r.event);
                                    if (r.type == "insert") { inserts += 1; }
                                    if (r.type == "update") { updates += 1; }
                                } else {
                                    failures.push({event: r.event, err: r.err});
                                }
                            });
                            console.log("Successfully dealt with", successes.length,
                                "events (" + inserts, "new events,", updates, "existing events)");
                            console.log("Failed to deal with", failures.length, "events");
                            if (failures.length > 0) {
                                console.log("== Failures ==");
                                failures.forEach(function(f) {
                                    console.log("Event", f.event.summary,
                                        "(" + f.event.uid + ", " + f.event.birminghamIOCalendarID + ")",
                                        JSON.stringify(f.err));
                                });
                            }
                            console.log("Events present in the Google calendar but not present in sources:", deletedUpstream.length);
                            deletedUpstream.forEach(function(duev) {
                                console.log(duev.summary + " (" + duev.id + ")", duev.start.dateTime);
                            });
                        });
                    });

                    // temporarily create one event, just to see if inserting works. Remove this once we've made inserting work.
                    /*gcal.events.insert({
                        auth: jwt,
                        calendarId: 'movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com',
                        resource: {
                            start: { dateTime: "2014-11-16T14:00:01+00:00"},
                            end: { dateTime: "2014-11-16T15:00:01+00:00"},
                            id: "siltest00002",
                            description: "adescription",
                            location: "alocation",
                            summary: "ahack on calendar app"
                        }
                    }, function(err, resp) {
                        console.log("inserted, now delete.", err, resp);
                    });*/
                    // create a new calendar owned by the service account
                    //gcal.calendars.insert({auth:jwt, resource: { summary: "Calendar for Birmingham.IO"}}, function(err, resp) {
                    //    console.log("got response", err, resp);
                    //});
                    // delegate access to new calendar
                    //gcal.acl.insert({auth:jwt,  calendarId: "movdt8poi0t3gfedfd80u1kcak@group.calendar.google.com", resource: {
                    //    role: "owner", scope: {type: "user", value: "info@birmingham.io"}
                    //}}, function(err, resp) {
                    //    console.log("got response", err, resp);
                    //})
                });
            });
        });
    });
}

if (require.main === module) {
    mainJob();
}<|endoftext|>


<commit_before>var config = require('./config/config');
var amqp = require('amqplib');
var fs = require('fs');
var path = require('path');
var celllog = require('./celllog');
var logger = require('./utils/logger');
var zlib = require('zlib');
var i_cell = 0;
var i_wifi = 0;
var i = 0;
var split = require('split');

var watchFolder = __dirname + '/' + config.watch.folder;
var processedFolder = __dirname + '/' + config.watch.processed;
var files = [];
var amqpConnection;

// method that starts this processed
// 0. connect to RabbitMQ
// 1. reads current
// 2. start watch folder
// 3. start processNext method
function startup() {
	connectToRabbitMQ(processNext);
	logger.log('status', 'CellData parser is started');

	// read current files in dir
	fs.readdir(watchFolder, function(err, newFiles) {
		files = files.concat(newFiles);
	});

	// start watching dir for new files
	logger.log('status', 'Watching folder ' + watchFolder);
	fs.watch(watchFolder, function(event, filename) {
		if (filename) {
			if (event === 'rename') {
				files.push(filename);
			}
		}
	});
}

function mkCallback(i) {

	return function(err) {
		if (err !== null) {
			logger.log('error', 'Message' + i + 'failed');
		}
	}

}

// processFile opens the given filename, split it, parse it and send to
// the WIFI or cell queue.
function processFile(filename) {
	var readStream = fs.createReadStream(watchFolder + filename);

	readStream.on('open', function() {
		logger.log('info', 'Open file: ' + filename);
		clparser = celllog.createCellog();

		var cl = readStream.pipe(split()).pipe(clparser);

		cl.on('data', function(chunk) {
			i++;
			var send = JSON.stringify(chunk);
			if (chunk.bssid) {
				i_wifi++;
				sendToQueue('WIFI', send);
			} else {
				i_cell++;
				sendToQueue('cell', send);
			}

		});
		cl.on('end', function() {
			logger.log('status', 'FINISHED PROCESSING ' + filename);
			processNext();
		});

	});

	readStream.on('end', function() {
		logger.log('info', 'Messages processed:' + i + ' total (' + i_cell + ' cells, ' + i_wifi + ' wifi)');

		fs.rename(watchFolder + filename, processedFolder + filename, function(err) {
			if (err) {
				console.log(err);
			}
		});
	})
}

function processZippedFile(filename) {
	logger.log('status', 'Unzipping file ' + filename);
	const gunzip = zlib.createGunzip();
	const inp = fs.createReadStream(watchFolder + filename);

	clparser = celllog.createCellog();

	var cl = inp.pipe(gunzip).pipe(split()).pipe(clparser);

	cl.on('data', function(chunk) {
		i++;
		var send = JSON.stringify(chunk);
		if (chunk.bssid) {
			i_wifi++;
			ch.publish(config.amqp.queue, 'WIFI', new Buffer(send), {}, mkCallback(i));
		} else {
			i_cell++;
			ch.publish(config.amqp.queue, 'cell', new Buffer(send), {}, mkCallback(i));
		}

	});
	cl.on('end', function() {
		logger.log('status', 'FINISHED PROCESSING ' + filename);
		logger.log('info', 'Messages processed:' + i + ' total (' + i_cell + ' cells, ' + i_wifi + ' wifi)');

		fs.rename(watchFolder + filename, processedFolder + filename, function(err) {
			if (err) {
				console.log(err);
			}
		});
		processNext();
	});
}

function connectToRabbitMQ(whenConnected) {
	amqp.connect(config.amqp.server, function(err, conn) {
		if (err) {
      console.error("[AMQP] couldn't connect to RabbitMQ ", err.message);
			// retry in 1 second
      return setTimeout(connectToRabbitMQ, 1000);
    }
		conn.on("error", function(err) {
			if (err.message !== "Connection closing") {
				 console.error("[AMQP] conn error", err.message);
			}
	 	});
	 	conn.on("close", function() {
			 console.error("[AMQP] reconnecting");
			 // retry in 1 second
			 return setTimeout(connectToRabbitMQ, 1000);
	 	});
	 	console.log("[AMQP] connected");
	 	amqpConnection = conn;
		// ready to start processing, wait for 1 sec
		setTimeout(whenConnected, 1000);
	});
}

// closing connection when there is an error
function closeOnErr(err) {
  if (!err) return false;
  console.error("[AMQP] error", err);
  amqpConnection.close();
  return true;
}

// send data to queue name, makes a connection first and closes it afterwards.
function sendToQueue(routingKey, data) {
	amqpConnection.createConfirmChannel(function(err, ch) {
		if (closeOnErr(err)) return;
    ch.on("error", function(err) {
      console.error("[AMQP] channel error", err.message);
    });
    ch.on("close", function() {
      console.log("[AMQP] channel closed");
    });
		ch.publish(config.amqp.queue, routingKey, new Buffer(data), {}, mkCallback(i));
	});
}

function processNext() {
	var file = files.pop();
	if (typeof file !== 'undefined' && file) {
		logger.log('status', 'processNext file: ' + file);
		fs.stat(watchFolder + file, function(err, stats) {
			if (err) {
				// File does not exist
				processNext();
			} else if (stats.isFile() && path.extname(file) === '.log') {
				processFile(file);
			} else if (stats.isFile() && path.extname(file) === '.gz') {
				processZippedFile(file);
			} else {
				processNext();
			}
		});
	} else {
		setTimeout(processNext, 10000);
	}
}

// End yes we're up and running
startup();
<commit_msg>Restart fixes, other way of setting up AMQP connections<commit_after>var config = require('./config/config');
var amqp = require('amqplib/callback_api');
var fs = require('fs');
var path = require('path');
var celllog = require('./celllog');
var logger = require('./utils/logger');
var zlib = require('zlib');
var i_cell = 0;
var i_wifi = 0;
var i = 0;
var split = require('split');

var watchFolder = __dirname + '/' + config.watch.folder;
var processedFolder = __dirname + '/' + config.watch.processed;
var files = [];
var amqpConnection;
var channel;

// method that starts this processed
// 0. connect to RabbitMQ
// 1. reads current
// 2. start watch folder
// 3. start processNext method
function startup() {
	connectToRabbitMQ();
	logger.log('status', 'CellData parser is started');

	// read current files in dir
	fs.readdir(watchFolder, function(err, newFiles) {
		files = files.concat(newFiles);
	});

	// start watching dir for new files
	logger.log('status', 'Watching folder ' + watchFolder);
	fs.watch(watchFolder, function(event, filename) {
		if (filename) {
			if (event === 'rename') {
				files.push(filename);
			}
		}
	});

}

function mkCallback(i) {

	return function(err) {
		if (err !== null) {
			logger.log('error', 'Message' + i + 'failed');
		}
	}

}

// processFile opens the given filename, split it, parse it and send to
// the WIFI or cell queue.
function processFile(filename) {
	var readStream = fs.createReadStream(watchFolder + filename);

	readStream.on('open', function() {
		logger.log('info', 'Open file: ' + filename);
		clparser = celllog.createCellog();

		var cl = readStream.pipe(split()).pipe(clparser);

		cl.on('data', function(chunk) {
			i++;
			var send = JSON.stringify(chunk);
			if (chunk.bssid) {
				i_wifi++;
				sendToQueue('WIFI', send);
			} else {
				i_cell++;
				sendToQueue('cell', send);
			}

		});
		cl.on('end', function() {
			logger.log('status', 'FINISHED PROCESSING ' + filename);
			processNext();
		});

	});

	readStream.on('end', function() {
		logger.log('info', 'Messages processed:' + i + ' total (' + i_cell + ' cells, ' + i_wifi + ' wifi)');

		fs.rename(watchFolder + filename, processedFolder + filename, function(err) {
			if (err) {
				logger.log(err);
			}
		});
	})
}

function processZippedFile(filename) {
	logger.log('status', 'Unzipping file ' + filename);
	const gunzip = zlib.createGunzip();
	const inp = fs.createReadStream(watchFolder + filename);

	clparser = celllog.createCellog();

	var cl = inp.pipe(gunzip).pipe(split()).pipe(clparser);

	cl.on('data', function(chunk) {
		i++;
		var send = JSON.stringify(chunk);
		if (chunk.bssid) {
			i_wifi++;
			sendToQueue('WIFI', send);
		} else {
			i_cell++;
			sendToQueue('cell', send);
		}

	});
	cl.on('end', function() {
		logger.log('status', 'FINISHED PROCESSING ' + filename);
		logger.log('info', 'Messages processed:' + i + ' total (' + i_cell + ' cells, ' + i_wifi + ' wifi)');

		fs.rename(watchFolder + filename, processedFolder + filename, function(err) {
			if (err) {
				logger.log(err);
			}
		});
		processNext();
	});
}

function connectToRabbitMQ() {
	logger.log('status', "[AMQP] start connect");
	amqp.connect(config.amqp.server, function(err, conn) {
		if (err) {
      logger.log('error', "[AMQP] couldn't connect to RabbitMQ " + err.message);
			// retry in 1 second
      return setTimeout(connectToRabbitMQ, 1000);
    }
		conn.on("error", function(err) {
			if (err.message !== "Connection closing") {
				 logger.log('error', "[AMQP] conn error " + err.message);
			}
	 	});
	 	conn.on("close", function() {
			 logger.log('error', "[AMQP] reconnecting");
			 // retry in 1 second
			 return setTimeout(connectToRabbitMQ, 1000);
	 	});
	 	logger.log('status', "[AMQP] connected");
	 	amqpConnection = conn;

		createConfirmChannel();
	});
}

// closing connection when there is an error
function closeOnErr(err) {
  if (!err) return false;
  logger.log('error', "[AMQP] error " + err);
  amqpConnection.close();
  return true;
}

function createConfirmChannel() {
	amqpConnection.createConfirmChannel(function(err, ch) {
		if (closeOnErr(err)) return;
    ch.on("error", function(err) {
      logger.log('error', "[AMQP] channel error", err.message);
    });
    ch.on("close", function() {
      logger.log('status', "[AMQP] channel closed");
			setTimeout(createConfirmChannel, 1000);
    });
		channel = ch;
		logger.log('status', "[AMQP] channel created");
		setTimeout(processNext, 1000);
	});
}

// send data to queue name, makes a connection first and closes it afterwards.
function sendToQueue(routingKey, data) {
	channel.publish(config.amqp.queue, routingKey, new Buffer(data), {}, mkCallback(i));
}

function processNext() {
	var file = files.pop();
	if (typeof file !== 'undefined' && file) {
		logger.log('status', 'processNext file: ' + file);
		fs.stat(watchFolder + file, function(err, stats) {
			if (err) {
				// File does not exist
				processNext();
			} else if (stats.isFile() && path.extname(file) === '.log') {
				processFile(file);
			} else if (stats.isFile() && path.extname(file) === '.gz') {
				processZippedFile(file);
			} else {
				processNext();
			}
		});
	} else {
		setTimeout(processNext, 10000);
	}
}

// End yes we're up and running
startup();
<|endoftext|>


<commit_before>
const ASSET_PREFIX = "asset-cache-v";
const ASSET_CACHE = ASSET_PREFIX + versionHash;
const COVER_ART_CACHE = "cover-art-cache";
const COVER_ART_HOSTNAME = "coverartarchive.org";
const THIRD_PARTY_ASSET_CACHE = "asset-cache-3rd-party";
const IS_DEVELOPMENT = location.hostname === "vm";

const delay = function(ms) {
    return new Promise(function(resolve) {
        setTimeout(resolve, ms);
    });
};

const isCors = function(url) {
    return new URL(url).origin !== location.origin;
};

self.addEventListener("install", function(e) {
    if (IS_DEVELOPMENT) return;

    var assetsCached = caches.open(ASSET_CACHE).then(function (cache) {
        return Promise.all(assets.map(function loop(url) {
            url = new URL(url, location.origin + (location.pathname || "")).toString();

            if (isCors(url)) {
                return cache.add(url);
            }

            return (function loop(retries) {
                return fetch(url, {
                    credentials: "include",
                    mode: "no-cors",
                    cache: "reload"
                }).then(function(response) {
                    if (response.status != 200) {
                        throw new Error("http");
                    }
                    return cache.put(url, response);
                }).catch(function(e) {
                    if (retries <= 5) {
                        return delay(1000).then(function() {
                            return loop(url, retries + 1);
                        });
                    } else {
                        throw e;
                    }
                });
            })(0);
        }));
    });
    e.waitUntil(assetsCached);
}, false);

self.addEventListener("activate", function(e) {
    var oldAssetsRemoved = caches.keys().then(function(keys) {
        return Promise.all(keys.filter(function(key) {
            // Never delete cover art.
            if (key === COVER_ART_CACHE) {
                return false;
            // Delete everything in development
            } else if (IS_DEVELOPMENT) {
                return true;
            // In production, only delete old assets.
            } else if (key !== ASSET_CACHE && key.indexOf(ASSET_PREFIX) >= 0) {
                return true;
            } else {
                return false;
            }
        }).map(function(key) {
            return caches.delete(key);
        }));
    });
    e.waitUntil(oldAssetsRemoved);
}, false);

var rfonts = /\.(woff2?)$/
self.addEventListener("fetch", function(e) {
    var request = e.request;

    if (request.method !== "GET") {
        return;
    }

    var requestURL = new URL(e.request.url);
    var isCors = location.origin !== requestURL.origin;
    var isHttp = requestURL.protocol.toLowerCase().indexOf("http") >= 0;
    var isCoverArt = COVER_ART_HOSTNAME === requestURL.hostname;
    var isQuery = requestURL.search && requestURL.search.length > 1 && isCors;
    var isCorsFont = isCors && rfonts.test(request.url);

    if (!isHttp || (!isCoverArt && (isCorsFont || isQuery))) {
        return;
    }

    // Cache coverart even in development.
    if (IS_DEVELOPMENT && !isCoverArt) {
        return;
    }

    var result = caches.match(e.request).then(function(response) {
        if (response) return response;

        var fetchRequest = e.request.clone();
        return fetch(fetchRequest, {mode: "no-cors"}).then(function(response) {
            var cacheName = null;
            if (!IS_DEVELOPMENT && response.type === "basic" && response.status < 300) {
                cacheName = ASSET_CACHE;
            } else if (COVER_ART_HOSTNAME === requestURL.hostname) {
                cacheName = COVER_ART_CACHE;
            }

            if (cacheName) {
                var responseToCache = response.clone();
                caches.open(cacheName).then(function(cache) {
                    cache.put(fetchRequest, responseToCache)
                });
            }
            return response;
        }).catch(function(e) {
            console.log(e.message);
        });
    });

    e.respondWith(result);
}, false);

self.addEventListener('message', function(e) {
    if (e.data.action === 'skipWaiting') {
        self.skipWaiting();
    }
});

self.addEventListener("notificationclick", function(event) {
    event.waitUntil(clients.matchAll({type: "window"}).then(function(clientList) {
        for (var i = 0; i < clientList.length; i++) {
            var client = clientList[i];
            if ('focus' in client) {
                return client.postMessage({
                    eventType: "swEvent",
                    type: "notificationClick",
                    action: event.action,
                    data: event.notification.data,
                    tag: event.notification.tag
                });
            }
        }
    }));
});
<commit_msg>don't wait for unnecessary icons to start the app<commit_after>
const ASSET_PREFIX = "asset-cache-v";
const ASSET_CACHE = ASSET_PREFIX + versionHash;
const COVER_ART_CACHE = "cover-art-cache";
const COVER_ART_HOSTNAME = "coverartarchive.org";
const THIRD_PARTY_ASSET_CACHE = "asset-cache-3rd-party";
const IS_DEVELOPMENT = location.hostname === "vm";

const delay = function(ms) {
    return new Promise(function(resolve) {
        setTimeout(resolve, ms);
    });
};

const isCors = function(url) {
    return new URL(url).origin !== location.origin;
};

const ricon = /(?:android-chrome|icon|safari-pinned|mstile)/;
const isUnnecessary = function(url) {
    return url.indexOf("dist/images") >= 0 && ricon.test(url);
};

self.addEventListener("install", function(e) {
    if (IS_DEVELOPMENT) return;

    var assetsCached = caches.open(ASSET_CACHE).then(function (cache) {
        return Promise.all(assets.map(function loop(url) {
            url = new URL(url, location.origin + (location.pathname || "")).toString();

            if (isCors(url)) {
                return cache.add(url);
            }

            if (isUnnecessary(url)) {
                return Promise.resolve();
            }

            return (function loop(retries) {
                return fetch(url, {
                    credentials: "include",
                    mode: "no-cors",
                    cache: "reload"
                }).then(function(response) {
                    if (response.status != 200) {
                        throw new Error("http");
                    }
                    return cache.put(url, response);
                }).catch(function(e) {
                    if (retries <= 5) {
                        return delay(1000).then(function() {
                            return loop(url, retries + 1);
                        });
                    } else {
                        throw e;
                    }
                });
            })(0);
        }));
    });
    e.waitUntil(assetsCached);
}, false);

self.addEventListener("activate", function(e) {
    var oldAssetsRemoved = caches.keys().then(function(keys) {
        return Promise.all(keys.filter(function(key) {
            // Never delete cover art.
            if (key === COVER_ART_CACHE) {
                return false;
            // Delete everything in development
            } else if (IS_DEVELOPMENT) {
                return true;
            // In production, only delete old assets.
            } else if (key !== ASSET_CACHE && key.indexOf(ASSET_PREFIX) >= 0) {
                return true;
            } else {
                return false;
            }
        }).map(function(key) {
            return caches.delete(key);
        }));
    });
    e.waitUntil(oldAssetsRemoved);
}, false);

var rfonts = /\.(woff2?)$/
self.addEventListener("fetch", function(e) {
    var request = e.request;

    if (request.method !== "GET") {
        return;
    }

    var requestURL = new URL(e.request.url);
    var isCors = location.origin !== requestURL.origin;
    var isHttp = requestURL.protocol.toLowerCase().indexOf("http") >= 0;
    var isCoverArt = COVER_ART_HOSTNAME === requestURL.hostname;
    var isQuery = requestURL.search && requestURL.search.length > 1 && isCors;
    var isCorsFont = isCors && rfonts.test(request.url);

    if (!isHttp || (!isCoverArt && (isCorsFont || isQuery))) {
        return;
    }

    // Cache coverart even in development.
    if (IS_DEVELOPMENT && !isCoverArt) {
        return;
    }

    var result = caches.match(e.request).then(function(response) {
        if (response) return response;

        var fetchRequest = e.request.clone();
        return fetch(fetchRequest, {mode: "no-cors"}).then(function(response) {
            var cacheName = null;
            if (!IS_DEVELOPMENT && response.type === "basic" && response.status < 300) {
                cacheName = ASSET_CACHE;
            } else if (COVER_ART_HOSTNAME === requestURL.hostname) {
                cacheName = COVER_ART_CACHE;
            }

            if (cacheName) {
                var responseToCache = response.clone();
                caches.open(cacheName).then(function(cache) {
                    cache.put(fetchRequest, responseToCache)
                });
            }
            return response;
        }).catch(function(e) {
            console.log(e.message);
        });
    });

    e.respondWith(result);
}, false);

self.addEventListener('message', function(e) {
    if (e.data.action === 'skipWaiting') {
        self.skipWaiting();
    }
});

self.addEventListener("notificationclick", function(event) {
    event.waitUntil(clients.matchAll({type: "window"}).then(function(clientList) {
        for (var i = 0; i < clientList.length; i++) {
            var client = clientList[i];
            if ('focus' in client) {
                return client.postMessage({
                    eventType: "swEvent",
                    type: "notificationClick",
                    action: event.action,
                    data: event.notification.data,
                    tag: event.notification.tag
                });
            }
        }
    }));
});
<|endoftext|>


<commit_before>/**
 * Component definition for frost-scroll component
 */
import Component from './frost-component'
import Ember from 'ember'
const {deprecate, run, typeOf} = Ember

export default Component.extend({

  // == Dependencies ==========================================================

  // == Keyword Properties ====================================================

  // == PropTypes =============================================================

  // == Computed Properties ===================================================

  // == Functions =============================================================

  /* eslint-disable complexity */
  /**
   * Setup the perfect-scrollbar plugin and events
   *
   * @private
   * @returns {undefined}
   */
  _setupPerfectScroll () {
    const debouncePeriod = 150

    run.scheduleOnce('afterRender', this, () => {
      if (!this.isDestroying && !this.isDestroyed) {
        window.Ps.initialize(this.$()[0], this.get('psOptions') || {})
      }
    })

    this._legacyScrollYEndHandler = () => {
      run.debounce(this, this['on-scroll-y-end'], debouncePeriod, true)
    }

    this._scrollDownHandler = () => {
      run.debounce(this, this.onScrollDown, debouncePeriod, true)
    }

    this._scrollUpHandler = () => {
      run.debounce(this, this.onScrollUp, debouncePeriod, true)
    }

    this._scrollYEndHandler = () => {
      run.debounce(this, this.onScrollYEnd, debouncePeriod, true)
    }

    this._scrollYStartHandler = () => {
      run.debounce(this, this.onScrollYStart, debouncePeriod, true)
    }

    this._scrollXHandler = () => {
      run.debounce(this, this.onScrollX, debouncePeriod, true)
    }

    this._scrollRightHandler = () => {
      run.debounce(this, this.onScrollRight, debouncePeriod, true)
    }

    this._scrollLeftHandler = () => {
      run.debounce(this, this.onScrollLeft, debouncePeriod, true)
    }

    if (typeOf(this.onScrollUp) === 'function') {
      this.$().on('ps-scroll-up', this._scrollUpHandler)
    }

    if (typeOf(this.onScrollDown) === 'function') {
      this.$().on('ps-scroll-down', this._scrollDownHandler)
    }

    if (typeOf(this.onScrollYStart) === 'function') {
      this.$().on('ps-y-reach-start', this._scrollYStartHandler)
    }

    if (typeOf(this.onScrollYEnd) === 'function') {
      this.$().on('ps-y-reach-end', this._scrollYEndHandler)
    }

    if (typeOf(this.onScrollX) === 'function') {
      this.$().on('ps-scroll-x', this._scrollXHandler)
    }

    if (typeOf(this.onScrollRight) === 'function') {
      this.$().on('ps-scroll-right', this._scrollRightHandler)
    }

    if (typeOf(this.onScrollLeft) === 'function') {
      this.$().on('ps-scroll-left', this._scrollLeftHandler)
    }

    if (typeOf(this['on-scroll-y-end']) === 'function') {
      deprecate('on-scroll-y-end has been deprecated in favor of onScrollYEnd',
        false,
        {
          id: 'frost-debug.deprecate-on-scroll-y-end',
          until: '1.0.0',
          url: 'http://ciena-frost.github.io/ember-frost-core/#/scroll'
        }
      )
    }
  },

  /**
   * Remove perfect-scrollbar plugin and events
   *
   * @private
   * @returns {undefined}
   */
  _unregisterEvents () {
    window.Ps.destroy(this.$()[0])

    if (typeOf(this.onScrollUp) === 'function') {
      this.$().off('ps-scroll-up', this._scrollUpHandler)
    }

    if (typeOf(this.onScrollDown) === 'function') {
      this.$().off('ps-scroll-down', this._scrollDownHandler)
    }

    if (typeOf(this.onScrollYStart) === 'function') {
      this.$().off('ps-y-reach-start', this._scrollYStartHandler)
    }

    if (typeOf(this.onScrollYEnd) === 'function') {
      this.$().off('ps-y-reach-end', this._scrollYEndHandler)
    }

    if (typeOf(this.onScrollX) === 'function') {
      this.$().off('ps-scroll-x', this._scrollXHandler)
    }

    if (typeOf(this.onScrollRight) === 'function') {
      this.$().off('ps-scroll-right', this._scrollRightHandler)
    }

    if (typeOf(this.onScrollLeft) === 'function') {
      this.$().off('ps-scroll-left', this._scrollLeftHandler)
    }
  },
  /* eslint-enable complexity */

  // == DOM Events ============================================================

  // == Lifecycle Hooks =======================================================

  didInsertElement () {
    this._super(...arguments)
    this._setupPerfectScroll()
  },

  willDestroyElement () {
    this._super(...arguments)
    this._unregisterEvents()
  },
  didRender () {
    window.Ps.update(this.$()[0])
  },
  // == Actions ===============================================================

  actions: {
  }
})
<commit_msg>Added option to refresh the scrollbar on mouse enter<commit_after>/**
 * Component definition for frost-scroll component
 */
import Component from './frost-component'
import Ember from 'ember'
const {deprecate, run, typeOf} = Ember
import {PropTypes} from 'ember-prop-types'

export default Component.extend({

  // == Dependencies ==========================================================

  // == Keyword Properties ====================================================

  // == PropTypes =============================================================
  propTypes: {
    refreshOnMouseEnter: PropTypes.bool.optional
  },

  getDefaultProps () {
    return {
      refreshOnMouseEnter: false
    }
  },
  // == Computed Properties ===================================================

  // == Functions =============================================================

  /* eslint-disable complexity */
  /**
   * Setup the perfect-scrollbar plugin and events
   *
   * @private
   * @returns {undefined}
   */
  _setupPerfectScroll () {
    const debouncePeriod = 150

    run.scheduleOnce('afterRender', this, () => {
      if (!this.isDestroying && !this.isDestroyed) {
        window.Ps.initialize(this.$()[0], this.get('psOptions') || {})
      }
    })

    this._legacyScrollYEndHandler = () => {
      run.debounce(this, this['on-scroll-y-end'], debouncePeriod, true)
    }

    this._scrollDownHandler = () => {
      run.debounce(this, this.onScrollDown, debouncePeriod, true)
    }

    this._scrollUpHandler = () => {
      run.debounce(this, this.onScrollUp, debouncePeriod, true)
    }

    this._scrollYEndHandler = () => {
      run.debounce(this, this.onScrollYEnd, debouncePeriod, true)
    }

    this._scrollYStartHandler = () => {
      run.debounce(this, this.onScrollYStart, debouncePeriod, true)
    }

    this._scrollXHandler = () => {
      run.debounce(this, this.onScrollX, debouncePeriod, true)
    }

    this._scrollRightHandler = () => {
      run.debounce(this, this.onScrollRight, debouncePeriod, true)
    }

    this._scrollLeftHandler = () => {
      run.debounce(this, this.onScrollLeft, debouncePeriod, true)
    }

    this._mouseEnterHandler = () => {
      window.Ps.update(this.get('element'))
    }

    if (typeOf(this.onScrollUp) === 'function') {
      this.$().on('ps-scroll-up', this._scrollUpHandler)
    }

    if (typeOf(this.onScrollDown) === 'function') {
      this.$().on('ps-scroll-down', this._scrollDownHandler)
    }

    if (typeOf(this.onScrollYStart) === 'function') {
      this.$().on('ps-y-reach-start', this._scrollYStartHandler)
    }

    if (typeOf(this.onScrollYEnd) === 'function') {
      this.$().on('ps-y-reach-end', this._scrollYEndHandler)
    }

    if (typeOf(this.onScrollX) === 'function') {
      this.$().on('ps-scroll-x', this._scrollXHandler)
    }

    if (typeOf(this.onScrollRight) === 'function') {
      this.$().on('ps-scroll-right', this._scrollRightHandler)
    }

    if (typeOf(this.onScrollLeft) === 'function') {
      this.$().on('ps-scroll-left', this._scrollLeftHandler)
    }

    if (this.refreshOnMouseEnter) {
      this.$().on('mouseenter', this._mouseEnterHandler)
    }

    if (typeOf(this['on-scroll-y-end']) === 'function') {
      deprecate('on-scroll-y-end has been deprecated in favor of onScrollYEnd',
        false,
        {
          id: 'frost-debug.deprecate-on-scroll-y-end',
          until: '1.0.0',
          url: 'http://ciena-frost.github.io/ember-frost-core/#/scroll'
        }
      )
    }
  },

  /**
   * Remove perfect-scrollbar plugin and events
   *
   * @private
   * @returns {undefined}
   */
  _unregisterEvents () {
    window.Ps.destroy(this.$()[0])

    if (typeOf(this.onScrollUp) === 'function') {
      this.$().off('ps-scroll-up', this._scrollUpHandler)
    }

    if (typeOf(this.onScrollDown) === 'function') {
      this.$().off('ps-scroll-down', this._scrollDownHandler)
    }

    if (typeOf(this.onScrollYStart) === 'function') {
      this.$().off('ps-y-reach-start', this._scrollYStartHandler)
    }

    if (typeOf(this.onScrollYEnd) === 'function') {
      this.$().off('ps-y-reach-end', this._scrollYEndHandler)
    }

    if (typeOf(this.onScrollX) === 'function') {
      this.$().off('ps-scroll-x', this._scrollXHandler)
    }

    if (typeOf(this.onScrollRight) === 'function') {
      this.$().off('ps-scroll-right', this._scrollRightHandler)
    }

    if (typeOf(this.onScrollLeft) === 'function') {
      this.$().off('ps-scroll-left', this._scrollLeftHandler)
    }

    if (this.refreshOnMouseEnter) {
      this.$().off('mouseenter', this._mouseEnterHandler)
    }
  },
  /* eslint-enable complexity */

  // == DOM Events ============================================================

  // == Lifecycle Hooks =======================================================

  didInsertElement () {
    this._super(...arguments)
    this._setupPerfectScroll()
  },

  willDestroyElement () {
    this._super(...arguments)
    this._unregisterEvents()
  },
  didRender () {
    window.Ps.update(this.$()[0])
  },
  // == Actions ===============================================================

  actions: {
  }
})
<|endoftext|>


<commit_before>var BUFFER_LENGTH = 1024 * 8;
var MAX_STACK_SIZE = 100;

var parser = require('./parser');
var nextTick = (typeof setImmediate == 'function') ? setImmediate : process.nextTick;

exports.render = render;

function render(tokens, context, partials, stream, callback) {
  if (!Array.isArray(context)) {
    context = [context];
  }

  return _render(tokens, context, partials, stream, callback);
}

function _render(tokens, context, partials, stream, callback) {
  if (tokens[0] !== 'multi') {
    throw new Error('Mu - WTF did you give me? I expected mustache tokens.');
  }

  var i = 1
    , stackSize = 0;

  function next() {
    try {

      if (stream.paused) {
        stream.once('resumed', function () {
          nextTick(next);
        });
        return;
      }

      if (++stackSize % MAX_STACK_SIZE == 0) {
        nextTick(next);
        return;
      }

      var token = tokens[i++];

      if (!token) {
        return callback ? callback() : true;
      }

      switch (token[0]) {
      case 'static':
        stream.emit('data', token[2]);
        return next();

      case 'mustache':
        switch (token[1]) {
//        case 'utag': // Unescaped Tag
//          stream.emit('data', s(normalize(context, token[2]),stream,token[2]));
//          return next();

        case 'etag': // Escaped Tag
        case 'utag': // Unescaped Tag (forces escape below)
          stream.emit('data', escape(s(normalize(context, token[2]),stream,token[2])));
          return next();

        case 'section':
          var res = normalize(context, token[2], token[3]);
          if (res) {
            return section(context, token[2], res, token[4], partials, stream, next);
          } else {
            return next();
          }

        case 'inverted_section':
          var res = normalize(context, token[2], token[3]);
          if (!res || res.length === 0) {
            return section(context, token[2], true, token[4], partials, stream, next);
          } else {
            return next();
          }

        case 'partial':
          var partial = partials[token[2]];
          // console.log(require('util').inspect(partials));
          if (partial) {
            return render(partial[0].tokens, context, partials, stream, next);
          } else {
            return next();
          }
        }

      }

    } catch (err) {
      stream.emit('error', err);
      next();
    }
  }

  next();
}

function s(val,stream,token) {
  if (val === null || typeof val === 'undefined') {
    if(stream && token){
      stream.emit('warn',{token:token,category:'token.undefined'});
    }
    return '';
  } else {
    return val.toString();
  }
}

function normalize(context, name, body) {
  var val = walkToFind(context, name);

  if (typeof(val) === 'function') {
    val = val.call(smashContext(context), body);
  }

  return val;
}

function walkToFind(context, name) {
  var i = context.length;

  while (i--) {
    var result = contextLevelContains(context[i], name);

    if (result !== undefined) {
      return result;
    }
  }

  return undefined;
}

function contextLevelContains(context, fullPath) {
  var pathParts = fullPath.split('.');
  var obj = context;

  for (var i = 0; i < pathParts.length; i++) {
    var part = pathParts[i];

    if (typeof obj == 'object' && part in obj) {
      obj = obj[part];
    } else {
      obj = undefined;
      break;
    }
  }

  return obj;
}

// TODO: if Proxy, make more efficient
// TODO: cache?
function smashContext(context) {
  var obj = {};

  for (var i = 0; i < context.length; i++) {
    var level = context[i];

    if (level instanceof Date) {
      obj.__date = level;
    } else {
      for (var k in level) {
        obj[k] = level[k];
      }
    }
  }

  return obj;
}

function section(context, name, val, tokens, partials, stream, callback) {
  if (val instanceof Array) {
    var i = 0;

    (function next() {
      var item = val[i++];

      if (item) {
        context.push(item);
        _render(tokens, context, partials, stream, function () {
          context.pop();

          if (i % MAX_STACK_SIZE == 0) {
            return nextTick(next);
          } else {
            next();
          }
        });
      } else {
        callback();
      }

    }());

    return;
  }

  if (typeof val === 'object') {
    context.push(val);
    _render(tokens, context, partials, stream, function () {
      context.pop();
      callback();
    });
    return;
  }

  if (val) {
    return _render(tokens, context, partials, stream, callback);
  }

  return callback();
}

//
//
//
function findInContext(context, key) {
  var i = context.length;
  while (i--) {
    if (context[i][key]) {
      return context[i][key];
    }
  }

  return undefined;
}

function escape(string) {
  return string.replace(/(@!)?[\W_]/igm, escapeReplace);
}
function escapeReplace(char) {
  if (char.length == 3) { return char.charAt(2); } //ignore the @! (literal) character
  else {
    switch (char) {
      case '{': return "\\{";
      case '}': return "\\}";
      case '_': return "\\_";
      case '#': return "\\#";
      case '&': return "\\&";
      case '$': return "\\$";
      case '%': return "\\%";
      case '@': return "\\at ";
      case '\n': return "\\\\";
      //NOTE below may require 'textcomp' package
      case '~': return "\\textasciitilde ";
      case '\\': return "\\textbackslash ";
      case '^': return "\\textasciicircum ";
      case '>': return "\\textgreater ";
      case '<': return "\\textless ";
      case '¢': return "\\textcent ";
      case '°': return "\\textdegree ";
      case '¶': return "\\P ";
      case '†': return "\\dag ";
      case '‡': return "\\ddag ";
      case '|': return "\\textbar ";
      case '™': return "\\texttrademark ";
      case '¡': return "\\textexclamdown ";
      case '£': return "\\pounds ";
      case '§': return "\\S ";
      case '®': return "\\textregistered ";
      case '¿': return "\\textquestiondown ";
      case '©': return "\\textcopyright ";
      default: return char;
    }
  }
}
<commit_msg>Remove @ case<commit_after>var BUFFER_LENGTH = 1024 * 8;
var MAX_STACK_SIZE = 100;

var parser = require('./parser');
var nextTick = (typeof setImmediate == 'function') ? setImmediate : process.nextTick;

exports.render = render;

function render(tokens, context, partials, stream, callback) {
  if (!Array.isArray(context)) {
    context = [context];
  }

  return _render(tokens, context, partials, stream, callback);
}

function _render(tokens, context, partials, stream, callback) {
  if (tokens[0] !== 'multi') {
    throw new Error('Mu - WTF did you give me? I expected mustache tokens.');
  }

  var i = 1
    , stackSize = 0;

  function next() {
    try {

      if (stream.paused) {
        stream.once('resumed', function () {
          nextTick(next);
        });
        return;
      }

      if (++stackSize % MAX_STACK_SIZE == 0) {
        nextTick(next);
        return;
      }

      var token = tokens[i++];

      if (!token) {
        return callback ? callback() : true;
      }

      switch (token[0]) {
      case 'static':
        stream.emit('data', token[2]);
        return next();

      case 'mustache':
        switch (token[1]) {
//        case 'utag': // Unescaped Tag
//          stream.emit('data', s(normalize(context, token[2]),stream,token[2]));
//          return next();

        case 'etag': // Escaped Tag
        case 'utag': // Unescaped Tag (forces escape below)
          stream.emit('data', escape(s(normalize(context, token[2]),stream,token[2])));
          return next();

        case 'section':
          var res = normalize(context, token[2], token[3]);
          if (res) {
            return section(context, token[2], res, token[4], partials, stream, next);
          } else {
            return next();
          }

        case 'inverted_section':
          var res = normalize(context, token[2], token[3]);
          if (!res || res.length === 0) {
            return section(context, token[2], true, token[4], partials, stream, next);
          } else {
            return next();
          }

        case 'partial':
          var partial = partials[token[2]];
          // console.log(require('util').inspect(partials));
          if (partial) {
            return render(partial[0].tokens, context, partials, stream, next);
          } else {
            return next();
          }
        }

      }

    } catch (err) {
      stream.emit('error', err);
      next();
    }
  }

  next();
}

function s(val,stream,token) {
  if (val === null || typeof val === 'undefined') {
    if(stream && token){
      stream.emit('warn',{token:token,category:'token.undefined'});
    }
    return '';
  } else {
    return val.toString();
  }
}

function normalize(context, name, body) {
  var val = walkToFind(context, name);

  if (typeof(val) === 'function') {
    val = val.call(smashContext(context), body);
  }

  return val;
}

function walkToFind(context, name) {
  var i = context.length;

  while (i--) {
    var result = contextLevelContains(context[i], name);

    if (result !== undefined) {
      return result;
    }
  }

  return undefined;
}

function contextLevelContains(context, fullPath) {
  var pathParts = fullPath.split('.');
  var obj = context;

  for (var i = 0; i < pathParts.length; i++) {
    var part = pathParts[i];

    if (typeof obj == 'object' && part in obj) {
      obj = obj[part];
    } else {
      obj = undefined;
      break;
    }
  }

  return obj;
}

// TODO: if Proxy, make more efficient
// TODO: cache?
function smashContext(context) {
  var obj = {};

  for (var i = 0; i < context.length; i++) {
    var level = context[i];

    if (level instanceof Date) {
      obj.__date = level;
    } else {
      for (var k in level) {
        obj[k] = level[k];
      }
    }
  }

  return obj;
}

function section(context, name, val, tokens, partials, stream, callback) {
  if (val instanceof Array) {
    var i = 0;

    (function next() {
      var item = val[i++];

      if (item) {
        context.push(item);
        _render(tokens, context, partials, stream, function () {
          context.pop();

          if (i % MAX_STACK_SIZE == 0) {
            return nextTick(next);
          } else {
            next();
          }
        });
      } else {
        callback();
      }

    }());

    return;
  }

  if (typeof val === 'object') {
    context.push(val);
    _render(tokens, context, partials, stream, function () {
      context.pop();
      callback();
    });
    return;
  }

  if (val) {
    return _render(tokens, context, partials, stream, callback);
  }

  return callback();
}

//
//
//
function findInContext(context, key) {
  var i = context.length;
  while (i--) {
    if (context[i][key]) {
      return context[i][key];
    }
  }

  return undefined;
}

function escape(string) {
  return string.replace(/(@!)?[\W_]/igm, escapeReplace);
}
function escapeReplace(char) {
  if (char.length == 3) { return char.charAt(2); } //ignore the @! (literal) character
  else {
    switch (char) {
      case '{': return "\\{";
      case '}': return "\\}";
      case '_': return "\\_";
      case '#': return "\\#";
      case '&': return "\\&";
      case '$': return "\\$";
      case '%': return "\\%";
      //case '@': return "\\at ";
      case '\n': return "\\\\";
      //NOTE below may require 'textcomp' package
      case '~': return "\\textasciitilde ";
      case '\\': return "\\textbackslash ";
      case '^': return "\\textasciicircum ";
      case '>': return "\\textgreater ";
      case '<': return "\\textless ";
      case '¢': return "\\textcent ";
      case '°': return "\\textdegree ";
      case '¶': return "\\P ";
      case '†': return "\\dag ";
      case '‡': return "\\ddag ";
      case '|': return "\\textbar ";
      case '™': return "\\texttrademark ";
      case '¡': return "\\textexclamdown ";
      case '£': return "\\pounds ";
      case '§': return "\\S ";
      case '®': return "\\textregistered ";
      case '¿': return "\\textquestiondown ";
      case '©': return "\\textcopyright ";
      default: return char;
    }
  }
}
<|endoftext|>


<commit_before>import { FETCH_STORY_INFO } from '../../actions/storyActions';
import { createAsyncReducer } from '../../lib/reduxHelpers';

const entities = createAsyncReducer({
  initialState: {
    info: [],
  },
  action: FETCH_STORY_INFO,
});

export default entities;
<commit_msg>#845 storyinfo- is this what we want<commit_after>import moment from 'moment';
import { FETCH_STORY, SELECT_STORY } from '../../actions/storyActions';
// FETCH_STORY_INFO
import { createAsyncReducer } from '../../lib/reduxHelpers';
import { TAG_SET_DATE_GUESS_METHOD, TAG_SET_EXTRACTOR_VERSION, TAG_SET_GEOCODER_VERSION, TAG_SET_NYT_THEMES_VERSION }
  from '../../lib/tagUtil';

// returns undefined if it is not found
function tagWithTagSetsId(tags, tagSetsId) {
  if (tags) {
    const tag = tags.find(t => t.tag_sets_id === tagSetsId);
    return (tag) ? tag.tag : undefined;
  }
  return undefined;
}

const info = createAsyncReducer({
  initialState: {
    id: null,
  },
  action: FETCH_STORY,
  handleSuccess: payload => ({
    ...payload.info,
    publishDateObj: moment(payload.publish_date).toDate(),
    dateGuessMethod: tagWithTagSetsId(payload.story_tags, TAG_SET_DATE_GUESS_METHOD),
    extractorVersion: tagWithTagSetsId(payload.story_tags, TAG_SET_EXTRACTOR_VERSION),
    geocoderVersion: tagWithTagSetsId(payload.story_tags, TAG_SET_GEOCODER_VERSION),
    nytThemesVersion: tagWithTagSetsId(payload.story_tags, TAG_SET_NYT_THEMES_VERSION),
  }),
  [SELECT_STORY]: payload => ({ id: payload }),
});

export default info;
<|endoftext|>


<commit_before>'use strict';

var util = require('util');
var EventEmitter = require('events').EventEmitter;
var debug = require('./util/debug')('batch');
var forever = require('./util/forever');
var queue = require('queue-async');
var Locker = require('./leader/locker');
var HostUserQueueMover = require('./maintenance/host-user-queue-mover');

function Batch(name, jobSubscriber, jobQueue, jobRunner, jobService, jobPublisher, redisPool, logger) {
    EventEmitter.call(this);
    this.name = name || 'batch';
    this.jobSubscriber = jobSubscriber;
    this.jobQueue = jobQueue;
    this.jobRunner = jobRunner;
    this.jobService = jobService;
    this.jobPublisher = jobPublisher;
    this.logger = logger;
    this.locker = Locker.create('redis-distlock', { pool: redisPool });
    this.hostUserQueueMover = new HostUserQueueMover(jobQueue, jobService, this.locker, redisPool);

    // map: host => map{user}. Useful to determine pending queued users.
    this.workingHosts = {};

    // map: user => jobId. Will be used for draining jobs.
    this.workInProgressJobs = {};
}
util.inherits(Batch, EventEmitter);

module.exports = Batch;

Batch.prototype.start = function () {
    this.hostUserQueueMover.moveOldJobs(function() {
        this.subscribe();
    }.bind(this));
};

Batch.prototype.subscribe = function () {
    var self = this;

    this.jobSubscriber.subscribe(
        function onJobHandler(user, host) {
            debug('onJobHandler(%s, %s)', user, host);
            if (self.isProcessing(host, user)) {
                return debug('%s is already processing host=%s user=%s', self.name, host, user);
            }

            self.setProcessing(host, user);

            // do forever, it does not throw a stack overflow
            forever(
                function (next) {
                    self.locker.lock(host, function(err) {
                        // we didn't get the lock for the host
                        if (err) {
                            debug('Could not lock host=%s from %s. Reason: %s', host, self.name, err.message);
                            return next(err);
                        }
                        debug('Locked host=%s from %s', host, user, self.name);
                        self.processNextJob(user, next);
                    });
                },
                function (err) {
                    if (err) {
                        debug(err.name === 'EmptyQueue' ? err.message : err);
                    }

                    self.clearProcessing(host, user);
                    if (!self.hasPendingJobs(host)) {
                        self.locker.unlock(host, debug);
                    }
                }
            );
        },
        function onJobSubscriberReady(err) {
            if (err) {
                return self.emit('error', err);
            }

            self.emit('ready');
        }
    );
};

Batch.prototype.processNextJob = function (user, callback) {
    // This is missing the logic for processing several users within the same host
    // It requires to:
    //  - Take care of number of jobs running at the same time per host.
    //  - Execute user jobs in order.
    var self = this;
    self.jobQueue.dequeue(user, function (err, jobId) {
        if (err) {
            return callback(err);
        }

        if (!jobId) {
            var emptyQueueError = new Error('Queue for user="' + user + '" is empty');
            emptyQueueError.name = 'EmptyQueue';
            return callback(emptyQueueError);
        }

        self.setWorkInProgressJob(user, jobId);
        self.jobRunner.run(jobId, function (err, job) {
            self.clearWorkInProgressJob(user);

            if (err) {
                debug(err);
                if (err.name === 'JobNotRunnable') {
                    return callback();
                }
                return callback(err);
            }

            debug('Job=%s status=%s user=%s (failed_reason=%s)', jobId, job.data.status, user, job.failed_reason);

            self.logger.log(job);

            self.emit('job:' + job.data.status, jobId);

            callback();
        });
    });
};

Batch.prototype.drain = function (callback) {
    var self = this;
    var workingUsers = this.getWorkInProgressUsers();
    var batchQueues = queue(workingUsers.length);

    workingUsers.forEach(function (user) {
        batchQueues.defer(self._drainJob.bind(self), user);
    });

    batchQueues.awaitAll(function (err) {
        if (err) {
            debug('Something went wrong draining', err);
        } else {
            debug('Drain complete');
        }

        callback();
    });
};

Batch.prototype._drainJob = function (user, callback) {
    var self = this;
    var job_id = this.getWorkInProgressJob(user);

    if (!job_id) {
        return process.nextTick(function () {
            return callback();
        });
    }

    this.jobService.drain(job_id, function (err) {
        if (err && err.name === 'CancelNotAllowedError') {
            return callback();
        }

        if (err) {
            return callback(err);
        }

        self.jobQueue.enqueueFirst(user, job_id, callback);
    });
};

Batch.prototype.stop = function (callback) {
    this.removeAllListeners();
    this.jobSubscriber.unsubscribe(callback);
};

/* Processing hosts => users */

Batch.prototype.setProcessing = function(host, user) {
    if (!this.workingHosts.hasOwnProperty(host)) {
        this.workingHosts[host] = {};
    }
    this.workingHosts[host][user] = true;
};

Batch.prototype.clearProcessing = function(host, user) {
    if (this.workingHosts.hasOwnProperty(host)) {
        delete this.workingHosts[host][user];
        if (!this.hasPendingJobs(host)) {
            delete this.workingHosts[host];
        }
    }
};

Batch.prototype.isProcessing = function(host, user) {
    return this.workingHosts.hasOwnProperty(host) && this.workingHosts[host].hasOwnProperty(user);
};

Batch.prototype.hasPendingJobs = function(host) {
    return this.workingHosts.hasOwnProperty(host) && Object.keys(this.workingHosts[host]).length > 0;
};

/* Work in progress jobs */

Batch.prototype.setWorkInProgressJob = function(user, jobId) {
    this.workInProgressJobs[user] = jobId;
};

Batch.prototype.getWorkInProgressJob = function(user) {
    return this.workInProgressJobs[user];
};

Batch.prototype.clearWorkInProgressJob = function(user) {
    delete this.workInProgressJobs[user];
};

Batch.prototype.getWorkInProgressUsers = function() {
    return Object.keys(this.workInProgressJobs);
};
<commit_msg>Do not emit job:status from batch<commit_after>'use strict';

var util = require('util');
var EventEmitter = require('events').EventEmitter;
var debug = require('./util/debug')('batch');
var forever = require('./util/forever');
var queue = require('queue-async');
var Locker = require('./leader/locker');
var HostUserQueueMover = require('./maintenance/host-user-queue-mover');

function Batch(name, jobSubscriber, jobQueue, jobRunner, jobService, jobPublisher, redisPool, logger) {
    EventEmitter.call(this);
    this.name = name || 'batch';
    this.jobSubscriber = jobSubscriber;
    this.jobQueue = jobQueue;
    this.jobRunner = jobRunner;
    this.jobService = jobService;
    this.jobPublisher = jobPublisher;
    this.logger = logger;
    this.locker = Locker.create('redis-distlock', { pool: redisPool });
    this.hostUserQueueMover = new HostUserQueueMover(jobQueue, jobService, this.locker, redisPool);

    // map: host => map{user}. Useful to determine pending queued users.
    this.workingHosts = {};

    // map: user => jobId. Will be used for draining jobs.
    this.workInProgressJobs = {};
}
util.inherits(Batch, EventEmitter);

module.exports = Batch;

Batch.prototype.start = function () {
    this.hostUserQueueMover.moveOldJobs(function() {
        this.subscribe();
    }.bind(this));
};

Batch.prototype.subscribe = function () {
    var self = this;

    this.jobSubscriber.subscribe(
        function onJobHandler(user, host) {
            debug('onJobHandler(%s, %s)', user, host);
            if (self.isProcessing(host, user)) {
                return debug('%s is already processing host=%s user=%s', self.name, host, user);
            }

            self.setProcessing(host, user);

            // do forever, it does not throw a stack overflow
            forever(
                function (next) {
                    self.locker.lock(host, function(err) {
                        // we didn't get the lock for the host
                        if (err) {
                            debug('Could not lock host=%s from %s. Reason: %s', host, self.name, err.message);
                            return next(err);
                        }
                        debug('Locked host=%s from %s', host, user, self.name);
                        self.processNextJob(user, next);
                    });
                },
                function (err) {
                    if (err) {
                        debug(err.name === 'EmptyQueue' ? err.message : err);
                    }

                    self.clearProcessing(host, user);
                    if (!self.hasPendingJobs(host)) {
                        self.locker.unlock(host, debug);
                    }
                }
            );
        },
        function onJobSubscriberReady(err) {
            if (err) {
                return self.emit('error', err);
            }

            self.emit('ready');
        }
    );
};

Batch.prototype.processNextJob = function (user, callback) {
    // This is missing the logic for processing several users within the same host
    // It requires to:
    //  - Take care of number of jobs running at the same time per host.
    //  - Execute user jobs in order.
    var self = this;
    self.jobQueue.dequeue(user, function (err, jobId) {
        if (err) {
            return callback(err);
        }

        if (!jobId) {
            var emptyQueueError = new Error('Queue for user="' + user + '" is empty');
            emptyQueueError.name = 'EmptyQueue';
            return callback(emptyQueueError);
        }

        self.setWorkInProgressJob(user, jobId);
        self.jobRunner.run(jobId, function (err, job) {
            self.clearWorkInProgressJob(user);

            if (err) {
                debug(err);
                if (err.name === 'JobNotRunnable') {
                    return callback();
                }
                return callback(err);
            }

            debug('Job=%s status=%s user=%s (failed_reason=%s)', jobId, job.data.status, user, job.failed_reason);

            self.logger.log(job);

            callback();
        });
    });
};

Batch.prototype.drain = function (callback) {
    var self = this;
    var workingUsers = this.getWorkInProgressUsers();
    var batchQueues = queue(workingUsers.length);

    workingUsers.forEach(function (user) {
        batchQueues.defer(self._drainJob.bind(self), user);
    });

    batchQueues.awaitAll(function (err) {
        if (err) {
            debug('Something went wrong draining', err);
        } else {
            debug('Drain complete');
        }

        callback();
    });
};

Batch.prototype._drainJob = function (user, callback) {
    var self = this;
    var job_id = this.getWorkInProgressJob(user);

    if (!job_id) {
        return process.nextTick(function () {
            return callback();
        });
    }

    this.jobService.drain(job_id, function (err) {
        if (err && err.name === 'CancelNotAllowedError') {
            return callback();
        }

        if (err) {
            return callback(err);
        }

        self.jobQueue.enqueueFirst(user, job_id, callback);
    });
};

Batch.prototype.stop = function (callback) {
    this.removeAllListeners();
    this.jobSubscriber.unsubscribe(callback);
};

/* Processing hosts => users */

Batch.prototype.setProcessing = function(host, user) {
    if (!this.workingHosts.hasOwnProperty(host)) {
        this.workingHosts[host] = {};
    }
    this.workingHosts[host][user] = true;
};

Batch.prototype.clearProcessing = function(host, user) {
    if (this.workingHosts.hasOwnProperty(host)) {
        delete this.workingHosts[host][user];
        if (!this.hasPendingJobs(host)) {
            delete this.workingHosts[host];
        }
    }
};

Batch.prototype.isProcessing = function(host, user) {
    return this.workingHosts.hasOwnProperty(host) && this.workingHosts[host].hasOwnProperty(user);
};

Batch.prototype.hasPendingJobs = function(host) {
    return this.workingHosts.hasOwnProperty(host) && Object.keys(this.workingHosts[host]).length > 0;
};

/* Work in progress jobs */

Batch.prototype.setWorkInProgressJob = function(user, jobId) {
    this.workInProgressJobs[user] = jobId;
};

Batch.prototype.getWorkInProgressJob = function(user) {
    return this.workInProgressJobs[user];
};

Batch.prototype.clearWorkInProgressJob = function(user) {
    delete this.workInProgressJobs[user];
};

Batch.prototype.getWorkInProgressUsers = function() {
    return Object.keys(this.workInProgressJobs);
};
<|endoftext|>


<commit_before>/*
 * This file is part of the Spludo Framework.
 * Copyright (c) 2009 DracoBlue, http://dracoblue.net/
 *
 * Licensed under the terms of MIT License. For the full copyright and license
 * information, please see the LICENSE file in the root folder.
 */

var sys = require("sys");

/**
 * @class A class for a connection to an dbslayer database.
 *
 * @extends Logging
 *
 * @since 0.1
 * @author DracoBlue
 */
DbslayerDatabaseConnection = function(name, options) {

    var port = options.port || 9090;
    var host = options.host || '127.0.0.1';
    var timeout = options.timeout || 0;

    delete options.timeout;
    delete options.port;
    delete options.host;

    var http = require("http");

    var db_escape_string = function(string) {
        return string.replace(/'/g, "''");
    };

    var query = function(sql) {
        var connection = http.createClient(port, host);

        var promise = new process.Promise();
        var request = connection.request('GET', '/db?' + escape(JSON.stringify({"SQL":sql})), {'host': host});
//        var request = connection.request('GET', "/", {'host': host});

        if (timeout !== 0) {
            promise.timeout(timeout);
        }

        request.finish(function(response){
            response.setBodyEncoding("binary");

            var body = [];

            response.addListener("body", function (chunk) {
                body.push(chunk);
            });

            response.addListener("complete", function () {
                var data = null;

                try {
                    data = JSON.parse(body.join(""));
                } catch(e){
                    promise.emitError(null, e.message);
                    return ;
                }

                if (typeof data.MYSQL_ERROR !== "undefined") {
                    promise.emitError(data.MYSQL_ERROR, data.MYSQL_ERRNO);
                } else if (typeof data.ERROR !== "undefined"){
                    promise.emitError(data.ERROR);
                } else {
                    var query_result = data.RESULT;

                    if (!query_result.ROWS) {
                        if (query_result.SUCCESS) {
                            promise.emitSuccess();
                        } else {
                            promise.emitError();
                        }
                        return ;
                    }

                    var headers = query_result.HEADER;

                    var headers_length = headers.length;

                    // var types = query_result.TYPES;

                    var rows = query_result.ROWS;
                    var rows_length = rows.length;

                    var results = [];

                    for (r=0; r < rows_length; r++) {
                        var row = {};
                        var raw_row = rows[r];
                        for (h=0; h < headers_length; h++) {
                            row[headers[h]] = raw_row[h];
                        }
                        results.push(row);
                    }

                    promise.emitSuccess(results);
                }
            });
        });

        return promise;
    };

    this.retrieveById = function(element_type, id) {
        var elements = null;
        var p = query("SELECT * FROM " + element_type + " WHERE id = '" + id + "' LIMIT 1");

        p.addCallback(function(data) {
            elements = data;
        });

        p.addErrback(function(data) {
            elements = null;
        });

        p.wait();

        if (elements && elements.length === 1) {
            var element = elements[0];
            element._table = element_type;
            element._server = name;
            return element;
        }

        return null;
    };

    this.retrieve = function(element_type, offset, limit) {
        var elements = null;
        offset = offset || 0;
        // FIXME: how to fix the limit problem, when we just need an offset?
        limit = limit || 230585;

        var p = query("SELECT * FROM " + element_type + " ORDER BY id DESC LIMIT " + offset + ", " + limit + "");

        p.addCallback(function(data) {
            elements = data;
        });

        p.addErrback(function(data) {
            elements = null;
        });

        p.wait();

        if (elements) {
            var elements_length = elements.length;
            for (i = 0; i < elements_length; i++) {
                var element = elements[i];
                element._table = element_type;
                element._server = name;
            }
        }

        return elements || [];
    };

    this.deleteById = function(element_type, id) {
        var had_error = false;
        var p = query("DELETE FROM " + element_type + " WHERE id = '" + id + "' LIMIT 1");

        p.addCallback(function(data) {
            had_error = false;
        });

        p.addErrback(function(data) {
            had_error = true;
        });

        p.wait();

        if (had_error) {
            throw new Error("Cannot delete entry by id " + id);
        }
    };

    this.create = function(element) {
        sys.debug(sys.inspect(element));
        if (typeof element.id !== "undefined") {
            throw new Error("element id given! We want to create an element, the id should not be given!");
        }

        var table_name = element._table;
        delete element._table;

        var keys = [];
        var values = [];
        for (key in element) {
            // FIXME: SQL ESCAPING FOR KEYS!
            keys.push('`' + key + '`');
            values.push('\'' + db_escape_string(element[key]) + '\'');
        }

        p = query("INSERT INTO " + table_name + " (" + keys.join(', ') + ") VALUES (" + values.join(', ') + ")");
        p.wait();
    };

    this.store = function(element) {
        if (!element || typeof element.id === "undefined") {
            throw new Error("element id not given!");
        }

        var table_name = element._table;
        delete element._table;

        var setters = [];
        for (key in element) {
            // FIXME: SQL ESCAPING FOR KEYS!
            setters.push('`' + key + '` = \'' + db_escape_string(element[key]) + '\'');
        }

        var p = null;
        try {
            p = query("INSERT INTO " + table_name + " (`id`) VALUES ('" + element.id + "')");
            p.wait();
        } catch (e) {
            /*
             * We do not care if that fails ;). The fact is, that we just want to secure that the entry exists.
             */
        }
        p = query("UPDATE " + table_name + " SET " + setters.join(', ') + " WHERE `id` = '" + element.id + "'");
        p.wait();
    };
};

process.mixin(true, DbslayerDatabaseConnection.prototype, DatabaseConnection.prototype);<commit_msg>sql escaping for keys and table names.<commit_after>/*
 * This file is part of the Spludo Framework.
 * Copyright (c) 2009 DracoBlue, http://dracoblue.net/
 *
 * Licensed under the terms of MIT License. For the full copyright and license
 * information, please see the LICENSE file in the root folder.
 */

var sys = require("sys");

/**
 * @class A class for a connection to an dbslayer database.
 *
 * @extends Logging
 *
 * @since 0.1
 * @author DracoBlue
 */
DbslayerDatabaseConnection = function(name, options) {

    var port = options.port || 9090;
    var host = options.host || '127.0.0.1';
    var timeout = options.timeout || 0;

    delete options.timeout;
    delete options.port;
    delete options.host;

    var http = require("http");

    var db_escape_string = function(string) {
        return string.replace(/([\n\r\'\"])/g, "\\$&");
    };

    var query = function(sql) {
        var connection = http.createClient(port, host);

        var promise = new process.Promise();
        var request = connection.request('GET', '/db?' + escape(JSON.stringify({"SQL":sql})), {'host': host});
//        var request = connection.request('GET', "/", {'host': host});

        if (timeout !== 0) {
            promise.timeout(timeout);
        }

        request.finish(function(response){
            response.setBodyEncoding("binary");

            var body = [];

            response.addListener("body", function (chunk) {
                body.push(chunk);
            });

            response.addListener("complete", function () {
                var data = null;

                try {
                    data = JSON.parse(body.join(""));
                } catch(e){
                    promise.emitError(null, e.message);
                    return ;
                }

                if (typeof data.MYSQL_ERROR !== "undefined") {
                    promise.emitError(data.MYSQL_ERROR, data.MYSQL_ERRNO);
                } else if (typeof data.ERROR !== "undefined"){
                    promise.emitError(data.ERROR);
                } else {
                    var query_result = data.RESULT;

                    if (!query_result.ROWS) {
                        if (query_result.SUCCESS) {
                            promise.emitSuccess();
                        } else {
                            promise.emitError();
                        }
                        return ;
                    }

                    var headers = query_result.HEADER;

                    var headers_length = headers.length;

                    // var types = query_result.TYPES;

                    var rows = query_result.ROWS;
                    var rows_length = rows.length;

                    var results = [];

                    for (r=0; r < rows_length; r++) {
                        var row = {};
                        var raw_row = rows[r];
                        for (h=0; h < headers_length; h++) {
                            row[headers[h]] = raw_row[h];
                        }
                        results.push(row);
                    }

                    promise.emitSuccess(results);
                }
            });
        });

        return promise;
    };

    this.retrieveById = function(element_type, id) {
        var elements = null;
        var p = query("SELECT * FROM " + db_escape_string(element_type) + " WHERE id = '" + id + "' LIMIT 1");

        p.addCallback(function(data) {
            elements = data;
        });

        p.addErrback(function(data) {
            elements = null;
        });

        p.wait();

        if (elements && elements.length === 1) {
            var element = elements[0];
            element._table = element_type;
            element._server = name;
            return element;
        }

        return null;
    };

    this.retrieve = function(element_type, offset, limit) {
        var elements = null;
        offset = offset || 0;
        // FIXME: how to fix the limit problem, when we just need an offset?
        limit = limit || 230585;

        var p = query("SELECT * FROM " + db_escape_string(element_type) + " ORDER BY id DESC LIMIT " + offset + ", " + limit + "");

        p.addCallback(function(data) {
            elements = data;
        });

        p.addErrback(function(data) {
            elements = null;
        });

        p.wait();

        if (elements) {
            var elements_length = elements.length;
            for (i = 0; i < elements_length; i++) {
                var element = elements[i];
                element._table = element_type;
                element._server = name;
            }
        }

        return elements || [];
    };

    this.deleteById = function(element_type, id) {
        var had_error = false;
        var p = query("DELETE FROM " + db_escape_string(element_type) + " WHERE id = '" + id + "' LIMIT 1");

        p.addCallback(function(data) {
            had_error = false;
        });

        p.addErrback(function(data) {
            had_error = true;
        });

        p.wait();

        if (had_error) {
            throw new Error("Cannot delete entry by id " + id);
        }
    };

    this.create = function(element) {
        sys.debug(sys.inspect(element));
        if (typeof element.id !== "undefined") {
            throw new Error("element id given! We want to create an element, the id should not be given!");
        }

        var table_name = element._table;
        delete element._table;

        var keys = [];
        var values = [];
        for (key in element) {
            keys.push('`' + db_escape_string(key) + '`');
            values.push('\'' + db_escape_string(element[key]) + '\'');
        }

        p = query("INSERT INTO " + db_escape_string(table_name) + " (" + keys.join(', ') + ") VALUES (" + values.join(', ') + ")");
        p.wait();
    };

    this.store = function(element) {
        if (!element || typeof element.id === "undefined") {
            throw new Error("element id not given!");
        }

        var table_name = element._table;
        delete element._table;

        var setters = [];
        for (key in element) {
            setters.push('`' + db_escape_string(key) + '` = \'' + db_escape_string(element[key]) + '\'');
        }

        var p = null;
        try {
            p = query("INSERT INTO " + db_escape_string(table_name) + " (`id`) VALUES ('" + element.id + "')");
            p.wait();
        } catch (e) {
            /*
             * We do not care if that fails ;). The fact is, that we just want to secure that the entry exists.
             */
        }
        p = query("UPDATE " + db_escape_string(table_name) + " SET " + setters.join(', ') + " WHERE `id` = '" + element.id + "'");
        p.wait();
    };
};

process.mixin(true, DbslayerDatabaseConnection.prototype, DatabaseConnection.prototype);<|endoftext|>