<commit_before>var file = require('fs');
var path = require('path');
var util = require("util");
var http = require("http");

var Sandbox = require("./lib/sandbox");
var FactoidServer = require("./lib/factoidserv");
var FeelingLucky = require("./lib/feelinglucky");
var CanIUseServer = require("./lib/caniuse");

var Bot = require("./lib/irc");
var Shared = require("./shared");
var scrapeMdn = require('scrape-mdn');

var JSBot = function(profile) {
	this.sandbox = new Sandbox(path.join(__dirname, "ecmabot-utils.js"));
	this.factoids = new FactoidServer(path.join(__dirname, "ecmabot-factoids.json"));
	this.caniuse_server = new CanIUseServer;
	this.executeRegex = /^((?:sm?|v8|js?|b|n|>>?|>>>>|\|)>)([^>].*)+/;

	Bot.call(this, profile);
	this.set_log_level(this.LOG_ALL);
	this.set_trigger("!"); // Exclamation
};

util.inherits(JSBot, Bot);

JSBot.prototype.init = function() {
	Bot.prototype.init.call(this);

	this.register_listener(this.executeRegex, Shared.execute_js);

	//this.register_listener(/^(\S+)(\+\+|--);?$/, this.do_beers);

	this.register_command("g", Shared.google, {
		help: "Run this command with a search query to return the first Google result. Usage: !g kitten images"});

	this.register_command("google", this.google, {
		help: "Returns a link to a Google search page of the search term. Usage: !google opencourseware computational complexity"});

	this.register_command("mdn", this.mdn, {
		help: "Search the Mozilla Developer Network. Usage: !mdn bitwise operators"});
	this.register_command("mdc", "mdn");

	this.register_command("ecma", this.ecma, {
		help: "Lookup a section from the ECMAScript spec. Usage: !ecma null value"});

	this.register_command("re", this.re, {
		help: "Usage: !re Your text here /expression/gi || FLAGS: (g: global match, i: ignore case)"});

	this.register_command("caniuse", this.caniuse, {
		help: "Search the caniuse.com database. Usage: !caniuse webgl"});
	this.register_command("ciu", "caniuse");

	this.register_command("find", Shared.find);

	this.register_command("help", this.help);

	this.register_command("auth", Shared.reauthenticate, {
		allow_intentions: false,
		help: "Attempt to re-authenticate with NickServ."});

	this.register_command("learn", Shared.learn, {
		allow_intentions: false,
		help: "Add factoid to bot. Usage: !learn ( [alias] foo = bar | foo =~ s/expression/replace/gi )"});

	this.register_command("forget", Shared.forget, {
		allow_intentions: false,
		help: "Remove factoid from bot. Usage: !forget foo"});

	this.register_command("commands", Shared.commands);

	this.on('command_not_found', this.command_not_found);

	this.load_ecma_ref();

};

JSBot.prototype.google = function(context, text) {

	if (!text) {
		context.channel.send_reply (context.sender, this.get_command_help("google"));
		return;
	}

	context.channel.send_reply (context.intent, "Google search: \""+text+"\" <http://www.google.com/search?q="+encodeURIComponent(text)+">");
};

JSBot.prototype.there_is_no_try = function(context, text) {
	var hours = 1000*60*60;
	var now = +new Date();

	if (now > arguments.callee.last_invocation + 3*hours ||
		typeof arguments.callee.last_invocation === "undefined") {

		context.channel.send_reply(context.sender, "Do or do not; there is no try. --Yoda");
		arguments.callee.last_invocation = now;

	}
};

JSBot.prototype.do_beers = function(context, text, nick, operation) {
	/**
	 * /(\S+)\s*(?:(\+\+|--)|=\s*(?:\1)\s*(\+|-)\s*1);?/
	 * TODO: More advanced beer management
	 **/
	if (operation === "++") {
		if (nick.toLowerCase() !== "c") {
			context.channel.send_reply(context.sender, "Even if " + nick +
				" deserves any beer, I don't have any to spare.");
		} else {
			context.channel.send_reply(context.sender, "C doesn't deserve beer.");
		}
	} else {
		context.channel.send_action(
			"steals a beer a from " + nick + ", since we're taking 'em.");
	}
};

JSBot.prototype.re = function(context, msg) {
	// Okay first we need to check for the regex literal at the end
	// The regular expression to match a real js regex literal
	// is too long, so we need to use a simpler one.
	var regexmatches, regexliteral = /\/((?:[^\\\/]|\\.)*)\/([gi]*)$/;

	if (regexmatches = msg.match(regexliteral)) {
		try {
			var regexpobj = new RegExp(regexmatches[1], regexmatches[2]);
		} catch (e) {
			/* We have an invalid regular expression */
			context.channel.send_reply(context.sender, e.message);
			return;
		}

		var texttomatch = msg.slice(0, -regexmatches[0].length).trim();
		var result = texttomatch.match(regexpobj);
		if (result === null) {
			context.channel.send_reply(context.intent, "No matches found.");
			return;
		}

		var reply = [];
		for (var i = 0, len = result.length; i < len; i++) {
			reply.push(typeof result[i] !== "undefined" ?
				"'"+result[i]+"'" :
				"[undefined]");
		}

		context.channel.send_reply(context.intent, "Matches: "+reply.join(", "), {truncate: true});
	} else {
		context.channel.send_reply(context.sender, this.get_command_help("re"));
	}
};

JSBot.prototype.help = function(context, text) {

	try {
		if (!text) {
			return this.command_not_found (context, "help");
		}

		context.channel.send_reply(context.intent, this.get_command_help(text));
	} catch(e) {
		context.channel.send_reply(context.sender, e);
	}
};

JSBot.prototype.mdn = function(context, text, command) {
	if (!text) {
		return;
	}

	scrapeMdn.search(text).then((results) => {
		var result = results[0];
		context.channel.send_reply(
			context.intent,
			'\x02' + result.title + '\x0F \x032< ' + result.url + '>\x0F',
			{color: true}
		);
	})
	.catch((err) => {
		console.error('Error with command "!' + command + ' ' + text + '"');
		console.error(err);
	});
};

JSBot.prototype.command_not_found = function(context, text) {
	Shared.findPlus.call(this, context, text, !context.priv);
};

// JSON.stringify([].slice.call(document.querySelectorAll('#toc-full a')).map(function(v) {return {title: v.firstChild.textContent, id: v.href.replace(/.+#/, '')};}));
// Use that to generate the required JSON from es5.github.io with Firefox

JSBot.prototype.ecma = function(context, text) {
	try {

	if (typeof this.ecma_ref === "undefined") {
		context.channel.send_reply(context.sender, "The ECMA-262 reference is not loaded.");
		return;
	}

	text = text.toLowerCase();
	var ref = this.ecma_ref, ch = text.charCodeAt(0);

	// If text begins with a number, the search must match at the beginning of the string
	var muststart = ch >= 48 && ch <= 57;

	for (var i = 0, len = ref.length; i < len; i++) {
		var item = ref[i], title = item.title.toLowerCase();
		if (muststart ? title.substring(0, text.length) === text : ~title.indexOf(text)) {
			context.channel.send_reply(context.intent,
				"Found: " + item.title + " <http://es5.github.io/#" + item.id + ">");
			return;
		}
	}

	throw new Error("Could not find text '"+text+"' in the ECMAScript 5.1 Table of Contents.");

	} catch (e) { context.channel.send_reply(context.sender, e); }
};

JSBot.prototype.load_ecma_ref = function() {
	var filename = path.join(__dirname, "ecmabot-reference.json");
	util.puts("Loading ECMA-262 reference...");
	var bot = this;
	file.readFile(filename, function (err, data) {
		if (err) util.puts(util.inspect(err));
		try {
			bot.ecma_ref = JSON.parse(data);
		} catch (e) {
			util.puts("ECMA-262 Error: "+e.name+": "+e.message);
		}
	});
	if (typeof this.ecma_ref_watching === "undefined") {
		this.ecma_ref_watching = true;
		file.watchFile(filename, function (curr, prev) {
			util.puts("ECMA-262 reference file has changed.");
			bot.load_ecma_ref();
		});
	}
};

JSBot.prototype.caniuse = function(context, text) {
	try {
		var text = this.caniuse_server.search(text);
		context.channel.send_reply(context.intent, text, {color: true});
	} catch(e) {
		context.channel.send_reply(context.sender, e);
	}
};

var profile;
if (process.env.ECMABOT_PROFILE) {
		profile = require(path.resolve(process.env.ECMABOT_PROFILE));
} else {
	profile = require("./ecmabot-profile.js");
}

(new JSBot(profile)).init();

<commit_msg>use secure google<commit_after>var file = require('fs');
var path = require('path');
var util = require("util");
var http = require("http");

var Sandbox = require("./lib/sandbox");
var FactoidServer = require("./lib/factoidserv");
var FeelingLucky = require("./lib/feelinglucky");
var CanIUseServer = require("./lib/caniuse");

var Bot = require("./lib/irc");
var Shared = require("./shared");
var scrapeMdn = require('scrape-mdn');

var JSBot = function(profile) {
	this.sandbox = new Sandbox(path.join(__dirname, "ecmabot-utils.js"));
	this.factoids = new FactoidServer(path.join(__dirname, "ecmabot-factoids.json"));
	this.caniuse_server = new CanIUseServer;
	this.executeRegex = /^((?:sm?|v8|js?|b|n|>>?|>>>>|\|)>)([^>].*)+/;

	Bot.call(this, profile);
	this.set_log_level(this.LOG_ALL);
	this.set_trigger("!"); // Exclamation
};

util.inherits(JSBot, Bot);

JSBot.prototype.init = function() {
	Bot.prototype.init.call(this);

	this.register_listener(this.executeRegex, Shared.execute_js);

	//this.register_listener(/^(\S+)(\+\+|--);?$/, this.do_beers);

	this.register_command("g", Shared.google, {
		help: "Run this command with a search query to return the first Google result. Usage: !g kitten images"});

	this.register_command("google", this.google, {
		help: "Returns a link to a Google search page of the search term. Usage: !google opencourseware computational complexity"});

	this.register_command("mdn", this.mdn, {
		help: "Search the Mozilla Developer Network. Usage: !mdn bitwise operators"});
	this.register_command("mdc", "mdn");

	this.register_command("ecma", this.ecma, {
		help: "Lookup a section from the ECMAScript spec. Usage: !ecma null value"});

	this.register_command("re", this.re, {
		help: "Usage: !re Your text here /expression/gi || FLAGS: (g: global match, i: ignore case)"});

	this.register_command("caniuse", this.caniuse, {
		help: "Search the caniuse.com database. Usage: !caniuse webgl"});
	this.register_command("ciu", "caniuse");

	this.register_command("find", Shared.find);

	this.register_command("help", this.help);

	this.register_command("auth", Shared.reauthenticate, {
		allow_intentions: false,
		help: "Attempt to re-authenticate with NickServ."});

	this.register_command("learn", Shared.learn, {
		allow_intentions: false,
		help: "Add factoid to bot. Usage: !learn ( [alias] foo = bar | foo =~ s/expression/replace/gi )"});

	this.register_command("forget", Shared.forget, {
		allow_intentions: false,
		help: "Remove factoid from bot. Usage: !forget foo"});

	this.register_command("commands", Shared.commands);

	this.on('command_not_found', this.command_not_found);

	this.load_ecma_ref();

};

JSBot.prototype.google = function(context, text) {

	if (!text) {
		context.channel.send_reply (context.sender, this.get_command_help("google"));
		return;
	}

	context.channel.send_reply (context.intent, "Google search: \""+text+"\" <https://www.google.com/search?q="+encodeURIComponent(text)+">");
};

JSBot.prototype.there_is_no_try = function(context, text) {
	var hours = 1000*60*60;
	var now = +new Date();

	if (now > arguments.callee.last_invocation + 3*hours ||
		typeof arguments.callee.last_invocation === "undefined") {

		context.channel.send_reply(context.sender, "Do or do not; there is no try. --Yoda");
		arguments.callee.last_invocation = now;

	}
};

JSBot.prototype.do_beers = function(context, text, nick, operation) {
	/**
	 * /(\S+)\s*(?:(\+\+|--)|=\s*(?:\1)\s*(\+|-)\s*1);?/
	 * TODO: More advanced beer management
	 **/
	if (operation === "++") {
		if (nick.toLowerCase() !== "c") {
			context.channel.send_reply(context.sender, "Even if " + nick +
				" deserves any beer, I don't have any to spare.");
		} else {
			context.channel.send_reply(context.sender, "C doesn't deserve beer.");
		}
	} else {
		context.channel.send_action(
			"steals a beer a from " + nick + ", since we're taking 'em.");
	}
};

JSBot.prototype.re = function(context, msg) {
	// Okay first we need to check for the regex literal at the end
	// The regular expression to match a real js regex literal
	// is too long, so we need to use a simpler one.
	var regexmatches, regexliteral = /\/((?:[^\\\/]|\\.)*)\/([gi]*)$/;

	if (regexmatches = msg.match(regexliteral)) {
		try {
			var regexpobj = new RegExp(regexmatches[1], regexmatches[2]);
		} catch (e) {
			/* We have an invalid regular expression */
			context.channel.send_reply(context.sender, e.message);
			return;
		}

		var texttomatch = msg.slice(0, -regexmatches[0].length).trim();
		var result = texttomatch.match(regexpobj);
		if (result === null) {
			context.channel.send_reply(context.intent, "No matches found.");
			return;
		}

		var reply = [];
		for (var i = 0, len = result.length; i < len; i++) {
			reply.push(typeof result[i] !== "undefined" ?
				"'"+result[i]+"'" :
				"[undefined]");
		}

		context.channel.send_reply(context.intent, "Matches: "+reply.join(", "), {truncate: true});
	} else {
		context.channel.send_reply(context.sender, this.get_command_help("re"));
	}
};

JSBot.prototype.help = function(context, text) {

	try {
		if (!text) {
			return this.command_not_found (context, "help");
		}

		context.channel.send_reply(context.intent, this.get_command_help(text));
	} catch(e) {
		context.channel.send_reply(context.sender, e);
	}
};

JSBot.prototype.mdn = function(context, text, command) {
	if (!text) {
		return;
	}

	scrapeMdn.search(text).then((results) => {
		var result = results[0];
		context.channel.send_reply(
			context.intent,
			'\x02' + result.title + '\x0F \x032< ' + result.url + '>\x0F',
			{color: true}
		);
	})
	.catch((err) => {
		console.error('Error with command "!' + command + ' ' + text + '"');
		console.error(err);
	});
};

JSBot.prototype.command_not_found = function(context, text) {
	Shared.findPlus.call(this, context, text, !context.priv);
};

// JSON.stringify([].slice.call(document.querySelectorAll('#toc-full a')).map(function(v) {return {title: v.firstChild.textContent, id: v.href.replace(/.+#/, '')};}));
// Use that to generate the required JSON from es5.github.io with Firefox

JSBot.prototype.ecma = function(context, text) {
	try {

	if (typeof this.ecma_ref === "undefined") {
		context.channel.send_reply(context.sender, "The ECMA-262 reference is not loaded.");
		return;
	}

	text = text.toLowerCase();
	var ref = this.ecma_ref, ch = text.charCodeAt(0);

	// If text begins with a number, the search must match at the beginning of the string
	var muststart = ch >= 48 && ch <= 57;

	for (var i = 0, len = ref.length; i < len; i++) {
		var item = ref[i], title = item.title.toLowerCase();
		if (muststart ? title.substring(0, text.length) === text : ~title.indexOf(text)) {
			context.channel.send_reply(context.intent,
				"Found: " + item.title + " <http://es5.github.io/#" + item.id + ">");
			return;
		}
	}

	throw new Error("Could not find text '"+text+"' in the ECMAScript 5.1 Table of Contents.");

	} catch (e) { context.channel.send_reply(context.sender, e); }
};

JSBot.prototype.load_ecma_ref = function() {
	var filename = path.join(__dirname, "ecmabot-reference.json");
	util.puts("Loading ECMA-262 reference...");
	var bot = this;
	file.readFile(filename, function (err, data) {
		if (err) util.puts(util.inspect(err));
		try {
			bot.ecma_ref = JSON.parse(data);
		} catch (e) {
			util.puts("ECMA-262 Error: "+e.name+": "+e.message);
		}
	});
	if (typeof this.ecma_ref_watching === "undefined") {
		this.ecma_ref_watching = true;
		file.watchFile(filename, function (curr, prev) {
			util.puts("ECMA-262 reference file has changed.");
			bot.load_ecma_ref();
		});
	}
};

JSBot.prototype.caniuse = function(context, text) {
	try {
		var text = this.caniuse_server.search(text);
		context.channel.send_reply(context.intent, text, {color: true});
	} catch(e) {
		context.channel.send_reply(context.sender, e);
	}
};

var profile;
if (process.env.ECMABOT_PROFILE) {
		profile = require(path.resolve(process.env.ECMABOT_PROFILE));
} else {
	profile = require("./ecmabot-profile.js");
}

(new JSBot(profile)).init();

<|endoftext|>


<commit_before>/// <reference types="cypress" />

context('Core', () => {
  beforeEach(() => {
    cy.swiperPage();
  });
  describe('centeredSlides & slidesPerView', () => {
    beforeEach(() => {
      cy.initSwiper({
        centeredSlides: true,
        slidesPerView: 3,
        speed: 0,
        spaceBetween: 10,
        navigation: true,
      });
    });
    it('should have slide with active class', () => {
      cy.getSlide(0).expectToBeActiveSlide().and('be.visible');
    });

    it('should position slides center', () => {
      cy.getActiveSlide().should(($el) => {
        expect($el[0].getBoundingClientRect().x).to.be.greaterThan(100);
      });
    });

    it('should not has slide with prev class', () => {
      cy.getSlides().should('not.have.class', 'swiper-slide-prev');
    });

    it('should have slide with next class', () => {
      cy.getSlide(1).should('have.class', 'swiper-slide-next').and('be.visible');
    });
    describe('visible slides', () => {
      it('should have 2 visible slides', () => {
        cy.getSlides().filter(':visible').its('length').should('be.eq', 2);
      });
      it('should have 3 visible slides after navigation', () => {
        cy.swipeLeft();
        cy.getSlides().filter(':visible').its('length').should('be.eq', 3);
      });
    });

    it('spaceBetween', () => {
      cy.getSlides()
        .should('have.attr', 'style')
        .and('match', /margin-right:\s+10px/);
    });
  });

  describe('methods', () => {
    beforeEach(() => {
      cy.initSwiper().as('swiper');
    });
    it('should slide next', function slideNext() {
      this.swiper.slideNext();
      cy.getSlideContains('Slide 2').expectToBeActiveSlide();
    });
    it('should slide prev', function slidePrev() {
      this.swiper.slideNext();
      this.swiper.slidePrev();
      cy.getSlideContains('Slide 1').expectToBeActiveSlide();
    });
    it('should slide to slide 3', function slideTo3() {
      this.swiper.slideTo(2);
      cy.getSlideContains('Slide 3').expectToBeActiveSlide();
    });
    // it('should slide to first slide when slideTo called with number bigger then slides amount', function () {
    //   this.swiper.slideTo(14);
    //   cy.getSlideContains('Slide 1').expectToBeActiveSlide();
    // });
    it('Add slide at index', function slideTo() {
      this.swiper.addSlide(1, '
Add slide
');
      cy.getSlide(1).should('contain', 'Add slide');
      this.swiper.addSlide(4, '
Add slide 4
');
      cy.getSlide(4).should('contain', 'Add slide 4');
    });

    // it('Add multiple slides at index', function () {
    //   this.swiper.addSlide(0, [
    //     '
Add slide 0
',
    //     '
Add slide 1
',
    //     '
Add slide 2
',
    //   ]);
    //   // cy.getSlide(2).should('contain', 'Add slide 0');
    //   // cy.getSlide(1).should('contain', 'Add slide 1');
    //   cy.getSlide(0).should('contain', 'Add slide 2');
    // });

    it('Add slide to the end', function slideToEnd() {
      this.swiper.appendSlide('
Add slide at the end
');
      cy.getSlide(10).should('contain', 'Add slide at the end');
      this.swiper.appendSlide([
        '
END 1
',
        '
END 2
',
      ]);
      cy.getSlide(11).should('contain', 'END 1');
      cy.getSlide(12).should('contain', 'END 2');
    });
  });

  describe('properties', () => {
    beforeEach(() => {
      cy.initSwiper().as('swiper');
    });

    it('allowSlideNext', function () {
      this.swiper.allowSlideNext = false;
      cy.swipeLeft();
      cy.getSlide(1).expectToBeActiveSlide();
    });
  });

  it('initialSlide', () => {
    cy.initSwiper({
      initialSlide: 2,
    });
    cy.getSlideContains('Slide 3').expectToBeActiveSlide();
    cy.reinitSwiper({
      initialSlide: 4,
    });
    cy.getSlideContains('Slide 5').expectToBeActiveSlide();
    cy.reinitSwiper({
      initialSlide: 0,
    });
    cy.getSlideContains('Slide 1').expectToBeActiveSlide();
  });

  it('containerModifierClass', () => {
    cy.initSwiper({
      containerModifierClass: 'unique-test-',
    });
    cy.get('.unique-test-horizontal').should('exist');
    cy.get('.swiper-container-horizontal').should('not.exist');
  });

  it('slideActiveClass', () => {
    cy.initSwiper({
      slideActiveClass: 'unique-test-active',
    });
    cy.get('.unique-test-active').should('exist');
    cy.get('.swiper-slide-active').should('not.exist');
  });

  it('slideBlankClass & loopFillGroupWithBlank', () => {
    cy.initSwiper({
      slideBlankClass: 'unique-test-invisible-blank',
      loop: true,
      slidesPerGroup: 3,
      loopFillGroupWithBlank: true,
    });
    cy.get('.unique-test-invisible-blank').should('exist');
    cy.get('.swiper-slide-invisible-blank').should('not.exist');
  });

  it('slideDuplicateActiveClass', () => {
    cy.initSwiper({
      loop: true,
      slideDuplicateActiveClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-active').should('not.exist');
  });

  it('slideDuplicateNextClass', () => {
    cy.initSwiper({
      loop: true,
      slidesPerView: 2,
      slideDuplicateNextClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-next').should('not.exist');
  });

  it('slideDuplicatePrevClass', () => {
    cy.initSwiper({
      loop: true,
      slidesPerView: 2,
      slideDuplicatePrevClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-prev').should('not.exist');
  });

  it('slideNextClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      slideNextClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-next').should('not.exist');
  });

  it('slidePrevClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      initialSlide: 2,
      slidePrevClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-prev').should('not.exist');
  });

  it('slideVisibleClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      watchSlidesVisibility: true,
      slideVisibleClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-visible').should('not.exist');
  });

  it('slideToClickedSlide', () => {
    cy.initSwiper({
      slideToClickedSlide: true,
      slidesPerView: 5,
    });
    cy.getSlide(3).click().expectToBeActiveSlide();
    cy.getSlide(4).click().expectToBeActiveSlide();
  });

  it('grabCursor', () => {
    cy.initSwiper({
      grabCursor: true,
    });
    cy.getSliderContainer()
      .should('have.attr', 'style')
      .and('match', /cursor:\s+grab/);
  });

  it('allowSlideNext', () => {
    cy.initSwiper({
      initialSlide: 2,
      allowSlideNext: false,
    });
    cy.swipeLeft();
    cy.getSlide(2).expectToBeActiveSlide();
  });

  it('direction horizontal', () => {
    cy.initSwiper(); // check default
    cy.get('.swiper-container-horizontal').should('exist');
    cy.get('.swiper-container-vertical').should('not.exist');
    cy.reinitSwiper({
      direction: 'horizontal',
    });
    cy.get('.swiper-container-horizontal').should('exist');
    cy.get('.swiper-container-vertical').should('not.exist');
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().x).to.be.greaterThan(100);
    });
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().y).to.be.not.greaterThan(100);
    });
  });

  it('direction vertical', () => {
    cy.initSwiper({
      direction: 'vertical',
    });
    cy.get('.swiper-container-vertical').should('exist');
    cy.get('.swiper-container-horizontal').should('not.exist');
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().y).to.be.greaterThan(100);
    });
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().x).to.be.not.greaterThan(100);
    });
  });

  it('loop', () => {
    cy.initSwiper({
      loop: true,
    });
    cy.getSlide(0).should('contain', 'Slide 10');
    cy.swipeRight();
    cy.getSlideContains('Slide 10').expectToBeActiveSlide();
  });

  it('Swipe left & right', () => {
    cy.initSwiper({
      speed: 0,
    });
    cy.swipeLeft();
    cy.getSlide(1).expectToBeActiveSlide();
    cy.swipeLeft();
    cy.swipeLeft();
    cy.getSlide(3).expectToBeActiveSlide();
    cy.swipeRight();
    cy.getSlide(2).expectToBeActiveSlide();
    cy.swipeRight();
    cy.swipeRight();
    cy.getSlide(0).expectToBeActiveSlide();
  });
});
<commit_msg>tests(core): allowSlideNext & allowSlidePrev<commit_after>/// <reference types="cypress" />

context('Core', () => {
  beforeEach(() => {
    cy.swiperPage();
  });
  describe('centeredSlides & slidesPerView', () => {
    beforeEach(() => {
      cy.initSwiper({
        centeredSlides: true,
        slidesPerView: 3,
        speed: 0,
        spaceBetween: 10,
        navigation: true,
      });
    });
    it('should have slide with active class', () => {
      cy.getSlide(0).expectToBeActiveSlide().and('be.visible');
    });

    it('should position slides center', () => {
      cy.getActiveSlide().should(($el) => {
        expect($el[0].getBoundingClientRect().x).to.be.greaterThan(100);
      });
    });

    it('should not has slide with prev class', () => {
      cy.getSlides().should('not.have.class', 'swiper-slide-prev');
    });

    it('should have slide with next class', () => {
      cy.getSlide(1).should('have.class', 'swiper-slide-next').and('be.visible');
    });
    describe('visible slides', () => {
      it('should have 2 visible slides', () => {
        cy.getSlides().filter(':visible').its('length').should('be.eq', 2);
      });
      it('should have 3 visible slides after navigation', () => {
        cy.swipeLeft();
        cy.getSlides().filter(':visible').its('length').should('be.eq', 3);
      });
    });

    it('spaceBetween', () => {
      cy.getSlides()
        .should('have.attr', 'style')
        .and('match', /margin-right:\s+10px/);
    });
  });

  describe('methods', () => {
    beforeEach(() => {
      cy.initSwiper().as('swiper');
    });
    it('should slide next', function slideNext() {
      this.swiper.slideNext();
      cy.getSlideContains('Slide 2').expectToBeActiveSlide();
    });
    it('should slide prev', function slidePrev() {
      this.swiper.slideNext();
      this.swiper.slidePrev();
      cy.getSlideContains('Slide 1').expectToBeActiveSlide();
    });
    it('should slide to slide 3', function slideTo3() {
      this.swiper.slideTo(2);
      cy.getSlideContains('Slide 3').expectToBeActiveSlide();
    });
    // it('should slide to first slide when slideTo called with number bigger then slides amount', function () {
    //   this.swiper.slideTo(14);
    //   cy.getSlideContains('Slide 1').expectToBeActiveSlide();
    // });
    it('Add slide at index', function slideTo() {
      this.swiper.addSlide(1, '
Add slide
');
      cy.getSlide(1).should('contain', 'Add slide');
      this.swiper.addSlide(4, '
Add slide 4
');
      cy.getSlide(4).should('contain', 'Add slide 4');
    });

    // it('Add multiple slides at index', function () {
    //   this.swiper.addSlide(0, [
    //     '
Add slide 0
',
    //     '
Add slide 1
',
    //     '
Add slide 2
',
    //   ]);
    //   // cy.getSlide(2).should('contain', 'Add slide 0');
    //   // cy.getSlide(1).should('contain', 'Add slide 1');
    //   cy.getSlide(0).should('contain', 'Add slide 2');
    // });

    it('Add slide to the end', function slideToEnd() {
      this.swiper.appendSlide('
Add slide at the end
');
      cy.getSlide(10).should('contain', 'Add slide at the end');
      this.swiper.appendSlide([
        '
END 1
',
        '
END 2
',
      ]);
      cy.getSlide(11).should('contain', 'END 1');
      cy.getSlide(12).should('contain', 'END 2');
    });
  });

  describe('properties', () => {
    beforeEach(() => {
      cy.initSwiper({ speed: 0 }).as('swiper');
    });

    // it('allowSlideNext', function () {
    //   cy.getSlide(0).expectToBeActiveSlide();
    //   cy.swipeLeft();
    //   cy.getSlide(1).expectToBeActiveSlide();
    //   cy.swipeLeft();
    //   cy.getSlide(2).expectToBeActiveSlide();
    //   this.swiper.allowSlideNext = false;
    //   cy.swipeLeft();
    //   cy.getSlide(2).expectToBeActiveSlide();
    // });

    // it('allowSlidePrev', function () {
    //   this.swiper.allowSlidePrev = false;
    //   cy.swipeRight();
    //   cy.swipeLeft();
    //   cy.getSlide(1).expectToBeActiveSlide();
    // });
  });

  it('initialSlide', () => {
    cy.initSwiper({
      initialSlide: 2,
    });
    cy.getSlideContains('Slide 3').expectToBeActiveSlide();
    cy.reinitSwiper({
      initialSlide: 4,
    });
    cy.getSlideContains('Slide 5').expectToBeActiveSlide();
    cy.reinitSwiper({
      initialSlide: 0,
    });
    cy.getSlideContains('Slide 1').expectToBeActiveSlide();
  });

  it('containerModifierClass', () => {
    cy.initSwiper({
      containerModifierClass: 'unique-test-',
    });
    cy.get('.unique-test-horizontal').should('exist');
    cy.get('.swiper-container-horizontal').should('not.exist');
  });

  it('slideActiveClass', () => {
    cy.initSwiper({
      slideActiveClass: 'unique-test-active',
    });
    cy.get('.unique-test-active').should('exist');
    cy.get('.swiper-slide-active').should('not.exist');
  });

  it('slideBlankClass & loopFillGroupWithBlank', () => {
    cy.initSwiper({
      slideBlankClass: 'unique-test-invisible-blank',
      loop: true,
      slidesPerGroup: 3,
      loopFillGroupWithBlank: true,
    });
    cy.get('.unique-test-invisible-blank').should('exist');
    cy.get('.swiper-slide-invisible-blank').should('not.exist');
  });

  it('slideDuplicateActiveClass', () => {
    cy.initSwiper({
      loop: true,
      slideDuplicateActiveClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-active').should('not.exist');
  });

  it('slideDuplicateNextClass', () => {
    cy.initSwiper({
      loop: true,
      slidesPerView: 2,
      slideDuplicateNextClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-next').should('not.exist');
  });

  it('slideDuplicatePrevClass', () => {
    cy.initSwiper({
      loop: true,
      slidesPerView: 2,
      slideDuplicatePrevClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-duplicate-prev').should('not.exist');
  });

  it('slideNextClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      slideNextClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-next').should('not.exist');
  });

  it('slidePrevClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      initialSlide: 2,
      slidePrevClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-prev').should('not.exist');
  });

  it('slideVisibleClass', () => {
    cy.initSwiper({
      slidesPerView: 2,
      watchSlidesVisibility: true,
      slideVisibleClass: 'uniqueTestClass',
    });
    cy.get('.uniqueTestClass').should('exist');
    cy.get('.swiper-slide-visible').should('not.exist');
  });

  it('slideToClickedSlide', () => {
    cy.initSwiper({
      slideToClickedSlide: true,
      slidesPerView: 5,
    });
    cy.getSlide(3).click().expectToBeActiveSlide();
    cy.getSlide(4).click().expectToBeActiveSlide();
  });

  it('grabCursor', () => {
    cy.initSwiper({
      grabCursor: true,
    });
    cy.getSliderContainer()
      .should('have.attr', 'style')
      .and('match', /cursor:\s+grab/);
  });

  it('allowSlideNext', () => {
    cy.initSwiper({
      initialSlide: 2,
      allowSlideNext: false,
    });
    cy.swipeRight();
    cy.swipeLeft();
    cy.getSlide(1).expectToBeActiveSlide();
  });

  it('allowSlidePrev', () => {
    cy.initSwiper({
      initialSlide: 2,
      allowSlidePrev: false,
    });
    cy.swipeLeft();
    cy.swipeRight();
    cy.getSlide(3).expectToBeActiveSlide();
  });

  it('direction horizontal', () => {
    cy.initSwiper(); // check default
    cy.get('.swiper-container-horizontal').should('exist');
    cy.get('.swiper-container-vertical').should('not.exist');
    cy.reinitSwiper({
      direction: 'horizontal',
    });
    cy.get('.swiper-container-horizontal').should('exist');
    cy.get('.swiper-container-vertical').should('not.exist');
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().x).to.be.greaterThan(100);
    });
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().y).to.be.not.greaterThan(100);
    });
  });

  it('direction vertical', () => {
    cy.initSwiper({
      direction: 'vertical',
    });
    cy.get('.swiper-container-vertical').should('exist');
    cy.get('.swiper-container-horizontal').should('not.exist');
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().y).to.be.greaterThan(100);
    });
    cy.getSlide(2).should(($el) => {
      expect($el[0].getBoundingClientRect().x).to.be.not.greaterThan(100);
    });
  });

  it('loop', () => {
    cy.initSwiper({
      loop: true,
    });
    cy.getSlide(0).should('contain', 'Slide 10');
    cy.swipeRight();
    cy.getSlideContains('Slide 10').expectToBeActiveSlide();
  });

  it('Swipe left & right', () => {
    cy.initSwiper({
      speed: 0,
    });
    cy.swipeLeft();
    cy.getSlide(1).expectToBeActiveSlide();
    cy.swipeLeft();
    cy.swipeLeft();
    cy.getSlide(3).expectToBeActiveSlide();
    cy.swipeRight();
    cy.getSlide(2).expectToBeActiveSlide();
    cy.swipeRight();
    cy.swipeRight();
    cy.getSlide(0).expectToBeActiveSlide();
  });
});
<|endoftext|>


<commit_before>// UI code.
"use strict";

function s(selector, opt_silent) {
  var el = document.querySelector(selector);
  if (!el && !opt_silent) {
    throw new Error("Could not find " + selector);
  }
  return el;
}

var textarea = s('textarea');
var errlog = s('#errlog');

function toArray(nodeList) {
  return Array.prototype.slice.call(nodeList);
}

function showError(err) {
  console.error(err, err.stack);
  if (!errlog.textContent) {
    errlog.textContent = 'Error: ';
  }
  errlog.textContent += err + '\n';
}

///////////////
// Resizable //
///////////////

(function() {
  var resizeEl = s('#noteList');

  function handleMove(e) {
    var w = e.clientX - resizeEl.offsetLeft - 4;
    resizeEl.style.width = w + 'px';
  }

  s('#dragbar')
      .addEventListener('mousedown', function(e) {
        document.body.classList.add('resizing');
        e.preventDefault();
        document.addEventListener('mousemove', handleMove);
      });

  document.addEventListener('mouseup', function(e) {
    document.body.classList.remove('resizing');
    document.removeEventListener('mousemove', handleMove);
  });
})();

///////////
// Dirty //
///////////

var dirtMonitor = (function() {
  var isDirty = false;
  var titlebar = s('.titlebar');
  var setDirty = function() {
    if (!isDirty) {
      isDirty = true;
      titlebar.classList.add('dirty');
    }
  };
  var setClean = function() {
    if (isDirty) {
      isDirty = false;
      titlebar.classList.remove('dirty');
    }
  };
  return {
    isDirty: function() { return isDirty; },
    setDirty: setDirty,
    setClean: setClean
  };
})();

////////////
// Search //
////////////

var highlight = (function() {
  var div = document.createElement('div');

  function safeHtml(text) {
    div.textContent = text;
    return div.innerHTML;
  }

  function escapeRegExp(str) {
    return str.replace(/[.^$*+?()[{\\|\]-]/g, '\\$&');
  }

  function highlight(terms, text) {
    text = safeHtml(text);
    if (terms && terms.length) {
      var regexStr = terms.map(escapeRegExp).join('|');
      var regex = new RegExp(regexStr, 'gi');
      text = text.replace(regex, '<q>$&</q>');
    }
    return text;
  }
  return highlight;
})();

var search = (function() {

  var index;
  var selected = null;
  var searchEl = s('#search');

  function createIndex(notes) {
    index = lunr(function() {
      this.field('title', {boost: 10});
      this.field('text');
      this.ref('id');
    });
    notes.forEach(function(n) { index.add(n); });
  }

  // Store any edits in the textarea.
  function storeEdits() {
    if (typeof selected === "number") {
      var old = model.getNote(selected);
      if (old.text === textarea.value) {
        return;
      }
      var updatedNote = DB.parseNote(textarea.value, selected);
      model.setNote(selected, updatedNote);
      // Update the listed item.
      display();
    }
  }

  function resetSelected() {
    searchEl.value = '';
    searchEl.focus();
    setSelected(null);
  }

  function setSelected(i) {
    if (typeof selected === 'number') {
      storeEdits();
      var old = s('.selected', true);
      if (old) {
        old.classList.remove('selected');
      }
    }
    selected = i;
    if (typeof selected === 'number') {
      var li = s('li[data-id="' + i + '"]');
      li.classList.add('selected');
      li.scrollIntoViewIfNeeded();
      // Set textarea to current note.
      var focus = false;
      updateTextArea(model.getNote(selected), focus);
    } else {
      textarea.value = '';
    }
  }

  var noteListEl = s('#noteList');

  // TODO(wdm) modify summary to show highlighted terms?
  function htmlDisplayNote(terms, note) {
    if (typeof note.id !== 'number') {
      throw new Error("note without id", note);
    }
    var ID = note.id;
    var TITLE = highlight(terms, note.title);
    var SUMMARY = highlight(terms, note.summary);
    var html = '
' + TITLE + '' + SUMMARY +
               '
';
    return html;
  }

  // highlight terms.
  function displayMatches(terms, results) {
    var html;
    var visibleNotes = {};

    if (!results || !results.length) {
      // Display all notes.
      html = model.getNotes().map(function(note) {
        visibleNotes[note.id] = true;
        return htmlDisplayNote(terms, note);
      });
    } else {
      // Filter by term and order by score.
      html = results.map(function(r) {
        var id = parseInt(r.ref, 10);
        var note = model.getNote(id);
        visibleNotes[id] = true;
        return htmlDisplayNote(terms, note);
      });
    }
    noteList.innerHTML = html.join('\n');
    if (visibleNotes[selected]) {
      setSelected(selected);
    }
  }

  function onInput() {
    setSelected(null);
    display();
  }

  // Display notes that match the current search terms (or all).
  // TOOD(wdm) Do something with score?
  function display() {
    var q = searchEl.value;
    var results;
    var terms;

    if (q.length) {
      results = index.search(q);
      terms = q.split(/\s+/);
    }
    displayMatches(terms, results);
  }

  function moveSelection(isDown) {
    var toSelect;
    var old = s('.selected', true);
    if (!old) {
      toSelect = noteListEl.firstChild;
    } else {
      toSelect = isDown ? old.nextElementSibling : old.previousElementSibling;
    }
    if (toSelect) {
      var id = parseInt(toSelect.dataset.id, 10);
      setSelected(id);
    }
  }

  function moveSelectionUp() { moveSelection(false); }

  function moveSelectionDown() { moveSelection(true); }

  function click(e) {
    var t = e.target;
    // Find the enclosing
.
    for (var i = 0; t.nodeName !== 'LI' && i < 3; i++) {
      t = t.parentNode;
    }
    if (t.nodeName !== 'LI') {
      console.error('Could not find
', e.target, t);
      return;
    }
    var id = parseInt(t.dataset.id, 10);
    setSelected(id);
  }

  function selectOrCreateNote() {
    if (typeof selected !== 'number') {
      // One newline is required to mark end of title. The next newline is
      // optional but looks nicer.
      var rawText = searchEl.value + '\n\n';
      textarea.value = rawText;  // HACK: required because otherwise storeEdits
                                 // will wipe out our new note.
      var note = model.newNote(rawText);
      index.add(note);
      display();  // HACK: need to add listitem to ul.
      setSelected(note.id);
    }
    var focus = true;
    updateTextArea(model.getNote(selected), focus);
  }

  function init(notes) {
    createIndex(notes);
    displayMatches();
    return notes;
  }

  return {
    onInput: onInput,
    selectOrCreateNote: selectOrCreateNote,
    moveSelectionUp: moveSelectionUp,
    moveSelectionDown: moveSelectionDown,
    resetSelected: resetSelected,
    click: click,
    storeEdits: storeEdits,
    init: init
  };
})();

// TODO(wdm) Does this actually buy us anything over a global 'notes'?
var model = (function() {

  // In memory note objects.
  var notes = [];

  return {
    init: function init(notes_) {
      notes = notes_;
      return notes;
    },
    newNote: function(rawText) {
      var id = notes.length;
      var note = DB.parseNote(rawText, id);
      notes.push(note);
      return note;
    },
    setNote: function setNote(i, note) { notes[i] = note; },
    getNote: function getNote(i) { return notes[i]; },
    getNotes: function getNotes() { return notes; }
  };

})();

////////
// DB //
////////

var DB = (function() {

  // TODO(wdm) Check that SEPERATOR does not appear in notes.
  var SEPERATOR = '\n[//]: # (NVC: Do not edit this line)\n\n';

  function parse(rawText) {
    var rawNotes = rawText.split(SEPERATOR);
    var notes = rawNotes.map(parseNote);
    return notes;
  }

  function parseNote(rawNote, i) {
    if (typeof i !== 'number') {
      throw new Error('tried to create note without index');
    }
    var firstLine = rawNote.indexOf('\n');
    return {
      id: i,
      title: rawNote.substring(0, firstLine),
      summary: rawNote.substring(firstLine + 1, 150),
      text: rawNote
    };
  }

  // TODO(wdm) Better name!
  function notesAsBlob() {
    search.storeEdits();  // Grabs any changes in textarea.
    var texts = model.getNotes().map(function(note) { return note.text; });
    var rawText = texts.join(SEPERATOR);
    return new Blob([rawText], { 'type' : 'text/plain' });
  }
  return {
    parse: parse,
    parseNote: parseNote,
    notesAsBlob: notesAsBlob,
  };
})();

//////////////
// TextArea //
//////////////

function updateTextArea(note, focus) {
  var text = note.text;
  textarea.value = text;
  if (focus) {
    textarea.focus();
    // Move to first line after title.
    var pos = note.title.length + 1;
    if (text[pos] === '\n') {
      pos++;
    }
    textarea.setSelectionRange(pos, pos);
    textarea.scrollTop = 0;
  }
}

function setCaret(elem, pos) {
  var range = elem.createTextRange();
  range.collapse(true);
  range.moveEnd('character', pos);
  range.moveStart('character', pos);
  range.select();
}

///////////////
/// File UI ///
///////////////

var fileEntry;

function onChooseFile() {
  chooseFile().then(showFilename).catch(showError);
}

function showFilename(fileEntry_) {
  fileEntry = fileEntry_;
  s('#filename').textContent = fileEntry.name;
  return fileEntry;
}

function doSave() {
  if (!dirtMonitor.isDirty) {
    console.log('save skipped: not dirty');
    return Promise.resolve();
  }
  var blob = DB.notesAsBlob();
  return writeFileEntry(fileEntry, blob)
      .then(dirtMonitor.setClean)
      .catch(showError);
}

function doClose() {
  doSave().then(window.close);
}

//////////////////
// Key Handling //
//////////////////

var keyHandler = (function() {
  var KEY_W = 'W'.charCodeAt(0);
  var KEY_S = 'S'.charCodeAt(0);
  var KEY_1 = '1'.charCodeAt(0);
  var KEY_ESC = 27;
  var KEY_ENTER = 13;
  var KEY_UP = 38;
  var KEY_DOWN = 40;

  // return values don't matter.
  function cmd(e, func) {
    e.preventDefault();
    func();
  }

  return function keyHandler(e) {
    var Alt = e.altKey;
    var Ctrl = e.ctrlKey;
    var key = e.keyCode;

    // Dismiss (with save)
    if ((Alt && key === KEY_1) || (Ctrl && key === KEY_W)) {
      return cmd(e, doClose);
    }

    // Reset (deselect note and start new search)
    if (key === KEY_ESC) {
      return cmd(e, search.resetSelected);
    }

    // Save: Ctrl-S
    if (Ctrl && key === KEY_S) {
      return cmd(e, doSave);
    }

    if (e.target !== textarea) {
      if (key == KEY_ENTER) {
        return cmd(e, search.selectOrCreateNote);
      }
      if (key === KEY_DOWN) {
        return cmd(e, search.moveSelectionDown);
      }
      if (key === KEY_UP) {
        return cmd(e, search.moveSelectionUp);
      }
    }
  }
})();

function init() {
  restoreChosenFile()
      .then(showFilename)
      .then(loadFileEntry)
      .then(DB.parse)
      .then(model.init)
      .then(search.init)
      .catch(showError);

  window.addEventListener('keydown', keyHandler);
  textarea.addEventListener('input', dirtMonitor.setDirty);
  s('#chooseFile').addEventListener('click', onChooseFile);
  s('#noteList').addEventListener('click', search.click);
  s('#search').addEventListener('input', search.onInput);
}

init();
<commit_msg>Don't highlight 1 or 2 char words.<commit_after>// UI code.
"use strict";

function s(selector, opt_silent) {
  var el = document.querySelector(selector);
  if (!el && !opt_silent) {
    throw new Error("Could not find " + selector);
  }
  return el;
}

var textarea = s('textarea');
var errlog = s('#errlog');

function toArray(nodeList) {
  return Array.prototype.slice.call(nodeList);
}

function showError(err) {
  console.error(err, err.stack);
  if (!errlog.textContent) {
    errlog.textContent = 'Error: ';
  }
  errlog.textContent += err + '\n';
}

///////////////
// Resizable //
///////////////

(function() {
  var resizeEl = s('#noteList');

  function handleMove(e) {
    var w = e.clientX - resizeEl.offsetLeft - 4;
    resizeEl.style.width = w + 'px';
  }

  s('#dragbar')
      .addEventListener('mousedown', function(e) {
        document.body.classList.add('resizing');
        e.preventDefault();
        document.addEventListener('mousemove', handleMove);
      });

  document.addEventListener('mouseup', function(e) {
    document.body.classList.remove('resizing');
    document.removeEventListener('mousemove', handleMove);
  });
})();

///////////
// Dirty //
///////////

var dirtMonitor = (function() {
  var isDirty = false;
  var titlebar = s('.titlebar');
  var setDirty = function() {
    if (!isDirty) {
      isDirty = true;
      titlebar.classList.add('dirty');
    }
  };
  var setClean = function() {
    if (isDirty) {
      isDirty = false;
      titlebar.classList.remove('dirty');
    }
  };
  return {
    isDirty: function() { return isDirty; },
    setDirty: setDirty,
    setClean: setClean
  };
})();

////////////
// Search //
////////////

var highlight = (function() {
  var div = document.createElement('div');

  function safeHtml(text) {
    div.textContent = text;
    return div.innerHTML;
  }

  function escapeRegExp(str) {
    return str.replace(/[.^$*+?()[{\\|\]-]/g, '\\$&');
  }

  // Don't hightlight 1 and 2 char words.
  function keepTerm(term) {
    return term.length>2;
  }

  function highlight(terms, text) {
    text = safeHtml(text);
    if (terms && terms.length) {
      var regexStr = terms.filter(keepTerm).map(escapeRegExp).join('|');
      var regex = new RegExp(regexStr, 'gi');
      text = text.replace(regex, '<q>$&</q>');
    }
    return text;
  }
  return highlight;
})();

var search = (function() {

  var index;
  var selected = null;
  var searchEl = s('#search');

  function createIndex(notes) {
    index = lunr(function() {
      this.field('title', {boost: 10});
      this.field('text');
      this.ref('id');
    });
    notes.forEach(function(n) { index.add(n); });
  }

  // Store any edits in the textarea.
  function storeEdits() {
    if (typeof selected === "number") {
      var old = model.getNote(selected);
      if (old.text === textarea.value) {
        return;
      }
      var updatedNote = DB.parseNote(textarea.value, selected);
      model.setNote(selected, updatedNote);
      // Update the listed item.
      display();
    }
  }

  function resetSelected() {
    searchEl.value = '';
    searchEl.focus();
    setSelected(null);
  }

  function setSelected(i) {
    if (typeof selected === 'number') {
      storeEdits();
      var old = s('.selected', true);
      if (old) {
        old.classList.remove('selected');
      }
    }
    selected = i;
    if (typeof selected === 'number') {
      var li = s('li[data-id="' + i + '"]');
      li.classList.add('selected');
      li.scrollIntoViewIfNeeded();
      // Set textarea to current note.
      var focus = false;
      updateTextArea(model.getNote(selected), focus);
    } else {
      textarea.value = '';
    }
  }

  var noteListEl = s('#noteList');

  // TODO(wdm) modify summary to show highlighted terms?
  function htmlDisplayNote(terms, note) {
    if (typeof note.id !== 'number') {
      throw new Error("note without id", note);
    }
    var ID = note.id;
    var TITLE = highlight(terms, note.title);
    var SUMMARY = highlight(terms, note.summary);
    var html = '
' + TITLE + '' + SUMMARY +
               '
';
    return html;
  }

  // highlight terms.
  function displayMatches(terms, results) {
    var html;
    var visibleNotes = {};

    if (!results || !results.length) {
      // Display all notes.
      html = model.getNotes().map(function(note) {
        visibleNotes[note.id] = true;
        return htmlDisplayNote(terms, note);
      });
    } else {
      // Filter by term and order by score.
      html = results.map(function(r) {
        var id = parseInt(r.ref, 10);
        var note = model.getNote(id);
        visibleNotes[id] = true;
        return htmlDisplayNote(terms, note);
      });
    }
    noteList.innerHTML = html.join('\n');
    if (visibleNotes[selected]) {
      setSelected(selected);
    }
  }

  function onInput() {
    setSelected(null);
    display();
  }

  // Display notes that match the current search terms (or all).
  // TOOD(wdm) Do something with score?
  function display() {
    var q = searchEl.value;
    var results;
    var terms;

    if (q.length) {
      results = index.search(q);
      terms = q.split(/\s+/);
    }
    displayMatches(terms, results);
  }

  function moveSelection(isDown) {
    var toSelect;
    var old = s('.selected', true);
    if (!old) {
      toSelect = noteListEl.firstChild;
    } else {
      toSelect = isDown ? old.nextElementSibling : old.previousElementSibling;
    }
    if (toSelect) {
      var id = parseInt(toSelect.dataset.id, 10);
      setSelected(id);
    }
  }

  function moveSelectionUp() { moveSelection(false); }

  function moveSelectionDown() { moveSelection(true); }

  function click(e) {
    var t = e.target;
    // Find the enclosing
.
    for (var i = 0; t.nodeName !== 'LI' && i < 3; i++) {
      t = t.parentNode;
    }
    if (t.nodeName !== 'LI') {
      console.error('Could not find
', e.target, t);
      return;
    }
    var id = parseInt(t.dataset.id, 10);
    setSelected(id);
  }

  function selectOrCreateNote() {
    if (typeof selected !== 'number') {
      // One newline is required to mark end of title. The next newline is
      // optional but looks nicer.
      var rawText = searchEl.value + '\n\n';
      textarea.value = rawText;  // HACK: required because otherwise storeEdits
                                 // will wipe out our new note.
      var note = model.newNote(rawText);
      index.add(note);
      display();  // HACK: need to add listitem to ul.
      setSelected(note.id);
    }
    var focus = true;
    updateTextArea(model.getNote(selected), focus);
  }

  function init(notes) {
    createIndex(notes);
    displayMatches();
    return notes;
  }

  return {
    onInput: onInput,
    selectOrCreateNote: selectOrCreateNote,
    moveSelectionUp: moveSelectionUp,
    moveSelectionDown: moveSelectionDown,
    resetSelected: resetSelected,
    click: click,
    storeEdits: storeEdits,
    init: init
  };
})();

// TODO(wdm) Does this actually buy us anything over a global 'notes'?
var model = (function() {

  // In memory note objects.
  var notes = [];

  return {
    init: function init(notes_) {
      notes = notes_;
      return notes;
    },
    newNote: function(rawText) {
      var id = notes.length;
      var note = DB.parseNote(rawText, id);
      notes.push(note);
      return note;
    },
    setNote: function setNote(i, note) { notes[i] = note; },
    getNote: function getNote(i) { return notes[i]; },
    getNotes: function getNotes() { return notes; }
  };

})();

////////
// DB //
////////

var DB = (function() {

  // TODO(wdm) Check that SEPERATOR does not appear in notes.
  var SEPERATOR = '\n[//]: # (NVC: Do not edit this line)\n\n';

  function parse(rawText) {
    var rawNotes = rawText.split(SEPERATOR);
    var notes = rawNotes.map(parseNote);
    return notes;
  }

  function parseNote(rawNote, i) {
    if (typeof i !== 'number') {
      throw new Error('tried to create note without index');
    }
    var firstLine = rawNote.indexOf('\n');
    return {
      id: i,
      title: rawNote.substring(0, firstLine),
      summary: rawNote.substring(firstLine + 1, 150),
      text: rawNote
    };
  }

  // TODO(wdm) Better name!
  function notesAsBlob() {
    search.storeEdits();  // Grabs any changes in textarea.
    var texts = model.getNotes().map(function(note) { return note.text; });
    var rawText = texts.join(SEPERATOR);
    return new Blob([rawText], { 'type' : 'text/plain' });
  }
  return {
    parse: parse,
    parseNote: parseNote,
    notesAsBlob: notesAsBlob,
  };
})();

//////////////
// TextArea //
//////////////

function updateTextArea(note, focus) {
  var text = note.text;
  textarea.value = text;
  if (focus) {
    textarea.focus();
    // Move to first line after title.
    var pos = note.title.length + 1;
    if (text[pos] === '\n') {
      pos++;
    }
    textarea.setSelectionRange(pos, pos);
    textarea.scrollTop = 0;
  }
}

function setCaret(elem, pos) {
  var range = elem.createTextRange();
  range.collapse(true);
  range.moveEnd('character', pos);
  range.moveStart('character', pos);
  range.select();
}

///////////////
/// File UI ///
///////////////

var fileEntry;

function onChooseFile() {
  chooseFile().then(showFilename).catch(showError);
}

function showFilename(fileEntry_) {
  fileEntry = fileEntry_;
  s('#filename').textContent = fileEntry.name;
  return fileEntry;
}

function doSave() {
  if (!dirtMonitor.isDirty) {
    console.log('save skipped: not dirty');
    return Promise.resolve();
  }
  var blob = DB.notesAsBlob();
  return writeFileEntry(fileEntry, blob)
      .then(dirtMonitor.setClean)
      .catch(showError);
}

function doClose() {
  doSave().then(window.close);
}

//////////////////
// Key Handling //
//////////////////

var keyHandler = (function() {
  var KEY_W = 'W'.charCodeAt(0);
  var KEY_S = 'S'.charCodeAt(0);
  var KEY_1 = '1'.charCodeAt(0);
  var KEY_ESC = 27;
  var KEY_ENTER = 13;
  var KEY_UP = 38;
  var KEY_DOWN = 40;

  // return values don't matter.
  function cmd(e, func) {
    e.preventDefault();
    func();
  }

  return function keyHandler(e) {
    var Alt = e.altKey;
    var Ctrl = e.ctrlKey;
    var key = e.keyCode;

    // Dismiss (with save)
    if ((Alt && key === KEY_1) || (Ctrl && key === KEY_W)) {
      return cmd(e, doClose);
    }

    // Reset (deselect note and start new search)
    if (key === KEY_ESC) {
      return cmd(e, search.resetSelected);
    }

    // Save: Ctrl-S
    if (Ctrl && key === KEY_S) {
      return cmd(e, doSave);
    }

    if (e.target !== textarea) {
      if (key == KEY_ENTER) {
        return cmd(e, search.selectOrCreateNote);
      }
      if (key === KEY_DOWN) {
        return cmd(e, search.moveSelectionDown);
      }
      if (key === KEY_UP) {
        return cmd(e, search.moveSelectionUp);
      }
    }
  }
})();

function init() {
  restoreChosenFile()
      .then(showFilename)
      .then(loadFileEntry)
      .then(DB.parse)
      .then(model.init)
      .then(search.init)
      .catch(showError);

  window.addEventListener('keydown', keyHandler);
  textarea.addEventListener('input', dirtMonitor.setDirty);
  s('#chooseFile').addEventListener('click', onChooseFile);
  s('#noteList').addEventListener('click', search.click);
  s('#search').addEventListener('input', search.onInput);
}

init();
<|endoftext|>


<commit_before>// Node.js API.
var assert = require('assert')
var util = require('util')

var Procession = require('./procession')
var Deferred = require('./deferred')

var abend = require('abend')

function Window () {
    Procession.call(this)
    this.trailer = this.shifter(this, '_shift')
    this._header = this.pump(this, '_push', abend)
    this._undecorated = []
    this._listeners = []
}
util.inherits(Window, Procession)

// Add a listener that can track values as they are enqueued and dequeued. The
// listener's push and shift methods  will only be invoked for values enqueued
// after it is added to to the procession.
//

//

// We decorate the listener with a deferred decorator. We keep a parallel arrays
// of undecorated listeners do we can find its array index if we're asked to
// remove it before it undecorates itself.
Window.prototype.addListener = function (listener) {
    this._undecorated.push(listener)
    this._listeners.push(new Deferred(this._listeners, listener))
}

// Remove a listener.
//

//
Window.prototype.removeListener = function (listener) {
    var index = this._undecorated.indexOf(this)
    this._undecorated.splice(index, 1)
    this._listeners.splice(index, 1)
}

Window.prototype._push = function (value) {
    if (value != null) {
        assert(this._header.node == this.head)
        for (var i = 0, I = this._listeners.length; i < I; i++) {
            this._listeners[i].pushed(this._header.node)
        }
    }
}

Window.prototype._shift = function (node) {
    if (node.body != null) {
        for (var i = 0, I = this._listeners.length; i < I; i++) {
            this._listeners[i].shifted(node)
        }
    }
}

module.exports = Window
<commit_msg>Use new pump with window.<commit_after>// Node.js API.
var assert = require('assert')
var util = require('util')

var Procession = require('./procession')
var Deferred = require('./deferred')

var abend = require('abend')

var Pump = require('./pump')

function Window () {
    Procession.call(this)
    this.trailer = this.shifter(this, '_shift')
    this._header = this.shifter()
    new Pump(this._header, this, '_push').run(abend)
    this._undecorated = []
    this._listeners = []
}
util.inherits(Window, Procession)

// Add a listener that can track values as they are enqueued and dequeued. The
// listener's push and shift methods  will only be invoked for values enqueued
// after it is added to to the procession.
//

//

// We decorate the listener with a deferred decorator. We keep a parallel arrays
// of undecorated listeners do we can find its array index if we're asked to
// remove it before it undecorates itself.
Window.prototype.addListener = function (listener) {
    this._undecorated.push(listener)
    this._listeners.push(new Deferred(this._listeners, listener))
}

// Remove a listener.
//

//
Window.prototype.removeListener = function (listener) {
    var index = this._undecorated.indexOf(this)
    this._undecorated.splice(index, 1)
    this._listeners.splice(index, 1)
}

Window.prototype._push = function (value) {
    if (value != null) {
        assert(this._header.node == this.head)
        for (var i = 0, I = this._listeners.length; i < I; i++) {
            this._listeners[i].pushed(this._header.node)
        }
    }
}

Window.prototype._shift = function (node) {
    if (node.body != null) {
        for (var i = 0, I = this._listeners.length; i < I; i++) {
            this._listeners[i].shifted(node)
        }
    }
}

module.exports = Window
<|endoftext|>


<commit_before>// worker process, handles twitter stream
'use strict';

var fs = require('fs'),
    url = require('url'),
    kue = require('kue'),
    redis = require('redis'),
    embedly = require('embedly'),
    twitter = require('twitter'),
    mongoose = require('mongoose');

process.env.NODE_ENV = process.env.NODE_ENV || 'development';

console.log('Initializing worker...');

var config = require('./config/config'),
    mailer = require('./config/mailer');

// kue settings
if (config.redis) {
    kue.redis.createClient = function() {
        var rtg = url.parse(config.redis);
        var client = redis.createClient(rtg.port, rtg.hostname);
        client.auth(rtg.auth.split(':')[1]);
        return client;
    };
}

// connect with the DB
mongoose.connect(config.db);

// Bootstrap models
var models_path = __dirname + '/app/models';
var walk = function(path) {
        fs.readdirSync(path).forEach(function(file) {
            var newPath = path + '/' + file;
            var stat = fs.statSync(newPath);
            if (stat.isFile()) {
                if (/(.*)\.(js$|coffee$)/.test(file)) {
                    require(newPath);
                }
            } else if (stat.isDirectory()) {
                walk(newPath);
            }
        });
    };
walk(models_path);

var jobs = kue.createQueue(),
    processPost = require('./config/process');

jobs.promote();

jobs.on('job complete', function(id) {
    kue.Job.get(id, function(err, job) {
        if (err) return;
        job.remove(function(err) {
            if (err) return;
        });
    });
});

jobs.process('twitterStream', 100, function(job, done) {
    console.log('Stream opened!');
    // Don't close this job, we need to close it when user changes email address
    var user = job.data;
    /*
     *  User
     *      -token
     *      -tokenSecret
     *      -profile
     */
    var twit = new twitter({
        consumer_key: config.twitter.clientID,
        consumer_secret: config.twitter.clientSecret,
        access_token_key: user.twitter.token,
        access_token_secret: user.twitter.tokenSecret
    });

    twit.stream('statuses/filter', {
        follow: user.twitter.profile.id_str,
        track: 'readly'
    }, function(stream) {
        stream.on('data', function(data) {
            if (data.entities) {
                mongoose
                .model('User')
                .findOne({
                    'twitter.profile.id': data.user.id
                }, function(err, readlyUser) {
                    var urls = [];
                    for (var iter = 0; iter < data.entities.urls.length; iter++) {
                        urls.push(data.entities.urls[iter].expanded_url);
                    }

                    var hashtags = [];
                    for (var iter = 0; iter < data.entities.hashtags.length; iter++) {
                        hashtags.push(data.entities.hashtags[iter].text);
                    }

                    processPost(urls, hashtags, readlyUser);
                });
            }
        });
    });

    done();
});

jobs.process('emailPost', 100, function(job, done) {
    var post = job.data;

    new embedly({
        key: config.embedlyKey
    }, function(err, api) {
        if (err) {
            console.error('Error creating Embedly api');
            return;
        }

        api.extract({
            url: post.url,
            maxwidth: 500
        }, function(err, results) {
            if (err) {
                console.log('Embedly request failed.');
                return;
            }

            var result = results[0];
            var description = result.description;
            var title = result.title;

            var mailOptions = {
                to: post.user.email,
                subject: 'Readly: ' + title,
                html:
                '
' +
                '
Title: ' + title +
                '
Description: ' + description +
                '
Visit link' +
                '
' +
                '
' +
                '
' +
                '
Unsubscribe from Readly' +
                '
- Readly Team
',
                text: 'Readly.io' +
                '\nTitle:' + title +
                '\nDescription: ' + description +
                '\nLink: ' + post.url +
                '\n' +
                '\n' +
                '\n' +
                '\nUnsubscribe from Readly: ' + config.url + '/unsubscribe/' + post.user.verificationCode +
                '\n- Readly Team'
            };
            mailer(mailOptions);

            // this is a hack, but due to the race condition preventing complete handler
            // look at /model/post.js
            mongoose.model('Post').findById(post._id).select('prev_reminders next_reminder jobId').exec(function(err, post) {
                post.prev_reminders.push(post.next_reminder);
                post.next_reminder = null;
                post.jobId = null;
                post.save();
            });
        })
    });

    done();
});<commit_msg>Twitter stream changes.<commit_after>// worker process, handles twitter stream
'use strict';

var fs = require('fs'),
    url = require('url'),
    kue = require('kue'),
    redis = require('redis'),
    embedly = require('embedly'),
    twitter = require('twitter'),
    mongoose = require('mongoose');

process.env.NODE_ENV = process.env.NODE_ENV || 'development';

console.log('Initializing worker...');

var config = require('./config/config'),
    mailer = require('./config/mailer');

// kue settings
if (config.redis) {
    kue.redis.createClient = function() {
        var rtg = url.parse(config.redis);
        var client = redis.createClient(rtg.port, rtg.hostname);
        client.auth(rtg.auth.split(':')[1]);
        return client;
    };
}

// connect with the DB
mongoose.connect(config.db);

// Bootstrap models
var models_path = __dirname + '/app/models';
var walk = function(path) {
        fs.readdirSync(path).forEach(function(file) {
            var newPath = path + '/' + file;
            var stat = fs.statSync(newPath);
            if (stat.isFile()) {
                if (/(.*)\.(js$|coffee$)/.test(file)) {
                    require(newPath);
                }
            } else if (stat.isDirectory()) {
                walk(newPath);
            }
        });
    };
walk(models_path);

var jobs = kue.createQueue(),
    processPost = require('./config/process');

jobs.promote();

jobs.on('job complete', function(id) {
    kue.Job.get(id, function(err, job) {
        if (err) return;
        job.remove(function(err) {
            if (err) return;
        });
    });
});

/*
var twit = new twitter({
    consumer_key: config.twitter.clientID,
    consumer_secret: config.twitter.clientSecret,
    access_token_key: user.twitter.token,
    access_token_secret: user.twitter.tokenSecret
});

twit.stream('statuses/filter', {
    follow: user.twitter.profile.id_str,
    track: 'readly'
}, function(stream) {
    stream.on('data', function(data) {
        if (data.entities) {
            mongoose.model('User').findOne({
                'twitter.profile.id': data.user.id
            }, function(err, readlyUser) {
                if (err) return;

                var urls = [];
                for (var iter = 0; iter < data.entities.urls.length; iter++) {
                    urls.push(data.entities.urls[iter].expanded_url);
                }

                var hashtags = [];
                for (var iter = 0; iter < data.entities.hashtags.length; iter++) {
                    hashtags.push(data.entities.hashtags[iter].text);
                }

                processPost(urls, hashtags, readlyUser);
            });
        }
    });
});
*/

jobs.process('twitterStream', 100, function(job, done) {
    console.log('Stream opened!');
    // Don't close this job, we need to close it when user changes email address
    var user = job.data;
    /*
     *  User
     *      -token
     *      -tokenSecret
     *      -profile
     */
    var twit = new twitter({
        consumer_key: config.twitter.clientID,
        consumer_secret: config.twitter.clientSecret,
        access_token_key: user.twitter.token,
        access_token_secret: user.twitter.tokenSecret
    });

    twit.stream('statuses/filter', {
        follow: user.twitter.profile.id_str,
        track: 'readly'
    }, function(stream) {
        stream.on('data', function(data) {
            if (data.entities && data.user) {
                if(data.user.id !== user.twitter.profile.id) {
                    return;
                }

                mongoose
                .model('User')
                .findOne({
                    'twitter.profile.id': data.user.id
                }, function(err, readlyUser) {
                    if(err) return;

                    var urls = [];
                    for (var iter = 0; iter < data.entities.urls.length; iter++) {
                        urls.push(data.entities.urls[iter].expanded_url);
                    }

                    var hashtags = [];
                    for (var iter = 0; iter < data.entities.hashtags.length; iter++) {
                        hashtags.push(data.entities.hashtags[iter].text);
                    }

                    processPost(urls, hashtags, readlyUser);
                });
            }
        });
    });

    done();
});

jobs.process('emailPost', 100, function(job, done) {
    var post = job.data;

    new embedly({
        key: config.embedlyKey
    }, function(err, api) {
        if (err) {
            console.error('Error creating Embedly api');
            return;
        }

        api.extract({
            url: post.url,
            maxwidth: 500
        }, function(err, results) {
            if (err) {
                console.log('Embedly request failed.');
                return;
            }

            var result = results[0];
            var description = result.description;
            var title = result.title;

            var mailOptions = {
                to: post.user.email,
                subject: 'Readly: ' + title,
                html: '
' + '
Title: ' + title + '
Description: ' + description + '
Visit link' + '
' + '
' + '
' + '
Unsubscribe from Readly' + '
- Readly Team
',
                text: 'Readly.io' + '\nTitle:' + title + '\nDescription: ' + description + '\nLink: ' + post.url + '\n' + '\n' + '\n' + '\nUnsubscribe from Readly: ' + config.url + '/unsubscribe/' + post.user.verificationCode + '\n- Readly Team'
            };
            mailer(mailOptions);

            // this is a hack, but due to the race condition preventing complete handler
            // look at /model/post.js
            mongoose.model('Post').findById(post._id).select('prev_reminders next_reminder jobId').exec(function(err, post) {
                post.prev_reminders.push(post.next_reminder);
                post.next_reminder = null;
                post.jobId = null;
                post.save();
            });
        })
    });

    done();
});<|endoftext|>


<commit_before>'use strict';

var jschardet = require('jschardet');
var iconv = require('iconv-lite');
var htmlparser = require('htmlparser2');
var http = require('http');
var url = require('url');
var fs = require('fs');

var options = {
	agent: false
};

var buffers, buf, encoding, body, req, id, fsWriteStream;

process.on('message', handleMessage);

function handleMessage (msg) {
	if (msg.saveId) {
		id = msg.saveId;
		fsWriteStream = fs.createWriteStream(__dirname + "/pages/"+id+".txt", {flag: "a+"});
		console.log("logfile:", __dirname + "/pages/"+id+".txt");
	}
	if (msg.url) {
		crawl(msg.url, msg.id);
	}
};

var parser = new htmlparser.Parser({
	onopentag: function (name, attribs) {
		if(name === "a" && attribs.href) {
			if(attribs.href.indexOf('jd.com') > 0) {
				// console.log(attribs.href);
				process.send({cmd: 'url', url: attribs.href});
			}
		}
	},
	ontext: function (text) {
		text = innerText(text);
		if(text)
			fsWriteStream.write(text, function (err) {
				if(err)
					console.log("######id:%s, error:", id, err);
			})
	}
});

function innerText (text) {
	if(!/[\n\(\/\sfp)]/.test(text[0]))
		return text;
	return null;
};

var crawl = function (crawlURL) {
	if (!crawlURL) {
		console.log('#### crawlURL is null');
		return setTimeout(function(){
			process.send({});
		}, 5000);
	}
	var parsedUrl = url.parse(crawlURL);
	if ("https:" === parsedUrl.protocol || !parsedUrl.protocol)
		return process.send({});
	//extract url
	options.hostname = parsedUrl.hostname;
	options.path = parsedUrl.path;
	buffers = [];

	req = http.request(options, function (res) {
		res.on('data', function (chunk) {
			// try {
			if(!buffers)
				return process.send({});
			buffers.push(chunk);
			// } catch (e) {
			// 	console.log('url:', crawlURL);
			// 	console.log(e);
			// 	console.log('buffers', buffers);
			// 	console.log(chunk);
			// }
		});

		res.on('end', function () {
			if(!buffers)
				return process.send({});
			try {
				buf = Buffer.concat(buffers);
			} catch (e) {
				console.log("buf:", buf);
				console.log("buffers:", buffers);
				return process.send({});
			}
			encoding = jschardet.detect(buf).encoding;
			if(!encoding)
				return process.send({});
			body = iconv.decode(buf, encoding);
			// console.log(body);
			// handleBody(body);
			parser.write(body);
			parser.end();
			buf = null;
			buffers = null;
			process.send({});
		});

		res.on('error', function(err){
			console.log(err);
		});
	});

	// req.setTimeout(30000, function(){
	// 	console.log(new Date(), 'request timeout, skip url:', crawlURL);
	// 	process.send({});
	// });

	req.on('error', function (e) {
		console.log(new Date() , ' connection problem:', e, crawlURL);
		process.send({cmd: 'error'});
	});

	req.end();
};

// var fs = require('fs');
// fs.readFile('./out.txt', {encoding: 'utf8'}, function(err, data){
// 	// console.log(data);
// 	console.log(data.match(/<body((.|\r\n)*)\/body>/g))
// 	// console.log(data.match(/<body>(.*)<\/body>/));
// })
// var str = '';
// console.log(str.match(/\<body\>.*\<\/body\>/g))

<commit_msg>add network control.:<commit_after>'use strict';

var jschardet = require('jschardet');
var iconv = require('iconv-lite');
var htmlparser = require('htmlparser2');
var http = require('http');
var url = require('url');
var fs = require('fs');

var options = {
	agent: false
};

var DOWNLOAD_SPEED = 1024*200; // 200 kb/s

var buffers, buf, encoding, body, req, id, fsWriteStream;

process.on('message', handleMessage);

function handleMessage (msg) {
	if (msg.saveId) {
		id = msg.saveId;
		fsWriteStream = fs.createWriteStream(__dirname + "/pages/"+id+".txt", {flag: "a+"});
		console.log("logfile:", __dirname + "/pages/"+id+".txt");
	}
	if (msg.url) {
		crawl(msg.url, msg.id);
	}
};

var parser = new htmlparser.Parser({
	onopentag: function (name, attribs) {
		if(name === "a" && attribs.href) {
			if(attribs.href.indexOf('jd.com') > 0) {
				// console.log(attribs.href);
				process.send({cmd: 'url', url: attribs.href});
			}
		}
	},
	ontext: function (text) {
		text = innerText(text);
		if(text)
			fsWriteStream.write(text, function (err) {
				if(err)
					console.log("######id:%s, error:", id, err);
			})
	}
});

function innerText (text) {
	if(!/[\n\(\/\sfp)]/.test(text[0]))
		return text;
	return null;
};

var crawl = function (crawlURL) {
	if (!crawlURL) {
		console.log('#### crawlURL is null');
		return setTimeout(function(){
			process.send({});
		}, 5000);
	}
	var parsedUrl = url.parse(crawlURL);
	if ("https:" === parsedUrl.protocol || !parsedUrl.protocol)
		return process.send({});
	//extract url
	options.hostname = parsedUrl.hostname;
	options.path = parsedUrl.path;
	buffers = [];

	req = http.request(options, function (res) {
		res.on('readable', function () {
			var chunk;
			while (null != (chunk = res.read(DOWNLOAD_SPEED))) {
				if(!buffers)
				return process.send({});
				try {
					buf = Buffer.concat(buffers);
				} catch (e) {
					console.log("buf:", buf);
					console.log("buffers:", buffers);
					return process.send({});
				}
				encoding = jschardet.detect(buf).encoding;
				if(!encoding)
					return process.send({});
				body = iconv.decode(buf, encoding);
				// console.log(body);
				// handleBody(body);
				parser.write(body);
				parser.end();
				buf = null;
				buffers = null;
				process.send({});
			}

		});
		// res.on('data', function (chunk) {
		// 	// try {
		// 	if(!buffers)
		// 		return process.send({});
		// 	buffers.push(chunk);
		// 	// } catch (e) {
		// 	// 	console.log('url:', crawlURL);
		// 	// 	console.log(e);
		// 	// 	console.log('buffers', buffers);
		// 	// 	console.log(chunk);
		// 	// }
		// });

		// res.on('end', function () {
		// 	if(!buffers)
		// 		return process.send({});
		// 	try {
		// 		buf = Buffer.concat(buffers);
		// 	} catch (e) {
		// 		console.log("buf:", buf);
		// 		console.log("buffers:", buffers);
		// 		return process.send({});
		// 	}
		// 	encoding = jschardet.detect(buf).encoding;
		// 	if(!encoding)
		// 		return process.send({});
		// 	body = iconv.decode(buf, encoding);
		// 	// console.log(body);
		// 	// handleBody(body);
		// 	parser.write(body);
		// 	parser.end();
		// 	buf = null;
		// 	buffers = null;
		// 	process.send({});
		// });

		res.on('error', function(err){
			console.log(err);
		});
	});

	// req.setTimeout(30000, function(){
	// 	console.log(new Date(), 'request timeout, skip url:', crawlURL);
	// 	process.send({});
	// });

	req.on('error', function (e) {
		console.log(new Date() , ' connection problem:', e, crawlURL);
		process.send({cmd: 'error'});
	});

	req.end();
};

// var fs = require('fs');
// fs.readFile('./out.txt', {encoding: 'utf8'}, function(err, data){
// 	// console.log(data);
// 	console.log(data.match(/<body((.|\r\n)*)\/body>/g))
// 	// console.log(data.match(/<body>(.*)<\/body>/));
// })
// var str = '';
// console.log(str.match(/\<body\>.*\<\/body\>/g))

<|endoftext|>


<commit_before>/**
 *  File worker that is started by the main server process. It downloads and anylyzes the zile in a different process
 *  leaving the main process for the server
 *
 *  We could kick off more processes for the analyzation, eg. each per file. I think this program to be mostly web application that server
 *  analyzation, but offers a way for the mainteiner to download the file again.
 *
 *  Call chaing in this solution after creating http.createServer if file does not exist
 *  getZipZile -> unZipData -> analyzeData
 */

var path = require('path');
var fs = require('fs');
var admZip = require('adm-zip');
var http = require('http');
var winston = require('winston');

// Configuration file
var config = require(path.join(__dirname, 'config.js'));

var logger = new winston.Logger({
	transports : [new winston.transports.Console({
		level : config.console,
		handleExceptions : true,
		colorize : true
	}), new winston.transports.File({
		filename : config.pathLog,
		level : config.file,
		handleExceptions : true
	})],
});

function analyzeJSON(filesJSON) {
	logger.info("Data structure ready, starting to analyze data...");

	var result = {
		from : undefined,
		to : undefined,
		days : 0,
		categories : {},
		locations : {},
		incidents : {}
	};

	var fileCounter = filesJSON.length;
	filesJSON.forEach(function(fileJSON) {
		fileCounter--;

		var dayCounter = fileJSON.autoreporter.opendata.length;
		fileJSON.autoreporter.opendata.forEach(function(day) {
			dayCounter--;

			// Capturing start and end dates
			var dateFrom = new Date(day.date.from);
			if (result.from == undefined || result.from.getTime() > dateFrom.getTime())
				result.from = dateFrom;

			var dateTo = new Date(day.date.to);
			if (result.to == undefined || result.to.getTime() < dateTo.getTime())
				result.to = dateTo;

			// Creating new year and month to object if does not exist
			if (!result.incidents.hasOwnProperty(dateFrom.getFullYear()))
				result.incidents[dateFrom.getFullYear()] = {};

			if (!result.incidents[dateFrom.getFullYear()].hasOwnProperty(dateFrom.getMonth()))
				result.incidents[dateFrom.getFullYear()][dateFrom.getMonth()] = 0;

			var occurrencyCounter = day.asn.length;
			day.asn.forEach(function(occurrency) {
				occurrencyCounter--;

				var ipaddressCounter = occurrency.ipaddress.length;
				occurrency.ipaddress.forEach(function(ipaddress) {
					ipaddressCounter--;

					var location = ipaddress.city + '-' + ipaddress.cc;

					// Creating new location to object if does not exist
					if (result.locations.hasOwnProperty(location)) {
						result.locations[location]++;
					} else {
						result.locations[location] = 1;
					}

					var incidentCounter = ipaddress.incident.length;
					ipaddress.incident.forEach(function(incident) {
						incidentCounter--;

						// Counting every incident under a month
						result.incidents[dateFrom.getFullYear()][dateFrom.getMonth()]++;

						// Creating new category to object if does not exist
						if (result.categories.hasOwnProperty(incident.category.main)) {
							result.categories[incident.category.main]++;
						} else {
							result.categories[incident.category.main] = 1;
						}

						// After every loop is done, print the result and setting analyzation done
						if (fileCounter == 0 && dayCounter == 0 && occurrencyCounter == 0 && ipaddressCounter == 0 && incidentCounter == 0) {
							var oneDay = (1000 * 60 * 60 * 24);
							result.days = Math.abs((result.from.getTime() - result.to.getTime()) / (oneDay));

							result.done = true;

							fs.writeFile(config.pathAnalyzation, JSON.stringify(result), function(err) {
								if (err) {
									logger.error('Error while result file to ' + config.pathAnalyzation + ', Error: ', err);
									process.exit(1);
								}

								logger.info('Analyzation saved to file ' + config.pathAnalyzation);

								// Pass results back to parent process
								process.send(result);
							});
						}
					});
				});
			});
		});
	});
};

var unZipData = function unZipData() {

	try {
		var zip = new admZip(config.pathJSON);
		var zipEntries = zip.getEntries();

		logger.info(config.JSON + " contains " + zipEntries.length + " files, extracting to array");
		var filesJSON = new Array(zipEntries.length);

		// Unzippping everything to data array as JSON objects then exec callback
		var index = 0;

		zipEntries.forEach(function(entry) {

			var entryName = entry.entryName;

			zip.readAsTextAsync(entry, function(data) {
				logger.info('Decompressing ' + entryName + ' as a JSON object to array index ' + index);

				try {
					filesJSON[index++] = JSON.parse(data);
					if (index == zipEntries.length)
						analyzeJSON(filesJSON);
				} catch(err) {
					fs.unlink(config.pathJSON, function() {
						logger.error('JSON parsing failed, deleting existing file, please try again. Error: ', err);
						process.exit(1);
					});
				}
			});
		});
	} catch(err) {
		fs.unlink(config.pathJSON, function() {
			logger.error('Reading ' + config.JSON + ' failed, deleting existing file, please try again. Error:', err);
			process.exit(1);
		});
	}
};

function getZipfile() {
	logger.info('Checking if ' + config.pathFolder + ' exists, creating if does not');

	fs.mkdir(config.pathFolder, function() {
		var lenght = 0;
		var file = fs.createWriteStream(config.pathJSON);
		var request = http.get(config.JSON_url, function(response) {

			var change = -1;
			logger.info('Downloading file.. ' + config.JSON_url);

			response.on('data', function(chunk) {
				file.write(chunk);
				lenght += chunk.length;

				var percent = Math.round((lenght / response.headers['content-length']) * 100);

				if (change != percent) {
					change = percent;
					process.stdout.write(" " + percent + "%");
				}
			});

			response.on('end', function() {
				console.log();
				file.close();
			});

			file.on('close', function() {
				process.nextTick(unZipData);
			});
		});
	});
};

// Worker process starts: analyzation file does not exist, checking json.zip
process.on('message', function(data) {
	if (data === 'start') {
		fs.exists(config.pathJSON, function(exists) {
			if (exists) {
				logger.warn(config.JSON + ' exists, but there is no ' + config.analyzation + ', starting unzip');
				unZipData();
			} else {
				logger.warn(config.JSON + ' file does not exist...');
				getZipfile();
			}
		});
	} else if (data === 'reset') {
		getZipfile();
	} else if (data === 'exit') {
		process.exit(0);
	}
});
<commit_msg>better flow control<commit_after>/**
 *  File worker that is started by the main server process. It downloads and anylyzes the zile in a different process
 *  leaving the main process for the server
 *
 *  We could kick off more processes for the analyzation, eg. each per file. I think this program to be mostly web application that server
 *  analyzation, but offers a way for the mainteiner to download the file again.
 *
 *  Call chaing in this solution after creating http.createServer if file does not exist
 *  getZipZile -> unZipData -> analyzeData
 */

var path = require('path');
var fs = require('fs');
var admZip = require('adm-zip');
var http = require('http');
var winston = require('winston');

// Configuration file
var config = require(path.join(__dirname, 'config.js'));

var logger = new winston.Logger({
	transports : [new winston.transports.Console({
		level : config.console,
		handleExceptions : true,
		colorize : true
	}), new winston.transports.File({
		filename : config.pathLog,
		level : config.file,
		handleExceptions : true
	})],
});

function saveAndSendResult(result) {
	fs.writeFile(config.pathAnalyzation, JSON.stringify(result), function(err) {
		if (err) {
			logger.error('Error while result file to ' + config.pathAnalyzation + ', Error: ', err);
			process.exit(1);
		}

		logger.info('Analyzation saved to file ' + config.pathAnalyzation);

		// Pass results back to parent process
		process.send(result);
	});
}

function analyzeJSON(filesJSON) {
	logger.info("Data structure ready, starting to analyze data...");

	var result = {
		from : undefined,
		to : undefined,
		days : 0,
		categories : {},
		locations : {},
		incidents : {}
	};

	var fileCounter = filesJSON.length;
	filesJSON.forEach(function(fileJSON) {
		fileCounter--;

		var dayCounter = fileJSON.autoreporter.opendata.length;
		fileJSON.autoreporter.opendata.forEach(function(day) {
			dayCounter--;

			// Capturing start and end dates
			var dateFrom = new Date(day.date.from);
			if (result.from == undefined || result.from.getTime() > dateFrom.getTime())
				result.from = dateFrom;

			var dateTo = new Date(day.date.to);
			if (result.to == undefined || result.to.getTime() < dateTo.getTime())
				result.to = dateTo;

			// Creating new year and month to object if does not exist
			if (!result.incidents.hasOwnProperty(dateFrom.getFullYear()))
				result.incidents[dateFrom.getFullYear()] = {};

			if (!result.incidents[dateFrom.getFullYear()].hasOwnProperty(dateFrom.getMonth()))
				result.incidents[dateFrom.getFullYear()][dateFrom.getMonth()] = 0;

			var occurrencyCounter = day.asn.length;
			day.asn.forEach(function(occurrency) {
				occurrencyCounter--;

				var ipaddressCounter = occurrency.ipaddress.length;
				occurrency.ipaddress.forEach(function(ipaddress) {
					ipaddressCounter--;

					var location = ipaddress.city + '-' + ipaddress.cc;

					// Creating new location to object if does not exist
					if (result.locations.hasOwnProperty(location)) {
						result.locations[location]++;
					} else {
						result.locations[location] = 1;
					}

					var incidentCounter = ipaddress.incident.length;
					ipaddress.incident.forEach(function(incident) {
						incidentCounter--;

						// Counting every incident under a month
						result.incidents[dateFrom.getFullYear()][dateFrom.getMonth()]++;

						// Creating new category to object if does not exist
						if (result.categories.hasOwnProperty(incident.category.main)) {
							result.categories[incident.category.main]++;
						} else {
							result.categories[incident.category.main] = 1;
						}

						// After every loop is done, print the result and setting analyzation done
						if (fileCounter == 0 && dayCounter == 0 && occurrencyCounter == 0 && ipaddressCounter == 0 && incidentCounter == 0) {
							var oneDay = (1000 * 60 * 60 * 24);
							result.days = Math.abs((result.from.getTime() - result.to.getTime()) / (oneDay));

							result.done = true;
							saveAndSendResult(result);
						}
					});
				});
			});
		});
	});
};

var unZipData = function unZipData() {

	try {
		var zip = new admZip(config.pathJSON);
		var zipEntries = zip.getEntries();

		logger.info(config.JSON + " contains " + zipEntries.length + " files, extracting to array");
		var filesJSON = new Array(zipEntries.length);

		// Unzippping everything to data array as JSON objects then exec callback
		var index = 0;

		zipEntries.forEach(function(entry) {

			var entryName = entry.entryName;

			zip.readAsTextAsync(entry, function(data) {
				logger.info('Decompressing ' + entryName + ' as a JSON object to array index ' + index);

				try {
					filesJSON[index++] = JSON.parse(data);
					if (index == zipEntries.length)
						analyzeJSON(filesJSON);
				} catch(err) {
					fs.unlink(config.pathJSON, function() {
						logger.error('JSON parsing failed, deleting existing file, please try again. Error: ', err);
						process.exit(1);
					});
				}
			});
		});
	} catch(err) {
		fs.unlink(config.pathJSON, function() {
			logger.error('Reading ' + config.JSON + ' failed, deleting existing file, please try again. Error:', err);
			process.exit(1);
		});
	}
};

function getZipfile() {
	logger.info('Checking if ' + config.pathFolder + ' exists, creating if does not');

	fs.mkdir(config.pathFolder, function() {
		var lenght = 0;
		var file = fs.createWriteStream(config.pathJSON);
		var request = http.get(config.JSON_url, function(response) {

			var change = -1;
			logger.info('Downloading file.. ' + config.JSON_url);

			response.on('data', function(chunk) {
				file.write(chunk);
				lenght += chunk.length;

				var percent = Math.round((lenght / response.headers['content-length']) * 100);

				if (change != percent) {
					change = percent;
					process.stdout.write(" " + percent + "%");
				}
			});

			response.on('end', function() {
				console.log();
				file.close();
			});

			file.on('close', function() {
				// We give some time for the writer to finnish
				process.nextTick(unZipData);
			});
		});
	});
};

// Worker process starts: analyzation file does not exist, checking json.zip
process.on('message', function(data) {
	if (data === 'start') {
		fs.exists(config.pathJSON, function(exists) {
			if (exists) {
				logger.warn(config.JSON + ' exists, but there is no ' + config.analyzation + ', starting unzip');
				unZipData();
			} else {
				logger.warn(config.JSON + ' file does not exist...');
				getZipfile();
			}
		});
	} else if (data === 'reset') {
		getZipfile();
	} else if (data === 'exit') {
		process.exit(0);
	}
});
<|endoftext|>