<commit_before>var animate=function(){var h=function(a){var b=a.length;return function d(){for(var e=arguments.length,f=Array(e),g=0;g<e;g++)f[g]=arguments[g];return f.length<b?function(){for(var a=arguments.length,b=Array(a),e=0;e<a;e++)b[e]=arguments[e];return d.apply(void 0,f.concat(b))}:a.apply(void 0,f)}},p=function(){for(var a=arguments.length,b=Array(a),c=0;c<a;c++)b[c]=arguments[c];return function(a){return b.reduce(function(a,b){return b(a)},a)}},q=function(a){return function(){return!a.apply(void 0,arguments)}},
E={linear:function(a,b,c,d){return b+a/d*c},easeInQuad:function(a,b,c,d){return c*(a/=d)*a+b},easeInCubic:function(a,b,c,d){return c*(a/=d)*a*a+b},easeInQuart:function(a,b,c,d){return c*(a/=d)*a*a*a+b},easeInQuint:function(a,b,c,d){return c*(a/=d)*a*a*a*a+b},easeInSine:function(a,b,c,d){return-c*Math.cos(a/d*(Math.PI/2))+c+b},easeInExpo:function(a,b,c,d){return 0==a?b:c*Math.pow(2,10*(a/d-1))+b},easeInCirc:function(a,b,c,d){return-c*(Math.sqrt(1-(a/=d)*a)-1)+b},easeInElastic:function(a,b,c,d){var e=
4>=arguments.length||void 0===arguments[4]?500:arguments[4];if(0==a)return b;if(1==(a/=d))return b+c;var e=d*(1-Math.min(e,999)/1E3),f=c<Math.abs(c)?e/4:e/(2*Math.PI)*Math.asin(c/c);return-(c*Math.pow(2,10*--a)*Math.sin(2*(a*d-f)*Math.PI/e))+b},easeInBack:function(a,b,c,d){return c*(a/=d)*a*(2.70158*a-1.70158)+b},easeOutQuad:function(a,b,c,d){return-c*(a/=d)*(a-2)+b},easeOutCubic:function(a,b,c,d){return c*((a=a/d-1)*a*a+1)+b},easeOutQuart:function(a,b,c,d){return-c*((a=a/d-1)*a*a*a-1)+b},easeOutQuint:function(a,
b,c,d){return c*((a=a/d-1)*a*a*a*a+1)+b},easeOutSine:function(a,b,c,d){return c*Math.sin(a/d*(Math.PI/2))+b},easeOutExpo:function(a,b,c,d){return a==d?b+c:c*(-Math.pow(2,-10*a/d)+1)+b},easeOutCirc:function(a,b,c,d){return c*Math.sqrt(1-(a=a/d-1)*a)+b},easeOutElastic:function(a,b,c,d){var e=4>=arguments.length||void 0===arguments[4]?500:arguments[4];if(0==a)return b;if(1==(a/=d))return b+c;e=d*(1-Math.min(e,999)/1E3);return c*Math.pow(2,-10*a)*Math.sin(2*(a*d-(c<Math.abs(c)?e/4:e/(2*Math.PI)*Math.asin(c/
c)))*Math.PI/e)+c+b},easeOutBack:function(a,b,c,d){return c*((a=a/d-1)*a*(2.70158*a+1.70158)+1)+b},easeOutBounce:function(a,b,c,d){return(a/=d)<1/2.75?7.5625*c*a*a+b:a<2/2.75?c*(7.5625*(a-=1.5/2.75)*a+.75)+b:a<2.5/2.75?c*(7.5625*(a-=2.25/2.75)*a+.9375)+b:c*(7.5625*(a-=2.625/2.75)*a+.984375)+b},easeInOutQuad:function(a,b,c,d){return 1>(a/=d/2)?c/2*a*a+b:-c/2*(--a*(a-2)-1)+b},easeInOutCubic:function(a,b,c,d){return 1>(a/=d/2)?c/2*a*a*a+b:c/2*((a-=2)*a*a+2)+b},easeInOutQuart:function(a,b,c,d){return 1>
(a/=d/2)?c/2*a*a*a*a+b:-c/2*((a-=2)*a*a*a-2)+b},easeInOutQuint:function(a,b,c,d){return 1>(a/=d/2)?c/2*a*a*a*a*a+b:c/2*((a-=2)*a*a*a*a+2)+b},easeInOutSine:function(a,b,c,d){return-c/2*(Math.cos(Math.PI*a/d)-1)+b},easeInOutExpo:function(a,b,c,d){return 0==a?b:a==d?b+c:1>(a/=d/2)?c/2*Math.pow(2,10*(a-1))+b:c/2*(-Math.pow(2,-10*--a)+2)+b},easeInOutCirc:function(a,b,c,d){return 1>(a/=d/2)?-c/2*(Math.sqrt(1-a*a)-1)+b:c/2*(Math.sqrt(1-(a-=2)*a)+1)+b},easeInOutElastic:function(a,b,c,d){var e=4>=arguments.length||
void 0===arguments[4]?500:arguments[4];if(0==a)return b;if(2==(a/=d/2))return b+c;var e=d*(1-Math.min(e,999)/1E3)*1.5,f=c<Math.abs(c)?e/4:e/(2*Math.PI)*Math.asin(c/c);return 1>a?-.5*c*Math.pow(2,10*--a)*Math.sin(2*(a*d-f)*Math.PI/e)+b:c*Math.pow(2,-10*--a)*Math.sin(2*(a*d-f)*Math.PI/e)*.5+c+b},easeInOutBack:function(a,b,c,d){var e=1.70158;return 1>(a/=d/2)?c/2*a*a*(((e*=1.525)+1)*a-e)+b:c/2*((a-=2)*a*(((e*=1.525)+1)*a+e)+2)+b}},w=function(a){return a[0]},F=function(a){return a.reduce(function(a,c){return a.concat(c)})},
m=function(){return Array.prototype.includes?function(a,b){return a.includes(b)}:function(a,b){return a.some(function(a){return a===b})}}(),x=function(a){for(var b=arguments.length,c=Array(1<b?b-1:0),d=1;d<b;d++)c[d-1]=arguments[d];var e=F(c);return a.filter(function(a){return q(m)(e,a)})},G=function(){var a=function(a){var c=new Map;Object.keys(a).forEach(function(d){return c.set(d,a[d])});return c};return function(b){return b instanceof Map?b:a(b)}}(),H=function(a){return/^#/.test(a)},y=function(a){return/^rgb/.test(a)},
I=function(){var a=function(a){return 7>a.length?a.split("").reduce(function(a,b){return a+b+b}):a},b=function(a){return a.match(/[\d\w]{2}/g).map(function(a){return parseInt(a,16)})};return function(c){if(y(c))return c;c=p(a,b)(c);return"rgb("+c[0]+", "+c[1]+", "+c[2]+")"}}(),z=function(a){return J("string"==typeof a?document.querySelectorAll(a):a)},J=function(){var a=[NodeList,HTMLCollection,Set];return function(b){return Array.isArray(b)?b:a.some(function(a){return b instanceof a})?Array.from(b):
b.nodeType?[b]:b.get()}}(),l=new Map;"el delay begin complete loop direction".split(" ").forEach(function(a){return l.set(a,null)});l.set("duration",1E3);l.set("easing","easeOutElastic");var K=function(){var a=Array.from(l.keys()).filter(function(a){return l.get(a)}),b=function(b){return a.every(function(a){return b.has(a)})},c=function(b){var c=new Map(b);a.forEach(function(a){c.has(a)||c.set(a,l.get(a))});return c};return function(a){return b(a)?a:c(a)}}(),L=function(){var a=h(function(a,b){return Array.isArray(a.get(b))}),
b=function(b){return n(b).every(a(b))},c=function(b){return n(b).filter(q(a(b)))};return function(a){if(b(a))return a;var e=new Map(a);c(e).forEach(function(a){return e.set(a,[A.get(a),e.get(a)])});return e}}(),M=function(){var a=function(a){return/\D$/.test(a)},b=h(function(b,c){return a(c)||/scale/.test(b)?c:/rotate|skew/.test(b)?c+"deg":c+"px"}),c=function(b,c){return c.every(function(c){return b.get(c).every(a)})};return function(a){var e=n(a).filter(r);if(c(a,e))return a;var f=new Map(a);e.forEach(function(c){return f.set(c,
a.get(c).map(b(c)))});return f}}(),N=function(){var a=h(function(a,b){return a.get(b).some(H)}),b=function(b){return!B(b).some(a(b))},c=function(b){return B(b).filter(a(b))};return function(a){if(b(a))return a;var e=new Map(a);c(a).forEach(function(a){return e.set(a,e.get(a).map(I))});return e}}(),C=function(a){var b=new Map(a);t(a).forEach(function(a){return b.set(a,b.get(a).slice().reverse())});return b},O=p(G,K,L,M,N,function(a){return(new Map(a)).set("el",z(a.get("el")))},function(a){return"reverse"==
a.get("direction")?C(a):a}),t=function(){var a=Array.from(l.keys()),b=function(b){return q(m)(a,b)};return function(a){return Array.from(a.keys()).filter(b)}}(),Q=function(){var a=p(w,y),b=h(function(b,d){var e=b.get(d).map(P),f=e[0],g=e[1],e=new Map;e.set("prop",d);e.set("from",f);e.set("to",g);e.set("isTransformFunction",r(d));e.set("isColor",a(b.get(d)));/\d$/.test(b.get("easing"))?(f=b.get("easing").split(" "),g=f[1],e.set("easing",f[0]),e.set("frequency",g)):e.set("easing",b.get("easing"));return e});
return function(a,d){return t(a).map(b(a))}}(),n=function(){var a=function(a){return m(u,a)};return function(b){return Array.from(b.keys()).filter(a)}}(),B=function(a){return x(t(a),n(a))},u="opacity translateX translateY scale rotate scaleX scaleY rotateX rotateY perspective skewX skewY translateZ rotateZ scaleZ".split(" "),A=new Map;(function(){var a=["opacity","scale","scaleX","scaleY"];u.forEach(function(b){return A.set(b,m(a,b)?1:0)})})();var r=function(){var a=u.filter(function(a){return"opacity"!=
a});return function(b){return m(a,b)}}(),R=function(a){var b=n(a);if(b.length){var c=[];b.some(r)&&c.push("transform");m(b,"opacity")&&c.push("opacity");var d=c.join();a.get("el").forEach(function(a){a.style.willChange||(a.style.willChange=d)})}},S=function(a,b){return b.reduce(function(b,d,e){return b+a[e-1]+d})},P=function(){var a=/-?\d*\.?\d+/g;return function(b){var c=new Map;c.set("digits",("string"==typeof b?b:String(b)).match(a).map(Number));c.set("others",("string"==typeof b?b:String(b)).split(a));
return c}}(),T=h(function(a,b,c){var d=c.get("to").get("digits").map(function(d,f){var g=c.get("from").get("digits")[f];if(g==d)return g;var D=d-g,g=E[c.get("easing")](b,g,D,a.get("duration"),c.get("frequency"));return c.get("isColor")?Math.round(g):g});return S(d,c.get("to").get("others"))}),U=h(function(a,b){var c=a.get(b.get("prop"));return w(c.slice(-1))}),V=h(function(a,b,c){var d=void 0;a.forEach(function(a,f){a.get("isTransformFunction")?(d||(d=[]),d.push(a.get("prop")+"("+b[f]+")")):"opacity"==
a.get("prop")?c.style.opacity=b[f]:c.setAttribute(a.get("prop"),b[f])});d&&(c.style.transform=d.join(" "))}),W=function(){var a=function(a,c){c.get("begin")&&c.get("begin")(c.get("el"));requestAnimationFrame(a)};return function(b,c){return c.get("delay")?setTimeout(function(){return a(b,c)},c.get("delay")):a(b,c)}}(),X=function(a){return v(function(){if("alternate"==a.get("direction"))return C(a);if("reverse"==a.get("direction")){var b=new Map(a);b["delete"]("direction");return b}return a}())},k=
new Map,Y=function(){var a=0;return function(b){var c=a++;k=(new Map(k)).set(c,b);return c}}(),v=function(a){var b=O(a),c=Q(b),d=Y(b.get("el")),e=new Map;R(b);W(function g(a){if(k.has(d)){e.has("start")||e.set("start",a);e.set("elapsed",a-e.get("start"));a=e.get("elapsed")<b.get("duration");var h=c.map(a?T(b,e.get("elapsed")):U(b));k.get(d).forEach(V(c,h));a?requestAnimationFrame(g):(a=new Map(k),a["delete"](d),k=a,b.get("complete")&&b.get("complete")(b.get("el")),b.get("loop")&&X(b))}},b)};v.stop=
function(a){var b=z(a),c=new Map(k);c.forEach(function(a,e){var f=x(a,b);f.length?c.set(e,f):c["delete"](e)});k=c};return v}();"undefined"!=typeof module&&module.exports&&(module.exports=animate);<commit_msg>Delete animate.min.js<commit_after><|endoftext|>


<commit_before>'use strict';
/**
 * an class to generate options for ECharts
 * @module option-generator
 * @see module:index
 */
const lang = require('zero-lang');
const util = require('../util/index');
const constant = require('../constant');

const MIN_DATA_ZOOM_LIMIT = 20;

const DEFAULT_OPTION = {
  theme: constant.DEFAULT_THEME_NAME,
  tooltip: {},
  toolbox: {
    feature: {},
  },
};

class OptionGenerator {
  constructor(extraOption) {
    const me = this;
    extraOption = extraOption || {};
    me._option = {};
    lang.merge(me._option, DEFAULT_OPTION);
    lang.merge(me._option, util.parseExtraOption(extraOption));
    this._configLayout();
  }

  configCoordinates(coordinates, horizontal, extraOption) {
    const me = this;
    const option = me._option;
    extraOption = extraOption || {};
    let dimensionAxis = 'xAxis';
    let measuresAxis = 'yAxis';
    if (horizontal) {
      const temp = measuresAxis;
      measuresAxis = dimensionAxis;
      dimensionAxis = temp;
    }
    option[dimensionAxis] = lang.extend({
      type: 'category',
      data: coordinates,
    }, extraOption.category);
    option[measuresAxis] = lang.extend({
      type: 'value',
    }, extraOption.value);
    lang.extend(option.xAxis, this._configXLabel(coordinates, option.grid.bottom))
    const coordinatesLen = coordinates.length;
    if (coordinatesLen > MIN_DATA_ZOOM_LIMIT) {
      option.dataZoom = [{
        show: true,
        realtime: true,
        start: 0,
        end: 100 * MIN_DATA_ZOOM_LIMIT / coordinatesLen, // show 20 records
      }];
    }
    return me;
  }

  configMeasures(type, measures, dataSet, extraSeriesOption) {
    const me = this;
    const option = me._option;
    option.series = [];
    lang.each(measures, (measure) => {
      const colInfo = dataSet.colByName[measure];
      option.series.push(lang.extend({
        name: colInfo.comments,
        type,
        data: dataSet.colValuesByName[measure],
      }, extraSeriesOption));
    });
    return me;
  }

  configLegend(colNames, dataSet) {
    const me = this;
    me._option.legend = {
      data: lang.map(colNames, colName => dataSet.colByName[colName].comments)
    };
    return me;
  }

  configLegendByData(data) {
    const me = this;
    me._option.legend = {
      data,
    };
    return me;
  }

  toOption() {
    return this._option;
  }

  _configLayout() {
    switch (this._option.padding) {
      case 'large':
        this._option.grid = {
          left: '12%',
          right: '12%',
          top: 90,
          bottom: 90,
        };
        break;
      case 'small':
        this._option.grid = {
          left: '1%',
          right: '1%',
          containLabel: true,
        };
        break;
      case 'none':
        this._option.grid = {
          left: 0,
          right: 0,
          containLabel: true,
        };
        break;
      default:
        break;
    }
  }

  _configXLabel(data, bottom = 60) {
    const lineWidth = bottom * Math.sqrt(2) - 8;
    return {
      axisLabel: {
        interval: 0,
        rotate: 45,
        formatter: (v) => {
          const defaultWidth = util.guessTextWidth(v);
          if (defaultWidth > lineWidth) {
            const guessLen = Math.floor(lineWidth / 12);
            let start = guessLen;
            let first = v.substr(0, start);
            while (util.guessTextWidth(`${first}...`) <= lineWidth) {
              first += v[start];
              start++;
            }
            return `${first}...`;
            // while (util.guessTextWidth(first) <= lineWidth) {
            //   first += v[start];
            //   start++;
            // }
            // let second = v.slice(start);
            // if (defaultWidth > lineWidth * 2) {
            //   second = v.substr(start, guessLen);
            //   while (util.guessTextWidth(`${second}...`) <= lineWidth) {
            //     second += v[start];
            //     start++;
            //   }
            //   second += '...';
            // }
            // return `${first}\n${second}`;
          }
          return v;
        },
      },
    };
  }
}

module.exports = OptionGenerator;
<commit_msg>eslint<commit_after>'use strict';
/**
 * an class to generate options for ECharts
 * @module option-generator
 * @see module:index
 */
const lang = require('zero-lang');
const util = require('../util/index');
const constant = require('../constant');

const MIN_DATA_ZOOM_LIMIT = 20;

const DEFAULT_OPTION = {
  theme: constant.DEFAULT_THEME_NAME,
  tooltip: {},
  toolbox: {
    feature: {},
  },
};

class OptionGenerator {
  constructor(extraOption) {
    const me = this;
    extraOption = extraOption || {};
    me._option = {};
    lang.merge(me._option, DEFAULT_OPTION);
    lang.merge(me._option, util.parseExtraOption(extraOption));
    this._configLayout();
  }

  configCoordinates(coordinates, horizontal, extraOption) {
    const me = this;
    const option = me._option;
    extraOption = extraOption || {};
    let dimensionAxis = 'xAxis';
    let measuresAxis = 'yAxis';
    if (horizontal) {
      const temp = measuresAxis;
      measuresAxis = dimensionAxis;
      dimensionAxis = temp;
    }
    option[dimensionAxis] = lang.extend({
      type: 'category',
      data: coordinates,
    }, extraOption.category);
    option[measuresAxis] = lang.extend({
      type: 'value',
    }, extraOption.value);
    lang.extend(option.xAxis, this._configXLabel(coordinates, option.grid.bottom));
    const coordinatesLen = coordinates.length;
    if (coordinatesLen > MIN_DATA_ZOOM_LIMIT) {
      option.dataZoom = [{
        show: true,
        realtime: true,
        start: 0,
        end: 100 * MIN_DATA_ZOOM_LIMIT / coordinatesLen, // show 20 records
      }];
    }
    return me;
  }

  configMeasures(type, measures, dataSet, extraSeriesOption) {
    const me = this;
    const option = me._option;
    option.series = [];
    lang.each(measures, (measure) => {
      const colInfo = dataSet.colByName[measure];
      option.series.push(lang.extend({
        name: colInfo.comments,
        type,
        data: dataSet.colValuesByName[measure],
      }, extraSeriesOption));
    });
    return me;
  }

  configLegend(colNames, dataSet) {
    const me = this;
    me._option.legend = {
      data: lang.map(colNames, colName => dataSet.colByName[colName].comments)
    };
    return me;
  }

  configLegendByData(data) {
    const me = this;
    me._option.legend = {
      data,
    };
    return me;
  }

  toOption() {
    return this._option;
  }

  _configLayout() {
    switch (this._option.padding) {
      case 'large':
        this._option.grid = {
          left: '12%',
          right: '12%',
          top: 90,
          bottom: 90,
        };
        break;
      case 'small':
        this._option.grid = {
          left: '1%',
          right: '1%',
          containLabel: true,
        };
        break;
      case 'none':
        this._option.grid = {
          left: 0,
          right: 0,
          containLabel: true,
        };
        break;
      default:
        break;
    }
  }

  _configXLabel(data, bottom = 60) {
    const lineWidth = bottom * Math.sqrt(2) - 8;
    return {
      axisLabel: {
        interval: 0,
        rotate: 45,
        formatter: (v) => {
          const defaultWidth = util.guessTextWidth(v);
          if (defaultWidth > lineWidth) {
            const guessLen = Math.floor(lineWidth / 12);
            let start = guessLen;
            let first = v.substr(0, start);
            while (util.guessTextWidth(`${first}...`) <= lineWidth) {
              first += v[start];
              start++;
            }
            return `${first}...`;
            // while (util.guessTextWidth(first) <= lineWidth) {
            //   first += v[start];
            //   start++;
            // }
            // let second = v.slice(start);
            // if (defaultWidth > lineWidth * 2) {
            //   second = v.substr(start, guessLen);
            //   while (util.guessTextWidth(`${second}...`) <= lineWidth) {
            //     second += v[start];
            //     start++;
            //   }
            //   second += '...';
            // }
            // return `${first}\n${second}`;
          }
          return v;
        },
      },
    };
  }
}

module.exports = OptionGenerator;
<|endoftext|>


<commit_before>const Discord = require('discord.js');
module.exports = async function (message, hotline){
    let me = message.embeds.shift();
    const embed = new Discord.RichEmbed();
    if(message.webhookID || message.webhookID !=null)
    {
        const webhook = await message.fetchWebhook();
        embed.setAuthor(webhook.name, webhook.avatar)
    }
    else
    {
        embed.setAuthor(message.author.username, message.author.avatarURL)
    }
    if (!hotline)
        embed.setTitle((message.guild)?(message.channel.name + " in " + message.guild.name):(" in a DM with " + message.channel.recipient.tag));
    if (!message.content.trim() == "")
        embed.setDescription(message.cleanContent);
    else
        embed.setDescription("**" + (( me.author && me.author!=null)?(me.author.name+ " : "):"")  + *(me.title && me.title!=null)?me.title:"") + "**\n	" + (me.description || "") + ((me.footer) ? ("\n*At " + me.footer.text + "*") : ""));
    if (!hotline)
        embed.setTimestamp(message.createdAt)
    let color = message.member;
    if (color) {
        color = color.roles.filter((r) => {
            if (r.color !== 0)
                return r;
        }).array().sort((r1, r2) => {
            if (r1.position < r2.position)
                return 1;
        }).shift();
        if (color)
            color = color.color;
    }
    else
        color = 0;
    embed.setColor(color);
    //embed.setTitle("Quote");
    return embed;
}
<commit_msg>fix typo<commit_after>const Discord = require('discord.js');
module.exports = async function (message, hotline){
    let me = message.embeds.shift();
    const embed = new Discord.RichEmbed();
    if(message.webhookID || message.webhookID !=null)
    {
        const webhook = await message.fetchWebhook();
        embed.setAuthor(webhook.name, webhook.avatar)
    }
    else
    {
        embed.setAuthor(message.author.username, message.author.avatarURL)
    }
    if (!hotline)
        embed.setTitle((message.guild)?(message.channel.name + " in " + message.guild.name):(" in a DM with " + message.channel.recipient.tag));
    if (!message.content.trim() == "")
        embed.setDescription(message.cleanContent);
    else
        embed.setDescription("**" + (( me.author && me.author!=null)?(me.author.name+ " : "):"")  + (me.title && me.title!=null)?me.title:"") + "**\n	" + (me.description || "") + ((me.footer) ? ("\n*At " + me.footer.text + "*") : ""));
    if (!hotline)
        embed.setTimestamp(message.createdAt)
    let color = message.member;
    if (color) {
        color = color.roles.filter((r) => {
            if (r.color !== 0)
                return r;
        }).array().sort((r1, r2) => {
            if (r1.position < r2.position)
                return 1;
        }).shift();
        if (color)
            color = color.color;
    }
    else
        color = 0;
    embed.setColor(color);
    //embed.setTitle("Quote");
    return embed;
}
<|endoftext|>


<commit_before>(function (Math) {
    /*global CryptoJS:true */

    'use strict';

    // Shortcuts
    var C = CryptoJS;
    var C_LIB = C.lib;
    var WordArray = C_LIB.WordArray;
    var Hasher = C_LIB.Hasher;
    var C_X64 = C.x64;
    var X64Word = C_X64.Word;

    // Constants tables
    var RHO_OFFSETS = [];
    var PI_INDEXES  = [];
    var ROUND_CONSTANTS = [];

    // Compute Constants
    (function () {
        // Compute rho offset constants
        var x = 1, y = 0;
        for (var t = 0; t < 24; t++) {
            RHO_OFFSETS[x + 5 * y] = ((t + 1) * (t + 2) / 2) % 64;

            var newX = y % 5;
            var newY = (2 * x + 3 * y) % 5;
            x = newX;
            y = newY;
        }

        // Compute pi index constants
        for (var x = 0; x < 5; x++) {
            for (var y = 0; y < 5; y++) {
                PI_INDEXES[x + 5 * y] = y + ((2 * x + 3 * y) % 5) * 5;
            }
        }

        // Compute round constants
        var LFSR = 0x01;
        for (var i = 0; i < 24; i++) {
            var roundConstantMsw = 0;
            var roundConstantLsw = 0;

            for (var j = 0; j < 7; j++) {
                if (LFSR & 0x01) {
                    var bitPosition = (1 << j) - 1;
                    if (bitPosition < 32) {
                        roundConstantLsw ^= 1 << bitPosition;
                    } else /* if (bitPosition >= 32) */ {
                        roundConstantMsw ^= 1 << (bitPosition - 32);
                    }
                }

                // Compute next LFSR
                if (LFSR & 0x80) {
                    // Primitive polynomial over GF(2): x^8 + x^6 + x^5 + x^4 + 1
                    LFSR = (LFSR << 1) ^ 0x71;
                } else {
                    LFSR <<= 1;
                }
            }

            ROUND_CONSTANTS[i] = new X64Word(roundConstantMsw, roundConstantLsw);
        }
    }());

    // Reusable objects for temporary values
    var T = [];
    (function () {
        for (var i = 0; i < 25; i++) {
            T[i] = new X64Word();
        }
    }());

    /**
     * SHA-3 hash algorithm.
     */
    var SHA3 = C.SHA3 = Hasher.extend({
        /**
         * Configuration options.
         *
         * @property {number} outputLength
         *   The desired number of bits in the output hash.
         *   The output length must be one of 224, 256, 384, or 512.
         *   Default: 512
         */
        cfg: Hasher.prototype.cfg.extend({
            outputLength: 512
        }),

        _doInit: function () {
            // <?php if ($debug): ?>
            {
                var outputLength = this.cfg.outputLength;
                if (
                    outputLength !== 224 && outputLength !== 256 &&
                    outputLength !== 384 && outputLength !== 512
                ) {
                    throw new OutputLengthError('The output length must be one of 224, 256, 384, or 512.');
                }
            }
            // <?php endif ?>
        },

        _doReset: function () {
            var state = this._state = []
            for (var i = 0; i < 25; i++) {
                state[i] = new X64Word();
            }

            this.blockSize = (1600 - 2 * this.cfg.outputLength) / 32;
        },

        _doProcessBlock: function (M) {
            // Shortcuts
            var state = this._state;
            var nBlockSizeLanes = this.blockSize / 2;

            // Absorb
            for (var i = 0; i < nBlockSizeLanes; i++) {
                // Shortcuts
                var M2i  = M[2 * i];
                var M2i1 = M[2 * i + 1];

                // Swap endian
                M2i = (
                    (((M2i << 8)  | (M2i >>> 24)) & 0x00ff00ff) |
                    (((M2i << 24) | (M2i >>> 8))  & 0xff00ff00)
                );
                M2i1 = (
                    (((M2i1 << 8)  | (M2i1 >>> 24)) & 0x00ff00ff) |
                    (((M2i1 << 24) | (M2i1 >>> 8))  & 0xff00ff00)
                );

                // Absorb message into state
                var lane = state[i];
                lane.msw ^= M2i1;
                lane.lsw ^= M2i;
            }

            // Rounds
            for (var round = 0; round < 24; round++) {
                // Theta
                for (var x = 0; x < 5; x++) {
                    // Mix column lanes
                    var tMsw = 0, tLsw = 0;
                    for (var y = 0; y < 5; y++) {
                        var lane = state[x + 5 * y];
                        tMsw ^= lane.msw;
                        tLsw ^= lane.lsw;
                    }

                    // Temporary values
                    var Tx = T[x];
                    Tx.msw = tMsw;
                    Tx.lsw = tLsw;
                }
                for (var x = 0; x < 5; x++) {
                    // Shortcuts
                    var Tx4 = T[(x + 4) % 5];
                    var Tx1 = T[(x + 1) % 5];
                    var Tx1Msw = Tx1.msw;
                    var Tx1Lsw = Tx1.lsw;

                    // Mix surrounding columns
                    var tMsw = Tx4.msw ^ ((Tx1Msw << 1) | (Tx1Lsw >>> 31));
                    var tLsw = Tx4.lsw ^ ((Tx1Lsw << 1) | (Tx1Msw >>> 31));
                    for (var y = 0; y < 5; y++) {
                        var lane = state[x + 5 * y];
                        lane.msw ^= tMsw;
                        lane.lsw ^= tLsw;
                    }
                }

                // Rho Pi
                for (var laneIndex = 1; laneIndex < 25; laneIndex++) {
                    // Shortcuts
                    var lane = state[laneIndex];
                    var laneMsw = lane.msw;
                    var laneLsw = lane.lsw;
                    var rhoOffset = RHO_OFFSETS[laneIndex];

                    // Rotate lanes
                    if (rhoOffset < 32) {
                        var tMsw = (laneMsw << rhoOffset) | (laneLsw >>> (32 - rhoOffset));
                        var tLsw = (laneLsw << rhoOffset) | (laneMsw >>> (32 - rhoOffset));
                    } else /* if (rhoOffset >= 32) */ {
                        var tMsw = (laneLsw << (rhoOffset - 32)) | (laneMsw >>> (64 - rhoOffset));
                        var tLsw = (laneMsw << (rhoOffset - 32)) | (laneLsw >>> (64 - rhoOffset));
                    }

                    // Transpose lanes
                    var TPiLane = T[PI_INDEXES[laneIndex]];
                    TPiLane.msw = tMsw;
                    TPiLane.lsw = tLsw;
                }

                // Rho pi at x = y = 0
                var T0 = T[0];
                var state0 = state[0];
                T0.msw = state0.msw;
                T0.lsw = state0.lsw;

                // Chi
                for (var x = 0; x < 5; x++) {
                    for (var y = 0; y < 5; y++) {
                        // Shortcuts
                        var laneIndex = x + 5 * y;
                        var lane = state[laneIndex];
                        var TLane = T[laneIndex];
                        var Tx1Lane = T[((x + 1) % 5) + 5 * y];
                        var Tx2Lane = T[((x + 2) % 5) + 5 * y];

                        // Mix rows
                        lane.msw = TLane.msw ^ (~Tx1Lane.msw & Tx2Lane.msw);
                        lane.lsw = TLane.lsw ^ (~Tx1Lane.lsw & Tx2Lane.lsw);
                    }
                }

                // Iota
                var lane = state[0];
                var roundConstant = ROUND_CONSTANTS[round];
                lane.msw ^= roundConstant.msw;
                lane.lsw ^= roundConstant.lsw;
            }
        },

        _doFinalize: function () {
            // Shortcuts
            var data = this._data;
            var dataWords = data.words;
            var nBitsTotal = this._nDataBytes * 8;
            var nBitsLeft = data.sigBytes * 8;
            var blockSizeBits = this.blockSize * 32;

            // Add padding
            dataWords[nBitsLeft >>> 5] |= 0x1 << (24 - nBitsLeft % 32);
            dataWords[((Math.ceil((nBitsLeft + 1) / blockSizeBits) * blockSizeBits) >>> 5) - 1] |= 0x80;
            data.sigBytes = dataWords.length * 4;

            // Hash final blocks
            this._process();

            // Shortcuts
            var state = this._state;
            var outputLengthBytes = this.cfg.outputLength / 8;
            var outputLengthLanes = outputLengthBytes / 8;

            // Squeeze
            var hashWords = [];
            for (var i = 0; i < outputLengthLanes; i++) {
                // Shortcuts
                var lane = state[i];
                var laneMsw = lane.msw;
                var laneLsw = lane.lsw;

                // Swap endian
                laneMsw = (
                    (((laneMsw << 8)  | (laneMsw >>> 24)) & 0x00ff00ff) |
                    (((laneMsw << 24) | (laneMsw >>> 8))  & 0xff00ff00)
                );
                laneLsw = (
                    (((laneLsw << 8)  | (laneLsw >>> 24)) & 0x00ff00ff) |
                    (((laneLsw << 24) | (laneLsw >>> 8))  & 0xff00ff00)
                );

                // Squeeze state to retrieve hash
                hashWords.push(laneLsw);
                hashWords.push(laneMsw);
            }

            // Return final computed hash
            return new WordArray(hashWords, outputLengthBytes);
        },

        clone: function () {
            var clone = SHA3.$super.prototype.clone.call(this);

            var state = clone._state = clone._state.slice(0);
            for (var i = 0; i < 25; i++) {
                state[i] = state[i].clone();
            }

            return clone;
        }
    });

    // <?php if ($debug): ?>
    {
        // Shortcut
        var C_ERR = C.err;

        /**
         * Output length error.
         */
        var OutputLengthError = C_ERR.OutputLengthError = Hasher.extend.call(Error);
    }
    // <?php endif ?>
}(Math));
<commit_msg>Optimized performance, and other cosmetic changes.<commit_after>(function (Math) {
    /*global CryptoJS:true */

    'use strict';

    // Shortcuts
    var C = CryptoJS;
    var C_LIB = C.lib;
    var WordArray = C_LIB.WordArray;
    var Hasher = C_LIB.Hasher;

    // Constants tables
    var RHO_OFFSETS = [];
    var PI_INDEXES  = [];
    var ROUND_CONSTANTS = [];

    // Compute rho offsets
    (function () {
        var x = 1;
        var y = 0;
        for (var t = 0; t < 24; t++) {
            RHO_OFFSETS[(x + 5 * y) * 2] = ((t + 1) * (t + 2) / 2) % 64;

            var newX = y % 5;
            var newY = (2 * x + 3 * y) % 5;
            x = newX;
            y = newY;
        }
    }());

    // Compute pi indexes
    (function () {
        for (var x = 0; x < 5; x++) {
            for (var y = 0; y < 5; y++) {
                PI_INDEXES[(x + 5 * y) * 2] = (y + ((2 * x + 3 * y) % 5) * 5) * 2;
            }
        }
    }());

    // Compute round constants
    (function () {
        var lfsr = 0x01;
        for (var i = 0; i < 48; i += 2) {
            var roundConstantMsw = 0;
            var roundConstantLsw = 0;

            for (var j = 0; j < 7; j++) {
                if (lfsr & 0x01) {
                    var bitPosition = (1 << j) - 1;
                    if (bitPosition < 32) {
                        roundConstantLsw ^= 1 << bitPosition;
                    } else {
                        roundConstantMsw ^= 1 << (bitPosition - 32);
                    }
                }

                // Compute next LFSR
                if (lfsr & 0x80) {
                    // Primitive polynomial over GF(2)
                    // x^8 + x^6 + x^5 + x^4 + 1
                    lfsr = (lfsr << 1) ^ 0x71;
                } else {
                    lfsr <<= 1;
                }
            }

            ROUND_CONSTANTS[i] = roundConstantMsw;
            ROUND_CONSTANTS[i + 1] = roundConstantLsw;
        }
    }());

    // Reusable objects for temporary values
    var T = [];

    /**
     * SHA-3 hash algorithm.
     */
    var SHA3 = C.SHA3 = Hasher.extend({
        /**
         * Configuration options.
         *
         * @property {number} outputLength
         *   The desired number of bits in the output hash.
         *   The output length must be one of 224, 256, 384, or 512.
         *   Default: 512
         */
        cfg: Hasher.prototype.cfg.extend({
            outputLength: 512
        }),

        _doInit: function () {
            // Shortcut
            var outputLength = this.cfg.outputLength;

            // <?php if ($debug): ?>
            {
                if (
                    outputLength !== 224 && outputLength !== 256 &&
                    outputLength !== 384 && outputLength !== 512
                ) {
                    throw new OutputLengthError('The output length must be one of 224, 256, 384, or 512.');
                }
            }
            // <?php endif ?>

            this.blockSize = 50 - outputLength / 16;
        },

        _doReset: function () {
            this._state = [
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0
            ];
        },

        _doProcessBlock: function (m) {
            // Shortcut
            var s = this._state;

            // Absorb
            var blockSize = this.blockSize;
            for (var i = 0; i < blockSize; i += 2) {
                // Shortcuts
                var mi  = m[i];
                var mi1 = m[i + 1];

                // Swap endian
                mi = (
                    (((mi << 8)  | (mi >>> 24)) & 0x00ff00ff) |
                    (((mi << 24) | (mi >>> 8))  & 0xff00ff00)
                );
                mi1 = (
                    (((mi1 << 8)  | (mi1 >>> 24)) & 0x00ff00ff) |
                    (((mi1 << 24) | (mi1 >>> 8))  & 0xff00ff00)
                );

                // Absorb message into state
                s[i] ^= mi1;
                s[i + 1] ^= mi;
            }

            // Rounds
            for (var round = 0; round < 48; round += 2) {
                // Theta
                for (var x = 0; x < 5; x++) {
                    // Mix column lanes
                    var tMsw = 0;
                    var tLsw = 0;
                    for (var y = 0; y < 5; y++) {
                        var laneIndex = (x + 5 * y) * 2;
                        tMsw ^= s[laneIndex];
                        tLsw ^= s[laneIndex + 1];
                    }

                    // Temporary values
                    var TIndex = x * 2;
                    T[TIndex] = tMsw;
                    T[TIndex + 1] = tLsw;
                }
                for (var x = 0; x < 5; x++) {
                    // Shortcuts
                    var Tx1Index = ((x + 1) % 5) * 2;
                    var Tx4Index = ((x + 4) % 5) * 2;
                    var Tx1Msw = T[Tx1Index];
                    var Tx1Lsw = T[Tx1Index + 1];
                    var Tx4Msw = T[Tx4Index];
                    var Tx4Lsw = T[Tx4Index + 1];

                    // Mix surrounding columns
                    var tMsw = Tx4Msw ^ ((Tx1Msw << 1) | (Tx1Lsw >>> 31));
                    var tLsw = Tx4Lsw ^ ((Tx1Lsw << 1) | (Tx1Msw >>> 31));
                    for (var y = 0; y < 5; y++) {
                        var laneIndex = (x + 5 * y) * 2;
                        s[laneIndex] ^= tMsw;
                        s[laneIndex + 1] ^= tLsw;
                    }
                }

                // Rho Pi
                T[0] = s[0];
                T[1] = s[1];
                for (var laneIndex = 2; laneIndex < 50; laneIndex += 2) {
                    // Shortcuts
                    var laneMsw = s[laneIndex];
                    var laneLsw = s[laneIndex + 1];
                    var rhoOffset = RHO_OFFSETS[laneIndex];
                    var piIndex = PI_INDEXES[laneIndex];

                    // Rotate lanes
                    if (rhoOffset < 32) {
                        var tMsw = (laneMsw << rhoOffset) | (laneLsw >>> (32 - rhoOffset));
                        var tLsw = (laneLsw << rhoOffset) | (laneMsw >>> (32 - rhoOffset));
                    } else {
                        var tMsw = (laneLsw << (rhoOffset - 32)) | (laneMsw >>> (64 - rhoOffset));
                        var tLsw = (laneMsw << (rhoOffset - 32)) | (laneLsw >>> (64 - rhoOffset));
                    }

                    // Transpose lanes
                    T[piIndex] = tMsw;
                    T[piIndex + 1] = tLsw;
                }

                // Chi
                for (var x = 0; x < 5; x++) {
                    for (var y = 0; y < 5; y++) {
                        // Shortcuts
                        var laneIndex = (x + 5 * y) * 2;
                        var Tx1LaneIndex = (((x + 1) % 5) + 5 * y) * 2;
                        var Tx2LaneIndex = (((x + 2) % 5) + 5 * y) * 2;

                        // Mix rows
                        s[laneIndex]     = T[laneIndex]     ^ (~T[Tx1LaneIndex]     & T[Tx2LaneIndex]);
                        s[laneIndex + 1] = T[laneIndex + 1] ^ (~T[Tx1LaneIndex + 1] & T[Tx2LaneIndex + 1]);
                    }
                }

                // Iota
                s[0] ^= ROUND_CONSTANTS[round];
                s[1] ^= ROUND_CONSTANTS[round + 1];
            }
        },

        _doFinalize: function () {
            // Shortcuts
            var data = this._data;
            var dataWords = data.words;
            var nBitsLeft = data.sigBytes * 8;
            var blockSizeBits = this.blockSize * 32;

            // Add padding
            dataWords[nBitsLeft >>> 5] |= 0x1 << (24 - nBitsLeft % 32);
            dataWords[((Math.ceil((nBitsLeft + 1) / blockSizeBits) * blockSizeBits) >>> 5) - 1] |= 0x80;
            data.sigBytes = dataWords.length * 4;

            // Hash final blocks
            this._process();

            // Shortcuts
            var s = this._state;
            var outputLengthBytes = this.cfg.outputLength / 8;
            var outputLengthLanes = outputLengthBytes / 8;

            // Squeeze
            var hashWords = [];
            for (var i = 0; i < outputLengthLanes; i++) {
                // Shortcuts
                var laneMsw = s[i * 2];
                var laneLsw = s[i * 2 + 1];

                // Swap endian
                laneMsw = (
                    (((laneMsw << 8)  | (laneMsw >>> 24)) & 0x00ff00ff) |
                    (((laneMsw << 24) | (laneMsw >>> 8))  & 0xff00ff00)
                );
                laneLsw = (
                    (((laneLsw << 8)  | (laneLsw >>> 24)) & 0x00ff00ff) |
                    (((laneLsw << 24) | (laneLsw >>> 8))  & 0xff00ff00)
                );

                // Squeeze state to retrieve hash
                hashWords.push(laneLsw);
                hashWords.push(laneMsw);
            }

            // Return final computed hash
            return new WordArray(hashWords, outputLengthBytes);
        },

        clone: function () {
            var clone = SHA3.$super.prototype.clone.call(this);
            clone._state = clone._state.slice(0);

            return clone;
        },

        // The block size varies depending on the output length
        blockSize: null
    });

    // <?php if ($debug): ?>
    {
        // Shortcut
        var C_ERR = C.err;

        /**
         * Output length error.
         */
        var OutputLengthError = C_ERR.OutputLengthError = Hasher.extend.call(Error);
    }
    // <?php endif ?>
}(Math));
<|endoftext|>


<commit_before>/*jslint node: true */
/* global -Promise */

"use strict";

var url = require('url'),
  _ = require('lodash'),
  request = require('../utils/request'),
  cheerio = require('cheerio'),
  microformats = require('microformat-node'),
  Promise = require('promise'),
  moment = require('moment'),
  knex = require('../knex'),
  options = require('../config'),
  urlTools = require('../utils/url-tools'),
  throttledKeys = {},
  throttleTimestamp,
  throttleByKey,
  WebMentionPing;

//TODO: Sync throttle between workers
//TODO: Ensure that the throttle persists between reboots of the app (enable eg. quick shutdowns on SIGTERM)
throttleByKey = function (key, onWaitCallback) {
  return new Promise(function (resolve, reject) {
    var now = Date.now(), time = Math.floor(now / options.throttleSpan), wait, err;

    if (
      (options.env === 'development' || options.env === 'test') &&
      !options.dev.throttling &&
      (key === '127.0.0.1' || key === 'localhost' || key === 'example.com')
    ) {
      // Don't throttle local pings in development and test environments
    } else if (throttleTimestamp !== time) {
      // Divide the time into timeslots and open up new requests whenever there's a new timeslot
      throttleTimestamp = time;
      throttledKeys = {};
    } else if (throttledKeys[key]) {
      if (!options.throttleCap || throttledKeys[key] < options.throttleCap) {
        // We're throttled – lets wait and try again when there's a new timeslot!

        //TODO: What is happening here? The throttleSpan tells the minimum gap between each request – yet all requests are scheduled to practially retry at the same time? Is that because we need to reinitialize throttledKeys[key] variable on each timeSpan and therefore need all requests to reregister again? Surely there can be a more elegant solution to that.
        wait = (time + 1) * options.throttleSpan - now + throttledKeys[key];
        console.log('Throttled! In', wait, 'milliseconds retrying key:', key);

        setTimeout(function () {
          throttleByKey(key).then(resolve, reject);
        }, wait);
        throttledKeys[key] += 1;

        if (onWaitCallback) {
          // If we need to wait, then we might want to eg. respond to the browser early
          onWaitCallback();
        }

        return;
      } else {
        // We're throttled – but there's too many waiting already so we won't get in line

        console.log('Reached throttle cap of', options.throttleCap, 'for key:', key);
        err = new Error('Too many mentions from source host at the moment.');
        err.status = 503;
        // Suggest that they ping again once we have emptied the queue so that we can fill it back up again
        // The requests will get throttled right away if they do this, but it will ensure that we utilize
        // the defined throttleSpan to the fullest and never stand idle for a millisecond.
        //
        // This should perhaps be set to just the time when there's any free spot in the queue at all, even
        // if that would mean that the throttleCap would be reached immediately again. Would follow the spec
        // more closely though: http://tools.ietf.org/html/rfc7231#section-7.1.3
        err.retryAfter = Math.ceil(((time + throttledKeys[key] - 1) * options.throttleSpan - now) / 1000);
        reject(err);

        return;
      }
    }

    // Nothing is stopping us – all lights are green – go go go!
    throttledKeys[key] = 1;
    resolve();
  });
};

WebMentionPing = function (source, target) {
  this.source = source;
  this.target = target;

  this.normalizedSource = urlTools.normalizeUrl(source);
  this.normalizedTarget = urlTools.normalizeUrl(target);

  this.sourceHost = url.parse(this.normalizedSource).hostname;
  this.targetHost = url.parse(this.normalizedTarget).hostname;
};

WebMentionPing.prototype.interactionMapping = {
  'reply': 'in-reply-to',
  'like': 'like-of',
  'repost': 'repost-of',
};

WebMentionPing.prototype.isValidTarget = function () {
  return knex('sites').first('aid').where('hostname', this.targetHost).then(function (row) {
    if (!row) {
      console.log('Invalid target site:', this.targetHost);

      var err = new Error('Invalid target site');
      err.status = 400;
      throw(err);
    }
  }.bind(this));
};

WebMentionPing.prototype.throttleFetches = function (onWaitCallback) {
  return throttleByKey(this.sourceHost, onWaitCallback);
};

WebMentionPing.prototype.fetchSourcePage = function () {
  return new Promise(function (resolve, reject) {
    request({ uri: this.source }, function (err, response, body) {
      if (err) {
        reject(err);
      } else if (response.statusCode !== 200) {
        reject(new Error('Expected HTTP code 200'));
      } else {
        resolve(body);
      }
    });
  }.bind(this));
};

WebMentionPing.prototype.parseSourcePage = function (body) {
  var $, links, linked, i, length, href, sourceParts;

  $ = cheerio.load(body);
  links = $('a');
  linked = false;

  for (i = 0, length = links.length; i < length; i += 1) {
    href = links.eq(i).attr('href');
    try {
      if (href && urlTools.normalizeUrl(href, { relativeTo: this.normalizedSource }) === this.normalizedTarget) {
        linked = true;
        break;
      }
    } catch (e) {}
  }

  if (!linked) {
    console.log("Couldn't find a link from source to target", this.source, this.target);
    return Promise.reject(new Error("Couldn't find a link from source to target (" + this.target + ")"));
  } else {
    sourceParts = url.parse(this.source);
    return Promise.denodeify(microformats.parseDom)($, $.root(), {
      filters : ['h-entry'],
      logLevel : 0,
      baseUrl: sourceParts.protocol + '//' + sourceParts.host + sourceParts.path,
    });
  }
};

WebMentionPing.prototype.createMention = function (data) {
  var entry = {},
    now = Date.now(),
    item,
    author,
    published;

  if (data.items.length) {
    item = data.items[0].properties;
    author = item.author ? item.author[0].properties : {};
    if (item.published) {
      published = moment.utc(item.published[0]);
      published = published.isValid() ? published.valueOf() : null;
    } else {
      published = null;
    }

    entry.url = item.url ? url.resolve(this.source, item.url[0]) : this.source;
    entry.name = item.name ? item.name[0] : null;
    entry.published = published || now;
    entry.summary = item.summary ?
      item.summary[0] :
      (item.content ? item.content[0].value : null);

    entry.author = {
      name : author.name ? author.name[0] : null,
      photo : author.photo ? url.resolve(this.source, author.photo[0]) : null,
      url : author.url ? url.resolve(this.source, author.url[0]) : null
    };

    // Find the kind of interaction this is and what the interaction targets are. Stop at first find.
    _.some(this.interactionMapping, function (key, type) {
      if (item[key] && item[key].length) {
        entry.interactionType = type;
        entry.interactions = item[key];
        return true;
      }
    });

    if (entry.name === entry.summary) {
      entry.name = null;
    }

    if (entry.summary && entry.summary.length > 512) {
      entry.summary = entry.summary.substr(0, 512);
    }

    if (entry.interactions) {
      entry.interactions = _.compact(_.uniq(_.map(entry.interactions, function (target) {
        if (_.isObject(target) && target.properties.url) {
          target = target.properties.url[0];
        } else if (!_.isString(target)) {
          return null;
        }
        return url.resolve(this.source, target);
      }.bind(this))));
    }

    // XSS protection, stop non-HTTP protocols
    if (!urlTools.isHttpUrl.test(entry.url)) {
      entry.url = this.source;
    }
    if (!urlTools.isHttpUrl.test(entry.author.url)) {
      entry.author.url = null;
    }
  } else {
    entry.published = now;
  }

  return {
    url : this.source,
    normalizedUrl : this.normalizedSource,
    published : new Date(entry.published),
    fetched : new Date(now),
    type: entry.interactionType || null,
    data : entry,
    raw : data
  };
};

WebMentionPing.prototype.saveMention = function (mention) {
  return knex
    .table('entries').pluck('id').where('normalizedUrl', mention.normalizedUrl)
    .then(function (id) {
      return id.length ? id : knex.table('entries').insert(mention, 'id');
    })
    .catch(function (err) {
      // Unique constraint error
      if (err.code === '23505') {
        return knex.table('entries').pluck('id').where('normalizedUrl', mention.normalizedUrl);
      } else {
        throw err;
      }
    })
    .then(function (id) {
      return Promise.all([

        knex.table('mentions').insert({
          url : this.target,
          normalizedUrl : this.normalizedTarget,
          interaction : mention.type && _.some(mention.data.interactions, function (target) {
            return urlTools.normalizeUrl(target, { relativeTo: this.normalizedSource }) === this.normalizedTarget;
          }.bind(this)) ? true : false,
          eid : id[0],
          hostname : this.targetHost
        }),

        knex.table('sites').update('lastmention', knex.fn.now()).where('hostname', this.targetHost)

      ]).then(function () {
        knex.raw("NOTIFY mentions, '" + JSON.stringify({
          eid : id[0],
        }) + "'").exec();
      }.bind(this));
    }.bind(this));
};

module.exports = WebMentionPing;
<commit_msg>fix(parser): ignore duplicate WebMention pings<commit_after>/*jslint node: true */
/* global -Promise */

"use strict";

var url = require('url'),
  _ = require('lodash'),
  request = require('../utils/request'),
  cheerio = require('cheerio'),
  microformats = require('microformat-node'),
  Promise = require('promise'),
  moment = require('moment'),
  knex = require('../knex'),
  options = require('../config'),
  urlTools = require('../utils/url-tools'),
  throttledKeys = {},
  throttleTimestamp,
  throttleByKey,
  WebMentionPing;

//TODO: Sync throttle between workers
//TODO: Ensure that the throttle persists between reboots of the app (enable eg. quick shutdowns on SIGTERM)
throttleByKey = function (key, onWaitCallback) {
  return new Promise(function (resolve, reject) {
    var now = Date.now(), time = Math.floor(now / options.throttleSpan), wait, err;

    if (
      (options.env === 'development' || options.env === 'test') &&
      !options.dev.throttling &&
      (key === '127.0.0.1' || key === 'localhost' || key === 'example.com')
    ) {
      // Don't throttle local pings in development and test environments
    } else if (throttleTimestamp !== time) {
      // Divide the time into timeslots and open up new requests whenever there's a new timeslot
      throttleTimestamp = time;
      throttledKeys = {};
    } else if (throttledKeys[key]) {
      if (!options.throttleCap || throttledKeys[key] < options.throttleCap) {
        // We're throttled – lets wait and try again when there's a new timeslot!

        //TODO: What is happening here? The throttleSpan tells the minimum gap between each request – yet all requests are scheduled to practially retry at the same time? Is that because we need to reinitialize throttledKeys[key] variable on each timeSpan and therefore need all requests to reregister again? Surely there can be a more elegant solution to that.
        wait = (time + 1) * options.throttleSpan - now + throttledKeys[key];
        console.log('Throttled! In', wait, 'milliseconds retrying key:', key);

        setTimeout(function () {
          throttleByKey(key).then(resolve, reject);
        }, wait);
        throttledKeys[key] += 1;

        if (onWaitCallback) {
          // If we need to wait, then we might want to eg. respond to the browser early
          onWaitCallback();
        }

        return;
      } else {
        // We're throttled – but there's too many waiting already so we won't get in line

        console.log('Reached throttle cap of', options.throttleCap, 'for key:', key);
        err = new Error('Too many mentions from source host at the moment.');
        err.status = 503;
        // Suggest that they ping again once we have emptied the queue so that we can fill it back up again
        // The requests will get throttled right away if they do this, but it will ensure that we utilize
        // the defined throttleSpan to the fullest and never stand idle for a millisecond.
        //
        // This should perhaps be set to just the time when there's any free spot in the queue at all, even
        // if that would mean that the throttleCap would be reached immediately again. Would follow the spec
        // more closely though: http://tools.ietf.org/html/rfc7231#section-7.1.3
        err.retryAfter = Math.ceil(((time + throttledKeys[key] - 1) * options.throttleSpan - now) / 1000);
        reject(err);

        return;
      }
    }

    // Nothing is stopping us – all lights are green – go go go!
    throttledKeys[key] = 1;
    resolve();
  });
};

WebMentionPing = function (source, target) {
  this.source = source;
  this.target = target;

  this.normalizedSource = urlTools.normalizeUrl(source);
  this.normalizedTarget = urlTools.normalizeUrl(target);

  this.sourceHost = url.parse(this.normalizedSource).hostname;
  this.targetHost = url.parse(this.normalizedTarget).hostname;
};

WebMentionPing.prototype.interactionMapping = {
  'reply': 'in-reply-to',
  'like': 'like-of',
  'repost': 'repost-of',
};

WebMentionPing.prototype.isValidTarget = function () {
  return knex('sites').first('aid').where('hostname', this.targetHost).then(function (row) {
    if (!row) {
      console.log('Invalid target site:', this.targetHost);

      var err = new Error('Invalid target site');
      err.status = 400;
      throw(err);
    }
  }.bind(this));
};

WebMentionPing.prototype.throttleFetches = function (onWaitCallback) {
  return throttleByKey(this.sourceHost, onWaitCallback);
};

WebMentionPing.prototype.fetchSourcePage = function () {
  return new Promise(function (resolve, reject) {
    request({ uri: this.source }, function (err, response, body) {
      if (err) {
        reject(err);
      } else if (response.statusCode !== 200) {
        reject(new Error('Expected HTTP code 200'));
      } else {
        resolve(body);
      }
    });
  }.bind(this));
};

WebMentionPing.prototype.parseSourcePage = function (body) {
  var $, links, linked, i, length, href, sourceParts;

  $ = cheerio.load(body);
  links = $('a');
  linked = false;

  for (i = 0, length = links.length; i < length; i += 1) {
    href = links.eq(i).attr('href');
    try {
      if (href && urlTools.normalizeUrl(href, { relativeTo: this.normalizedSource }) === this.normalizedTarget) {
        linked = true;
        break;
      }
    } catch (e) {}
  }

  if (!linked) {
    console.log("Couldn't find a link from source to target", this.source, this.target);
    return Promise.reject(new Error("Couldn't find a link from source to target (" + this.target + ")"));
  } else {
    sourceParts = url.parse(this.source);
    return Promise.denodeify(microformats.parseDom)($, $.root(), {
      filters : ['h-entry'],
      logLevel : 0,
      baseUrl: sourceParts.protocol + '//' + sourceParts.host + sourceParts.path,
    });
  }
};

WebMentionPing.prototype.createMention = function (data) {
  var entry = {},
    now = Date.now(),
    item,
    author,
    published;

  if (data.items.length) {
    item = data.items[0].properties;
    author = item.author ? item.author[0].properties : {};
    if (item.published) {
      published = moment.utc(item.published[0]);
      published = published.isValid() ? published.valueOf() : null;
    } else {
      published = null;
    }

    entry.url = item.url ? url.resolve(this.source, item.url[0]) : this.source;
    entry.name = item.name ? item.name[0] : null;
    entry.published = published || now;
    entry.summary = item.summary ?
      item.summary[0] :
      (item.content ? item.content[0].value : null);

    entry.author = {
      name : author.name ? author.name[0] : null,
      photo : author.photo ? url.resolve(this.source, author.photo[0]) : null,
      url : author.url ? url.resolve(this.source, author.url[0]) : null
    };

    // Find the kind of interaction this is and what the interaction targets are. Stop at first find.
    _.some(this.interactionMapping, function (key, type) {
      if (item[key] && item[key].length) {
        entry.interactionType = type;
        entry.interactions = item[key];
        return true;
      }
    });

    if (entry.name === entry.summary) {
      entry.name = null;
    }

    if (entry.summary && entry.summary.length > 512) {
      entry.summary = entry.summary.substr(0, 512);
    }

    if (entry.interactions) {
      entry.interactions = _.compact(_.uniq(_.map(entry.interactions, function (target) {
        if (_.isObject(target) && target.properties.url) {
          target = target.properties.url[0];
        } else if (!_.isString(target)) {
          return null;
        }
        return url.resolve(this.source, target);
      }.bind(this))));
    }

    // XSS protection, stop non-HTTP protocols
    if (!urlTools.isHttpUrl.test(entry.url)) {
      entry.url = this.source;
    }
    if (!urlTools.isHttpUrl.test(entry.author.url)) {
      entry.author.url = null;
    }
  } else {
    entry.published = now;
  }

  return {
    url : this.source,
    normalizedUrl : this.normalizedSource,
    published : new Date(entry.published),
    fetched : new Date(now),
    type: entry.interactionType || null,
    data : entry,
    raw : data
  };
};

WebMentionPing.prototype.saveMention = function (mention) {
  return knex
    .table('entries').pluck('id').where('normalizedUrl', mention.normalizedUrl)
    .bind(this)
    .then(function (id) {
      //TODO: Update the entry content and notify all mentions about it
      return id.length ? id : knex.table('entries').insert(mention, 'id');
    })
    .catch(function (err) {
      // Unique constraint error
      if (err.code === '23505') {
        return knex.table('entries').pluck('id').where('normalizedUrl', mention.normalizedUrl);
      } else {
        throw err;
      }
    })
    .then(function (id) {
      return knex.table('mentions').insert({
        url : this.target,
        normalizedUrl : this.normalizedTarget,
        interaction : mention.type && _.some(mention.data.interactions, function (target) {
          return urlTools.normalizeUrl(target, { relativeTo: this.normalizedSource }) === this.normalizedTarget;
        }.bind(this)) ? true : false,
        eid : id[0],
        hostname : this.targetHost
      }).then(function () {
        return id;
      });
    })
    .then(function (id) {
      return Promise.all([
        knex.table('sites').update('lastmention', knex.fn.now()).where('hostname', this.targetHost),
        knex.raw("NOTIFY mentions, '" + JSON.stringify({ eid : id[0], }) + "'").exec(),
      ]);
    })
    .catch(function (err) {
      // Unique constraint error, ignore for now
      if (err.code === '23505') {
        //TODO: Add test for both correct update + duplicate ping behaviour
        //TODO: Update interaction-status of mention
        //TODO: Ensure that a notification is sent for new mentions and/or updated entry data (and maybe for updated mention interaction status)
      } else {
        throw err;
      }
    });
  };

module.exports = WebMentionPing;
<|endoftext|>


<commit_before>/*jslint node: true */
/* global -Promise */

"use strict";

var url = require('url'),
  _ = require('lodash'),
  request = require('../utils/request'),
  cheerio = require('cheerio'),
  microformats = require('microformat-node'),
  Promise = require('promise'),
  moment = require('moment'),
  knex = require('../knex'),
  options = require('../config'),
  urlTools = require('../utils/url-tools'),
  throttledKeys = {},
  throttleTimestamp,
  throttleByKey,
  WebMentionPing;

//TODO: Sync throttle between workers
//TODO: Ensure that the throttle persists between reboots of the app (enable eg. quick shutdowns on SIGTERM)
throttleByKey = function (key, onWaitCallback) {
  return new Promise(function (resolve, reject) {
    var now = Date.now(), time = Math.floor(now / options.throttleSpan), wait, err;

    if (
      (options.env === 'development' || options.env === 'test') &&
      !options.dev.throttling &&
      (key === '127.0.0.1' || key === 'localhost' || key === 'example.com')
    ) {
      // Don't throttle local pings in development and test environments
    } else if (throttleTimestamp !== time) {
      // Divide the time into timeslots and open up new requests whenever there's a new timeslot
      throttleTimestamp = time;
      throttledKeys = {};
    } else if (throttledKeys[key]) {
      if (!options.throttleCap || throttledKeys[key] < options.throttleCap) {
        // We're throttled – lets wait and try again when there's a new timeslot!

        //TODO: What is happening here? The throttleSpan tells the minimum gap between each request – yet all requests are scheduled to practially retry at the same time? Is that because we need to reinitialize throttledKeys[key] variable on each timeSpan and therefore need all requests to reregister again? Surely there can be a more elegant solution to that.
        wait = (time + 1) * options.throttleSpan - now + throttledKeys[key];
        console.log('Throttled! In', wait, 'milliseconds retrying key:', key);

        setTimeout(function () {
          throttleByKey(key).then(resolve, reject);
        }, wait);
        throttledKeys[key] += 1;

        if (onWaitCallback) {
          // If we need to wait, then we might want to eg. respond to the browser early
          onWaitCallback();
        }

        return;
      } else {
        // We're throttled – but there's too many waiting already so we won't get in line

        console.log('Reached throttle cap of', options.throttleCap, 'for key:', key);
        err = new Error('Too many mentions from source host at the moment.');
        err.status = 503;
        // Suggest that they ping again once we have emptied the queue so that we can fill it back up again
        // The requests will get throttled right away if they do this, but it will ensure that we utilize
        // the defined throttleSpan to the fullest and never stand idle for a millisecond.
        //
        // This should perhaps be set to just the time when there's any free spot in the queue at all, even
        // if that would mean that the throttleCap would be reached immediately again. Would follow the spec
        // more closely though: http://tools.ietf.org/html/rfc7231#section-7.1.3
        err.retryAfter = Math.ceil(((time + throttledKeys[key] - 1) * options.throttleSpan - now) / 1000);
        reject(err);

        return;
      }
    }

    // Nothing is stopping us – all lights are green – go go go!
    throttledKeys[key] = 1;
    resolve();
  });
};

WebMentionPing = function (source, target) {
  this.source = source;
  this.target = target;

  this.normalizedSource = urlTools.normalizeUrl(source);
  this.normalizedTarget = urlTools.normalizeUrl(target);

  this.sourceHost = url.parse(this.normalizedSource).hostname;
  this.targetHost = url.parse(this.normalizedTarget).hostname;

  this.sourceBase = this.source;
};

WebMentionPing.prototype.interactionMapping = {
  'reply': 'in-reply-to',
  'like': 'like-of',
  'repost': 'repost-of',
};

WebMentionPing.prototype.isValidTarget = function () {
  return knex('sites').first('aid').where('hostname', this.targetHost).then(function (row) {
    if (!row) {
      console.log('Invalid target site:', this.targetHost);

      var err = new Error('Invalid target site');
      err.status = 400;
      throw(err);
    }
  }.bind(this));
};

WebMentionPing.prototype.throttleFetches = function (onWaitCallback) {
  return throttleByKey(this.sourceHost, onWaitCallback);
};

WebMentionPing.prototype.fetchSourcePage = function () {
  return new Promise(function (resolve, reject) {
    request({ uri: this.source }, function (err, response, body) {
      if (err) {
        reject(err);
      } else if (response.statusCode !== 200) {
        reject(new Error('Expected HTTP code 200'));
      } else {
        resolve(body);
      }
    });
  }.bind(this));
};

WebMentionPing.prototype.findLink = function (normalizedTarget) {
  var i, length, href,
    links = this.$('a'),
    linked = false;

  for (i = 0, length = links.length; i < length; i += 1) {
    href = links.eq(i).attr('href');
    try {
      if (href && urlTools.normalizeUrl(href, { relativeTo: this.sourceBase }) === normalizedTarget) {
        linked = true;
        break;
      }
    } catch (e) {}
  }

  return linked;
};

WebMentionPing.prototype.parseSourcePage = function (body) {
  var $, $mf, base, linked, err, sourceParts;

  $ = cheerio.load(body);

  this.$ = $;

  base = $('base').attr('href');

  if (base) {
    this.sourceBase = url.resolve(this.source, base);
  }

  linked = this.findLink(this.normalizedTarget);

  if (!linked) {
    console.log("Couldn't find a link from source to target", this.source, this.target);
    err = new Error("Couldn't find a link from source to target (" + this.target + ")");
    err.name = 'WebMentionPingError';
    return Promise.reject(err);
  } else {
    sourceParts = url.parse(this.source);
    $mf = cheerio.load(body);
    return Promise.denodeify(microformats.parseDom)($mf, $mf.root(), {
      filters : ['h-entry'],
      logLevel : 0,
      baseUrl: sourceParts.protocol + '//' + sourceParts.host + sourceParts.path,
    });
  }
};

WebMentionPing.prototype.createMention = function (data) {
  var entry = {},
    now = Date.now(),
    item,
    author,
    published;

  if (data.items.length) {
    item = data.items[0].properties;
    author = item.author ? item.author[0].properties : {};
    if (item.published) {
      published = moment.utc(item.published[0]);
      published = published.isValid() ? published.valueOf() : null;
    } else {
      published = null;
    }

    entry.url = item.url ? url.resolve(this.sourceBase, item.url[0]) : this.source;
    entry.name = item.name ? item.name[0] : null;
    entry.published = published || now;
    entry.summary = item.summary ?
      item.summary[0] :
      (item.content ? item.content[0].value : null);

    entry.author = {
      name : author.name ? author.name[0] : null,
      photo : author.photo ? url.resolve(this.sourceBase, author.photo[0]) : null,
      url : author.url ? url.resolve(this.sourceBase, author.url[0]) : null
    };

    // Find the kind of interaction this is and what the interaction targets are. Stop at first find.
    _.some(this.interactionMapping, function (key, type) {
      if (item[key] && item[key].length) {
        entry.interactionType = type;
        entry.interactions = item[key];
        return true;
      }
    });

    if (entry.name === entry.summary) {
      entry.name = null;
    }

    if (entry.summary && entry.summary.length > 512) {
      entry.summary = entry.summary.substr(0, 512);
    }

    if (entry.interactions) {
      entry.interactions = _.compact(_.uniq(_.map(entry.interactions, function (target) {
        if (_.isObject(target) && target.properties.url) {
          target = target.properties.url[0];
        } else if (!_.isString(target)) {
          return null;
        }
        return url.resolve(this.sourceBase, target);
      }.bind(this))));
    }

    // XSS protection, stop non-HTTP protocols
    if (!urlTools.isHttpUrl.test(entry.url)) {
      entry.url = this.source;
    }
    if (!urlTools.isHttpUrl.test(entry.author.url)) {
      entry.author.url = null;
    }
  } else {
    entry.published = now;
  }

  return {
    url : this.source,
    normalizedUrl : this.normalizedSource,
    published : new Date(entry.published),
    fetched : new Date(now),
    updated : new Date(now),
    type: entry.interactionType || null,
    data : entry,
    raw : data
  };
};

WebMentionPing.prototype._upsertEntry = function (entry) {
  var updated, updateMention;

  updateMention = function () {
    return knex.table('entries')
      .where('normalizedUrl', entry.normalizedUrl)
      .update({
        updated: entry.fetched,
        type: entry.type,
        data : entry.data,
        raw : entry.raw,
      }, 'id').then(function (id) {
        if (id[0]) {
          updated = true;
        }
        return id;
      });
  };

  return updateMention()
    .bind(this)
    .then(function (id) {
      return id.length ? id : knex.table('entries').insert(entry, 'id');
    })
    .catch(function (err) {
      // Unique constraint error
      if (err.code === '23505') {
        return updateMention();
      } else {
        throw err;
      }
    }).then(function (id) {
      return {
        id: id[0],
        entry: entry,
        updated: updated,
      };
    });
};

WebMentionPing.prototype._updateOrDeleteExistingMentions = function (data) {
  var id = data.id,
    entry = data.entry;

  // If new entry, then no existing mentions exists
  if (!data.updated) {
    return Promise.resolve({
      id: id,
      entry: entry,
      newMention: true,
    });
  }

  //TODO: Add test for entry updates and mention updates / removal
  return knex.table('mentions').where('eid', id).select().bind(this).then(function (existingMentions) {
    var newMention = true;

    var updates = _.map(existingMentions, function (existingMention) {
      var currentMention, interaction, update;

      if (newMention && existingMention.normalizedUrl === this.normalizedTarget) {
        currentMention = true;
        newMention = false;
      }

      if (!currentMention && !this.findLink(existingMention.normalizedUrl)) {
        console.log('Removing mention:', id, existingMention.normalizedUrl);
        update = knex.table('mentions')
          .where('eid', id)
          .whereIn('normalizedUrl', existingMention.normalizedUrl)
          .del();
      } else {
        interaction = entry.type && _.some(entry.data.interactions, function (target) {
          return urlTools.normalizeUrl(target, { relativeTo: this.sourceBase }) === existingMention.normalizedTarget;
        }.bind(this)) ? true : false;

        if (interaction !== existingMention.interaction) {
          //TODO: Ensure these updates are broadcast
          console.log('Updating mention:', id, existingMention.normalizedUrl);
          update = knex.table('mentions')
            .where('eid', id)
            .where('normalizedUrl', existingMention.normalizedUrl)
            .update({
              updated: entry.fetched,
              interaction: interaction,
            });
        }
      }

      return update;
    }, this);

    return Promise.all(updates).then(function () {
      return {
        id: id,
        entry: entry,
        newMention: newMention,
      };
    });
  });
};

WebMentionPing.prototype._insertMention = function (data) {
  var id = data.id,
    entry = data.entry,
    updates = [];

  if (!data.newMention) {
    return Promise.resolve(id);
  }

  updates.push(knex.table('sites').update('lastmention', knex.fn.now()).where('hostname', this.targetHost));
  updates.push(
    knex.table('mentions').insert({
      url : this.target,
      normalizedUrl : this.normalizedTarget,
      interaction : entry.type && _.some(entry.data.interactions, function (target) {
        return urlTools.normalizeUrl(target, { relativeTo: this.sourceBase }) === this.normalizedTarget;
      }.bind(this)) ? true : false,
      eid : id,
      hostname : this.targetHost
    })
    .catch(function (err) {
      if (err.code === '23505') {
        // Unique constraint error, ignore
      } else {
        throw err;
      }
    })
  );

  return Promise.all(updates).then(function () {
    return id;
  });
};

WebMentionPing.prototype.saveMention = function (mention) {
  return this._upsertEntry(mention)
    .then(this._updateOrDeleteExistingMentions.bind(this))
    .then(this._insertMention.bind(this))
    .then(function (id) {
      return knex.raw("NOTIFY mentions, '" + JSON.stringify({ eid : id, }) + "'").exec();
    });
};

module.exports = WebMentionPing;
<commit_msg>fix(parser): mention interaction status update<commit_after>/*jslint node: true */
/* global -Promise */

"use strict";

var url = require('url'),
  _ = require('lodash'),
  request = require('../utils/request'),
  cheerio = require('cheerio'),
  microformats = require('microformat-node'),
  Promise = require('promise'),
  moment = require('moment'),
  knex = require('../knex'),
  options = require('../config'),
  urlTools = require('../utils/url-tools'),
  throttledKeys = {},
  throttleTimestamp,
  throttleByKey,
  WebMentionPing;

//TODO: Sync throttle between workers
//TODO: Ensure that the throttle persists between reboots of the app (enable eg. quick shutdowns on SIGTERM)
throttleByKey = function (key, onWaitCallback) {
  return new Promise(function (resolve, reject) {
    var now = Date.now(), time = Math.floor(now / options.throttleSpan), wait, err;

    if (
      (options.env === 'development' || options.env === 'test') &&
      !options.dev.throttling &&
      (key === '127.0.0.1' || key === 'localhost' || key === 'example.com')
    ) {
      // Don't throttle local pings in development and test environments
    } else if (throttleTimestamp !== time) {
      // Divide the time into timeslots and open up new requests whenever there's a new timeslot
      throttleTimestamp = time;
      throttledKeys = {};
    } else if (throttledKeys[key]) {
      if (!options.throttleCap || throttledKeys[key] < options.throttleCap) {
        // We're throttled – lets wait and try again when there's a new timeslot!

        //TODO: What is happening here? The throttleSpan tells the minimum gap between each request – yet all requests are scheduled to practially retry at the same time? Is that because we need to reinitialize throttledKeys[key] variable on each timeSpan and therefore need all requests to reregister again? Surely there can be a more elegant solution to that.
        wait = (time + 1) * options.throttleSpan - now + throttledKeys[key];
        console.log('Throttled! In', wait, 'milliseconds retrying key:', key);

        setTimeout(function () {
          throttleByKey(key).then(resolve, reject);
        }, wait);
        throttledKeys[key] += 1;

        if (onWaitCallback) {
          // If we need to wait, then we might want to eg. respond to the browser early
          onWaitCallback();
        }

        return;
      } else {
        // We're throttled – but there's too many waiting already so we won't get in line

        console.log('Reached throttle cap of', options.throttleCap, 'for key:', key);
        err = new Error('Too many mentions from source host at the moment.');
        err.status = 503;
        // Suggest that they ping again once we have emptied the queue so that we can fill it back up again
        // The requests will get throttled right away if they do this, but it will ensure that we utilize
        // the defined throttleSpan to the fullest and never stand idle for a millisecond.
        //
        // This should perhaps be set to just the time when there's any free spot in the queue at all, even
        // if that would mean that the throttleCap would be reached immediately again. Would follow the spec
        // more closely though: http://tools.ietf.org/html/rfc7231#section-7.1.3
        err.retryAfter = Math.ceil(((time + throttledKeys[key] - 1) * options.throttleSpan - now) / 1000);
        reject(err);

        return;
      }
    }

    // Nothing is stopping us – all lights are green – go go go!
    throttledKeys[key] = 1;
    resolve();
  });
};

WebMentionPing = function (source, target) {
  this.source = source;
  this.target = target;

  this.normalizedSource = urlTools.normalizeUrl(source);
  this.normalizedTarget = urlTools.normalizeUrl(target);

  this.sourceHost = url.parse(this.normalizedSource).hostname;
  this.targetHost = url.parse(this.normalizedTarget).hostname;

  this.sourceBase = this.source;
};

WebMentionPing.prototype.interactionMapping = {
  'reply': 'in-reply-to',
  'like': 'like-of',
  'repost': 'repost-of',
};

WebMentionPing.prototype.isValidTarget = function () {
  return knex('sites').first('aid').where('hostname', this.targetHost).then(function (row) {
    if (!row) {
      console.log('Invalid target site:', this.targetHost);

      var err = new Error('Invalid target site');
      err.status = 400;
      throw(err);
    }
  }.bind(this));
};

WebMentionPing.prototype.throttleFetches = function (onWaitCallback) {
  return throttleByKey(this.sourceHost, onWaitCallback);
};

WebMentionPing.prototype.fetchSourcePage = function () {
  return new Promise(function (resolve, reject) {
    request({ uri: this.source }, function (err, response, body) {
      if (err) {
        reject(err);
      } else if (response.statusCode !== 200) {
        reject(new Error('Expected HTTP code 200'));
      } else {
        resolve(body);
      }
    });
  }.bind(this));
};

WebMentionPing.prototype.findLink = function (normalizedTarget) {
  var i, length, href,
    links = this.$('a'),
    linked = false;

  for (i = 0, length = links.length; i < length; i += 1) {
    href = links.eq(i).attr('href');
    try {
      if (href && urlTools.normalizeUrl(href, { relativeTo: this.sourceBase }) === normalizedTarget) {
        linked = true;
        break;
      }
    } catch (e) {}
  }

  return linked;
};

WebMentionPing.prototype.parseSourcePage = function (body) {
  var $, $mf, base, linked, err, sourceParts;

  $ = cheerio.load(body);

  this.$ = $;

  base = $('base').attr('href');

  if (base) {
    this.sourceBase = url.resolve(this.source, base);
  }

  linked = this.findLink(this.normalizedTarget);

  if (!linked) {
    console.log("Couldn't find a link from source to target", this.source, this.target);
    err = new Error("Couldn't find a link from source to target (" + this.target + ")");
    err.name = 'WebMentionPingError';
    return Promise.reject(err);
  } else {
    sourceParts = url.parse(this.source);
    $mf = cheerio.load(body);
    return Promise.denodeify(microformats.parseDom)($mf, $mf.root(), {
      filters : ['h-entry'],
      logLevel : 0,
      baseUrl: sourceParts.protocol + '//' + sourceParts.host + sourceParts.path,
    });
  }
};

WebMentionPing.prototype.createMention = function (data) {
  var entry = {},
    now = Date.now(),
    item,
    author,
    published;

  if (data.items.length) {
    item = data.items[0].properties;
    author = item.author ? item.author[0].properties : {};
    if (item.published) {
      published = moment.utc(item.published[0]);
      published = published.isValid() ? published.valueOf() : null;
    } else {
      published = null;
    }

    entry.url = item.url ? url.resolve(this.sourceBase, item.url[0]) : this.source;
    entry.name = item.name ? item.name[0] : null;
    entry.published = published || now;
    entry.summary = item.summary ?
      item.summary[0] :
      (item.content ? item.content[0].value : null);

    entry.author = {
      name : author.name ? author.name[0] : null,
      photo : author.photo ? url.resolve(this.sourceBase, author.photo[0]) : null,
      url : author.url ? url.resolve(this.sourceBase, author.url[0]) : null
    };

    // Find the kind of interaction this is and what the interaction targets are. Stop at first find.
    _.some(this.interactionMapping, function (key, type) {
      if (item[key] && item[key].length) {
        entry.interactionType = type;
        entry.interactions = item[key];
        return true;
      }
    });

    if (entry.name === entry.summary) {
      entry.name = null;
    }

    if (entry.summary && entry.summary.length > 512) {
      entry.summary = entry.summary.substr(0, 512);
    }

    if (entry.interactions) {
      entry.interactions = _.compact(_.uniq(_.map(entry.interactions, function (target) {
        if (_.isObject(target) && target.properties.url) {
          target = target.properties.url[0];
        } else if (!_.isString(target)) {
          return null;
        }
        return url.resolve(this.sourceBase, target);
      }.bind(this))));
    }

    // XSS protection, stop non-HTTP protocols
    if (!urlTools.isHttpUrl.test(entry.url)) {
      entry.url = this.source;
    }
    if (!urlTools.isHttpUrl.test(entry.author.url)) {
      entry.author.url = null;
    }
  } else {
    entry.published = now;
  }

  return {
    url : this.source,
    normalizedUrl : this.normalizedSource,
    published : new Date(entry.published),
    fetched : new Date(now),
    updated : new Date(now),
    type: entry.interactionType || null,
    data : entry,
    raw : data
  };
};

WebMentionPing.prototype._isInteractionTarget = function (entry, normalizedTarget) {
  return entry.type && _.some(entry.data.interactions, function (target) {
    return urlTools.normalizeUrl(target, { relativeTo: this.sourceBase }) === normalizedTarget;
  }.bind(this)) ? true : false;
};

WebMentionPing.prototype._upsertEntry = function (entry) {
  var updated, updateMention;

  updateMention = function () {
    return knex.table('entries')
      .where('normalizedUrl', entry.normalizedUrl)
      .update({
        updated: entry.fetched,
        type: entry.type,
        data : entry.data,
        raw : entry.raw,
      }, 'id').then(function (id) {
        if (id[0]) {
          updated = true;
        }
        return id;
      });
  };

  return updateMention()
    .bind(this)
    .then(function (id) {
      return id.length ? id : knex.table('entries').insert(entry, 'id');
    })
    .catch(function (err) {
      // Unique constraint error
      if (err.code === '23505') {
        return updateMention();
      } else {
        throw err;
      }
    }).then(function (id) {
      return {
        id: id[0],
        entry: entry,
        updated: updated,
      };
    });
};

WebMentionPing.prototype._updateOrDeleteExistingMentions = function (data) {
  var id = data.id,
    entry = data.entry;

  // If new entry, then no existing mentions exists
  if (!data.updated) {
    return Promise.resolve({
      id: id,
      entry: entry,
      newMention: true,
    });
  }

  //TODO: Add test for entry updates and mention updates / removal
  return knex.table('mentions').where('eid', id).select().bind(this).then(function (existingMentions) {
    var newMention = true;

    var updates = _.map(existingMentions, function (existingMention) {
      var currentMention, interaction, update;

      if (newMention && existingMention.normalizedUrl === this.normalizedTarget) {
        currentMention = true;
        newMention = false;
      }

      if (!currentMention && !this.findLink(existingMention.normalizedUrl)) {
        console.log('Removing mention:', id, existingMention.normalizedUrl);
        update = knex.table('mentions')
          .where('eid', id)
          .whereIn('normalizedUrl', existingMention.normalizedUrl)
          .del();
      } else {
        interaction = this._isInteractionTarget(entry, existingMention.normalizedUrl);

        if (interaction !== existingMention.interaction) {
          //TODO: Ensure these updates are broadcast
          console.log('Updating mention:', id, existingMention.normalizedUrl);
          update = knex.table('mentions')
            .where('eid', id)
            .where('normalizedUrl', existingMention.normalizedUrl)
            .update({
              updated: entry.fetched,
              interaction: interaction,
            });
        }
      }

      return update;
    }, this);

    return Promise.all(updates).then(function () {
      return {
        id: id,
        entry: entry,
        newMention: newMention,
      };
    });
  });
};

WebMentionPing.prototype._insertMention = function (data) {
  var id = data.id,
    entry = data.entry,
    updates = [];

  if (!data.newMention) {
    return Promise.resolve(id);
  }

  updates.push(knex.table('sites').update('lastmention', knex.fn.now()).where('hostname', this.targetHost));
  updates.push(
    knex.table('mentions').insert({
      url : this.target,
      normalizedUrl : this.normalizedTarget,
      interaction : this._isInteractionTarget(entry, this.normalizedTarget),
      eid : id,
      hostname : this.targetHost
    })
    .catch(function (err) {
      if (err.code === '23505') {
        // Unique constraint error, ignore
      } else {
        throw err;
      }
    })
  );

  return Promise.all(updates).then(function () {
    return id;
  });
};

WebMentionPing.prototype.saveMention = function (mention) {
  return this._upsertEntry(mention)
    .then(this._updateOrDeleteExistingMentions.bind(this))
    .then(this._insertMention.bind(this))
    .then(function (id) {
      return knex.raw("NOTIFY mentions, '" + JSON.stringify({ eid : id, }) + "'").exec();
    });
};

module.exports = WebMentionPing;
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/client/WebSocketClient.js                       *
 *                                                        *
 * Hprose WebSocket Client for HTML5.                     *
 *                                                        *
 * LastModified: Jul 17, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

/*jshint node:true, eqeqeq:true */
'use strict';

var util = require('util');
var parse = require('url').parse;
var WebSocket = require('ws');

var Client = global.hprose.Client;
var BytesIO = global.hprose.BytesIO;
var Completer = global.hprose.Completer;
var TimeoutError = require('../common/TimeoutError');

function noop(){}

function WebSocketClient(uri, functions) {
    if (this.constructor !== WebSocketClient) {
        return new WebSocketClient(uri, functions);
    }
    Client.call(this, uri, functions);
    this.timeout = 0;

    var _id = 0;
    var _count = 0;
    var _reqcount = 0;
    var _completers = Object.create(null);
    var _requests = Object.create(null);
    var _ready = false;
    var ws = null;

    var self = this;

    function getNextId() {
        return (_id < 0x7fffffff) ? ++_id : _id = 0;
    }

    function send(id, request) {
        var bytes = new BytesIO();
        bytes.writeByte((id >> 24) & 0xff);
        bytes.writeByte((id >> 16) & 0xff);
        bytes.writeByte((id >> 8) & 0xff);
        bytes.writeByte(id & 0xff);
        if (request.constructor === String) {
            bytes.writeString(request);
        }
        else {
            bytes.write(request);
        }
        var completer = _completers[id];
        try {
            ws.send(bytes.bytes);
        }
        catch (e) {
            completer.completeError(e);
        }
    }
    function onopen() {
        _ready = true;
        if (_reqcount > 0) {
            for (var id in _requests) {
                send(id, _requests[id]);
                delete _requests[id];
            }
        }
        _reqcount = 0;
    }
    function onmessage(data, flags) {
        var bytes = new BytesIO(data);
        var id = bytes.readByte() << 24 |
                 bytes.readByte() << 16 |
                 bytes.readByte() << 8  |
                 bytes.readByte();
        var completer = _completers[id];
        delete _completers[id];
        if (completer !== undefined) {
            --_count;
            completer.complete(bytes.read(bytes.length - 4));
        }
        if (_count === 0) {
            if (!self.keepAlive) close();
        }
    }
    function onclose(code, message) {
        onerror(new Error(code + ':' + message));
        ws = null;
    }
    function onerror(error) {
        for (var id in _completers) {
            var completer = _completers[id];
            if (completer !== undefined) {
                completer.completeError(error);
            }
            delete _completers[id];
            delete _requests[id];
        }
        _count = 0;
    }
    function connect() {
        _ready = false;
        self.setOption('perMessageDeflate', false);
        ws = new WebSocket(self.uri, self.options);
        ws.on('open', onopen);
        ws.on('message', onmessage);
        ws.on('error', onerror);
        ws.on('close', onclose);
    }
    function sendAndReceive(request) {
        if (ws === null ||
            ws.readyState === WebSocket.CLOSING ||
            ws.readyState === WebSocket.CLOSED) {
            connect();
        }
        ++_count;
        var id = getNextId();
        var completer = new Completer();
        var future = completer.future;
        if (self.timeout > 0) {
            future = future.timeout(self.timeout).catchError(function(e) {
                delete _completers[id];
                delete _requests[id];
                --_count;
                throw e;
            },
            function(e) {
                return e instanceof TimeoutError;
            });
        }
        _completers[id] = completer;
        if (_ready) {
            send(id, request);
        }
        else {
            _requests[id] = request;
            ++_reqcount;
        }
        return completer.future;
    }
    function close() {
        if (ws !== null) {
            ws.removeAllListeners('open');
            ws.removeAllListeners('message');
            ws.removeAllListeners('error');
            ws.removeAllListeners('close');
            ws.close();
        }
    }
    Object.defineProperties(this, {
        sendAndReceive: { value: sendAndReceive },
        close: { value: close },
    });
}

function create(uri, functions) {
    var protocol = parse(uri).protocol;
    if (protocol === 'ws:' ||
        protocol === 'wss:') {
        return new WebSocketClient(uri, functions);
    }
    throw new Error('This client desn\'t support ' + protocol + ' scheme.');
}

Object.defineProperty(WebSocketClient, 'create', { value: create });

util.inherits(WebSocketClient, Client);

global.hprose.WebSocketClient = WebSocketClient;
<commit_msg>Fixed a bug of WebSocketClient<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * hprose/client/WebSocketClient.js                       *
 *                                                        *
 * Hprose WebSocket Client for HTML5.                     *
 *                                                        *
 * LastModified: Jul 17, 2015                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

/*jshint node:true, eqeqeq:true */
'use strict';

var util = require('util');
var parse = require('url').parse;
var WebSocket = require('ws');

var Client = global.hprose.Client;
var BytesIO = global.hprose.BytesIO;
var Completer = global.hprose.Completer;
var TimeoutError = require('../common/TimeoutError');

function noop(){}

function WebSocketClient(uri, functions) {
    if (this.constructor !== WebSocketClient) {
        return new WebSocketClient(uri, functions);
    }
    Client.call(this, uri, functions);
    this.timeout = 0;

    var _id = 0;
    var _count = 0;
    var _reqcount = 0;
    var _completers = Object.create(null);
    var _requests = Object.create(null);
    var _ready = false;
    var ws = null;

    var self = this;

    function getNextId() {
        return (_id < 0x7fffffff) ? ++_id : _id = 0;
    }

    function send(id, request) {
        var bytes = new BytesIO();
        bytes.writeByte((id >> 24) & 0xff);
        bytes.writeByte((id >> 16) & 0xff);
        bytes.writeByte((id >> 8) & 0xff);
        bytes.writeByte(id & 0xff);
        if (request.constructor === String) {
            bytes.writeString(request);
        }
        else {
            bytes.write(request);
        }
        var completer = _completers[id];
        try {
            ws.send(bytes.bytes);
        }
        catch (e) {
            completer.completeError(e);
        }
    }
    function onopen() {
        _ready = true;
        if (_reqcount > 0) {
            for (var id in _requests) {
                send(id, _requests[id]);
                delete _requests[id];
            }
        }
        _reqcount = 0;
    }
    function onmessage(data, flags) {
        var bytes = new BytesIO(data);
        var id = bytes.readByte() << 24 |
                 bytes.readByte() << 16 |
                 bytes.readByte() << 8  |
                 bytes.readByte();
        var completer = _completers[id];
        delete _completers[id];
        if (completer !== undefined) {
            --_count;
            completer.complete(bytes.read(bytes.length - 4));
        }
        if (_count === 0) {
            if (!self.keepAlive) close();
        }
    }
    function onclose(code, message) {
        onerror(new Error(code + ':' + message));
        ws = null;
    }
    function onerror(error) {
        for (var id in _completers) {
            var completer = _completers[id];
            if (completer !== undefined) {
                completer.completeError(error);
            }
            delete _completers[id];
            delete _requests[id];
        }
        _count = 0;
    }
    function connect() {
        _ready = false;
        self.setOption('perMessageDeflate', false);
        ws = new WebSocket(self.uri, self.options);
        ws.on('open', onopen);
        ws.on('message', onmessage);
        ws.on('error', onerror);
        ws.on('close', onclose);
    }
    function sendAndReceive(request) {
        if (ws === null ||
            ws.readyState === WebSocket.CLOSING ||
            ws.readyState === WebSocket.CLOSED) {
            connect();
        }
        ++_count;
        var id = getNextId();
        var completer = new Completer();
        var future = completer.future;
        if (self.timeout > 0) {
            future = future.timeout(self.timeout).catchError(function(e) {
                delete _completers[id];
                delete _requests[id];
                --_count;
                throw e;
            },
            function(e) {
                return e instanceof TimeoutError;
            });
        }
        _completers[id] = completer;
        if (_ready) {
            send(id, request);
        }
        else {
            _requests[id] = request;
            ++_reqcount;
        }
        return future;
    }
    function close() {
        if (ws !== null) {
            ws.removeAllListeners('open');
            ws.removeAllListeners('message');
            ws.removeAllListeners('error');
            ws.removeAllListeners('close');
            ws.close();
        }
    }
    Object.defineProperties(this, {
        sendAndReceive: { value: sendAndReceive },
        close: { value: close },
    });
}

function create(uri, functions) {
    var protocol = parse(uri).protocol;
    if (protocol === 'ws:' ||
        protocol === 'wss:') {
        return new WebSocketClient(uri, functions);
    }
    throw new Error('This client desn\'t support ' + protocol + ' scheme.');
}

Object.defineProperty(WebSocketClient, 'create', { value: create });

util.inherits(WebSocketClient, Client);

global.hprose.WebSocketClient = WebSocketClient;
<|endoftext|>