<commit_before>/**
 * The Conversation Class
 * @class Conversation
 * @param {Object} document An object representing a conversation ususally a Mongo document
 */
Conversation = BaseModel.extendAndSetupCollection("conversations");

/**
 * Fetch list of participants in the conversation
 * @param   {Number}       limit     The maximum number of participants to return
 * @param   {Number}       skip      The number of records to skip
 * @param   {String}       sortBy    They key to sort on
 * @param   {Number}       sortOrder The order in which to sort 1 for ascending, -1 for descending
 * @returns {Mongo.Cursor} Cursor which upon iteration will return a Participant instance for each record
 */
Conversation.prototype.participants = function (limit, skip, sortBy, sortOrder) {
    var options = {};
    var sort = {};

    if(limit){
        options.limit = limit;
    }

    if(skip){
        options.skip = skip;
    }

    if(sortBy && sortOrder){
        sort[sortBy] = sortOrder;
        options.sort = sort;
    }
    return ParticipantsCollection.find({conversationId:this._id, userId:{$ne:Meteor.userId()}, deleted:{$exists:false}}, options);
};

/**
 * Check if the conversation has not been read by a the current user
 * @returns {Boolean} Whether the conversation is unread
 */
Conversation.prototype.isUnread = function () {
    var userId = Meteor.userId();
    return !!ParticipantsCollection.findOne({conversationId:this._id, userId:userId, read:false});
};

/**
 * Check if the conversation is read only
 * @returns {Boolean} Whether the conversation is read only
 */
Conversation.prototype.isReadOnly = function () {
    //Conversation is readOnly if there is one participant left and it is the current user
    return (this._participants.length == 1);
};

/**
 * Retrieve the list of messages for the conversation
 * @param   {Number}       limit     The maximum messages to retrieve
 * @param   {Number}       skip      The number of records to skip
 * @param   {String}       sortBy    The field to sort on
 * @param   {Number}       sortOrder The order in which to sort 1 for ascending, -1 for decending
 * @returns {Mongo.Cursor} Cursor which upon iteration will return a Message instance for each record
 */
Conversation.prototype.messages = function (limit, skip, sortBy, sortOrder) {
    var options = {};
    var sort = {};
    if(limit){
        options.limit = limit;
    }
    if(skip){
        options.skip = skip;
    }
    if(sortBy && sortOrder){
        sort[sortBy] = sortOrder;
        options.sort = sort;
    }
    return MessagesCollection.find({conversationId:this._id}, options);
};

/**
 * Retrieve the last message in the conversation
 * @returns {Message} An instance of Message which was the last sent message for the conversation
 */
Conversation.prototype.lastMessage = function () {
    return MessagesCollection.findOne({conversationId:this._id}, {sort:{date:-1}, limit:1});
};

/**
 * Add a new message to the conversation
 * @param {String}   body     The body of the message
 * @param {Function} callback The callback to run upon insertion of the document
 */
Conversation.prototype.sendMessage = function (body, callback) {
    new Message({body:body, conversationId:this._id, inFlight:true}).save(callback);
};

/**
 * Add participants to the conversation
 * @method addParticipants
 * @param {Array} participants An array of userId's to add as participants on the conversation
 */
Conversation.prototype.addParticipants = function(participants) {
    var self = this;
    _.each(participants, function(participant){
       new Participant({userId:participant._id, conversationId:self._id}).save();
    });
};

/**
 * Set the read state of the conversation for the current user
 *
 * This is set on the particpant object for the user
 * @param {[[Type]]} state [[Description]]
 */
Conversation.prototype.updateReadState = function (state) {
    var participant = ParticipantsCollection.findOne({conversationId:this._id, userId:Meteor.userId()});
    participant.update({$set:{read:state}});
};

/**
 * Get a serialized sentence of the users who have read the conversation
 * @returns {String} A string list of users that have read the current state of the conversation
 */
Conversation.prototype.readBy = function () {
    var readBy = [];

    this.participants().forEach(function(participant) {
        var user;
        if(participant.userId !== Meteor.userId() && participant.read){
            user = Meteor.users.findOne(participant.userId, {fields:{username:true}});

            if(user){
                readBy.push(user.username);
            }
        }
    });

    return readBy.length > 0 && "read by " + _.toSentenceSerial(readBy);
};

/**
 * Remove a participant from a conversation
 * @method removeParticipant
 * @param {User} user The user to remove, defaults to the currently logged in user
 */
Conversation.prototype.removeParticipant = function(user){
        var userId = user._id || Meteor.userId();
        var query = {conversationId:this._id, userId:userId};
        var modifier = {$set:{deleted:true, read:true}};

        if(Meteor.isClient){
            var participant = ParticipantsCollection.findOne(query);
            participant && participant.update(modifier);
        }else{
            ParticipantsCollection.update(query, modifier);
        }
};

ConversationsCollection = Conversation.collection;

//The Schema for a Converation
Conversation.appendSchema({
    "date":{
        type:Date,
        autoValue: function() {
            return new Date();
        },
        index: -1
    },
    "_participants":{
        type:[String],
        defaultValue:[],
        index:1
    },
    "_participants.$":{
        type:String,
        regEx:SimpleSchema.RegEx.Id
    }
});

<commit_msg>allow adding participants one at a time.<commit_after>/**
 * The Conversation Class
 * @class Conversation
 * @param {Object} document An object representing a conversation ususally a Mongo document
 */
Conversation = BaseModel.extendAndSetupCollection("conversations");

/**
 * Fetch list of participants in the conversation
 * @param   {Number}       limit     The maximum number of participants to return
 * @param   {Number}       skip      The number of records to skip
 * @param   {String}       sortBy    They key to sort on
 * @param   {Number}       sortOrder The order in which to sort 1 for ascending, -1 for descending
 * @returns {Mongo.Cursor} Cursor which upon iteration will return a Participant instance for each record
 */
Conversation.prototype.participants = function (limit, skip, sortBy, sortOrder) {
    var options = {};
    var sort = {};

    if(limit){
        options.limit = limit;
    }

    if(skip){
        options.skip = skip;
    }

    if(sortBy && sortOrder){
        sort[sortBy] = sortOrder;
        options.sort = sort;
    }
    return ParticipantsCollection.find({conversationId:this._id, userId:{$ne:Meteor.userId()}, deleted:{$exists:false}}, options);
};

/**
 * Check if the conversation has not been read by a the current user
 * @returns {Boolean} Whether the conversation is unread
 */
Conversation.prototype.isUnread = function () {
    var userId = Meteor.userId();
    return !!ParticipantsCollection.findOne({conversationId:this._id, userId:userId, read:false});
};

/**
 * Check if the conversation is read only
 * @returns {Boolean} Whether the conversation is read only
 */
Conversation.prototype.isReadOnly = function () {
    //Conversation is readOnly if there is one participant left and it is the current user
    return (this._participants.length == 1);
};

/**
 * Retrieve the list of messages for the conversation
 * @param   {Number}       limit     The maximum messages to retrieve
 * @param   {Number}       skip      The number of records to skip
 * @param   {String}       sortBy    The field to sort on
 * @param   {Number}       sortOrder The order in which to sort 1 for ascending, -1 for decending
 * @returns {Mongo.Cursor} Cursor which upon iteration will return a Message instance for each record
 */
Conversation.prototype.messages = function (limit, skip, sortBy, sortOrder) {
    var options = {};
    var sort = {};
    if(limit){
        options.limit = limit;
    }
    if(skip){
        options.skip = skip;
    }
    if(sortBy && sortOrder){
        sort[sortBy] = sortOrder;
        options.sort = sort;
    }
    return MessagesCollection.find({conversationId:this._id}, options);
};

/**
 * Retrieve the last message in the conversation
 * @returns {Message} An instance of Message which was the last sent message for the conversation
 */
Conversation.prototype.lastMessage = function () {
    return MessagesCollection.findOne({conversationId:this._id}, {sort:{date:-1}, limit:1});
};

/**
 * Add a new message to the conversation
 * @param {String}   body     The body of the message
 * @param {Function} callback The callback to run upon insertion of the document
 */
Conversation.prototype.sendMessage = function (body, callback) {
    new Message({body:body, conversationId:this._id, inFlight:true}).save(callback);
};

/**
 * Add participants to the conversation
 * @method addParticipants
 * @param {Array} participants An array of userId's to add as participants on the conversation
 */
Conversation.prototype.addParticipants = function(participants) {
    var self = this;

    if(_.isArray(participants)){
        _.each(participants, function(participant){
            self.addParticipant(participant);
        });
    }else{
        self.addParticipant(participants);
    }
};

/**
 * Add participant to the conversation
 * @method addParticipant
 * @param {User} participant A instance of User to add as a participant in the conversation
 */
Conversation.prototype.addParticipant = function(participant) {
    if(participant instanceof User){
        new Participant({userId:participant._id, conversationId:this._id}).save();
    }else{
        throw new Meteor.Error("User Required", "Each participant must be an instance of User Class");
    }
};

/**
 * Set the read state of the conversation for the current user
 *
 * This is set on the particpant object for the user
 * @param {[[Type]]} state [[Description]]
 */
Conversation.prototype.updateReadState = function (state) {
    var participant = ParticipantsCollection.findOne({conversationId:this._id, userId:Meteor.userId()});
    participant.update({$set:{read:state}});
};

/**
 * Get a serialized sentence of the users who have read the conversation
 * @returns {String} A string list of users that have read the current state of the conversation
 */
Conversation.prototype.readBy = function () {
    var readBy = [];

    this.participants().forEach(function(participant) {
        var user;
        if(participant.userId !== Meteor.userId() && participant.read){
            user = Meteor.users.findOne(participant.userId, {fields:{username:true}});

            if(user){
                readBy.push(user.username);
            }
        }
    });

    return readBy.length > 0 && "read by " + _.toSentenceSerial(readBy);
};

/**
 * Remove a participant from a conversation
 * @method removeParticipant
 * @param {User} user The user to remove, defaults to the currently logged in user
 */
Conversation.prototype.removeParticipant = function(user){
        var userId = user._id || Meteor.userId();
        var query = {conversationId:this._id, userId:userId};
        var modifier = {$set:{deleted:true, read:true}};

        if(Meteor.isClient){
            var participant = ParticipantsCollection.findOne(query);
            participant && participant.update(modifier);
        }else{
            ParticipantsCollection.update(query, modifier);
        }
};

ConversationsCollection = Conversation.collection;

//The Schema for a Converation
Conversation.appendSchema({
    "date":{
        type:Date,
        autoValue: function() {
            return new Date();
        },
        index: -1
    },
    "_participants":{
        type:[String],
        defaultValue:[],
        index:1
    },
    "_participants.$":{
        type:String,
        regEx:SimpleSchema.RegEx.Id
    }
});

<|endoftext|>


<commit_before>const sqlite3 = require('sqlite3')
const conf = require('./configuration.json')
const {DirectedGraph} = require('graphology');
const {density} = require('graphology-metrics');
const {modularity} = require('graphology-metrics');
const pagerank = require('graphology-pagerank');
const louvain = require('graphology-communities-louvain');
const gexf = require('graphology-gexf/browser');
const async = require('async');
const fs = require('fs');

// RICardo database acces as singleton
const DB =  ( () => {
    var _db = null;
    return new function(){
        this.get = () => {
            if (_db == null){
                _db = new sqlite3.Database(`./data/${conf.RICardo_data}`);
            }
            return _db;
        }
    }
})();

const computeGraph = (year, done) =>{
  //Metrics output

  // {
  //   entities:{
  //     entitySlug:{
  //       label:"entity name",
  //       pagerank:value,
  //       ...
  //     }
  //   },
  //   network:{
  //     density:{year:value}...
  //   }
  // }

  DB.get().all(`SELECT * FROM flow_joined
    WHERE
      flow is not null and rate is not null AND
      year = ${year} AND
      (partner is not null AND (partner not LIKE 'world%' OR partner = 'World_best_guess'))
      `,
    function(err, rows) {

      const graph = new DirectedGraph()
      const totalTrade = {'Imp':0, 'Exp':0}
      const nbMirrorFlows = {'by_source_type':{},'by_reporting_type':{},'by_partner_type':{}}

      rows.forEach(r => {

        const w = r.flow*(r.unit || 1)/r.rate;
        if (w && w != 0){

          if (r.partner_slug === "Worldbestguess"){

            // store total flows as nodes params
            const p = {type: r.reporting_type, label: r.reporting, continent: r.reporting_continent};
            p[`world_${r.expimp}`] = w;
            totalTrade[r.expimp] += w;
            graph.mergeNode(r.reporting_slug,p);
          }
          else{

            graph.mergeNode(r.reporting_slug,{type:r.reporting_type, label:r.reporting, continent: r.reporting_continent, reporting:1})
            graph.mergeNode(r.partner_slug,{type:r.partner_type, label: r.partner, continent: r.partner_continent})
            let source = r.reporting_slug
            let target = r.partner_slug
            // swap
            if (r.expimp === 'Imp')
              [source,target] = [target,source]

            try{
              graph.addEdge(source, target, {weight:w, direction:r.expimp,source_type:r.type});
            }
            catch(error){
              // duplicated edge
              const dup_edge = graph.edge(source,target)
              const dup_source_type = graph.getEdgeAttribute(dup_edge, 'source_type')
              // add info that this edge is mirrored
              graph.setEdgeAttribute(dup_edge, 'mirrored', true)
              // storing extra meta for mirroflows for later stats analysis
              graph.setEdgeAttribute(dup_edge, 'source_types', [r.type,dup_source_type].sort())
              graph.setEdgeAttribute(dup_edge, 'reporting_types', [r.reporting_type,graph.getNodeAttribute(r.expimp === 'Imp' ? source : target, 'type')].sort())
              graph.setEdgeAttribute(dup_edge, 'partner_types', [r.partner_type,graph.getNodeAttribute(r.expimp === 'Imp' ? target : source, 'type')].sort())

              // rules to choose which information to keep
              if(
                // dup edege is from aggregation but the new is not, let's merge to give priority to source
                (dup_source_type === 'aggregation' && r.source_type !== 'aggregation') ||
                // mirror flows from sources, we prefer Exp on Imp in such cases
                (r.expimp==='Exp' && dup_source_type !== 'aggregation' && r.source_type !== 'aggregation')||
                // same rule of mirror flows if both edges are from aggregations
                (r.expimp==='Exp' && dup_source_type === 'aggregation' && r.source_type === 'aggregation')){
                  graph.mergeEdge(source, target, {weight:w, direction:r.expimp, source_type:r.type});
              }
            }
          }
        }

      })
      //prepare output metrics container
      const metrics = {entities:{},networks:{density:{},order:{},size:{}}};
      // compute metrics
      metrics.year = year;
      // network level
      metrics.networks.density = density(graph);
      metrics.networks.nb_reportings = graph.order;
      metrics.networks.nb_flows = graph.size;
      // Modularity score of Louvain partition
      louvain.assign(graph);
      metrics.networks.modularity = modularity(graph);

      // number of mirror Flows as defined by flows between reportings
      metrics.networks.mirrorFlows = graph.edges().reduce((acc,e) => {
        //if (graph.extremities(e).map(n => !!graph.getNodeAttribute(n, 'reporting')).filter(_ => _).length === 2)
        if (graph.getEdgeAttribute(e, 'mirrored')){
          if (graph.getEdgeAttribute(e, 'mirrored')){
            let st = graph.getEdgeAttribute(e, 'source_types');
            let rt = graph.getEdgeAttribute(e, 'reporting_types');
            let pt = graph.getEdgeAttribute(e, 'partner_types');
            nbMirrorFlows['by_source_type'][st] = (nbMirrorFlows['by_source_type'][st] + 1) || 1
            nbMirrorFlows['by_reporting_type'][rt] = (nbMirrorFlows['by_reporting_type'][rt] + 1) || 1
            nbMirrorFlows['by_partner_type'][pt] = (nbMirrorFlows['by_partner_type'][pt] + 1) || 1
          }
          return acc + 1;
        }
        else
          return acc;
      },0);
      metrics.nbMirrorFlows = nbMirrorFlows;

      //node level
      pagerank.assign(graph);
      graph.nodes().forEach(n => {
        const reportingType = graph.getNodeAttribute(n, 'type');
        metrics.networks[`nb_entities_${reportingType}`] = (metrics.networks[`nb_entities_${reportingType}`] || 0 ) + 1
        if (graph.getNodeAttribute(n, 'reporting'))
          metrics.networks[`nb_reportings_${reportingType}`] = (metrics.networks[`nb_reportings_${reportingType}`] || 0 ) + 1
        else
          metrics.networks[`nb_partners_${reportingType}`] = (metrics.networks[`nb_partners_${reportingType}`] || 0 ) + 1

        // herfindall index
        const inDegree = graph.inEdges(n).reduce((acc,e) => acc + graph.getEdgeAttribute(e,'weight'), 0);
        const outDegree = graph.outEdges(n).reduce((acc,e) => acc + graph.getEdgeAttribute(e,'weight'), 0);
        graph.setNodeAttribute(n, 'inHerfindahl', graph.inEdges(n).reduce((acc,e) => acc + Math.pow(graph.getEdgeAttribute(e,"weight")/inDegree,2), 0));
        graph.setNodeAttribute(n, 'outHerfindahl', graph.outEdges(n).reduce((acc,e) => acc + Math.pow(graph.getEdgeAttribute(e,"weight")/outDegree,2), 0));
        graph.setNodeAttribute(n, 'herfindahl', graph.neighbors(n).reduce((acc,neighbor) =>{
          // total trade of this neighbor
          let neighborTotal = graph.edges(n,neighbor).reduce((s, e) => s+graph.getEdgeAttribute(e,'weight'), 0);
          return acc + Math.pow(neighborTotal/(inDegree+outDegree),2);
        },0));
        graph.setNodeAttribute(n, 'weightedInDegree', inDegree);
        graph.setNodeAttribute(n, 'weightedOutDegree', outDegree);
        graph.setNodeAttribute(n, 'inDegree', graph.inDegree(n));
        graph.setNodeAttribute(n, 'outDegree', graph.outDegree(n));

        // store node attributes to metrics
        metrics.entities[n] = graph.getNodeAttributes(n);

        if (metrics.entities[n].world_Exp && metrics.entities[n].world_Imp){
          metrics.entities[n].worldTradePart = (metrics.entities[n].world_Imp + metrics.entities[n].world_Exp) / (totalTrade.Exp + totalTrade.Imp);
          graph.setNodeAttribute(n,'worldTradePart', metrics.entities[n].worldTradePart);
        }
      })
      //Write gexf
      if (conf.writeGexf){
        const gexfString = gexf.write(graph);
        fs.writeFile(`./data/networks/${year}.gexf`, gexfString, 'utf8', (err) => {
          if (err)
            done(err);
          else{
            done(null, metrics);
          }
        })
      }
      else
        done(null, metrics);
  });
}

// prepare list of years to compute from cn*onfig
const years = []
for (year = conf.startDate; year <= conf.endDate; year++) {
      years.push(year)
}
// throw computation in async mode
async.map(years, computeGraph, (err, metrics) =>{
   //merge metrics
   const gapMinderMetrics = []
   const networksMetrics =[]
   const metricsByYear = {networks:{density:{},nb_reportings:{},nb_flows:{},modularity:{}},entities:{}}
   const general_stat = {}
   const nbMirrorFlows = {'by_source_type':{},'by_reporting_type':{},'by_partner_type':{}}

   metrics.forEach(m => {

      // aggregating nbMirrorFlows
      for (by in m.nbMirrorFlows){
        for (p in m.nbMirrorFlows[by]){
          nbMirrorFlows[by][p] = (nbMirrorFlows[by][p] || 0) + m.nbMirrorFlows[by][p]
        }
      }
      general_stat['nbMirrorFlows'] = nbMirrorFlows

      // to be factorized later
      metricsByYear.networks.density[m.year] = m.networks.density;
      metricsByYear.networks.modularity[m.year] = m.networks.modularity;
      metricsByYear.networks.nb_reportings[m.year] = m.networks.nb_reportings;
      metricsByYear.networks.nb_flows[m.year] = m.networks.nb_flows;
      // networks metrics only
      networksMetric = {year:m.year};
      for (a in m.networks)
        networksMetric[a] = m.networks[a];
      networksMetrics.push(networksMetric);

      for (e in m.entities){

        // gapminder viz data formatting
        // check if entity is a reporting (i.e. has world trade reports)
        gapMinderMetrics.push({
            year: m.year,
            entity: m.entities[e].label,
            type: m.entities[e].type,
            isReporting: m.entities[e].reporting || false,
            continent: m.entities[e].continent,
            pagerank: m.entities[e].pagerank,
            worldTrade: m.entities[e].world_Imp || 0 + m.entities[e].world_Exp || 0,
            worldTradePart: m.entities[e].worldTradePart || null,
            herfindahl: m.entities[e].herfindahl,
            inHerfindahl: m.entities[e].inHerfindahl,
            outHerfindahl: m.entities[e].outHerfindahl,
            inDegree: m.entities[e].inDegree,
            outDegree: m.entities[e].outDegree,
            weightedInDegree: m.entities[e].weightedInDegree,
            weightedOutDegree: m.entities[e].weightedOutDegree
        })

        // generic output
        if(!metricsByYear.entities[e])
          metricsByYear.entities[e] = {}

        for (p in m.entities[e]){
          if(!metricsByYear.entities[e][p])
            metricsByYear.entities[e][p]={}
          if (['type','label','continent'].indexOf(p) !== -1)
            metricsByYear.entities[e][p] = m.entities[e][p]
          else
            metricsByYear.entities[e][p][m.year] = m.entities[e][p]
        }
      }
   })

   const output_filename = `metrics_${conf.startDate}_${conf.endDate}.json`

   fs.writeFile(`./data/${output_filename}`, JSON.stringify(metricsByYear,null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${output_filename}`);
      else console.log(`writing to ${output_filename}`);
    });
  fs.writeFile(`./data/${conf.gapMinder_metric_filename}`, JSON.stringify(gapMinderMetrics, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.gapMinder_metric_filename}`);
      else console.log(`writing to ${conf.gapMinder_metric_filename}`);
    });
  fs.writeFile(`./data/${conf.network_metric_filename}`, JSON.stringify(networksMetrics, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.network_metric_filename}`);
      else console.log(`writing to ${conf.network_metric_filename}`);
    });
  fs.writeFile(`./data/${conf.general_stat_filename}`, JSON.stringify(general_stat, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.general_stat_filename}`);
      else console.log(`writing to ${conf.general_stat_filename}`);
    });
})

DB.get().close();<commit_msg>network analysis on the aggregated flow table<commit_after>const sqlite3 = require('sqlite3')
const conf = require('./configuration.json')
const {DirectedGraph} = require('graphology');
const {density} = require('graphology-metrics');
const {modularity} = require('graphology-metrics');
const pagerank = require('graphology-pagerank');
const louvain = require('graphology-communities-louvain');
const gexf = require('graphology-gexf/browser');
const async = require('async');
const fs = require('fs');

// RICardo database acces as singleton
const DB =  ( () => {
    var _db = null;
    return new function(){
        this.get = () => {
            if (_db == null){
                _db = new sqlite3.Database(`./data/${conf.RICardo_data}`);
            }
            return _db;
        }
    }
})();

const computeGraph = (year, done) =>{
  //Metrics output

  // {
  //   entities:{
  //     entitySlug:{
  //       label:"entity name",
  //       pagerank:value,
  //       ...
  //     }
  //   },
  //   network:{
  //     density:{year:value}...
  //   }
  // }

  DB.get().all(`SELECT * FROM flow_aggregated
    WHERE
      flow is not null and rate is not null AND
      year = ${year} AND
      (partner is not null AND (partner not LIKE 'world%' OR partner = 'World_best_guess'))
      `,
    function(err, rows) {

      const graph = new DirectedGraph()
      const totalTrade = {'Imp':0, 'Exp':0}
      const nbMirrorFlows = {'by_source_type':{},'by_reporting_type':{},'by_partner_type':{}}

      rows.forEach(r => {

        const w = r.flow*(r.unit || 1)/r.rate;
        if (w && w != 0){

          if (r.partner_slug === "Worldbestguess"){

            // store total flows as nodes params
            const p = {type: r.reporting_type, label: r.reporting, continent: r.reporting_continent};
            p[`world_${r.expimp}`] = w;
            totalTrade[r.expimp] += w;
            graph.mergeNode(r.reporting_slug,p);
          }
          else{

            graph.mergeNode(r.reporting_slug,{type:r.reporting_type, label:r.reporting, continent: r.reporting_continent, reporting:1})
            graph.mergeNode(r.partner_slug,{type:r.partner_type, label: r.partner, continent: r.partner_continent})
            let source = r.reporting_slug
            let target = r.partner_slug
            // swap
            if (r.expimp === 'Imp')
              [source,target] = [target,source]

            try{
              graph.addEdge(source, target, {weight:w, direction:r.expimp,source_type:r.type});
            }
            catch(error){
              // duplicated edge
              const dup_edge = graph.edge(source,target)
              const dup_source_type = graph.getEdgeAttribute(dup_edge, 'source_type')
              // add info that this edge is mirrored
              graph.setEdgeAttribute(dup_edge, 'mirrored', true)
              // storing extra meta for mirroflows for later stats analysis
              graph.setEdgeAttribute(dup_edge, 'source_types', [r.type,dup_source_type].sort())
              graph.setEdgeAttribute(dup_edge, 'reporting_types', [r.reporting_type,graph.getNodeAttribute(r.expimp === 'Imp' ? source : target, 'type')].sort())
              graph.setEdgeAttribute(dup_edge, 'partner_types', [r.partner_type,graph.getNodeAttribute(r.expimp === 'Imp' ? target : source, 'type')].sort())

              // rules to choose which information to keep
              if(
                // dup edege is from aggregation but the new is not, let's merge to give priority to source
                (dup_source_type === 'aggregation' && r.source_type !== 'aggregation') ||
                // mirror flows from sources, we prefer Exp on Imp in such cases
                (r.expimp==='Exp' && dup_source_type !== 'aggregation' && r.source_type !== 'aggregation')||
                // same rule of mirror flows if both edges are from aggregations
                (r.expimp==='Exp' && dup_source_type === 'aggregation' && r.source_type === 'aggregation')){
                  graph.mergeEdge(source, target, {weight:w, direction:r.expimp, source_type:r.type});
              }
            }
          }
        }

      })
      //prepare output metrics container
      const metrics = {entities:{},networks:{density:{},order:{},size:{}}};
      // compute metrics
      metrics.year = year;
      // network level
      metrics.networks.density = density(graph);
      metrics.networks.nb_reportings = graph.order;
      metrics.networks.nb_flows = graph.size;
      // Modularity score of Louvain partition
      louvain.assign(graph);
      metrics.networks.modularity = modularity(graph);

      // number of mirror Flows as defined by flows between reportings
      metrics.networks.mirrorFlows = graph.edges().reduce((acc,e) => {
        //if (graph.extremities(e).map(n => !!graph.getNodeAttribute(n, 'reporting')).filter(_ => _).length === 2)
        if (graph.getEdgeAttribute(e, 'mirrored')){
          if (graph.getEdgeAttribute(e, 'mirrored')){
            let st = graph.getEdgeAttribute(e, 'source_types');
            let rt = graph.getEdgeAttribute(e, 'reporting_types');
            let pt = graph.getEdgeAttribute(e, 'partner_types');
            nbMirrorFlows['by_source_type'][st] = (nbMirrorFlows['by_source_type'][st] + 1) || 1
            nbMirrorFlows['by_reporting_type'][rt] = (nbMirrorFlows['by_reporting_type'][rt] + 1) || 1
            nbMirrorFlows['by_partner_type'][pt] = (nbMirrorFlows['by_partner_type'][pt] + 1) || 1
          }
          return acc + 1;
        }
        else
          return acc;
      },0);
      metrics.nbMirrorFlows = nbMirrorFlows;

      //node level
      pagerank.assign(graph);
      graph.nodes().forEach(n => {
        const reportingType = graph.getNodeAttribute(n, 'type');
        metrics.networks[`nb_entities_${reportingType}`] = (metrics.networks[`nb_entities_${reportingType}`] || 0 ) + 1
        if (graph.getNodeAttribute(n, 'reporting'))
          metrics.networks[`nb_reportings_${reportingType}`] = (metrics.networks[`nb_reportings_${reportingType}`] || 0 ) + 1
        else
          metrics.networks[`nb_partners_${reportingType}`] = (metrics.networks[`nb_partners_${reportingType}`] || 0 ) + 1

        // herfindall index
        const inDegree = graph.inEdges(n).reduce((acc,e) => acc + graph.getEdgeAttribute(e,'weight'), 0);
        const outDegree = graph.outEdges(n).reduce((acc,e) => acc + graph.getEdgeAttribute(e,'weight'), 0);
        graph.setNodeAttribute(n, 'inHerfindahl', graph.inEdges(n).reduce((acc,e) => acc + Math.pow(graph.getEdgeAttribute(e,"weight")/inDegree,2), 0));
        graph.setNodeAttribute(n, 'outHerfindahl', graph.outEdges(n).reduce((acc,e) => acc + Math.pow(graph.getEdgeAttribute(e,"weight")/outDegree,2), 0));
        graph.setNodeAttribute(n, 'herfindahl', graph.neighbors(n).reduce((acc,neighbor) =>{
          // total trade of this neighbor
          let neighborTotal = graph.edges(n,neighbor).reduce((s, e) => s+graph.getEdgeAttribute(e,'weight'), 0);
          return acc + Math.pow(neighborTotal/(inDegree+outDegree),2);
        },0));
        graph.setNodeAttribute(n, 'weightedInDegree', inDegree);
        graph.setNodeAttribute(n, 'weightedOutDegree', outDegree);
        graph.setNodeAttribute(n, 'inDegree', graph.inDegree(n));
        graph.setNodeAttribute(n, 'outDegree', graph.outDegree(n));

        // store node attributes to metrics
        metrics.entities[n] = graph.getNodeAttributes(n);

        if (metrics.entities[n].world_Exp && metrics.entities[n].world_Imp){
          metrics.entities[n].worldTradePart = (metrics.entities[n].world_Imp + metrics.entities[n].world_Exp) / (totalTrade.Exp + totalTrade.Imp);
          graph.setNodeAttribute(n,'worldTradePart', metrics.entities[n].worldTradePart);
        }
      })
      //Write gexf
      if (conf.writeGexf){
        const gexfString = gexf.write(graph);
        fs.writeFile(`./data/networks/${year}.gexf`, gexfString, 'utf8', (err) => {
          if (err)
            done(err);
          else{
            done(null, metrics);
          }
        })
      }
      else
        done(null, metrics);
  });
}

// prepare list of years to compute from cn*onfig
const years = []
for (year = conf.startDate; year <= conf.endDate; year++) {
      years.push(year)
}
// throw computation in async mode
async.map(years, computeGraph, (err, metrics) =>{
   //merge metrics
   const gapMinderMetrics = []
   const networksMetrics =[]
   const metricsByYear = {networks:{density:{},nb_reportings:{},nb_flows:{},modularity:{}},entities:{}}
   const general_stat = {}
   const nbMirrorFlows = {'by_source_type':{},'by_reporting_type':{},'by_partner_type':{}}

   metrics.forEach(m => {

      // aggregating nbMirrorFlows
      for (by in m.nbMirrorFlows){
        for (p in m.nbMirrorFlows[by]){
          nbMirrorFlows[by][p] = (nbMirrorFlows[by][p] || 0) + m.nbMirrorFlows[by][p]
        }
      }
      general_stat['nbMirrorFlows'] = nbMirrorFlows

      // to be factorized later
      metricsByYear.networks.density[m.year] = m.networks.density;
      metricsByYear.networks.modularity[m.year] = m.networks.modularity;
      metricsByYear.networks.nb_reportings[m.year] = m.networks.nb_reportings;
      metricsByYear.networks.nb_flows[m.year] = m.networks.nb_flows;
      // networks metrics only
      networksMetric = {year:m.year};
      for (a in m.networks)
        networksMetric[a] = m.networks[a];
      networksMetrics.push(networksMetric);

      for (e in m.entities){

        // gapminder viz data formatting
        // check if entity is a reporting (i.e. has world trade reports)
        gapMinderMetrics.push({
            year: m.year,
            entity: m.entities[e].label,
            type: m.entities[e].type,
            isReporting: m.entities[e].reporting || false,
            continent: m.entities[e].continent,
            pagerank: m.entities[e].pagerank,
            worldTrade: m.entities[e].world_Imp || 0 + m.entities[e].world_Exp || 0,
            worldTradePart: m.entities[e].worldTradePart || null,
            herfindahl: m.entities[e].herfindahl,
            inHerfindahl: m.entities[e].inHerfindahl,
            outHerfindahl: m.entities[e].outHerfindahl,
            inDegree: m.entities[e].inDegree,
            outDegree: m.entities[e].outDegree,
            weightedInDegree: m.entities[e].weightedInDegree,
            weightedOutDegree: m.entities[e].weightedOutDegree
        })

        // generic output
        if(!metricsByYear.entities[e])
          metricsByYear.entities[e] = {}

        for (p in m.entities[e]){
          if(!metricsByYear.entities[e][p])
            metricsByYear.entities[e][p]={}
          if (['type','label','continent'].indexOf(p) !== -1)
            metricsByYear.entities[e][p] = m.entities[e][p]
          else
            metricsByYear.entities[e][p][m.year] = m.entities[e][p]
        }
      }
   })

   const output_filename = `metrics_${conf.startDate}_${conf.endDate}.json`

   fs.writeFile(`./data/${output_filename}`, JSON.stringify(metricsByYear,null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${output_filename}`);
      else console.log(`writing to ${output_filename}`);
    });
  fs.writeFile(`./data/${conf.gapMinder_metric_filename}`, JSON.stringify(gapMinderMetrics, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.gapMinder_metric_filename}`);
      else console.log(`writing to ${conf.gapMinder_metric_filename}`);
    });
  fs.writeFile(`./data/${conf.network_metric_filename}`, JSON.stringify(networksMetrics, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.network_metric_filename}`);
      else console.log(`writing to ${conf.network_metric_filename}`);
    });
  fs.writeFile(`./data/${conf.general_stat_filename}`, JSON.stringify(general_stat, null, 2), 'utf8', (err)=>{
      if (err) console.log(`error : couldn't write ${conf.general_stat_filename}`);
      else console.log(`writing to ${conf.general_stat_filename}`);
    });
})

DB.get().close();<|endoftext|>


<commit_before>$(function() {
    if (!$('.islive').length) return;

    // http://stackoverflow.com/a/2646441/205832
    function addCommas(nStr) {
        nStr += '';
        var x = nStr.split('.');
        var x1 = x[0];
        var x2 = x.length > 1 ? '.' + x[1] : '';
        var rgx = /(\d+)(\d{3})/;
        while (rgx.test(x1)) {
            x1 = x1.replace(rgx, '$1' + ',' + '$2');
        }
        return x1 + x2;
    }

    var submitted = false;
    var loop = setInterval(function() {
        var req, url = $('.islive').data('url');
        if (submitted) {
            req = $.getJSON(url);
        } else {
            var data = {
                csrfmiddlewaretoken:
                $('.islive input[name="csrfmiddlewaretoken"]').val()
            };
            req = $.post(url, data);
            submitted = true;
        }
        req.then(function(response) {
            if (response.hits) {
                $('.islive b').text(addCommas(response.hits));
                $('.islive').show();
            }
        })
        .fail(function() {
            clearInterval(loop);
        });
    }, 10 * 1000);
});
<commit_msg>bug 1105067 - show livehits immediately<commit_after>$(function() {
    if (!$('.islive').length) return;

    // http://stackoverflow.com/a/2646441/205832
    function addCommas(nStr) {
        nStr += '';
        var x = nStr.split('.');
        var x1 = x[0];
        var x2 = x.length > 1 ? '.' + x[1] : '';
        var rgx = /(\d+)(\d{3})/;
        while (rgx.test(x1)) {
            x1 = x1.replace(rgx, '$1' + ',' + '$2');
        }
        return x1 + x2;
    }

    function update(response) {
        // Doing this if because we don't want to display 0 views
        // when you're the first one to view and your own view hasn't
        // counted yet.
        if (response.hits) {
            $('.islive b').text(addCommas(response.hits));
            $('.islive').show();
        }
    }
    var url = $('.islive').data('url');
    // send a POST after some time to count this as a view
    setTimeout(function() {
        var data = {
            csrfmiddlewaretoken:
            $('.islive input[name="csrfmiddlewaretoken"]').val()
        };
        $.post(url, data).then(update);
    }, 10 * 1000);

    var loop = null;
    function fetch() {
        $.getJSON(url)
        .then(update)
        .fail(function() {
            if (loop !== null) {
                clearInterval(loop);
            }
        });
    }
    // first do an immediate AJAX get to get the number
    fetch();
    // then set up a loop
    setInterval(fetch, 10 * 1000);

});
<|endoftext|>


<commit_before>import { Actions } from '../constants';
import firebase from '../firebase';
import uuid from 'uuid';

export default {
  load() {
    return (dispatch) => {
      dispatch({ type: Actions.FetchingCharacters });
      firebase.database().ref('characters').on('value', (snapshot) => {
        dispatch({
          type: Actions.ReceiveCharacters,
          values: snapshot.val() || {}
        });
      });
    };
  },

  startNewCharacterCreation() {
    return (dispatch) => {
      dispatch({ type: Actions.StartCreatingNewCharacter });
    };
  },

  deleteCharacter(character) {
    return (dispatch) => {
      dispatch({ type: Actions.DeletingCharacter });

      // TODO: Delete portrait from storage here.
      var characterKey = character.name.toLowerCase();
      firebase.database().ref(`characters/${characterKey}`).remove().then(
        () => {
          dispatch({
            type: Actions.CharacterDeleted,
            character
          });
        },

        (err) => {
          console.error('Failed to delete character: ' + err);
        }
      );
    };
  },

  confirmNewCharacter(character) {
    return (dispatch) => {
      dispatch({ type: Actions.SavingNewCharacter });
      character = Object.assign({}, character, {
        id: uuid.v4()
      });

      const portraitMeta = {
        contentType: character.portrait.file.type
      };
      const file = character.portrait.file;

      const portraitPath = `portraits/${character.id}`;
      character.portraitPath = portraitPath;

      const uploadTask = firebase.storage().ref().child(portraitPath).put(file, portraitMeta);
      uploadTask.on('state_changed',
        (snapshot) => {

        },
        (error) => {
          console.error(error);
        },

        () => {

          const characterKey = character.name.toLowerCase();
          character.portraitURL = uploadTask.snapshot.downloadURL;
          firebase.database().ref(`characters/${characterKey}`).set(character).then(
            () => {
              dispatch({
                type: Actions.NewCharacterSaved,
                character
              });
            },
            (err) => {
              console.error('Failed to save character: ' + err);
            }
          )
        }
      );
    };
  },

  discardNewCharacter() {
    return (dispatch) =>{
      dispatch({ type: Actions.DiscardNewCharacter });
    };
  }
}
<commit_msg>Don't duplicate the base64<commit_after>import { Actions } from '../constants';
import firebase from '../firebase';
import uuid from 'uuid';
import { pick } from 'lodash';

export default {
  load() {
    return (dispatch) => {
      dispatch({ type: Actions.FetchingCharacters });
      firebase.database().ref('characters').on('value', (snapshot) => {
        dispatch({
          type: Actions.ReceiveCharacters,
          values: snapshot.val() || {}
        });
      });
    };
  },

  startNewCharacterCreation() {
    return (dispatch) => {
      dispatch({ type: Actions.StartCreatingNewCharacter });
    };
  },

  deleteCharacter(character) {
    return (dispatch) => {
      dispatch({ type: Actions.DeletingCharacter });

      // TODO: Delete portrait from storage here.
      var characterKey = character.name.toLowerCase();
      firebase.database().ref(`characters/${characterKey}`).remove().then(
        () => {
          dispatch({
            type: Actions.CharacterDeleted,
            character
          });
        },

        (err) => {
          console.error('Failed to delete character: ' + err);
        }
      );
    };
  },

  confirmNewCharacter(character) {
    return (dispatch) => {
      dispatch({ type: Actions.SavingNewCharacter });
      character = Object.assign({}, character, {
        id: uuid.v4()
      });

      const portraitMeta = {
        contentType: character.portrait.file.type
      };
      const file = character.portrait.file;

      const portraitPath = `portraits/${character.id}`;
      character.portraitPath = portraitPath;

      const uploadTask = firebase.storage().ref().child(portraitPath).put(file, portraitMeta);
      uploadTask.on('state_changed',
        (snapshot) => {

        },
        (error) => {
          console.error(error);
        },

        () => {

          const characterKey = character.name.toLowerCase();
          character.portraitURL = uploadTask.snapshot.downloadURL;
          const sanitizedCharacter = pick(character, [
            'id',
            'name',
            'portraitPath',
            'portraitURL'
          ]);
          firebase.database().ref(`characters/${characterKey}`).set(sanitizedCharacter).then(
            () => {
              dispatch({
                type: Actions.NewCharacterSaved,
                character
              });
            },
            (err) => {
              console.error('Failed to save character: ' + err);
            }
          )
        }
      );
    };
  },

  discardNewCharacter() {
    return (dispatch) =>{
      dispatch({ type: Actions.DiscardNewCharacter });
    };
  }
}
<|endoftext|>


<commit_before>/**
 * TTFB / TTLB metrics
 */
'use strict';

exports.version = '1.1';

exports.module = function(phantomas) {
	var measured = false,
		reqId = 1; // request ID to consider when calculating TTFB / TTLB

	phantomas.setMetric('timeToFirstByte'); // @desc time it took to receive the first byte of the first response (that was not a redirect)
	phantomas.setMetric('timeToLastByte'); // @desc time it took to receive the last byte of the first response (that was not a redirect)

	phantomas.on('recv', function(entry, res) {
		// metrics already calculated
		if (measured) {
			return;
		}

		if (entry.isRedirect) {
			// wait for the next request
			reqId = entry.id + 1;

			phantomas.log('Time to first byte: response #%d <%s> is a redirect (waiting for response #%d)', entry.id, entry.url, reqId);
			return;
		}

		// check the first response which is not a redirect (issue #74)
		if (entry.id === reqId) {
			phantomas.setMetric('timeToFirstByte', entry.timeToFirstByte, true);
			phantomas.setMetric('timeToLastByte', entry.timeToLastByte, true);

			measured = true;

			phantomas.log('Time to first byte: set to %d ms for #%d request to <%s> (HTTP %d)', entry.timeToFirstByte, entry.id, entry.url, entry.status);
			phantomas.log('Time to last byte: set to %d ms', entry.timeToLastByte);

			phantomas.emit('responseEnd', entry, res); // @desc the first response (that was not a redirect) fully received
		}
	});
};
<commit_msg>timeToFirstByte: emit "responseEdit" internally<commit_after>/**
 * TTFB / TTLB metrics
 */
'use strict';

exports.version = '1.1';

exports.module = function(phantomas) {
	var measured = false,
		reqId = 1; // request ID to consider when calculating TTFB / TTLB

	phantomas.setMetric('timeToFirstByte'); // @desc time it took to receive the first byte of the first response (that was not a redirect)
	phantomas.setMetric('timeToLastByte'); // @desc time it took to receive the last byte of the first response (that was not a redirect)

	phantomas.on('recv', function(entry, res) {
		// metrics already calculated
		if (measured) {
			return;
		}

		if (entry.isRedirect) {
			// wait for the next request
			reqId = entry.id + 1;

			phantomas.log('Time to first byte: response #%d <%s> is a redirect (waiting for response #%d)', entry.id, entry.url, reqId);
			return;
		}

		// check the first response which is not a redirect (issue #74)
		if (entry.id === reqId) {
			phantomas.setMetric('timeToFirstByte', entry.timeToFirstByte, true);
			phantomas.setMetric('timeToLastByte', entry.timeToLastByte, true);

			measured = true;

			phantomas.log('Time to first byte: set to %d ms for #%d request to <%s> (HTTP %d)', entry.timeToFirstByte, entry.id, entry.url, entry.status);
			phantomas.log('Time to last byte: set to %d ms', entry.timeToLastByte);

			phantomas.emitInternal('responseEnd', entry, res); // @desc the first response (that was not a redirect) fully received
		}
	});
};
<|endoftext|>


<commit_before>'use strict';

/**
 * @module devices
 */

/**
 * Fired when a new device is connected.
 *
 * @event MESSAGES.AUTHENTICATED
 * @param {String} The device HTTP address
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when the status of a device has been updated.
 *
 * @event MESSAGES.SESSION_STATUS_UPDATED
 * @param {String} The new session status
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when a new recording index has been created.
 *
 * @event MESSAGES.NEW_SESSION_INDEX
 * @param {String} The index type ("image" or "tag")
 * @param {Number} The index timecode (in ms)
 * @param {Mixed} The index associated data
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when the name of a device has been updated.
 *
 * @event MESSAGES.NAME_UPDATED
 * @param {String} The device's name
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when configured presets of a device have changed.
 *
 * @event MESSAGES.PRESETS_UPDATED
 * @param {Object} The list of configured presets
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when storage of a device has been updated.
 *
 * @event MESSAGES.STORAGE_UPDATED
 * @param {Number} Number of free Bytes
 * @param {Number} Number of used Bytes
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when inputs of a device have changed.
 *
 * @event MESSAGES.INPUTS_UPDATED
 * @param {Object} Camera input status
 * @param {Object} Slides input status
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when a device has been disconnected.
 *
 * @event MESSAGES.DISCONNECTED
 * @param {String} The id of the device
 */

/**
 * Fired when something went wrong on the connection with a device.
 *
 * @event MESSAGES.ERROR
 * @param {Error} The socket.io error
 * @param {String} The id of the device
 */

var util = require('util');
var shortid = require('shortid');
var async = require('async');
var Pilot = require('@openveo/api').socket.Pilot;
var DevicePilot = process.requireDevicesApi('app/server/devices/DevicePilot.js');
var DeviceError = process.requireDevicesApi('app/server/devices/DeviceError.js');
var DEVICES_TYPES = process.requireDevicesApi('app/server/devices/types.js');
var VEOBOX_MESSAGES = process.requireDevicesApi('app/server/devices/veobox/messages.js');
var VEOBOX_STATUSES = process.requireDevicesApi('app/server/devices/veobox/statuses.js');

/**
 * Asks a device.
 *
 * @method askForSettings
 * @private
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {String} messageId The id of the message to send to the device
 * @param {Object} data Data to send to the device
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
function ask(ids, name, data, callback) {
  var self = this;
  var actions = [];

  ids.forEach(function(id) {
    actions.push(function(callback) {
      var device = self.getClient(id);

      if (!device)
        return callback(new DeviceError('Device not connected', id));

      process.logger.debug('Say "' + name + '" to device', {id: id, data: data});
      device.socket.emit(name, data, function(response) {
        if (response && response.error)
          return callback(new DeviceError('Asking for "' + name + '" failed', id, response.error.code));

        callback();
      });
    });
  });

  async.parallel(async.reflectAll(actions), callback);
}

/**
 * Asks a device for its settings (status, storage, inputs and presets).
 *
 * @method askForSettings
 * @private
 * @async
 * @param {String} id The device's id
 * @param {Function} callback Function to call when it's done with :
 *  - **Error** An error if something went wrong, null otherwise
 */
function askForSettings(id, callback) {
  var settings = ['session.status', 'storage', 'inputs', 'settings.presets'];
  var actions = [];
  var device = this.getClient(id);

  settings.forEach(function(setting) {
    actions.push(function(callback) {
      if (!device)
        return callback(new DeviceError('Device not connected', id));

      var data = {
        event: setting
      };

      process.logger.debug('Say "get" to device', {id: id, data: data});
      device.socket.emit('get', data, function(response) {
        if (response && response.error)
          return callback(new DeviceError('Getting setting "' + setting + '" failed', id, response.error.code));

        callback();
      });
    });
  });

  async.parallel(actions, callback);
}

/**
 * Defines a pilot to interact with [Veobox devices](http://www.veo-labs.com/veobox).
 *
 * @class VeoboxPilot
 * @extends DevicePilot
 * @constructor
 * @param {AdvancedEmitter} devicesEmitter The devices' emitter to listen to devices' messages
 * @param {SocketNamespace} namespace The namespace associated to the devices
 */
function VeoboxPilot(devicesEmitter, namespace) {
  var self = this;
  VeoboxPilot.super_.call(this, DEVICES_TYPES.VEOBOX, devicesEmitter, namespace);

  Object.defineProperties(this, {

    /**
     * Available messages.
     *
     * @property MESSAGES
     * @type Object
     * @final
     */
    MESSAGES: {value: VEOBOX_MESSAGES},

    /**
     * Available statuses.
     *
     * @property STATUSES
     * @type Object
     * @final
     */
    STATUSES: {value: VEOBOX_STATUSES}

  });

  // Listen to device's disconnection
  this.clientEmitter.on(VEOBOX_MESSAGES.DISCONNECTED, function(eventName, socket) {
    var client = self.removeClientBySocketId(socket.id);

    if (client)
      self.emit(eventName, client.id);
  });

  // Listen to device's connection errors
  this.clientEmitter.on(VEOBOX_MESSAGES.ERROR, function(eventName, error, socket) {
    var client = self.getClientBySocketId(socket.id);

    if (client)
      self.emit(eventName, error, client.id);
  });

  // Listen to device's greeting message
  this.clientEmitter.on(VEOBOX_MESSAGES.AUTHENTICATED, function(eventName, id, socket, callback) {
    self.addClient(id, socket);
    self.emit(eventName, socket.handshake.headers['x-forwarded-for'] || socket.handshake.address, id, callback);
  });

  // Emits devices' messages
  this.clientEmitter.on(VEOBOX_MESSAGES.CONNECTED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.SESSION_STATUS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.NEW_SESSION_INDEX, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.NAME_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.PRESETS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.STORAGE_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.INPUTS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
}

module.exports = VeoboxPilot;
util.inherits(VeoboxPilot, DevicePilot);

/**
 * Asks for devices' settings.
 *
 * Asks devices for status, storage, inputs and presets.
 *
 * @method askForSettings
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForSettings = function(ids, callback) {
  var self = this;
  var actions = [];

  ids.forEach(function(id) {
    actions.push(function(callback) {
      askForSettings.call(self, id, callback);
    });
  });

  async.parallel(async.reflectAll(actions), callback);
};

/**
 * Asks for devices' names.
 *
 * @method askForName
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForName = function(ids, callback) {
  ask.call(this, ids, 'get', {event: 'settings.name'}, callback);
};

/**
 * Asks a device to update its name.
 *
 * @method askForUpdateName
 * @async
 * @param {String} id The device's id
 * @param {String} name The new name of the device
 * @param {Function} callback Function to call when it's done with :
 *  - **Error** An error if something went wrong, null otherwise
 */
VeoboxPilot.prototype.askForUpdateName = function(id, name, callback) {
  var device = this.getClient(id);

  if (!device)
    return callback(new DeviceError('Device not connected', id));

  var data = {
    name: name
  };
  process.logger.debug('Say "settings.name" to device', {id: id, data: data});
  device.socket.emit('settings.name', data, function(response) {
    if (response && response.error)
      return callback(new DeviceError('Updating device name failed (' + name + ')', id, response.error.code));

    callback();
  });
};

/**
 * Asks devices to start a record.
 *
 * @method askForStartRecord
 * @async
 * @param {Array} ids The list of connected devices' ids to start
 * @param {Number|Null} presetId The id of the preset to use for the recording session
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForStartRecord = function(ids, presetId, callback) {
  ask.call(this, ids, 'session.start', {
    id: shortid.generate(),
    preset: (presetId) ? presetId : null
  }, callback);
};

/**
 * Asks devices to stop recording.
 *
 * @method askForStopRecord
 * @async
 * @param {Array} ids The list of connected devices' ids to stop
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForStopRecord = function(ids, callback) {
  ask.call(this, ids, 'session.stop', {}, callback);
};

/**
 * Asks devices to index a record (tag).
 *
 * @method askForSessionIndex
 * @async
 * @param {Array} ids The list of connected devices' ids to index
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForSessionIndex = function(ids, callback) {
  ask.call(this, ids, 'session.index', {type: 'tag'}, callback);
};
<commit_msg>Veobox pilot now specifies the device id in callbacks<commit_after>'use strict';

/**
 * @module devices
 */

/**
 * Fired when a new device is connected.
 *
 * @event MESSAGES.AUTHENTICATED
 * @param {String} The device HTTP address
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when the status of a device has been updated.
 *
 * @event MESSAGES.SESSION_STATUS_UPDATED
 * @param {String} The new session status
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when a new recording index has been created.
 *
 * @event MESSAGES.NEW_SESSION_INDEX
 * @param {String} The index type ("image" or "tag")
 * @param {Number} The index timecode (in ms)
 * @param {Mixed} The index associated data
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when the name of a device has been updated.
 *
 * @event MESSAGES.NAME_UPDATED
 * @param {String} The device's name
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when configured presets of a device have changed.
 *
 * @event MESSAGES.PRESETS_UPDATED
 * @param {Object} The list of configured presets
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when storage of a device has been updated.
 *
 * @event MESSAGES.STORAGE_UPDATED
 * @param {Number} Number of free Bytes
 * @param {Number} Number of used Bytes
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when inputs of a device have changed.
 *
 * @event MESSAGES.INPUTS_UPDATED
 * @param {Object} Camera input status
 * @param {Object} Slides input status
 * @param {String} The id of the device
 * @param {Function} The function to respond to the device
 */

/**
 * Fired when a device has been disconnected.
 *
 * @event MESSAGES.DISCONNECTED
 * @param {String} The id of the device
 */

/**
 * Fired when something went wrong on the connection with a device.
 *
 * @event MESSAGES.ERROR
 * @param {Error} The socket.io error
 * @param {String} The id of the device
 */

var util = require('util');
var shortid = require('shortid');
var async = require('async');
var Pilot = require('@openveo/api').socket.Pilot;
var DevicePilot = process.requireDevicesApi('app/server/devices/DevicePilot.js');
var DeviceError = process.requireDevicesApi('app/server/devices/DeviceError.js');
var DEVICES_TYPES = process.requireDevicesApi('app/server/devices/types.js');
var VEOBOX_MESSAGES = process.requireDevicesApi('app/server/devices/veobox/messages.js');
var VEOBOX_STATUSES = process.requireDevicesApi('app/server/devices/veobox/statuses.js');

/**
 * Asks a device.
 *
 * @method askForSettings
 * @private
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {String} messageId The id of the message to send to the device
 * @param {Object} data Data to send to the device
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device with :
 *    - **DeviceError** error An error if something went wrong
 *    - **String** value The device's id
 */
function ask(ids, name, data, callback) {
  var self = this;
  var actions = [];

  ids.forEach(function(id) {
    actions.push(function(callback) {
      var device = self.getClient(id);

      if (!device)
        return callback(new DeviceError('Device not connected', id));

      process.logger.debug('Say "' + name + '" to device', {id: id, data: data});
      device.socket.emit(name, data, function(response) {
        if (response && response.error)
          return callback(new DeviceError('Asking for "' + name + '" failed', id, response.error.code), id);

        callback(null, id);
      });
    });
  });

  async.parallel(async.reflectAll(actions), callback);
}

/**
 * Asks a device for its settings (status, storage, inputs and presets).
 *
 * @method askForSettings
 * @private
 * @async
 * @param {String} id The device's id
 * @param {Function} callback Function to call when it's done with :
 *  - **Error** An error if something went wrong, null otherwise
 */
function askForSettings(id, callback) {
  var settings = ['session.status', 'storage', 'inputs', 'settings.presets'];
  var actions = [];
  var device = this.getClient(id);

  settings.forEach(function(setting) {
    actions.push(function(callback) {
      if (!device)
        return callback(new DeviceError('Device not connected', id));

      var data = {
        event: setting
      };

      process.logger.debug('Say "get" to device', {id: id, data: data});
      device.socket.emit('get', data, function(response) {
        if (response && response.error)
          return callback(new DeviceError('Getting setting "' + setting + '" failed', id, response.error.code));

        callback();
      });
    });
  });

  async.parallel(actions, callback);
}

/**
 * Defines a pilot to interact with [Veobox devices](http://www.veo-labs.com/veobox).
 *
 * @class VeoboxPilot
 * @extends DevicePilot
 * @constructor
 * @param {AdvancedEmitter} devicesEmitter The devices' emitter to listen to devices' messages
 * @param {SocketNamespace} namespace The namespace associated to the devices
 */
function VeoboxPilot(devicesEmitter, namespace) {
  var self = this;
  VeoboxPilot.super_.call(this, DEVICES_TYPES.VEOBOX, devicesEmitter, namespace);

  Object.defineProperties(this, {

    /**
     * Available messages.
     *
     * @property MESSAGES
     * @type Object
     * @final
     */
    MESSAGES: {value: VEOBOX_MESSAGES},

    /**
     * Available statuses.
     *
     * @property STATUSES
     * @type Object
     * @final
     */
    STATUSES: {value: VEOBOX_STATUSES}

  });

  // Listen to device's disconnection
  this.clientEmitter.on(VEOBOX_MESSAGES.DISCONNECTED, function(eventName, socket) {
    var client = self.removeClientBySocketId(socket.id);

    if (client)
      self.emit(eventName, client.id);
  });

  // Listen to device's connection errors
  this.clientEmitter.on(VEOBOX_MESSAGES.ERROR, function(eventName, error, socket) {
    var client = self.getClientBySocketId(socket.id);

    if (client)
      self.emit(eventName, error, client.id);
  });

  // Listen to device's greeting message
  this.clientEmitter.on(VEOBOX_MESSAGES.AUTHENTICATED, function(eventName, id, socket, callback) {
    self.addClient(id, socket);
    self.emit(eventName, socket.handshake.headers['x-forwarded-for'] || socket.handshake.address, id, callback);
  });

  // Emits devices' messages
  this.clientEmitter.on(VEOBOX_MESSAGES.CONNECTED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.SESSION_STATUS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.NEW_SESSION_INDEX, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.NAME_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.PRESETS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.STORAGE_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
  this.clientEmitter.on(VEOBOX_MESSAGES.INPUTS_UPDATED, Pilot.prototype.emitMessageWithId.bind(this));
}

module.exports = VeoboxPilot;
util.inherits(VeoboxPilot, DevicePilot);

/**
 * Asks for devices' settings.
 *
 * Asks devices for status, storage, inputs and presets.
 *
 * @method askForSettings
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForSettings = function(ids, callback) {
  var self = this;
  var actions = [];

  ids.forEach(function(id) {
    actions.push(function(callback) {
      askForSettings.call(self, id, callback);
    });
  });

  async.parallel(async.reflectAll(actions), callback);
};

/**
 * Asks for devices' names.
 *
 * @method askForName
 * @async
 * @param {Array} ids The list of devices' ids
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForName = function(ids, callback) {
  ask.call(this, ids, 'get', {event: 'settings.name'}, callback);
};

/**
 * Asks a device to update its name.
 *
 * @method askForUpdateName
 * @async
 * @param {String} id The device's id
 * @param {String} name The new name of the device
 * @param {Function} callback Function to call when it's done with :
 *  - **DeviceError** An error if something went wrong, null otherwise
 */
VeoboxPilot.prototype.askForUpdateName = function(id, name, callback) {
  var device = this.getClient(id);

  if (!device)
    return callback(new DeviceError('Device not connected', id));

  var data = {
    name: name
  };
  process.logger.debug('Say "settings.name" to device', {id: id, data: data});
  device.socket.emit('settings.name', data, function(response) {
    if (response && response.error)
      return callback(new DeviceError('Updating device name failed (' + name + ')', id, response.error.code));

    callback();
  });
};

/**
 * Asks devices to start a record.
 *
 * @method askForStartRecord
 * @async
 * @param {Array} ids The list of connected devices' ids to start
 * @param {Number|Null} presetId The id of the preset to use for the recording session
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForStartRecord = function(ids, presetId, callback) {
  ask.call(this, ids, 'session.start', {
    id: shortid.generate(),
    preset: (presetId) ? presetId : null
  }, callback);
};

/**
 * Asks devices to stop recording.
 *
 * @method askForStopRecord
 * @async
 * @param {Array} ids The list of connected devices' ids to stop
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForStopRecord = function(ids, callback) {
  ask.call(this, ids, 'session.stop', {}, callback);
};

/**
 * Asks devices to index a record (tag).
 *
 * @method askForSessionIndex
 * @async
 * @param {Array} ids The list of connected devices' ids to index
 * @param {Function} callback Function to call when it's done with :
 *  - **Null** Always null
 *  - **Array** Results for each device
 */
VeoboxPilot.prototype.askForSessionIndex = function(ids, callback) {
  ask.call(this, ids, 'session.index', {type: 'tag'}, callback);
};
<|endoftext|>


<commit_before>import React from 'react';
import { UnauthorizedError } from './UnauthorizedError';

export const GraphQLError = ({ error }) => {
  if (error && error.graphQLErrors && error.graphQLErrors.length > 0) {
    return
{error.message}
;
  }
  if (error && error.networkError) {
    switch (error.networkError.statusCode) {
      case 401:
        return <UnauthorizedError />;
      default:
        return

          {`${error.networkError.statusCode}: ${error.networkError.message}`}

;
    }
  }
  return
No error. Something wrong happened...
;
};
<commit_msg>:sparkles: Added session expired exception with button to re-login<commit_after>import React from 'react';
import { UnauthorizedError } from './UnauthorizedError';

export const GraphQLError = ({ error }) => {
  if (error && error.graphQLErrors && error.graphQLErrors.length > 0) {
    if (error.message === 'jwt expired') {
      return <UnauthorizedError />;
    }
    return
{error.message}
;
  }
  if (error && error.networkError) {
    switch (error.networkError.statusCode) {
      case 401:
        return <UnauthorizedError />;
      default:
        return

          {`${error.networkError.statusCode}: ${error.networkError.message}`}

;
    }
  }
  return
No error. Something wrong happened...
;
};
<|endoftext|>


<commit_before>var request = require('request');
var url = require('url');
var fs = require('fs');
var path = require('path');
var ContentService = require('../content-service');
var util = require('util');
var config = require('../../../config/secrets');
var crowdtangle_lists = require('../../../config/crowdtangle_list');
var Promise = require('promise');

var request = require('request');
var _ = require('underscore');

//options.lastReportDate is passed here through the content-service-factory and will be utilised in calling the api, but its actually only maintained by the parent content service, it is only utilised by the child
var CrowdTangleContentService = function(options) {
	this._nextPageUrl = undefined;
	this._nextPageUrlSavedSearch = undefined;
	this.reloadSettings();
	this._keywords = options.keywords;
	this._lastReportDate = options.lastReportDate;
	this._lastReportDateSavedSearch = options.lastReportDateSavedSearch;
	this.fetchType = 'pull';
	this.interval = 10000;
	this.detectHateSpeech = config.get().detectHateSpeech;
	ContentService.call(this, options); //associates the child service with its parent, which has the notion of lastreportdate
}

util.inherits(CrowdTangleContentService, ContentService);

//this method overwrites the _doFetch method in the content-service (which is the parent class)
//and then inside the content-service, we have a fetch method that is called by the pull-bot and is responsible for emitting (writing) the reports and updating the final value of lastReportDate
//options.maxCount (present but not used, used by the parent content service)
//what about types? -- discuss with Michael
CrowdTangleContentService.prototype._doFetch = function(options, callback) {
	var self = this;
 	//handle errors using process.nextTick
 	// this will depend mostly on the options that we expect to receive, and if there is issue with that

	// if (!this._url) {
	// 	process.nextTick(function() { self.emit('error', new Error('Missing URL')); });
	// 	return callback([]);
	// }

	// now we need to submit the request
	var  requestUrl = this._completeUrl(options)

	if (requestUrl !== null){
		this._httpRequest({url: requestUrl}, function(err, res, body) {
			if (err) {
				self.emit('error', new Error('HTTP error: ' + err.message));
				return callback([]);
			} else if (res.statusCode != 200) {
				console.error("Error response:", JSON.stringify(res, null, 2));
				self.emit('error', new Error.HTTP(res.statusCode));
				return callback([]);
			}

			//if no errors, parse the body..
			var responses;
			try {
				responses = JSON.parse(body).result.posts;
				var nextPageUrl = JSON.parse(body).result.pagination.nextPage;
				if (!(responses instanceof Array)) {
					self.emit('error', new Error('Wrong data'));
					return callback([]);
				}
				// any other error handling wrt the structure of responses
			} catch (e) {
				self.emit('error', new Error('Parse error: ' + e.message));
				return callback([]);
			}
			// the responses will be sorted by the content-service (parent method)
			// Parse response data and return them.
			if (self.detectHateSpeech){
				var hateSpeechRequests = responses.map(function(x) {return {"data" : x, "request" :ContentService.prototype.getHateSpeechRequest(x.message ? x.message : "")}; });
				Promise.all(hateSpeechRequests.map(function(x){return x.request})).then(
					function(hateSpeechResults){
						hateSpeechResults.forEach((hateSpeechResult, index)=>{
							hateSpeechRequests[index].data.hateSpeechScore = hateSpeechResult.result.hateSpeechScore;
						});
						return hateSpeechRequests;
					}
				).then(function(hateSpeechRequests) {
					callback({"nextPageUrl": nextPageUrl, "reportData": hateSpeechRequests.map(function(x){return self._parse(x.data, options);})});
				});
			} else{
				var reportData = responses.map(function(x) { return self._parse(x, options); });
				callback({"nextPageUrl": nextPageUrl, "reportData": reportData});
			}

		});
	} else {
		callback({});
	}

};

CrowdTangleContentService.prototype._httpRequest = function(params, callback) {
	request(params, callback);
};

CrowdTangleContentService.prototype._completeUrl = function(options) {
	// add one second to last report date to start fetching from
	// if it is the first time fetching data, initialize start date to an old date
	var initialStartDate = new Date();
	initialStartDate.setHours(initialStartDate.getHours() - 3);

	//assigning correct value to lastReportDate based on whether the request is for SAVED_SEARCH or LIST
	var lastReportDate =  this._lastReportDate;
	var nextPageUrl = this._nextPageUrl;
	var listIds = "";
	if(options.requestType == "SAVED_SEARCH"){
		lastReportDate = this._lastReportDateSavedSearch;
		nextPageUrl = this._nextPageUrlSavedSearch;
		listIds = Object.keys(crowdtangle_lists.crowdtangle_saved_searches).join(",");
		//this ensures that if a user doesn't have saved search configured, Aggie doesnt make a Saved search request to CT
		if(listIds == "") {
			return null;
		}
	}

	var startDate = lastReportDate ? new Date(lastReportDate.getTime() + 1000): initialStartDate;
	if(nextPageUrl !== undefined && nextPageUrl !== null) {
		// this._nextPageUrl contains all information about language, searchTerm, sortBy etc.
		var urlObj = url.format(nextPageUrl, {
			protocol: 'https'
		});
		return urlObj;
	} else {
		var urlObj = url.format({
			protocol: 'https',
			hostname: this._baseUrl,
			pathname: this._pathName,
			query: {
				token: this._apiToken,
				count: this._count,
				language: this._language,
				listIds: listIds,
				searchTerm: this._keywords,
				sortBy: this._sortParameter,
				startDate: startDate.toISOString(),
				endDate: new Date().toISOString()
			}
		});
		return urlObj;
	}
}

CrowdTangleContentService.prototype._parse = function(data, options) {
	var metadata = {
		sponsor: data.brandedContentSponsor	|| null,
		caption: data.caption || null,
		description: data.description || null, // this stores the content of sharedposts
		imageText: data.imageText || null,
		title: data.title || null,
		crowdtangleId: data.id || null,
		externalUrl: data.link || null,
		platform: data.platform || null,
		type: data.type || null,
		accountVerified: data.account ? data.account.verified : false,
		accountHandle: data.account ? data.account.handle : null,
		subscriberCount: data.account ? data.account.subscriberCount : 0,
		accountUrl: data.account ? data.account.url : null,
		mediaUrl: data.media? data.media.map(function(medium) {
			return {type: medium.type, url: medium.url}
		}) : null,
		actualStatistics: data.statistics.actual || null,
		expectedStatistics: data.statistics.expected || null,
		rawAPIResponse: data,
		hateSpeechScore: data.hateSpeechScore || null
	};

	var author = data.account ? data.account.name || data.account.handle : null;
	var content = data.message || 'No Content'

	// This code deals specifically with matching a crowdtangle list to a report's account id
	this.crowdtangle_lists = crowdtangle_lists.crowdtangle_list_account_pairs;
	this.crowdtangle_saved_searches = crowdtangle_lists.crowdtangle_saved_searches;
	// If the list is found and matched, then the ct_tag is the list name
	if (this.crowdtangle_lists[data.account.id]) {
		metadata.ct_tag = this.crowdtangle_lists[data.account.id];
	} else if (options.requestType == "SAVED_SEARCH") {
		metadata.ct_tag = []
		// This is a workaround to identify possible saved search names associated with a post using substring search.
		// We plan to remove this workaround when CT adds list IDs/account IDs to saved search API responses.
		for(var saved_search_id in this.crowdtangle_saved_searches){
			var saved_search = this.crowdtangle_saved_searches[saved_search_id];
			const doesMatch = this.doesReportMatchWords(saved_search.words, content, metadata)
			doesMatch && metadata.ct_tag.push(saved_search.name);
		}
		//if no matches are found, then add default "Saved Search" tag.
		if(metadata.ct_tag.length == 0){
			metadata.ct_tag.push("Saved Search");
		}
	}
	else {
		// If the list is not found and not matched, make the ct_tag the account.id so we can identify it later.
		metadata.ct_tag = data.account.id;
	}

	return {
	    authoredAt: new Date(data.date + " UTC") || new Date(),
	    fetchedAt: new Date(),
	    content: content,
	    author: author,
	    metadata: metadata,
	    url: data.postUrl,
	    //_sources: '', //need to get this info from somewhere
	    //_sourceNicknames: ''
  	};
}

//regex function used to check if words in saved searches match report data
CrowdTangleContentService.prototype.doesReportMatchWords = function(words, content, metadata) {
	try{
		var regex = new RegExp(words.join("|"), 'gi');
		if (words.length &&
			(regex.test(content)||
			regex.test(metadata.caption) ||
			regex.test(metadata.title) ||
			regex.test(metadata.description)||
			regex.test(metadata.imageText))) {
				return true;
		}
	} catch (err) {
		console.error("Error creating regex for saved searches");
	}
	return false;
}
CrowdTangleContentService.prototype.reloadSettings = function() {
	this._baseUrl = config.get().crowdtangle.baseUrl;
	this._pathName = config.get().crowdtangle.pathName;
	this._count = config.get().crowdtangle.count;
	const useLanguage = config.get().crowdtangle.useLanguage;
	this._language = useLanguage ? config.get().crowdtangle.language : undefined;
	this._sortParameter = config.get().crowdtangle.sortParam;
	this._apiToken = config.get().crowdtangle.apiToken;
};

CrowdTangleContentService.prototype.reloadCTList = function() {
	var listFile = path.resolve(__dirname + '../../../../config/crowdtangle_list.json');
	fs.readFile(listFile, 'utf8', function(err, data) {
			crowdtangle_lists = JSON.parse(data);
	});
};

module.exports = CrowdTangleContentService;
<commit_msg>Exclude CT gateway timeout errors<commit_after>var request = require('request');
var url = require('url');
var fs = require('fs');
var path = require('path');
var ContentService = require('../content-service');
var util = require('util');
var config = require('../../../config/secrets');
var crowdtangle_lists = require('../../../config/crowdtangle_list');
var Promise = require('promise');

var request = require('request');
var _ = require('underscore');

//options.lastReportDate is passed here through the content-service-factory and will be utilised in calling the api, but its actually only maintained by the parent content service, it is only utilised by the child
var CrowdTangleContentService = function(options) {
	this._nextPageUrl = undefined;
	this._nextPageUrlSavedSearch = undefined;
	this.reloadSettings();
	this._keywords = options.keywords;
	this._lastReportDate = options.lastReportDate;
	this._lastReportDateSavedSearch = options.lastReportDateSavedSearch;
	this.fetchType = 'pull';
	this.interval = 10000;
	this.detectHateSpeech = config.get().detectHateSpeech;
	ContentService.call(this, options); //associates the child service with its parent, which has the notion of lastreportdate
}

util.inherits(CrowdTangleContentService, ContentService);

//this method overwrites the _doFetch method in the content-service (which is the parent class)
//and then inside the content-service, we have a fetch method that is called by the pull-bot and is responsible for emitting (writing) the reports and updating the final value of lastReportDate
//options.maxCount (present but not used, used by the parent content service)
//what about types? -- discuss with Michael
CrowdTangleContentService.prototype._doFetch = function(options, callback) {
	var self = this;
 	//handle errors using process.nextTick
 	// this will depend mostly on the options that we expect to receive, and if there is issue with that

	// if (!this._url) {
	// 	process.nextTick(function() { self.emit('error', new Error('Missing URL')); });
	// 	return callback([]);
	// }

	// now we need to submit the request
	var  requestUrl = this._completeUrl(options)

	if (requestUrl !== null){
		this._httpRequest({url: requestUrl}, function(err, res, body) {
			if (err) {
				self.emit('error', new Error('HTTP error: ' + err.message));
				return callback([]);
			} else if (res.statusCode !== 200) {
				console.error("Error response:", JSON.stringify(res, null, 2));
				if (res.statusCode === 504) {
					// Ignore Gateway Timeout error, the CT API is flaky.
				} else {
					self.emit('error', new Error.HTTP(res.statusCode));
				}
				return callback([]);
			}

			//if no errors, parse the body..
			var responses;
			try {
				responses = JSON.parse(body).result.posts;
				var nextPageUrl = JSON.parse(body).result.pagination.nextPage;
				if (!(responses instanceof Array)) {
					self.emit('error', new Error('Wrong data'));
					return callback([]);
				}
				// any other error handling wrt the structure of responses
			} catch (e) {
				self.emit('error', new Error('Parse error: ' + e.message));
				return callback([]);
			}
			// the responses will be sorted by the content-service (parent method)
			// Parse response data and return them.
			if (self.detectHateSpeech){
				var hateSpeechRequests = responses.map(function(x) {return {"data" : x, "request" :ContentService.prototype.getHateSpeechRequest(x.message ? x.message : "")}; });
				Promise.all(hateSpeechRequests.map(function(x){return x.request})).then(
					function(hateSpeechResults){
						hateSpeechResults.forEach((hateSpeechResult, index)=>{
							hateSpeechRequests[index].data.hateSpeechScore = hateSpeechResult.result.hateSpeechScore;
						});
						return hateSpeechRequests;
					}
				).then(function(hateSpeechRequests) {
					callback({"nextPageUrl": nextPageUrl, "reportData": hateSpeechRequests.map(function(x){return self._parse(x.data, options);})});
				});
			} else{
				var reportData = responses.map(function(x) { return self._parse(x, options); });
				callback({"nextPageUrl": nextPageUrl, "reportData": reportData});
			}

		});
	} else {
		callback({});
	}

};

CrowdTangleContentService.prototype._httpRequest = function(params, callback) {
	request(params, callback);
};

CrowdTangleContentService.prototype._completeUrl = function(options) {
	// add one second to last report date to start fetching from
	// if it is the first time fetching data, initialize start date to an old date
	var initialStartDate = new Date();
	initialStartDate.setHours(initialStartDate.getHours() - 3);

	//assigning correct value to lastReportDate based on whether the request is for SAVED_SEARCH or LIST
	var lastReportDate =  this._lastReportDate;
	var nextPageUrl = this._nextPageUrl;
	var listIds = "";
	if(options.requestType == "SAVED_SEARCH"){
		lastReportDate = this._lastReportDateSavedSearch;
		nextPageUrl = this._nextPageUrlSavedSearch;
		listIds = Object.keys(crowdtangle_lists.crowdtangle_saved_searches).join(",");
		//this ensures that if a user doesn't have saved search configured, Aggie doesnt make a Saved search request to CT
		if(listIds == "") {
			return null;
		}
	}

	var startDate = lastReportDate ? new Date(lastReportDate.getTime() + 1000): initialStartDate;
	if(nextPageUrl !== undefined && nextPageUrl !== null) {
		// this._nextPageUrl contains all information about language, searchTerm, sortBy etc.
		var urlObj = url.format(nextPageUrl, {
			protocol: 'https'
		});
		return urlObj;
	} else {
		var urlObj = url.format({
			protocol: 'https',
			hostname: this._baseUrl,
			pathname: this._pathName,
			query: {
				token: this._apiToken,
				count: this._count,
				language: this._language,
				listIds: listIds,
				searchTerm: this._keywords,
				sortBy: this._sortParameter,
				startDate: startDate.toISOString(),
				endDate: new Date().toISOString()
			}
		});
		return urlObj;
	}
}

CrowdTangleContentService.prototype._parse = function(data, options) {
	var metadata = {
		sponsor: data.brandedContentSponsor	|| null,
		caption: data.caption || null,
		description: data.description || null, // this stores the content of sharedposts
		imageText: data.imageText || null,
		title: data.title || null,
		crowdtangleId: data.id || null,
		externalUrl: data.link || null,
		platform: data.platform || null,
		type: data.type || null,
		accountVerified: data.account ? data.account.verified : false,
		accountHandle: data.account ? data.account.handle : null,
		subscriberCount: data.account ? data.account.subscriberCount : 0,
		accountUrl: data.account ? data.account.url : null,
		mediaUrl: data.media? data.media.map(function(medium) {
			return {type: medium.type, url: medium.url}
		}) : null,
		actualStatistics: data.statistics.actual || null,
		expectedStatistics: data.statistics.expected || null,
		rawAPIResponse: data,
		hateSpeechScore: data.hateSpeechScore || null
	};

	var author = data.account ? data.account.name || data.account.handle : null;
	var content = data.message || 'No Content'

	// This code deals specifically with matching a crowdtangle list to a report's account id
	this.crowdtangle_lists = crowdtangle_lists.crowdtangle_list_account_pairs;
	this.crowdtangle_saved_searches = crowdtangle_lists.crowdtangle_saved_searches;
	// If the list is found and matched, then the ct_tag is the list name
	if (this.crowdtangle_lists[data.account.id]) {
		metadata.ct_tag = this.crowdtangle_lists[data.account.id];
	} else if (options.requestType == "SAVED_SEARCH") {
		metadata.ct_tag = []
		// This is a workaround to identify possible saved search names associated with a post using substring search.
		// We plan to remove this workaround when CT adds list IDs/account IDs to saved search API responses.
		for(var saved_search_id in this.crowdtangle_saved_searches){
			var saved_search = this.crowdtangle_saved_searches[saved_search_id];
			const doesMatch = this.doesReportMatchWords(saved_search.words, content, metadata)
			doesMatch && metadata.ct_tag.push(saved_search.name);
		}
		//if no matches are found, then add default "Saved Search" tag.
		if(metadata.ct_tag.length == 0){
			metadata.ct_tag.push("Saved Search");
		}
	}
	else {
		// If the list is not found and not matched, make the ct_tag the account.id so we can identify it later.
		metadata.ct_tag = data.account.id;
	}

	return {
	    authoredAt: new Date(data.date + " UTC") || new Date(),
	    fetchedAt: new Date(),
	    content: content,
	    author: author,
	    metadata: metadata,
	    url: data.postUrl,
	    //_sources: '', //need to get this info from somewhere
	    //_sourceNicknames: ''
  	};
}

//regex function used to check if words in saved searches match report data
CrowdTangleContentService.prototype.doesReportMatchWords = function(words, content, metadata) {
	try{
		var regex = new RegExp(words.join("|"), 'gi');
		if (words.length &&
			(regex.test(content)||
			regex.test(metadata.caption) ||
			regex.test(metadata.title) ||
			regex.test(metadata.description)||
			regex.test(metadata.imageText))) {
				return true;
		}
	} catch (err) {
		console.error("Error creating regex for saved searches");
	}
	return false;
}
CrowdTangleContentService.prototype.reloadSettings = function() {
	this._baseUrl = config.get().crowdtangle.baseUrl;
	this._pathName = config.get().crowdtangle.pathName;
	this._count = config.get().crowdtangle.count;
	const useLanguage = config.get().crowdtangle.useLanguage;
	this._language = useLanguage ? config.get().crowdtangle.language : undefined;
	this._sortParameter = config.get().crowdtangle.sortParam;
	this._apiToken = config.get().crowdtangle.apiToken;
};

CrowdTangleContentService.prototype.reloadCTList = function() {
	var listFile = path.resolve(__dirname + '../../../../config/crowdtangle_list.json');
	fs.readFile(listFile, 'utf8', function(err, data) {
			crowdtangle_lists = JSON.parse(data);
	});
};

module.exports = CrowdTangleContentService;
<|endoftext|>


<commit_before>(function() {
    angular
        .module("resourceStoreService", [])
        .factory("ResourceStore", ["Resource", ResourceStore]);

    function ResourceStore(Resource) {
        var service, _current, _created, _loading, _flags, _hasRequired, _cache, _dataTypes, _idbase;
        service = {
            getCurrent: getCurrent,
            pivot: pivot,
            pivotDone: pivotDone,
            getFlags: getFlags,
            getCreated: getCreated,
            addCreated: addCreated,
            setActiveTemplate: setActiveTemplate,
            getActiveTemplate: getActiveTemplate,
            getDataTypes: getDataTypes,
            getDataTypeByID: getDataTypeByID,
            setLoading: setLoading,
            getAllLoading: getAllLoading,
            isLoading: isLoading,
            hasRequired: hasRequired,
            setHasRequired: setHasRequired,
            addDataTypeHandler: addDataTypeHandler,
            newResource: newResource,
            setResourceTemplate: setResourceTemplate,
            cacheDropzone: cacheDropzone,
            reset: reset,
            clear: clear,
            setIDBase: setIDBase
        };

        _current = null;
        _parents = {
            current: [],
            loading: [],
            flags: [],
            hasRequired: []
        };
        _activeTemplate = null;
        _dataTypes = {};
        _created = [];
        _loading = {};
        _flags = {};
        _hasRequired = false;
        _cache = {
            dz: null
        };

        return service;

        function getCurrent() {
            return _current;
        }

        function pivot(r) {
            _parents.current.push(_current);
            _parents.loading.push(_loading);
            _parents.flags.push(_flags);
            _parents.hasRequired.push(_hasRequired);
            _hasRequired = false;
            _flags = {};
            _loading = {};
            _current = r;
        }

        function pivotDone() {
            var ret = _current;
            addCreated(_current);
            _current = _parents.current.pop();
            _loading = _parents.loading.pop();
            _flags = _parents.flags.pop();
            _hasRequired = _parents.hasRequired.pop();
            return ret;
        }

        function getFlags() {
            return _flags;
        }

        function getCreated() {
            return _created;
        }

        function addCreated(r) {
            _created.push(r);
        }

        function getActiveTemplate() {
            return _activeTemplate;
        }

        function getDataTypes() {
            return _dataTypes;
        }

        function getDataTypeByID(id) {
            return _dataTypes[id];
        }

        function setLoading(prop, loading) {
           _loading[prop] = loading;
        }

        function getAllLoading() {
            return _loading;
        }

        function isLoading(prop) {
            if (typeof _loading[prop] !== "undefined") {
                return _loading[prop];
            } else {
                return false;
            }
        }

        function hasRequired() {
            return _hasRequired;
        }

        function setHasRequired(req) {
            _hasRequired = req;
        }

        function isDirty() {
            return _flags.isDirty;
        }

        function setActiveTemplate(tmpl) {
            _activeTemplate = tmpl;
        }

        function addDataTypeHandler(id, handler) {
            _dataTypes[id] = handler;
        }

        function newResource() {
            _current = new Resource(_idbase);
        }

        function setResourceTemplate(tmpl) {
            _current.setTemplate(tmpl);
        }

        function cacheDropzone(dz) {
            _cache.dz = dz;
        }

        function reset() {
            _current.reset();
            if (_cache.dz) {
                _cache.dz.removeAllFiles();
            }
        }

        function clear() {
            _hasRequired = false;
            _flags = {};
            _loading = {};
            if (_cache.dz !== null) {
                _cache.dz.destroy();
                _cache.dz = null;
            }
            newResource();
        }

        function setIDBase(base) {
            _idbase = base;
        }
    }
})();
<commit_msg>remove unused method<commit_after>(function() {
    angular
        .module("resourceStoreService", [])
        .factory("ResourceStore", ["Resource", ResourceStore]);

    function ResourceStore(Resource) {
        var service, _current, _created, _loading, _flags, _hasRequired, _cache, _dataTypes, _idbase;
        service = {
            getCurrent: getCurrent,
            pivot: pivot,
            pivotDone: pivotDone,
            getFlags: getFlags,
            getCreated: getCreated,
            addCreated: addCreated,
            setActiveTemplate: setActiveTemplate,
            getActiveTemplate: getActiveTemplate,
            getDataTypes: getDataTypes,
            getDataTypeByID: getDataTypeByID,
            setLoading: setLoading,
            getAllLoading: getAllLoading,
            isLoading: isLoading,
            hasRequired: hasRequired,
            setHasRequired: setHasRequired,
            addDataTypeHandler: addDataTypeHandler,
            newResource: newResource,
            setResourceTemplate: setResourceTemplate,
            cacheDropzone: cacheDropzone,
            reset: reset,
            clear: clear,
            setIDBase: setIDBase
        };

        _current = null;
        _parents = {
            current: [],
            loading: [],
            flags: [],
            hasRequired: []
        };
        _activeTemplate = null;
        _dataTypes = {};
        _created = [];
        _loading = {};
        _flags = {};
        _hasRequired = false;
        _cache = {
            dz: null
        };

        return service;

        function getCurrent() {
            return _current;
        }

        function pivot(r) {
            _parents.current.push(_current);
            _parents.loading.push(_loading);
            _parents.flags.push(_flags);
            _parents.hasRequired.push(_hasRequired);
            _hasRequired = false;
            _flags = {};
            _loading = {};
            _current = r;
        }

        function pivotDone() {
            var ret = _current;
            addCreated(_current);
            _current = _parents.current.pop();
            _loading = _parents.loading.pop();
            _flags = _parents.flags.pop();
            _hasRequired = _parents.hasRequired.pop();
            return ret;
        }

        function getFlags() {
            return _flags;
        }

        function getCreated() {
            return _created;
        }

        function addCreated(r) {
            _created.push(r);
        }

        function getActiveTemplate() {
            return _activeTemplate;
        }

        function getDataTypes() {
            return _dataTypes;
        }

        function getDataTypeByID(id) {
            return _dataTypes[id];
        }

        function setLoading(prop, loading) {
           _loading[prop] = loading;
        }

        function getAllLoading() {
            return _loading;
        }

        function isLoading(prop) {
            if (typeof _loading[prop] !== "undefined") {
                return _loading[prop];
            } else {
                return false;
            }
        }

        function hasRequired() {
            return _hasRequired;
        }

        function setHasRequired(req) {
            _hasRequired = req;
        }

        function setActiveTemplate(tmpl) {
            _activeTemplate = tmpl;
        }

        function addDataTypeHandler(id, handler) {
            _dataTypes[id] = handler;
        }

        function newResource() {
            _current = new Resource(_idbase);
        }

        function setResourceTemplate(tmpl) {
            _current.setTemplate(tmpl);
        }

        function cacheDropzone(dz) {
            _cache.dz = dz;
        }

        function reset() {
            _current.reset();
            if (_cache.dz) {
                _cache.dz.removeAllFiles();
            }
        }

        function clear() {
            _hasRequired = false;
            _flags = {};
            _loading = {};
            if (_cache.dz !== null) {
                _cache.dz.destroy();
                _cache.dz = null;
            }
            newResource();
        }

        function setIDBase(base) {
            _idbase = base;
        }
    }
})();
<|endoftext|>