<commit_before>// alpha scroll event hook
// XScrollY: When X gets scrolled... do Y
// Set up a notification for when targets get scrolled to (top of the viewport).
// Trying to follow this scrollspy implementation as a rough guide:
// https://github.com/twitter/bootstrap/blob/master/js/bootstrap-scrollspy.js
//

/*jshint expr:true*/

// define(['jquery', 'underscore'], function($, _) {
  var defaultOptions = {
    updateOffsets: 0,      // force script to re-calculate offsets:
                           //   0 (default)   calculate only the first time
                           //   1             re-calculate after `unveil`
                           //   2             re-calculate after `change`
                           //   3             re-calculate every scroll
    offset: 0,             // pixels from the top of the page to set the origin
    targets: 'section',    // selector for the targets
    throttle: 200          // milliseconds to de-bounce the scroll event
  };

  function XScrollY(options) {
    var self = this;
    this._seen = [];
    if (options.container) {
      this.$scrollElement = $(options.container);
      delete options.container;
    } else {
      this.$scrollElement = $(window);
  }
    this.options = $.extend({}, defaultOptions, options || {});
    this.$targets = $(this.options.targets);
    this.updateOffsets();  // TODO assert offets are monotonic increasing
    this.$scrollElement.on('scroll.xscrolly',
      _.throttle(function() { self.process.call(self); }, this.options.throttle));
    this.process();
  }

  // update target offset lookup
  XScrollY.prototype.updateOffsets = function() {
    var self = this;
    this.offsets = [];
    this.offsetMap = {};
    this.$targets.each(function(idx, target) {
      var offset = $(target).position().top;
      if (!self.offsetMap[offset]) {
        self.offsetMap[offset] = [];
      }
      self.offsets.push(offset);
      self.offsetMap[offset].push(target);
    });
  };

  // get the active offset for the current scroll depth
  XScrollY.prototype.getActiveOffset = function() {
    var scrollTop = this.$scrollElement.scrollTop() + this.options.offset,
        i = 0, n = this.offsets.length;
    for (; i < n; ++i) {
      if (scrollTop < this.offsets[i]) {
        return this.offsets[Math.max(0, i - 1)];
      }
    }
    return this.offsets[n - 1];
  };

  // Find target(s) directly above the origin. If there are multiple targets at
  // the same vertical position, return all of them.
  XScrollY.prototype.getActive = function() {
    var offset = this.getActiveOffset();
    this.activeOffset = offset;
    return $(this.offsetMap[offset]);
  };

  // EVENTS

  // Callback for the scroll event
  XScrollY.prototype.process = function() {
    if (this.options.updateOffsets === 3) {
      this.updateOffsets();
    }
    var oldActiveOffset = this.activeOffset,
        $active = this.getActive();  // this.activeOffset gets set in here :(
    if (this.activeOffset != oldActiveOffset) {
      this.change($active);
      this.$active = $active;
    }
    this.options.scroll && this.options.scroll.call(this, $active);
  };

  // when target scope changes
  XScrollY.prototype.change = function($active) {
    if (this.options.updateOffsets === 2) {
      this.updateOffsets();
    }
    if (_.indexOf(this._seen, $active[0]) == -1) {
      this._seen.push($active[0]);
      this.unveil($active);
    }
    this.options.change && this.options.change.call(this, $active);
  };

  // when target scope changes for the first time
  XScrollY.prototype.unveil = function($active) {
    if (this.options.updateOffsets === 1) {
      this.updateOffsets();
    }
    this.options.unveil && this.options.unveil.call(this, $active);
  };

  // METHODS

  XScrollY.prototype.disable = function() {
    // TODO
    this.$scrollElement.off('.xscrolly');
  };
  XScrollY.prototype.enable = function() {
    // TODO
  };

  // HELPER METHODS
  //
  // to get elements based purely on internally stored offsets

  // get elements between `top` and `bottom` scroll depth.
  XScrollY.prototype.slice = function(top, bottom) {
    var offsets = this.offsets,
        offsetMap = this.offsetMap;
    var i = 0, n = offsets.length,
        $ret = $(),
        off;
    for (; i < n; ++i) {
      off = offsets[i];
      if (top <= off && off < bottom) {
        $ret = $ret.add(offsetMap[off]);
      }
    }
    return $ret;
  };

  // get all targets visible on screen
  //
  // arguments:
  //
  //   localOffset    (default: 0) shift the viewport down this many pixels.
  //                  Useful if you need to counter a global offset.
  //   bleed          (default: 0) Extend the viewport this many pixels. Just
  //                  like how bleed works in print.
  //
  XScrollY.prototype.visible = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var scrollTop = this.$scrollElement.scrollTop(),
        scrollBottom = scrollTop + this.$scrollElement.height();
    return this.slice(scrollTop + this.options.offset + localOffset - bleed,
        scrollBottom + this.options.offset + localOffset + bleed);
  };

  // get all targets above
  XScrollY.prototype.above = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop();
    return this.slice(0,
        origin + this.options.offset + localOffset + bleed);
  };

  // get all targets above origin + screen
  XScrollY.prototype.aboves = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop(),
        origins = origin + this.$scrollElement.height();
    return this.slice(0,
        origins + this.options.offset + localOffset + bleed);
  };

  // get all targets below
  XScrollY.prototype.below = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop();
    return this.slice(origin + this.options.offset + localOffset - bleed,
      Infinity);
  };

  // get all targets below screen
  XScrollY.prototype.belows = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop(),
        origins = origin + this.$scrollElement.height();
    return this.slice(origins + this.options.offset + localOffset - bleed,
      Infinity);
  };

  // set an option: `option('offset', 100)`
  // set several options: `option({offset: 100, throttle: 1000})`
  XScrollY.prototype.option = function(key, value) {
    if (typeof key === 'object' && typeof value === 'undefined') {
      $.extend(this.options, key);
    } else {
      this.options[key] = value;
    }
  };

  // jQuery plugin
  $.fn.xscrolly = function(options, key, value) {
    if (options === 'option') {
      this.data('xscrolly').option(key, value);
    } else {
      options.targets = this;
      this.data('xscrolly', new XScrollY(options));
    }
    return this;
  };
// });
<commit_msg>refactor meat of updateOffsets out<commit_after>// alpha scroll event hook
// XScrollY: When X gets scrolled... do Y
// Set up a notification for when targets get scrolled to (top of the viewport).
// Trying to follow this scrollspy implementation as a rough guide:
// https://github.com/twitter/bootstrap/blob/master/js/bootstrap-scrollspy.js
//

/*jshint expr:true*/

// define(['jquery', 'underscore'], function($, _) {
  var defaultOptions = {
    updateOffsets: 0,      // force script to re-calculate offsets:
                           //   0 (default)   calculate only the first time
                           //   1             re-calculate after `unveil`
                           //   2             re-calculate after `change`
                           //   3             re-calculate every scroll
    offset: 0,             // pixels from the top of the page to set the origin
    targets: 'section',    // selector for the targets
    throttle: 200          // milliseconds to de-bounce the scroll event
  };

  function XScrollY(options) {
    var self = this;
    this._seen = [];
    if (options.container) {
      this.$scrollElement = $(options.container);
      delete options.container;
    } else {
      this.$scrollElement = $(window);
  }
    this.options = $.extend({}, defaultOptions, options || {});
    this.$targets = $(this.options.targets);
    this.updateOffsets();  // TODO assert offsets are monotonic increasing
    this.$scrollElement.on('scroll.xscrolly',
      _.throttle(function() { self.process.call(self); }, this.options.throttle));
    this.process();
  }

  // update the offsets cache and offset-to-target map, passed in by reference.
  XScrollY.prototype._updateOffsets = function($target, _offsets, _map) {
    $target.each(function(idx, target) {
      var offset = $(target).position().top;
      if (!_map[offset]) {
        _map[offset] = [];
      }
      _offsets.push(offset);
      _map[offset].push(target);
    });
  };

  // update target offset lookup
  XScrollY.prototype.updateOffsets = function() {
    var self = this;
    this.offsets = [];
    this.offsetMap = {};
    this._updateOffsets(this.$targets, this.offsets, this.offsetMap);
  };

  // get the active offset for the current scroll depth
  XScrollY.prototype.getActiveOffset = function() {
    var scrollTop = this.$scrollElement.scrollTop() + this.options.offset,
        i = 0, n = this.offsets.length;
    for (; i < n; ++i) {
      if (scrollTop < this.offsets[i]) {
        return this.offsets[Math.max(0, i - 1)];
      }
    }
    return this.offsets[n - 1];
  };

  // Find target(s) directly above the origin. If there are multiple targets at
  // the same vertical position, return all of them.
  XScrollY.prototype.getActive = function() {
    var offset = this.getActiveOffset();
    this.activeOffset = offset;
    return $(this.offsetMap[offset]);
  };

  // EVENTS

  // Callback for the scroll event
  XScrollY.prototype.process = function() {
    if (this.options.updateOffsets === 3) {
      this.updateOffsets();
    }
    var oldActiveOffset = this.activeOffset,
        $active = this.getActive();  // this.activeOffset gets set in here :(
    if (this.activeOffset != oldActiveOffset) {
      this.change($active);
      this.$active = $active;
    }
    this.options.scroll && this.options.scroll.call(this, $active);
  };

  // when target scope changes
  XScrollY.prototype.change = function($active) {
    if (this.options.updateOffsets === 2) {
      this.updateOffsets();
    }
    if (_.indexOf(this._seen, $active[0]) == -1) {
      this._seen.push($active[0]);
      this.unveil($active);
    }
    this.options.change && this.options.change.call(this, $active);
  };

  // when target scope changes for the first time
  XScrollY.prototype.unveil = function($active) {
    if (this.options.updateOffsets === 1) {
      this.updateOffsets();
    }
    this.options.unveil && this.options.unveil.call(this, $active);
  };

  // METHODS

  XScrollY.prototype.disable = function() {
    // TODO
    this.$scrollElement.off('.xscrolly');
  };
  XScrollY.prototype.enable = function() {
    // TODO
  };

  // HELPER METHODS
  //
  // to get elements based purely on internally stored offsets

  // get elements between `top` and `bottom` scroll depth.
  XScrollY.prototype.slice = function(top, bottom) {
    var offsets = this.offsets,
        offsetMap = this.offsetMap;
    var i = 0, n = offsets.length,
        $ret = $(),
        off;
    for (; i < n; ++i) {
      off = offsets[i];
      if (top <= off && off < bottom) {
        $ret = $ret.add(offsetMap[off]);
      }
    }
    return $ret;
  };

  // get all targets visible on screen
  //
  // arguments:
  //
  //   localOffset    (default: 0) shift the viewport down this many pixels.
  //                  Useful if you need to counter a global offset.
  //   bleed          (default: 0) Extend the viewport this many pixels. Just
  //                  like how bleed works in print.
  //
  XScrollY.prototype.visible = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var scrollTop = this.$scrollElement.scrollTop(),
        scrollBottom = scrollTop + this.$scrollElement.height();
    return this.slice(scrollTop + this.options.offset + localOffset - bleed,
        scrollBottom + this.options.offset + localOffset + bleed);
  };

  // get all targets above
  XScrollY.prototype.above = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop();
    return this.slice(0,
        origin + this.options.offset + localOffset + bleed);
  };

  // get all targets above origin + screen
  XScrollY.prototype.aboves = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop(),
        origins = origin + this.$scrollElement.height();
    return this.slice(0,
        origins + this.options.offset + localOffset + bleed);
  };

  // get all targets below
  XScrollY.prototype.below = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop();
    return this.slice(origin + this.options.offset + localOffset - bleed,
      Infinity);
  };

  // get all targets below screen
  XScrollY.prototype.belows = function(localOffset, bleed) {
    localOffset = localOffset || 0;
    bleed = bleed || 0;
    var origin = this.$scrollElement.scrollTop(),
        origins = origin + this.$scrollElement.height();
    return this.slice(origins + this.options.offset + localOffset - bleed,
      Infinity);
  };

  // set an option: `option('offset', 100)`
  // set several options: `option({offset: 100, throttle: 1000})`
  XScrollY.prototype.option = function(key, value) {
    if (typeof key === 'object' && typeof value === 'undefined') {
      $.extend(this.options, key);
    } else {
      this.options[key] = value;
    }
  };

  // jQuery plugin
  $.fn.xscrolly = function(options, key, value) {
    if (options === 'option') {
      this.data('xscrolly').option(key, value);
    } else {
      options.targets = this;
      this.data('xscrolly', new XScrollY(options));
    }
    return this;
  };
// });
<|endoftext|>


<commit_before>/*The file to interface with the beam API */

//run 'npm i -S beam-client-node' to get these two things
const BeamClient = require('beam-client-node');
const BeamSocket = require('beam-client-node/lib/ws');
const config = require('./config.json');
const spawn = require('child_process').spawn;

const debug = config.debug;
var moderationModule = config.moderationOn;
const commandModule = config.commandsOn;

var deathblossomMode = false;

let userInfo;
let channelId = -1;

const client = new BeamClient();

//OAuth gives permision for all following actions
client.use('oauth',{
  tokens:{
    access: config.OAuth,
    expires: Date.now() + (365 * 24 *60 *60 *1000)
  },
});

//get following channel and join its chat
client.request('GET', 'users/current')
.then(response => {
    if(debug) console.log(response.body);
    userInfo = response.body;
    return client.request('GET','users/'+userInfo.id+'/follows');
})
.then(response =>{
  //TODO modify to join all followed channels
  if(debug){
    return client.chat.join(userInfo.channel.id);
  }else{
    if(response.body[0]){
      channelId = response.body[0].id;
      console.log("Joining",response.body[0].name);
      return client.chat.join(channelId);
    }else{
      return client.chat.join(userInfo.channel.id);
    }
  }
})
.then(response => {
    const body = response.body;
    if(debug) console.log(body);
    if(!body.roles.includes('Mod') && !body.roles.includes('Owner')){
      console.log("Cannot moderate this chat");
      moderationModule = false;
    }
    if(!debug && channelId !== -1){
      return createChatSocket(userInfo.id, channelId, body.endpoints, body.authkey);
    }else{
      return createChatSocket(userInfo.id, userInfo.channel.id, body.endpoints, body.authkey);
    }
})
.catch(error => {
      console.log("Something went wrong:", error);
});

/**
 * Creates a beam chat socket and sets up listeners to various chat events.
 * @param {number} userId The user to authenticate as
 * @param {number} channelId The channel id to join
 * @param {any} endpoints An endpoints array from a beam.chat.join call.
 * @param {any} authkey An authentication key from a beam.chat.join call.
 * @returns {Promise.<>}
 */
function createChatSocket (userId, channelId, endpoints, authkey) {
    const socket = new BeamSocket(endpoints).boot();

    socket.auth(channelId, userId, authkey)
    .then(() => {
        if(debug) console.log('You are now authenticated!');
        // Send a chat message
        console.log("Joined chatroom");
        return socket.call('msg', ['BOOTING UP...']);
    })
    .catch(error => {
        console.log('Oh no! An error occurred!', error);
    });

    // Listen to chat messages, note that you will also receive your own!
    socket.on('ChatMessage', data => {
        if(debug) console.log('We got a ChatMessage packet!');
        if(debug) console.log(data);
        if(debug) console.log(data.message); // lets take a closer look
        var roles = data.user_roles;
        if(!roles.includes('Mod') && !roles.includes('Owner')){
          if(!deathblossomMode && moderationModule){
            moderate(socket,data,roles);
          }else if(deathblossomMode){
            deathblossom(socket,data,roles);
          }
        }
        if(commandModule){
          commands(socket,data,roles);
        }
    });

    // Listen to socket errors, you'll need to handle these!
    socket.on('error', error => {
        console.error('Socket error', error);
    });
}

function moderate(socket,messageData,roles){
  //Prepare data before spawning thread
  if(debug) console.log("MODERATE: In moderate function");
  var msgArr = messageToString(messageData.message.message);
  var role = parseRoles(roles);
  var msg = msgArr[0];
  var hasLink = msgArr[1]; //TODO make link handler
  var toPython = messageData.user_name + " " + role + " " + msg;
  if(debug) console.log("\tMODERATE: Input to python:", toPython);

  if(hasLink){
    //This is how to send the data to be processed by the python
    var linkHandler = spawn('python3', ['ganon.py']);

    linkHandler.stdin.write(messageData.user_name + " " + role);
    linkHandler.stdin.end();

    linkHandler.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tLINKHANDLER: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tLINKHANDLER: User is:", user);
        console.log("\tLINKHANDLER: Action to take:",action);
        console.log("\tLINKHANDLER: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "nothing"){
        if(debug){console.log("\tLINKHANDLER:No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    linkHandler.stdout.on('end', function(){
      if(debug) console.log("MODERATE: Finished moderate parse");
    });

  }else{
    //This is how to send the data to be processed by the python
    var moderator = spawn('python3', ['hammer.py']);

    moderator.stdin.write(toPython);
    moderator.stdin.end();

    moderator.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tMODERATE: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tMODERATE: User is:", user);
        console.log("\tMODERATE: Action to take:",action);
        console.log("\tMODERATE: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "nothing"){
        if(debug){console.log("\tMODERATE:No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    moderator.stdout.on('end', function(){
      if(debug) console.log("MODERATE: Finished moderate parse");
    });
  }
}

function ban(socket,user,response,msg){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to ban",user);
  }else{
    socket.call('timeout',[user,config.banDuration]); //need to timeout for 1 to unban
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Banning",user,"for message:",msg);
  }
}

function purge(socket,user,response,msg){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to purge",user);
  }else{
    socket.call('purge',[user]);
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Purging",user,"for message:",msg);
  }
}

//To unban, use a duration of 1 with this function
function timeout(socket,user,response,msg,duration){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to timeout",user);
  }else{
    if(duration){
      socket.call('timeout',[user,duration]);
    }else{
      socket.call('timeout',[user,config.timeoutDuration]);
    }
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Timing out",user,"for message:",msg);
  }
}

function unban(socket,user){
  timeout(socket,user,"You have been unbanned", "<TO UNBAN>",1);
}

function messageToString(array){
  var result = "";
  var hasLink = false;
  for(var index in array){
    var element = array[index];
    result += element.text;
    if(element.type === "link"){
      hasLink = true;
    }
  }
  if(debug) console.log("\tMESSAGE2STRING: messageToString is:", result);
  return [result, hasLink];
}

function commands(socket,messageData,roles){
  if(debug) console.log("COMMANDS: In commands function");
  var msg = messageToString(messageData.message.message)[0];
  var role = "Normal";
  var username = messageData.user_name;

  //Send command to python script
  if(msg.charAt(0) != '!'){
    return;
  }else{
    role = parseRoles(roles);
    var handleCmds = spawn('python3', ['centurion.py']);
    var toPython = username + " " + role + " " + msg;
    if(debug) console.log("\tCOMMANDS: Input to python:", toPython);
    handleCmds.stdin.write(toPython);
    handleCmds.stdin.end();

    handleCmds.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tCOMMANDS: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tCOMMANDS: User is:", user);
        console.log("\tCOMMANDS: Action to take:",action);
        console.log("\tCOMMANDS: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "respond"){
        socket.call('msg', [response]);
      }else if(action === "unban"){
        unban(socket,user);
      }else if(action === "deathblossom"){
        if(!deathblossomMode){
          console.log("Activating DeathBlossom");
          socket.call('msg', ["Initializing the DeathBlossom..."]);
          socket.call('msg', ["Now Banning on sight. Lurking in the shadows is recommended"]);
          deathblossomMode = true;
        }else{
          console.log("Deactivating DeathBlossom");
          socket.call('msg', ["DeathBlossom Deactivated."]);
          deathblossomMode = false;
        }
      }else if(action === "nothing"){
        if(debug){console.log("\tCOMMANDS: No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    handleCmds.stdout.on('end', function(){
      if(debug) console.log("COMMANDS: Finished command parse");
    });
  }

}

function parseRoles(roleArr){
  var result = "Normal";
  for(var index in roleArr){
    var role = roleArr[index];
    //TODO parse the remaining Mixer roles'
    if(role === "Owner"){
      result = "Caster";
    }else if(role === "Mod" && result !== "Caster"){
      result = "Mod";
    }else if(role == "User" && result !== "Caster" && result !== "Mod"){
      result = "Normal";
    }
  }
  return result;
}

function deathblossom(socket, messageData, roles){
  var role = parseRoles(roles);
  var username = messageData.user_name;
  timeout(socket, username, null, null, 86400); //Time out for one day

} //TODO
<commit_msg>added preliminary log function (just stdout output for now)<commit_after>/*The file to interface with the beam API */

//run 'npm i -S beam-client-node' to get these two things
const BeamClient = require('beam-client-node');
const BeamSocket = require('beam-client-node/lib/ws');
const config = require('./config.json');
const spawn = require('child_process').spawn;

const debug = config.debug;
var moderationModule = config.moderationOn;
const commandModule = config.commandsOn;

var deathblossomMode = false;

let userInfo;
let channelId = -1;

const client = new BeamClient();

//OAuth gives permision for all following actions
client.use('oauth',{
  tokens:{
    access: config.OAuth,
    expires: Date.now() + (365 * 24 *60 *60 *1000)
  },
});

//get following channel and join its chat
client.request('GET', 'users/current')
.then(response => {
    if(debug) console.log(response.body);
    userInfo = response.body;
    return client.request('GET','users/'+userInfo.id+'/follows');
})
.then(response =>{
  //TODO modify to join all followed channels
  if(debug){
    return client.chat.join(userInfo.channel.id);
  }else{
    if(response.body[0]){
      channelId = response.body[0].id;
      console.log("Joining",response.body[0].name);
      return client.chat.join(channelId);
    }else{
      return client.chat.join(userInfo.channel.id);
    }
  }
})
.then(response => {
    const body = response.body;
    if(debug) console.log(body);
    if(!body.roles.includes('Mod') && !body.roles.includes('Owner')){
      console.log("Cannot moderate this chat");
      moderationModule = false;
    }
    if(!debug && channelId !== -1){
      return createChatSocket(userInfo.id, channelId, body.endpoints, body.authkey);
    }else{
      return createChatSocket(userInfo.id, userInfo.channel.id, body.endpoints, body.authkey);
    }
})
.catch(error => {
      console.log("Something went wrong:", error);
});

/**
 * Creates a beam chat socket and sets up listeners to various chat events.
 * @param {number} userId The user to authenticate as
 * @param {number} channelId The channel id to join
 * @param {any} endpoints An endpoints array from a beam.chat.join call.
 * @param {any} authkey An authentication key from a beam.chat.join call.
 * @returns {Promise.<>}
 */
function createChatSocket (userId, channelId, endpoints, authkey) {
    const socket = new BeamSocket(endpoints).boot();

    socket.auth(channelId, userId, authkey)
    .then(() => {
        if(debug) console.log('You are now authenticated!');
        // Send a chat message
        console.log("Joined chatroom");
        return socket.call('msg', ['BOOTING UP...']);
    })
    .catch(error => {
        console.log('Oh no! An error occurred!', error);
    });

    // Listen to chat messages, note that you will also receive your own!
    socket.on('ChatMessage', data => {
        if(debug) console.log('We got a ChatMessage packet!');
        if(debug) console.log(data);
        if(debug) console.log(data.message); // lets take a closer look
        var roles = data.user_roles;
        log(socket,data,roles);
        if(!roles.includes('Mod') && !roles.includes('Owner')){
          if(!deathblossomMode && moderationModule){
            moderate(socket,data,roles);
          }else if(deathblossomMode){
            deathblossom(socket,data,roles);
          }
        }
        if(commandModule){
          commands(socket,data,roles);
        }
    });

    // Listen to socket errors, you'll need to handle these!
    socket.on('error', error => {
        console.error('Socket error', error);
    });
}

function moderate(socket,messageData,roles){
  //Prepare data before spawning thread
  if(debug) console.log("MODERATE: In moderate function");
  var msgArr = messageToString(messageData.message.message);
  var role = parseRoles(roles);
  var msg = msgArr[0];
  var hasLink = msgArr[1]; //TODO make link handler
  var toPython = messageData.user_name + " " + role + " " + msg;
  if(debug) console.log("\tMODERATE: Input to python:", toPython);

  if(hasLink){
    //This is how to send the data to be processed by the python
    var linkHandler = spawn('python3', ['ganon.py']);

    linkHandler.stdin.write(messageData.user_name + " " + role);
    linkHandler.stdin.end();

    linkHandler.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tLINKHANDLER: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tLINKHANDLER: User is:", user);
        console.log("\tLINKHANDLER: Action to take:",action);
        console.log("\tLINKHANDLER: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "nothing"){
        if(debug){console.log("\tLINKHANDLER:No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    linkHandler.stdout.on('end', function(){
      if(debug) console.log("MODERATE: Finished moderate parse");
    });

  }else{
    //This is how to send the data to be processed by the python
    var moderator = spawn('python3', ['hammer.py']);

    moderator.stdin.write(toPython);
    moderator.stdin.end();

    moderator.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tMODERATE: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tMODERATE: User is:", user);
        console.log("\tMODERATE: Action to take:",action);
        console.log("\tMODERATE: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "nothing"){
        if(debug){console.log("\tMODERATE:No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    moderator.stdout.on('end', function(){
      if(debug) console.log("MODERATE: Finished moderate parse");
    });
  }
}

function ban(socket,user,response,msg){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to ban",user);
  }else{
    socket.call('timeout',[user,config.banDuration]); //need to timeout for 1 to unban
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Banning",user,"for message:",msg);
  }
}

function purge(socket,user,response,msg){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to purge",user);
  }else{
    socket.call('purge',[user]);
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Purging",user,"for message:",msg);
  }
}

//To unban, use a duration of 1 with this function
function timeout(socket,user,response,msg,duration){ //TODO check that the call completed successfully
  if(debug){
    console.log("Need to timeout",user);
  }else{
    if(duration){
      socket.call('timeout',[user,duration]);
    }else{
      socket.call('timeout',[user,config.timeoutDuration]);
    }
    if(!config.silentBans && response !== null) socket.call('msg',["@"+user+": "+response]);
    console.log("Timing out",user,"for message:",msg);
  }
}

function unban(socket,user){
  timeout(socket,user,"You have been unbanned", "<TO UNBAN>",1);
}

function messageToString(array){
  var result = "";
  var hasLink = false;
  for(var index in array){
    var element = array[index];
    result += element.text;
    if(element.type === "link"){
      hasLink = true;
    }
  }
  if(debug) console.log("\tMESSAGE2STRING: messageToString is:", result);
  return [result, hasLink];
}

function commands(socket,messageData,roles){
  if(debug) console.log("COMMANDS: In commands function");
  var msg = messageToString(messageData.message.message)[0];
  var role = "Normal";
  var username = messageData.user_name;

  //Send command to python script
  if(msg.charAt(0) != '!'){
    return;
  }else{
    role = parseRoles(roles);
    var handleCmds = spawn('python3', ['centurion.py']);
    var toPython = username + " " + role + " " + msg;
    if(debug) console.log("\tCOMMANDS: Input to python:", toPython);
    handleCmds.stdin.write(toPython);
    handleCmds.stdin.end();

    handleCmds.stdout.on('data', function(data){
      var pythonOut = data.toString().trim();
      if(debug) console.log("\tCOMMANDS: From python:",pythonOut);
      var action = pythonOut.split(" ")[0];
      var user = pythonOut.split(" ")[1];
      var indexOfSpace1 = pythonOut.indexOf(" ");
      var indexOfSpace2 = pythonOut.indexOf(" ",indexOfSpace1+1);
      var response = pythonOut.substr(indexOfSpace2);
      if(indexOfSpace2 === -1){
        response = "";
      }
      if(debug){
        console.log("\tCOMMANDS: User is:", user);
        console.log("\tCOMMANDS: Action to take:",action);
        console.log("\tCOMMANDS: Response is: ","@"+user+": "+response);
      }
      //Parse the actions and take action if needed
      if(action === "timeout"){
        timeout(socket,user,response,msg);
      }else if(action === "ban"){
        ban(socket,user,response,msg);
      }else if(action === "purge"){
        purge(socket,user,response,msg);
      }else if(action === "respond"){
        socket.call('msg', [response]);
      }else if(action === "unban"){
        unban(socket,user);
      }else if(action === "deathblossom"){
        if(!deathblossomMode){
          console.log("Activating DeathBlossom");
          socket.call('msg', ["Initializing the DeathBlossom..."]);
          socket.call('msg', ["Now Banning on sight. Lurking in the shadows is recommended"]);
          deathblossomMode = true;
        }else{
          console.log("Deactivating DeathBlossom");
          socket.call('msg', ["DeathBlossom Deactivated."]);
          deathblossomMode = false;
        }
      }else if(action === "nothing"){
        if(debug){console.log("\tCOMMANDS: No action to take");}
        //THIS SPACE INTENTIONALLY LEFT BLANK
      }
    });

    handleCmds.stdout.on('end', function(){
      if(debug) console.log("COMMANDS: Finished command parse");
    });
  }

}

function parseRoles(roleArr){
  var result = "Normal";
  for(var index in roleArr){
    var role = roleArr[index];
    //TODO parse the remaining Mixer roles'
    if(role === "Owner"){
      result = "Caster";
    }else if(role === "Mod" && result !== "Caster"){
      result = "Mod";
    }else if(role == "User" && result !== "Caster" && result !== "Mod"){
      result = "Normal";
    }
  }
  return result;
}

function log(socket,messageData,roles){
  if(debug) console.log("LOG: In logs function");
  var msg = messageToString(messageData.message.message)[0];
  var role = "Normal";
  var username = messageData.user_name;

  console.log("[CHAT]"+username + ": "+msg); //TODO add timestamp? or only in log?

  //TODO connect with python module
}

function deathblossom(socket, messageData, roles){
  var role = parseRoles(roles);
  var username = messageData.user_name;
  timeout(socket, username, null, null, 86400); //Time out for one day

} //TODO
<|endoftext|>


<commit_before>"use strict";

var Server = require( "socket.io" );
var log = require( "fm-log" ).module();

var WebSocketLibrary = (function() {
	function WebSocketLibrary() {
	}

	WebSocketLibrary.prototype.configure = function( configuration ) {
		// Did we receive a socket.io instance?
		if( configuration instanceof Server ) {
			this.io = configuration;
			return;
		}

		// Is the configuration just a port number?
		if( typeof configuration == "number" ) {
			this.io = require( "socket.io" )( configuration );
			return;
		}

		// Is it a configuration hash?
		if( typeof configuration == "object" ) {
			this.io = require( "socket.io" )( configuration.port );
			return;
		}
	};

	WebSocketLibrary.prototype.emit = function( name, payload ) {
		if( null !== this.io ) {
			this.io.sockets.emit( name, payload );
		} else {
			log.warn( "Websocket transport has no socket.io configured. emit() has no effect." );
		}
	};

	return WebSocketLibrary;
})();

module.exports = new WebSocketLibrary();
<commit_msg>[FEATURE] Warn when possible socket.io conflict was detected<commit_after>"use strict";

var Server = require( "socket.io" );
var log = require( "fm-log" ).module();

var WebSocketLibrary = (function() {
	function WebSocketLibrary() {
	}

	WebSocketLibrary.prototype.configure = function( configuration ) {
		// Did we receive a socket.io instance?
		if( configuration instanceof Server ) {
			this.io = configuration;
			return;

		} else if( configuration.constructor.name == "Server" ) {
			log.warn( "configuration is an instance of 'Server', but wasn't detected as socket.io instance. This indicates a possible socket.io version conflict between your application and absync! absync will probably not work." );
		}

		// Is the configuration just a port number?
		if( typeof configuration == "number" ) {
			this.io = require( "socket.io" )( configuration );
			return;
		}

		// Is it a configuration hash?
		if( typeof configuration == "object" ) {
			this.io = require( "socket.io" )( configuration.port );
			return;
		}
	};

	WebSocketLibrary.prototype.emit = function( name, payload ) {
		if( null !== this.io ) {
			this.io.sockets.emit( name, payload );
		} else {
			log.warn( "Websocket transport has no socket.io configured. emit() has no effect." );
		}
	};

	return WebSocketLibrary;
})();

module.exports = new WebSocketLibrary();
<|endoftext|>


<commit_before>/* This module uses a give ip to visit the whole urls in the bucket assigned to it
*/
module.exports = function(bucketExplorer , database) {
	//
    var ObjectId = require('mongodb').ObjectId;
	var Spooky = require('spooky');
	var bucket;

    //
    var Buckets = database.model('Buckets');

    //
    var setBucket =  function(bucketObj){
    	 console.log(bucketObj);
    	 bucket = bucketObj;
         if(bucket.urls.length>0){
            for(var i=0; i<bucket.urls.length; i++){
                bucket.urls[i].visited = 0;
                bucket.urls[i].statusText = 'No status yet';
            };
         }

    	 console.log(bucket);
    	 console.log('bucket set in visitor');
         startUpdateDaemon();
    };

    var getBucket = function(){
    	return bucket;
    };

    var updateBucket = function(bucketObj , meta){
    	console.log(meta);
    	if(bucket){
            if(bucketObj._id+'' == bucket._id){
	           if(meta.action == 0){
	           	   console.log('Deleting url in this bucket');
                   bucket.urls.splice(meta.index , 1);
	           }
	           else if(meta.action == 1) {
                   console.log('Updating url in this bucket');
                   bucket = bucketObj;
	           }
	    	}
	    	else{
	    	   console.log('Just passing by');
	    	}
    	}
    	else{
    		 console.log('Bucket empty here');
    	}
    }

    var visitWith = function(ip){
        runGhostProxy (ip , bucket.urls[i] , 0);
    }

    ////updateBucket after every 100 secs of activity
    function startUpdateDaemon(){
        console.log('Starting cron daemon');
        setInterval(function(){
            bucket.lastActive = Date.now()+'';
            Buckets.update(
                {_id : ObjectId(bucket._id)},
                bucket,
                function(err , result){
                    if(err){
                        console.log(err);
                        res.status(500).send('Database error during cron update in visitor');
                    }
                    else {
                        console.log('bucket updated in cron job');
                    }
                }
          );
        } , 100000);
    }

    //
    var exitWhenDone = function(){
         setTimeout(function(){
             console.log('Exiting process');
             process.exit(0);
         } , 10*60000);
    }

    //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

	function runGhostProxy (ip , url , index){
		console.log('starting ghost');
		console.log('process starting '+ip+' '+url.urlName+' '+url.selector);

	    var spooky = new Spooky({
			child: {
				transport: 'http',
				proxy: ip
			},
			casper: {
				logLevel: 'debug',
				verbose: true,
				options: {
				   clientScripts: ['http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js'],
				   pageSettings: {
				        Referer: 'https://www.google.com.ng'
				    }
				}
			}
		 },
		 function(err){
		   if (err) {
				e = new Error('Failed to initialize SpookyJS');
				e.details = err;
				throw e;
			}

			//
			spooky.start(url.urlName);
			spooky.then([{url:url.urlName , selector:url.selector} , function(){

                 this.viewport(1024, 768, function() {
					  console.log('Viewport size changed');

			          //
                      if(selector=='none'){
							 this.then(function(){
							 	  this.wait(5000 , function(){
								    this.emit('done', 'Hello, from ' + this.getCurrentUrl());
								    phantom.clearCookies();
							 	  	this.clear();
							 	  });

							 });
						}

						//General case
						else{
						   this.then(function(){
						   	   this.waitForSelector(selector , function(){
						   	   	  this.thenClick(selector , function() {
										this.wait(5000 , function(){
										    this.emit('done', 'Hello, from ' + this.getCurrentUrl());
										    phantom.clearCookies();
									 	  	this.clear();
									 	});
								    });

						   	   } , function(){
						   	   	     this.emit('done', 'The selector was not found');
								     phantom.clearCookies();
							 	  	 this.clear();
						   	   } , 20000);
						   });
						}

				    });

			}]);

            //
			spooky.run();
		});

		//logs and listeners
		spooky.on('error', function (e, stack) {
			console.log('here');//
			console.error(e);
			Greeting = e;

			if (stack) {
				console.log(stack);
				Greeting = stack;
			}

            //
            if(index < urlsArr.length - 1){
               index++;
               spooky.destroy();
               return runGhostProxy(ip , urlsArr , index);
			}
            else{
                console.log('This round done visiting with '+ip);
                return;
            }
		});

		//
		spooky.on('console', function (line) {
			console.log(line);
		});

		////
		spooky.on('done', function (greeting) {
			console.log(greeting);
			bucket.urls[index].statusText = greeting;
            bucket.urls[index].visited++;

			if(index < urlsArr.length - 1){
               index++;
               return runGhostProxy(ip , urlsArr , index);
			}
            else{
                console.log('This round done visiting with '+ip);
                return;
            }

		});
	};

    //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
	return {
	    visitWith:visitWith,
	    getBucket:getBucket,
	    setBucket:setBucket,
	    updateBucket:updateBucket,
	    exitWhenDone:exitWhenDone
	}

};<commit_msg>updagted<commit_after>/* This module uses a give ip to visit the whole urls in the bucket assigned to it
*/
module.exports = function(bucketExplorer , database) {
	//
    var ObjectId = require('mongodb').ObjectId;
	var Spooky = require('spooky');
	var bucket;

    //
    var Buckets = database.model('Buckets');

    //
    var setBucket =  function(bucketObj){
    	 console.log(bucketObj);
    	 bucket = bucketObj;
         if(bucket.urls.length>0){
            for(var i=0; i<bucket.urls.length; i++){
                bucket.urls[i].visited = 0;
                bucket.urls[i].statusText = 'No status yet';
            };
         }

    	 console.log(bucket);
    	 console.log('bucket set in visitor');
         startUpdateDaemon();
    };

    var getBucket = function(){
    	return bucket;
    };

    var updateBucket = function(bucketObj , meta){
    	console.log(meta);
    	if(bucket){
            if(bucketObj._id+'' == bucket._id){
	           if(meta.action == 0){
	           	   console.log('Deleting url in this bucket');
                   bucket.urls.splice(meta.index , 1);
	           }
	           else if(meta.action == 1) {
                   console.log('Updating url in this bucket');
                   bucket = bucketObj;
	           }
	    	}
	    	else{
	    	   console.log('Just passing by');
	    	}
    	}
    	else{
    		 console.log('Bucket empty here');
    	}
    }

    var visitWith = function(ip){
        runGhostProxy (ip , bucket.urls , 0);
    }

    ////updateBucket after every 100 secs of activity
    function startUpdateDaemon(){
        console.log('Starting cron daemon');
        setInterval(function(){
            bucket.lastActive = Date.now()+'';
            Buckets.update(
                {_id : ObjectId(bucket._id)},
                bucket,
                function(err , result){
                    if(err){
                        console.log(err);
                        res.status(500).send('Database error during cron update in visitor');
                    }
                    else {
                        console.log('bucket updated in cron job');
                    }
                }
          );
        } , 100000);
    }

    //
    var exitWhenDone = function(){
         setTimeout(function(){
             console.log('Exiting process');
             process.exit(0);
         } , 10*60000);
    }

    //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

	function runGhostProxy (ip , urls , index){
		console.log('starting ghost');
		console.log('process starting '+ip+' '+urls[index].urlName+' '+urls[index].selector);

	    var spooky = new Spooky({
			child: {
				transport: 'http',
				proxy: ip
			},
			casper: {
				logLevel: 'debug',
				verbose: true,
				options: {
				   clientScripts: ['http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js'],
				   pageSettings: {
				        Referer: 'https://www.google.com.ng'
				    }
				}
			}
		 },
		 function(err){
		   if (err) {
				e = new Error('Failed to initialize SpookyJS');
				e.details = err;
				throw e;
			}

			//
			spooky.start(urls[index].urlName);
			spooky.then([{url:urls[index].urlName , selector:urls[index].selector} , function(){

                 this.viewport(1024, 768, function() {
					  console.log('Viewport size changed');

			          //
                      if(selector=='none'){
							 this.then(function(){
							 	  this.wait(5000 , function(){
								    this.emit('done', 'Hello, from ' + this.getCurrentUrl());
								    phantom.clearCookies();
							 	  	this.clear();
							 	  });

							 });
						}

						//General case
						else{
						   this.then(function(){
						   	   this.waitForSelector(selector , function(){
						   	   	  this.thenClick(selector , function() {
										this.wait(5000 , function(){
										    this.emit('done', 'Hello, from ' + this.getCurrentUrl());
										    phantom.clearCookies();
									 	  	this.clear();
									 	});
								    });

						   	   } , function(){
						   	   	     this.emit('done', 'The selector was not found');
								     phantom.clearCookies();
							 	  	 this.clear();
						   	   } , 20000);
						   });
						}

				    });

			}]);

            //
			spooky.run();
		});

		//logs and listeners
		spooky.on('error', function (e, stack) {
			console.log('here');//
			console.error(e);
			Greeting = e;

			if (stack) {
				console.log(stack);
				Greeting = stack;
			}

            //
            if(index < urls.length - 1){
               index++;
               spooky.destroy();
               return runGhostProxy(ip , urls , index);
			}
            else{
                console.log('This round done visiting with '+ip);
                return spooky.destroy();
            }
		});

		//
		spooky.on('console', function (line) {
			console.log(line);
		});

		////
		spooky.on('done', function (greeting) {
			console.log(greeting);
			bucket.urls[index].statusText = greeting;
            bucket.urls[index].visited++;

			if(index < urls.length - 1){
               index++;
               spooky.destroy();
               return runGhostProxy(ip , urls , index);
			}
            else{
                console.log('This round done visiting with '+ip);
                return spooky.destroy();
            }

		});
	};

    //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
	return {
	    visitWith:visitWith,
	    getBucket:getBucket,
	    setBucket:setBucket,
	    updateBucket:updateBucket,
	    exitWhenDone:exitWhenDone
	}

};<|endoftext|>


<commit_before>/* This module uses a give ip to visit the whole urls in the bucket assigned to it
*/
'use strict';

module.exports = function(agent , database , ipTracker) {
  console.log(agent.getAgent())
	//
  var ObjectId = require('mongodb').ObjectId;
	var Spooky = require('spooky');
  var EventEmitter = require('events').EventEmitter;
  var domain = require('domain');

	var bucket;
  var ipQueue = [];
  var child_processes = 0;

    //
    var Buckets = database.model('Buckets');

    //
    var setBucket =  function(bucketObj){
    	 bucket = bucketObj;
         if(bucket.urls.length>0){
            for(let i=0; i < bucket.urls.length; i++){
                bucket.urls[i].visited = 0;
                bucket.urls[i].statusText = 'No status yet';
            };
         }

    	 console.log(bucket);
    	 console.log('bucket set in visitor');
    };

    //
    var getBucket = function(){
    	return bucket;
    };

    //
    var updateBucket = function(bucketObj , meta){
    	console.log(meta);
    	if(bucket){
            //
            if(bucketObj._id+'' == bucket._id){
	           if(meta.action == 0){
	           	   console.log('Deleting url in this bucket');
                   bucket.urls.splice(meta.index , 1);
	           }
	           else if(meta.action == 1) {
                   console.log('Updating url in this bucket');
                   bucket = bucketObj;
	           }
	    	}
	    	else{
	    	   console.log('Just passing by');
	    	}
    	}
    	else{
    		 console.log('Bucket empty here');
    	}
    }

    //
    var notifyDelete = function(_id){
         console.log('Bucket with'+ _id +'deleted');
         if(bucket){
             if(bucket._id == _id){
                console.log('This bucket was deleted');
                setTimeout(function(){
                    process.exit(0);
                } , 10000);
             }
             else{
                 console.log('Not this one 1..');
             }

         }
         else{
            console.log('Not this one..');
         }
    }

    //
    function visitWith(ip){
        console.log('Adding %s to queue' , ip);
        if(ipQueue.indexOf(ip)<0){
            ipQueue.push(ip);
        }
        else{
            console.log('ip alredy used or currently in use');
        }

        if(ipQueue.length == 1){
            startUpdateDaemon();
            startVisitingDaemon();
        }
    }

    //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    function V_WORKER(){
        let workerEvents = new EventEmitter;

        let visit = function(ip , urlObj , index , userAgent){
            console.log('visit starting with '+ip+' and '+urlObj.urlName);

            let spooky = Spooky.create({
                child: {
                    transport: 'http',
                    proxy: ip
                },

                casper: {
                    logLevel: 'debug',
                    verbose: true,
                    options: {
                        clientScripts: ['public/js/vendors/jquery.min.js'],
                        pageSettings: {
                            webSecurityEnabled: false,
                            userAgent:userAgent
                        }
                    }
                }
            },

            function(err){
               if (err) {
                    e = new Error('Failed to initialize SpookyJS');
                    e.details = err;
                    throw e;
                }

                //
                (function(url , selector , index){
                    //
                    console.log(userAgent);
                    spooky.start();
                    spooky.then(function () {
                        this.page.customHeaders = {
                            "Referer": 'https://www.plus.google.com'
                        };
                    });
                    spooky.thenOpen(url);
                    spooky.then([{url:url , selector:selector , index:index} , function(){
                        //
                        this.done = function(err , status){
                            if(err){
                                this.emit('done', {status:this.getCurrentUrl() , index:index});
                                return phantom.clearCookies();
                            }
                            else{
                                this.emit('done', {status:this.getCurrentUrl() , index:index});
                                return phantom.clearCookies();
                            }
                        }

                       if(selector=='none'){
                            this.then(function(){
                                this.wait(10000 , function(){
                                    this.done(null , true);
                                });
                            });
                        }

                        else{
                            this.then(function(){
                                this.waitForSelector(selector , function(){
                                    this.thenClick(selector , function() {
                                        this.done(null , true);
                                    });

                                } , function(){
                                    this.done(true , null);
                                } , 10000);
                            });
                        }
                    }]);
                })(urlObj.urlName , urlObj.selector , index);

                //
                spooky.run();
            });

            //
            spooky.on('console', function (line) {
                console.log(line);
            });

            //
            spooky.on('error', function (e, stack) {
                console.error(stack||e);
                workerEvents.emit('done' , {status:stack?stack:e , index:0});
            });

            //
            spooky.on('done', function (status) {
                workerEvents.emit('done' , status);
            });

            //self destruct this instance in 8 minutes
            setTimeout(function(){
                console.log('Instance destroyed');
                spooky.destroy();
                child_processes--;
            } , 8*60000);
        };

        return {
            visit:visit,
            status:workerEvents
        }
    }
    //++++++++++++++++++++++++++++++++END++++++++++++++++++++++++++++++++++++++++++

    //
    function startVisitingDaemon(){
       let v_worker = V_WORKER();

       //@TODO rewrite this function
       (function fillVisiting(currentIp){
            if(currentIp < ipQueue.length && child_processes<20){
               child_processes+=bucket.urls.length;
               console.log(child_processes+' child processes currently running');

               //spin all new workers for each urls
               (function validateUnique(urlIndex){
                    if(urlIndex<0){
                        console.log('ip round complete.. starting next round in 15 secs');
                        setTimeout(function(){
                             return fillVisiting(++currentIp);
                        } , 10000);
                    }
                    else{
                       ipTracker.isUsable(ipQueue[currentIp] , bucket.urls[urlIndex] , function(err , ip){
                           if(ip){
                               v_worker.visit(ip , bucket.urls[urlIndex] , urlIndex , agent.getAgent());
                           }
                           return validateUnique(--urlIndex);
                       });
                    }
               })(bucket.urls.length-1);
            }
            else{
                console.log('Good ip shortage trying again in 15 secs');
                setTimeout(function(){
                     return fillVisiting(currentIp);
                } , 10000);
            }
       })(0);

      //
      v_worker.status.on('done' , function(status){
            console.log(status);
            bucket.urls[status.index].statusText = status.status;
            bucket.urls[status.index].visited++;
      });
    }

    //updateBucket after every 120 secs of activity
    function startUpdateDaemon(){
        let timer = 0;
        console.log('Starting cron daemon');
        setInterval(function(){
            bucket.lastActive = Date.now()+'';
            Buckets.update(
                {_id : ObjectId(bucket._id)},
                bucket,
                function(err , result){
                    if(err){
                        console.log(err);
                        res.status(500).send('Database error during cron update in visitor');
                    }
                    else {
                        timer+=2;
                        if(timer >= 180){
                            console.log('Maximum uptime of three hours exceeded exiting....');
                            process.exit(0);
                        }
                        console.log('bucket updated in cron job');
                    }
                }
          );
        } , 120000);
    }

	  return {
	     visitWith:visitWith,
	     getBucket:getBucket,
	     setBucket:setBucket,
	     updateBucket:updateBucket,
       notifyDelete:notifyDelete
	  }
};
<commit_msg>Signed-off-by: codemuhammed <codemuhammed@gmail.com><commit_after>/* This module uses a give ip to visit the whole urls in the bucket assigned to it
*/
'use strict';

module.exports = function(agent , database , ipTracker) {
  console.log(agent.getAgent())
	//
  var ObjectId = require('mongodb').ObjectId;
	var Spooky = require('spooky');
  var EventEmitter = require('events').EventEmitter;
  var domain = require('domain');

	var bucket;
  var ipQueue = [];
  var child_processes = 0;

    //
    var Buckets = database.model('Buckets');

    //
    var setBucket =  function(bucketObj){
    	 bucket = bucketObj;
         if(bucket.urls.length>0){
            for(let i=0; i < bucket.urls.length; i++){
                bucket.urls[i].visited = 0;
                bucket.urls[i].statusText = 'No status yet';
            };
         }

    	 console.log(bucket);
    	 console.log('bucket set in visitor');
    };

    //
    var getBucket = function(){
    	return bucket;
    };

    //
    var updateBucket = function(bucketObj , meta){
    	console.log(meta);
    	if(bucket){
            //
            if(bucketObj._id+'' == bucket._id){
	           if(meta.action == 0){
	           	   console.log('Deleting url in this bucket');
                   bucket.urls.splice(meta.index , 1);
	           }
	           else if(meta.action == 1) {
                   console.log('Updating url in this bucket');
                   bucket = bucketObj;
	           }
	    	}
	    	else{
	    	   console.log('Just passing by');
	    	}
    	}
    	else{
    		 console.log('Bucket empty here');
    	}
    }

    //
    var notifyDelete = function(_id){
         console.log('Bucket with'+ _id +'deleted');
         if(bucket){
             if(bucket._id == _id){
                console.log('This bucket was deleted');
                setTimeout(function(){
                    process.exit(0);
                } , 10000);
             }
             else{
                 console.log('Not this one 1..');
             }

         }
         else{
            console.log('Not this one..');
         }
    }

    //
    function visitWith(ip){
        console.log('Adding %s to queue' , ip);
        if(ipQueue.indexOf(ip)<0){
            ipQueue.push(ip);
        }
        else{
            console.log('ip alredy used or currently in use');
        }

        if(ipQueue.length == 1){
            startUpdateDaemon();
            startVisitingDaemon();
        }
    }

    //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    function V_WORKER(){
        let workerEvents = new EventEmitter;

        let visit = function(ip , urlObj , index , userAgent){
            console.log('visit starting with '+ip+' and '+urlObj.urlName);

            let spooky = Spooky.create({
                child: {
                    transport: 'http',
                    proxy: ip
                },

                casper: {
                    logLevel: 'debug',
                    verbose: true,
                    options: {
                        clientScripts: ['public/js/vendors/jquery.min.js'],
                        pageSettings: {
                            webSecurityEnabled: false,
                            userAgent:userAgent
                        }
                    }
                }
            },

            function(err){
               if (err) {
                    e = new Error('Failed to initialize SpookyJS');
                    e.details = err;
                    throw e;
                }

                //
                (function(url , selector , index){
                    //
                    console.log(userAgent);
                    spooky.start();
                    spooky.then(function () {
                        this.page.customHeaders = {
                            "Referer": 'https://www.plus.google.com'
                        };
                    });
                    spooky.thenOpen(url);
                    spooky.then([{url:url , selector:selector , index:index} , function(){
                        //
                        this.done = function(err , status){
                            if(err){
                                this.emit('done', {status:this.getCurrentUrl() , index:index});
                                return phantom.clearCookies();
                            }
                            else{
                                this.emit('done', {status:this.getCurrentUrl() , index:index});
                                return phantom.clearCookies();
                            }
                        }

                       if(selector=='none'){
                            this.then(function(){
                                this.wait(10000 , function(){
                                    this.done(null , true);
                                });
                            });
                        }

                        else{
                            this.then(function(){
                                this.waitForSelector(selector , function(){
                                    this.thenClick(selector , function() {
                                        this.done(null , true);
                                    });

                                } , function(){
                                    this.done(true , null);
                                } , 10000);
                            });
                        }
                    }]);
                })(urlObj.urlName , urlObj.selector , index);

                //
                spooky.run();
            });

            //
            spooky.on('console', function (line) {
                console.log(line);
            });

            //
            spooky.on('error', function (e, stack) {
                console.error(stack||e);
                workerEvents.emit('done' , {status:stack?stack:e , index:0});
            });

            //
            spooky.on('done', function (status) {
                workerEvents.emit('done' , status);
            });

            //self destruct this instance in 8 minutes
            setTimeout(function(){
                console.log('Instance destroyed');
                spooky.destroy();
                child_processes--;
            } , 8*60000);
        };

        return {
            visit:visit,
            status:workerEvents
        }
    }
    //++++++++++++++++++++++++++++++++END++++++++++++++++++++++++++++++++++++++++++

    //
    function startVisitingDaemon(){
       let v_worker = V_WORKER();

       //@TODO rewrite this function
       (function fillVisiting(currentIp){
            if(currentIp < ipQueue.length && child_processes<20){
               child_processes+=bucket.urls.length;
               console.log(child_processes+' child processes currently running');

               //spin all new workers for each urls
               (function validateUnique(urlIndex){
                    if(urlIndex<0){
                        console.log('ip round complete.. starting next round in 15 secs');
                        setTimeout(function(){
                             return fillVisiting(++currentIp);
                        } , 10000);
                    }
                    else{
                       ipTracker.isUsable(ipQueue[currentIp] , bucket.urls[urlIndex] , function(err , ip){
                           if(ip){
                               v_worker.visit(ip , bucket.urls[urlIndex] , urlIndex , agent.getAgent());
                           }
                           return validateUnique(--urlIndex);
                       });
                    }
               })(bucket.urls.length-1);
            }
            else{
                console.log('Good ip shortage trying again in 15 secs');
                setTimeout(function(){
                     return fillVisiting(currentIp);
                } , 10000);
            }
       })(0);

      //
      v_worker.status.on('done' , function(status){
            console.log(status);
            bucket.urls[status.index].statusText = status.status;
            bucket.urls[status.index].visited++;
      });
    }

    //updateBucket after every 120 secs of activity
    function startUpdateDaemon(){
        let timer = 0;
        console.log('Starting cron daemon');
        setInterval(function(){
            bucket.lastActive = Date.now()+'';
            Buckets.update(
                {_id : ObjectId(bucket._id)},
                bucket,
                function(err , result){
                    if(err){
                        console.log(err);
                        res.status(500).send('Database error during cron update in visitor');
                    }
                    else {
                        timer+=2;
                        if(timer >= 120){
                            console.log('Maximum uptime of three hours exceeded exiting....');
                            process.exit(0);
                        }
                        console.log('bucket updated in cron job');
                    }
                }
          );
        } , 120000);
    }

	  return {
	     visitWith:visitWith,
	     getBucket:getBucket,
	     setBucket:setBucket,
	     updateBucket:updateBucket,
       notifyDelete:notifyDelete
	  }
};
<|endoftext|>


<commit_before>var http = require('http'),
  fs = require('fs'),
  path = require('path'),
  utile = require('utile'),
  libxml = require('libxmljs');

var stream = null;
var json_arr = [];
var srw_config = { // TODO: limit options
  start: 1,
  limit: 100
};

// args
var argv = require('minimist')(process.argv.slice(2), {
  string: ['q, O, d, f, h'],
  boolean: ['s', 'i', 'a'],
  alias: {
    'q': 'query',
    'O': 'output',
    'd': 'dir',
    'f': 'format',
    'h': 'host',
    's': 'stream',
		'i': 'include-all',
    'a': 'include-anno'
  },
  default: {
    'd': 'output',
    'O': 'output',
    'f': 'csv',
    'h': 'devtools.clarin.dk',
  }
});

var delimiter = (argv.f == 'tsv') ? '\t' : '%'; // TODO: custom delimiter

console.log('include all: ' + argv.i);
console.log('include anno: ' + argv.a);

var annotationsOnly = (!argv.i && argv.a);

// eSciDoc 1.3.x SRW Namespaces
var ns_obj = {
  'sru-zr': 'http://www.loc.gov/zing/srw/',
  'escidocItem': 'http://www.escidoc.de/schemas/item/0.10',
  'escidocMetadataRecords': 'http://www.escidoc.de/schemas/metadatarecords/0.5',
  'escidocContainer': 'http://wwww.escidoc.de/schemas/container/0.4',
  'escidocContentStreams': 'http://www.escidoc.de/schemas/contentstreams/0.7',
  'escidocComponents': 'http://www.escidoc.de/schemas/components/0.9',
  'version': 'http://escidoc.de/core/01/properties/version/',
  'release': 'http://escidoc.de/core/01/properties/release/',
  'prop': 'http://escidoc.de/core/01/properties/',
  'srel': 'http://escidoc.de/core/01/structural-relations/',
  'relations': 'http://www.escidoc.de/schemas/relations/0.3',
  'xlink': 'http://www.w3.org/1999/xlink'
};

var getItemProperties = function(item) {
  var props = {}; // Properties object

	var escidocID_href = item.attr('href').value();
  props.escidocID = escidocID_href.substring(escidocID_href.indexOf('dkclarin'), escidocID_href.length);
	if(props.escidocID.indexOf('properties') != -1) props.escidocID = props.escidocID.substring(0, props.escidocID.indexOf('properties')-1);

  var contentModelID_href = item.get('//srel:content-model', ns_obj).attr('href').value();
  props.contentModelID = contentModelID_href.substring(contentModelID_href.indexOf('dkclarin'), contentModelID_href.length);

  var obj_pid = item.get('//prop:pid', ns_obj);
	var ver_pid = item.get('//prop:version/version:pid', ns_obj);
  props.ver_no = item.get('//prop:latest-version/version:number', ns_obj).text();

  var last_date = item.attr('last-modification-date').value();

  props.pid = "none";

  if (ver_pid != null)
    props.pid = ver_pid.text();
  else if(obj_pid != null)
    props.pid = obj_pid.text();
	else
		console.error('No PID value found for item: ' + props.escidocID);

  return props;
}

// Parse and pull ID data from the XMLDocument
var parse = function(doc) {
  var totalRecords = doc.get('//sru-zr:numberOfRecords', ns_obj).text();
  var items = doc.find('//escidocItem:item', ns_obj);
  var containers = doc.find('//escidocContainer:container', ns_obj);

  console.log('Total records in query: ' + totalRecords);

  if (utile.isArray(items)) console.log('Found ' + items.length + ' items.');
  else console.error('Found 0 items in XMLDocument.');

  if (utile.isArray(containers)) console.log('Found ' + containers.length + ' containers.');
  else console.error('Found 0 containers in XMLDocument.');

  // Add the items to list
  utile.each(items, function(val, key) {
    // CSV format: item%{escidocID}%[{objectPID}}|{lastVersionPID}]%{versionNo}
    if (!annotationsOnly) {
      addMember(getItemProperties(val));
    }

		if(argv.a){
      var relations = val.find('relations:relations/relations:relation', ns_obj);
      relations.forEach(function(relation) {
        if (relation != undefined || relation != null) {
          relation_type = relation.attr('predicate').value();
          if (relation_type.indexOf('HasAnnotation') != -1) {
            var relation_href = relation.attr('href').value();
            var relationObjID = relation_href.substring(relation_href.indexOf('dkclarin'), relation.length);

            console.log('Found Annotation: ' + relationObjID);

            // retrieve properties for Annotation item from eSciDoc REST
            retrieveItemProperties(relationObjID, function(annoPropsItem) {
							console.log('props xml: ' + annoPropsItem.toString());
              var props = getItemProperties(annoPropsItem.root());
              addMember(props); // add annotation member to file
            });
          }
        }
      });
    }
  });

  // TODO: Handle containers array
  utile.each(containers, function(val, key) {});

  // iterate over complete SRW result set
  if(Number(totalRecords) >= (srw_config.limit + srw_config.start)) {
    srw_config.start += srw_config.limit;
    retrieveSRWResult(srw_config, function(xml) {
      parse(xml);
    });
  } else {
    if(argv.f == 'json')
      addMemberToFile(JSON.stringify(json_arr), stream); // write JSON object to output file
    //if(stream instanceof fs.WriteStream)
      //stream.destroySoon();
  }
}

/** SRW Query fetch */
var retrieveSRWResult = function(config, callback) {
  return execGET(srw_options(argv.q, config), callback);
}

/* Item Properties */
var retrieveItemProperties = function(escidocID, callback) {
  return execGET(ir_options(escidocID, '/properties'), callback);
}

var execGET = function(options, callback) {
  http.get(options, function(res) {
    var str = "";
    res.on('data', function(chunk) {
      str += chunk;
    });
    res.on('end', function() {
      if (res.statusCode == "200")
        callback(libxml.parseXmlString(str));
      else
        throw new Error("Error code: " + res.statusCode);
    });
  }).on('error', function(e) {
    console.error(e);
  });

  return true;
}

/** SRW query options */
var srw_options = function(queryCQL, record_vals) {
  var search_path_limits = "&maximumRecords=" + record_vals.limit + "&startRecord=" + record_vals.start;
  var targetUrl = argv.h;
  return {
    host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
    path: '/srw/search/escidoc_all?query=' + encodeURIComponent(queryCQL.replace(/\s/g, "%20")) + search_path_limits + '&d=' + Date.now()
  };
};

/* eSciDoc Item REST */
var ir_options = function(escidocID, path) {
  var targetUrl = argv.h;
  return {
    host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
    path: '/ir/item/' + escidocID + path
  };
};

// append props to output file
var addMember = function(props) {
  if (argv.f == 'csv' || argv.f == 'tsv')
    addMemberToFile(['item', props.escidocID, props.contentModelID, props.pid, props.ver_no].join(delimiter) + '\n');
  else if (argv.f == 'json') // write output to JSON object
    json_arr.push({
		  type: 'item',
		  systemID: props.escidocID,
		  contentModelID: props.contentModelID,
		  PID: props.pid,
		  versionNo: props.ver_no
		});
  else
    throw new Error('Unsupported format:' + argv.f);
}

// write output to file
var addMemberToFile = function(data) {
  if (stream != null)
    if (stream.write(data)) console.log('Data written: ' + data);
    else console.log('Buffer full waiting for drain..');
  else
    process.send(data);
}

var run = function() {
  var is_json = (argv.f == 'json');
  var is_csv = (argv.f == 'csv');
  var is_tsv = (argv.f == 'tsv');

  var file_name = argv.O;
  var file_dir = argv.d;
  var file_path = null;

  if (!argv.s) {
    console.log("Exists?: " + file_dir);
    if (fs.existsSync(file_dir)) {
      if (is_json)
        file_path = path.join(file_dir, file_name + '.json');
      else if (is_csv)
        file_path = path.join(file_dir, file_name + '.csv');
      else if (is_tsv)
        file_path = path.join(file_dir, file_name + '.tsv');

      stream = fs.createWriteStream(file_path, {
        encoding: 'utf8'
      });
      stream.on('error', function(err) {
        if (err) throw err;
      });
      stream.on('drain', function() {
        console.log('Data drained from buffer to file:' + file_path);
      	stream.destorySoon();
			});
      stream.on('close', function() {
        console.log('Closing file stream.');
      });

      console.log('Using query: ' + argv.q);
    } else {
      console.log('Target directory doesn\'t exist: ' + file_dir);
      fs.mkdir(file_dir, function(err) {
        if (err) throw err;
        console.log('Created directory: ' + file_dir);
        run();
      });
    }
  }

  if (argv.s || stream != null)
    retrieveSRWResult(srw_config, function(xml) {
      parse(xml);
    });
}

module.exports = run;

run();
<commit_msg>Added annotation options for inclusion in outputted members list.<commit_after>var http = require('http'),
  fs = require('fs'),
  path = require('path'),
  utile = require('utile'),
  libxml = require('libxmljs');

var stream = null;
var json_arr = [];
var srw_config = { // TODO: limit options
  start: 1,
  limit: 100
};

// args
var argv = require('minimist')(process.argv.slice(2), {
  string: ['q, O, d, f, h'],
  boolean: ['s', 'i', 'a'],
  alias: {
    'q': 'query',
    'O': 'output',
    'd': 'dir',
    'f': 'format',
    'h': 'host',
    's': 'stream',
		'i': 'include-all',
    'a': 'include-anno'
  },
  default: {
    'd': 'output',
    'O': 'output',
    'f': 'csv',
    'h': 'devtools.clarin.dk',
  }
});

var delimiter = (argv.f == 'tsv') ? '\t' : '%'; // TODO: custom delimiter

console.log('include all: ' + argv.i);
console.log('include anno: ' + argv.a);

var annotationsOnly = (!argv.i && argv.a);

// eSciDoc 1.3.x SRW Namespaces
var ns_obj = {
  'sru-zr': 'http://www.loc.gov/zing/srw/',
  'escidocItem': 'http://www.escidoc.de/schemas/item/0.10',
  'escidocMetadataRecords': 'http://www.escidoc.de/schemas/metadatarecords/0.5',
  'escidocContainer': 'http://wwww.escidoc.de/schemas/container/0.4',
  'escidocContentStreams': 'http://www.escidoc.de/schemas/contentstreams/0.7',
  'escidocComponents': 'http://www.escidoc.de/schemas/components/0.9',
  'version': 'http://escidoc.de/core/01/properties/version/',
  'release': 'http://escidoc.de/core/01/properties/release/',
  'prop': 'http://escidoc.de/core/01/properties/',
  'srel': 'http://escidoc.de/core/01/structural-relations/',
  'relations': 'http://www.escidoc.de/schemas/relations/0.3',
  'xlink': 'http://www.w3.org/1999/xlink'
};

var getItemProperties = function(item) {
  var props = {}; // Properties object

	var escidocID_href = item.attr('href').value();
  props.escidocID = escidocID_href.substring(escidocID_href.indexOf('dkclarin'), escidocID_href.length);
	if(props.escidocID.indexOf('properties') != -1) props.escidocID = props.escidocID.substring(0, props.escidocID.indexOf('properties')-1);

  var contentModelID_href = item.get('//srel:content-model', ns_obj).attr('href').value();
  props.contentModelID = contentModelID_href.substring(contentModelID_href.indexOf('dkclarin'), contentModelID_href.length);

  var obj_pid = item.get('//prop:pid', ns_obj);
	var ver_pid = item.get('//prop:version/version:pid', ns_obj);
  props.ver_no = item.get('//prop:latest-version/version:number', ns_obj).text();

  var last_date = item.attr('last-modification-date').value();

  props.pid = "none";

  if (ver_pid != null)
    props.pid = ver_pid.text();
  else if(obj_pid != null)
    props.pid = obj_pid.text();
	else
		console.error('No PID value found for item: ' + props.escidocID);

  return props;
}

// Parse and pull ID data from the XMLDocument
var parse = function(doc) {
  var totalRecords = doc.get('//sru-zr:numberOfRecords', ns_obj).text();
  var items = doc.find('//escidocItem:item', ns_obj);
  var containers = doc.find('//escidocContainer:container', ns_obj);

  console.log('Total records in query: ' + totalRecords);

  if (utile.isArray(items)) console.log('Found ' + items.length + ' items.');
  else console.error('Found 0 items in XMLDocument.');

  if (utile.isArray(containers)) console.log('Found ' + containers.length + ' containers.');
  else console.error('Found 0 containers in XMLDocument.');

  // Add the items to list
  utile.each(items, function(val, key) {
    // CSV format: item%{escidocID}%[{objectPID}}|{lastVersionPID}]%{versionNo}
    if (!annotationsOnly) {
      addMember(getItemProperties(val));
    }

		if(argv.a){
      var relations = val.find('relations:relations/relations:relation', ns_obj);
      relations.forEach(function(relation) {
        if (relation != undefined || relation != null) {
          relation_type = relation.attr('predicate').value();
          if (relation_type.indexOf('HasAnnotation') != -1) {
            var relation_href = relation.attr('href').value();
            var relationObjID = relation_href.substring(relation_href.indexOf('dkclarin'), relation.length);

            console.log('Found Annotation: ' + relationObjID);

            // retrieve properties for Annotation item from eSciDoc REST
            retrieveItemProperties(relationObjID, function(annoPropsItem) {
							console.log('props xml: ' + annoPropsItem.toString());
              var props = getItemProperties(annoPropsItem.root());
              addMember(props); // add annotation member to file
            });
          }
        }
      });
    }
  });

  // TODO: Handle containers array
  utile.each(containers, function(val, key) {});

  // iterate over complete SRW result set
  if(Number(totalRecords) >= (srw_config.limit + srw_config.start)) {
    srw_config.start += srw_config.limit;
    retrieveSRWResult(srw_config, function(xml) {
      parse(xml);
    });
  } else {
    if(argv.f == 'json')
      addMemberToFile(JSON.stringify(json_arr), stream); // write JSON object to output file
    //if(stream instanceof fs.WriteStream)
      //stream.destroySoon();
  }
}

/** SRW Query fetch */
var retrieveSRWResult = function(config, callback) {
  return execGET(srw_options(argv.q, config), callback);
}

/* Item Properties */
var retrieveItemProperties = function(escidocID, callback) {
  return execGET(ir_options(escidocID, '/properties'), callback);
}

var execGET = function(options, callback) {
  http.get(options, function(res) {
    var str = "";
    res.on('data', function(chunk) {
      str += chunk;
    });
    res.on('end', function() {
      if (res.statusCode == "200")
        callback(libxml.parseXmlString(str));
      else
        throw new Error("Error code: " + res.statusCode);
    });
  }).on('error', function(e) {
    console.error(e);
  });

  return true;
}

/** SRW query options */
var srw_options = function(queryCQL, record_vals) {
  var search_path_limits = "&maximumRecords=" + record_vals.limit + "&startRecord=" + record_vals.start;
  var targetUrl = argv.h;
  return {
    host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
    path: '/srw/search/escidoc_all?query=' + encodeURIComponent(queryCQL.replace(/\s/g, "%20")) + search_path_limits + '&d=' + Date.now()
  };
};

/* eSciDoc Item REST */
var ir_options = function(escidocID, path) {
  var targetUrl = argv.h;
  return {
    host: (targetUrl.indexOf('http://') != -1) ? targetUrl.replace('http://', '') : targetUrl,
    path: '/ir/item/' + escidocID + path
  };
};

// append props to output file
var addMember = function(props) {
  if (argv.f == 'csv' || argv.f == 'tsv')
    addMemberToFile(['item', props.escidocID, props.contentModelID, props.pid, props.ver_no].join(delimiter) + '\n');
  else if (argv.f == 'json') // write output to JSON object
    json_arr.push({
		  type: 'item',
		  systemID: props.escidocID,
		  contentModelID: props.contentModelID,
		  PID: props.pid,
		  versionNo: props.ver_no
		});
  else
    throw new Error('Unsupported format:' + argv.f);
}

// write output to file
var addMemberToFile = function(data) {
  if (stream != null)
    if (stream.write(data)) console.log('Data written: ' + data);
    else console.log('Buffer full waiting for drain..');
  else
    process.send(data);
}

var run = function() {
  var is_json = (argv.f == 'json');
  var is_csv = (argv.f == 'csv');
  var is_tsv = (argv.f == 'tsv');

  var file_name = argv.O;
  var file_dir = argv.d;
  var file_path = null;

  if (!argv.s) {
    console.log("Exists?: " + file_dir);
    if (fs.existsSync(file_dir)) {
      if (is_json)
        file_path = path.join(file_dir, file_name + '.json');
      else if (is_csv)
        file_path = path.join(file_dir, file_name + '.csv');
      else if (is_tsv)
        file_path = path.join(file_dir, file_name + '.tsv');

      stream = fs.createWriteStream(file_path, {
        encoding: 'utf8'
      });
      stream.on('error', function(err) {
        if (err) throw err;
      });
      stream.on('drain', function() {
        console.log('Data drained from buffer to file:' + file_path);
      	stream.end();
			});
      stream.on('close', function() {
        console.log('Closing file stream.');
      });

      console.log('Using query: ' + argv.q);
    } else {
      console.log('Target directory doesn\'t exist: ' + file_dir);
      fs.mkdir(file_dir, function(err) {
        if (err) throw err;
        console.log('Created directory: ' + file_dir);
        run();
      });
    }
  }

  if (argv.s || stream != null)
    retrieveSRWResult(srw_config, function(xml) {
      parse(xml);
    });
}

module.exports = run;

run();
<|endoftext|>