<commit_before>/**
 * A Bot for Slack!
 */

/**
 * Define a function for initiating a conversation on installation
 * With custom integrations, we don't have a way to find out who installed us, so we can't message them :(
 */
var runHappening = false
,   runList = [];
// ,   q = require('q')\
//runModel = require('../model/runModel')

function onInstallation(bot, installer) {
    if (installer) {
        bot.startPrivateConversation({user: installer}, function (err, convo) {
            if (err) {
                console.log(err);
            } else {
                convo.say('I am a bot that has just joined your team');
                convo.say('You must now /invite me to a channel so that I can be of use!');
            }
        });
    }
}

/**
 * Configure the persistence options
 */

var config = {};
if (process.env.MONGOLAB_URI) {
    var BotkitStorage = require('botkit-storage-mongo');
    config = {
        storage: BotkitStorage({mongoUri: process.env.MONGOLAB_URI}),
    };
} else {
    config = {
        json_file_store: ((process.env.TOKEN)?'./db_slack_bot_ci/':'./db_slack_bot_a/'), //use a different name if an app or CI
    };
}

/**
 * Are being run as an app or a custom integration? The initialization will differ, depending
 */

if (process.env.TOKEN || process.env.SLACK_TOKEN) {
    //Treat this as a custom integration
    var customIntegration = require('./lib/custom_integrations');
    var token = (process.env.TOKEN) ? process.env.TOKEN : process.env.SLACK_TOKEN;
    var controller = customIntegration.configure(token, config, onInstallation);
} else if (process.env.CLIENT_ID && process.env.CLIENT_SECRET && process.env.PORT) {
    //Treat this as an app
    var app = require('./lib/apps');
    var controller = app.configure(process.env.PORT, process.env.CLIENT_ID, process.env.CLIENT_SECRET, config, onInstallation);
} else {
    console.log('Error: If this is a custom integration, please specify TOKEN in the environment. If this is an app, please specify CLIENTID, CLIENTSECRET, and PORT in the environment');
    process.exit(1);
}

/**
 * A demonstration for how to handle websocket events. In this case, just log when we have and have not
 * been disconnected from the websocket. In the future, it would be super awesome to be able to specify
 * a reconnect policy, and do reconnections automatically. In the meantime, we aren't going to attempt reconnects,
 * WHICH IS A B0RKED WAY TO HANDLE BEING DISCONNECTED. So we need to fix this.
 *
 * TODO: fixed b0rked reconnect behavior
 */
// Handle events related to the websocket connection to Slack
controller.on('rtm_open', function (bot) {
    console.log('** The RTM api just connected!');
});

controller.on('rtm_close', function (bot) {
    console.log('** The RTM api just closed');
    // you may want to attempt to re-open
});

/**
 * Core bot logic goes here!
 */
// BEGIN EDITING HERE!

controller.on('bot_channel_join', function (bot, message) {
    bot.reply(message, "I'm here!")
});

controller.hears('hello', 'direct_message', function (bot, message) {
    bot.reply(message, 'Hello!');
});

controller.hears('lunch run', 'direct_message, direct_mention', function(bot, message) {

        bot.startConversation(message, function(err,convo) {

            convo.say('Oooh! I LOVE Pop-a-Top!');
            convo.ask('Would you like me to see if anyone else wants something?', [{
                pattern: bot.utterances.yes,
                callback: function(response, convo){
                    convo.say('Okay! What a sweetheart you are. :heart:');
                    convo.next();
                }
            },
            {
                pattern: bot.utterances.no,
                callback: function(response, convo){
                    convo.say('Alright! Well thanks for letting me know. :grin:');
                    convo.next();
                }
            }
        ]);
    });
});

controller.hears('start', 'direct_mention', function(bot, message) {

    if (runHappening) {
        return bot.reply(message, 'Already gathering requests for a run in' + runChannel);
    }

   /* if (!runModel.getSummaryChannel()) {
        runModel.setSummaryChannel(message.channel);
    }*/
    runChannel = message.channel;

    //No other users on team
    /*if (usersModel.list().length === 0) {
        return bot.reply(message, 'Looks like everyone is right here bud... Add some people to your team and we will talk.');
    }*/

    bot.reply(message, 'Holy La Croix! <@' + message.user + '> is going for a Pop-a-Top run. Who <@here> has a request? Tell me what you want and I will let them know!');
    runHappening = true;
    console.log('runHappening =' + runHappening);

    /*//notify first user and start a conversation
    userIterator = usersModel.iterator();
    currentUser = userIterator.next();
    promptUser(bot);*/

});

controller.hears('end', 'direct_mention', function(bot, message) {
    if (!runHappening) {
        return bot.reply(message, 'Nobody is on a run right now. Start a new run with `start`');
    }

    runHappening = false;

    bot.reply(message, 'The run is now over!');
    console.log('runHappening =' + runHappening);

    bot.startConversation(message, function(err,convo){
        convo.ask('<@' + message.user + '> want to see the list?', [{
            pattern: bot.utterances.yes,
            callback: function(response, convo) {
                bot.reply(message, runList);
                // return summarizeRun(bot);
                runList = [];
            }
        }, {
            pattern: bot.utterances.no,
            callback: function(response,convo) {
                bot.say({channel: message.channel, text: 'Okey Dokey. That was fun, see you later!'});
                runModel.clearStatuses();
                convo.next();
            }
        }]);
    });
});

controller.hears('(.*)', 'direct_mention, direct_message', function(bot, message) {
    if (runHappening) {
        gatherRequest(bot, message);
    }
});

function gatherRequest(bot, message) {
    console.log(message);
    bot.reply(message, 'Got it! Thanks ' + message.user);
    runList.push({user: message.user, request: message.text});
    console.log('Pushing request to runList');

   /* runModel.addRequest({
        request: message.text,
        user: message.user
    });
    */

}

<commit_msg>removed overwriting runList<commit_after>/**
 * A Bot for Slack!
 */

/**
 * Define a function for initiating a conversation on installation
 * With custom integrations, we don't have a way to find out who installed us, so we can't message them :(
 */
var runHappening = false
,   runList = [];
// ,   q = require('q')\
//runModel = require('../model/runModel')

function onInstallation(bot, installer) {
    if (installer) {
        bot.startPrivateConversation({user: installer}, function (err, convo) {
            if (err) {
                console.log(err);
            } else {
                convo.say('I am a bot that has just joined your team');
                convo.say('You must now /invite me to a channel so that I can be of use!');
            }
        });
    }
}

/**
 * Configure the persistence options
 */

var config = {};
if (process.env.MONGOLAB_URI) {
    var BotkitStorage = require('botkit-storage-mongo');
    config = {
        storage: BotkitStorage({mongoUri: process.env.MONGOLAB_URI}),
    };
} else {
    config = {
        json_file_store: ((process.env.TOKEN)?'./db_slack_bot_ci/':'./db_slack_bot_a/'), //use a different name if an app or CI
    };
}

/**
 * Are being run as an app or a custom integration? The initialization will differ, depending
 */

if (process.env.TOKEN || process.env.SLACK_TOKEN) {
    //Treat this as a custom integration
    var customIntegration = require('./lib/custom_integrations');
    var token = (process.env.TOKEN) ? process.env.TOKEN : process.env.SLACK_TOKEN;
    var controller = customIntegration.configure(token, config, onInstallation);
} else if (process.env.CLIENT_ID && process.env.CLIENT_SECRET && process.env.PORT) {
    //Treat this as an app
    var app = require('./lib/apps');
    var controller = app.configure(process.env.PORT, process.env.CLIENT_ID, process.env.CLIENT_SECRET, config, onInstallation);
} else {
    console.log('Error: If this is a custom integration, please specify TOKEN in the environment. If this is an app, please specify CLIENTID, CLIENTSECRET, and PORT in the environment');
    process.exit(1);
}

/**
 * A demonstration for how to handle websocket events. In this case, just log when we have and have not
 * been disconnected from the websocket. In the future, it would be super awesome to be able to specify
 * a reconnect policy, and do reconnections automatically. In the meantime, we aren't going to attempt reconnects,
 * WHICH IS A B0RKED WAY TO HANDLE BEING DISCONNECTED. So we need to fix this.
 *
 * TODO: fixed b0rked reconnect behavior
 */
// Handle events related to the websocket connection to Slack
controller.on('rtm_open', function (bot) {
    console.log('** The RTM api just connected!');
});

controller.on('rtm_close', function (bot) {
    console.log('** The RTM api just closed');
    // you may want to attempt to re-open
});

/**
 * Core bot logic goes here!
 */
// BEGIN EDITING HERE!

controller.on('bot_channel_join', function (bot, message) {
    bot.reply(message, "I'm here!")
});

controller.hears('hello', 'direct_message', function (bot, message) {
    bot.reply(message, 'Hello!');
});

controller.hears('lunch run', 'direct_message, direct_mention', function(bot, message) {

        bot.startConversation(message, function(err,convo) {

            convo.say('Oooh! I LOVE Pop-a-Top!');
            convo.ask('Would you like me to see if anyone else wants something?', [{
                pattern: bot.utterances.yes,
                callback: function(response, convo){
                    convo.say('Okay! What a sweetheart you are. :heart:');
                    convo.next();
                }
            },
            {
                pattern: bot.utterances.no,
                callback: function(response, convo){
                    convo.say('Alright! Well thanks for letting me know. :grin:');
                    convo.next();
                }
            }
        ]);
    });
});

controller.hears('start', 'direct_mention', function(bot, message) {

    if (runHappening) {
        return bot.reply(message, 'Already gathering requests for a run in' + runChannel);
    }

   /* if (!runModel.getSummaryChannel()) {
        runModel.setSummaryChannel(message.channel);
    }*/
    runChannel = message.channel;

    //No other users on team
    /*if (usersModel.list().length === 0) {
        return bot.reply(message, 'Looks like everyone is right here bud... Add some people to your team and we will talk.');
    }*/

    bot.reply(message, 'Holy La Croix! <@' + message.user + '> is going for a Pop-a-Top run. Who <@here> has a request? Tell me what you want and I will let them know!');
    runHappening = true;
    console.log('runHappening =' + runHappening);

    /*//notify first user and start a conversation
    userIterator = usersModel.iterator();
    currentUser = userIterator.next();
    promptUser(bot);*/

});

controller.hears('end', 'direct_mention', function(bot, message) {
    if (!runHappening) {
        return bot.reply(message, 'Nobody is on a run right now. Start a new run with `start`');
    }

    runHappening = false;

    bot.reply(message, 'The run is now over!');
    console.log('runHappening =' + runHappening);

    bot.startConversation(message, function(err,convo){
        convo.ask('<@' + message.user + '> want to see the list?', [{
            pattern: bot.utterances.yes,
            callback: function(response, convo) {
                bot.reply(message, runList);
                // return summarizeRun(bot);
            }
        }, {
            pattern: bot.utterances.no,
            callback: function(response,convo) {
                bot.say({channel: message.channel, text: 'Okey Dokey. That was fun, see you later!'});
                runModel.clearStatuses();
                convo.next();
            }
        }]);
    });
});

controller.hears('(.*)', 'direct_mention, direct_message', function(bot, message) {
    if (runHappening) {
        gatherRequest(bot, message);
    }
});

function gatherRequest(bot, message) {
    console.log(message);
    bot.reply(message, 'Got it! Thanks ' + message.user);
    runList.push({user: message.user, request: message.text});
    console.log('Pushing request to runList');

   /* runModel.addRequest({
        request: message.text,
        user: message.user
    });
    */

}

<|endoftext|>


<commit_before>var queryResp = '/q/zmw:02108.1.99999';
var coat = 36;
var jacket = 45;
var sweater = 60;

function query(location) {
    $.ajax({
        type: 'GET',
        url: 'https://api.wunderground.com/api/dba29ab1640cac5b/hourly' + location + '.json',
        dataType: 'json',
        success: function(data) {processData(data);}
    });
}

function processData(data) {
    $('#searchResp').empty();
    $('#rec').show();
    var entries = data.hourly_forecast;
    var feels = entries[0].feelslike.english;
    var feelsMin = feels;
    for (var i = 1; i < 12; i++) {
        if (feelsMin > entries[i].feelslike.english) {
            feelsMin = entries[i].feelslike.english;
        }
    }
    var simp = 'No';
    var rec = 'you don\'t need a jacket';
    if (feelsMin < coat) {
        simp = 'Yes';
        rec = 'it\'s cold out today, you need a coat';
    } else if (feelsMin < jacket) {
        simp = 'Yes';
        rec = 'you should wear a jacket today';
    } else if (feelsMin < sweater) {
        simp = 'No';
        rec = 'you only need a sweater';
    }
    $('title').text('Do I Need a Jacket? ' + simp + '.');
    $('#simpResp').text(simp + '. ');
    $('#longResp').text(rec + '. The coldest it will feel today is ' + feelsMin + ' F.');
}

function search() {
    var query = $('#searchField').val();
    $('#rec').hide();
    $('body').append("");
    return false;
}

function processQuery(data) {
    var entries = data.RESULTS;
    $('#searchResp').empty();
    if (entries.length == 0) {
        $('#searchResp').append('No results.')
    } else {
        $('#searchResp').append('

');
    }
    for (var i = 0; i < entries.length; i++) {
        var entry = entries[i];
        $('#searchResp ul').append('
' + entry.name + '
')
    }
}<commit_msg>Change to https<commit_after>var queryResp = '/q/zmw:02108.1.99999';
var coat = 36;
var jacket = 45;
var sweater = 60;

function query(location) {
    $.ajax({
        type: 'GET',
        url: 'https://api.wunderground.com/api/dba29ab1640cac5b/hourly' + location + '.json',
        dataType: 'json',
        success: function(data) {processData(data);}
    });
}

function processData(data) {
    $('#searchResp').empty();
    $('#rec').show();
    var entries = data.hourly_forecast;
    var feels = entries[0].feelslike.english;
    var feelsMin = feels;
    for (var i = 1; i < 12; i++) {
        if (feelsMin > entries[i].feelslike.english) {
            feelsMin = entries[i].feelslike.english;
        }
    }
    var simp = 'No';
    var rec = 'you don\'t need a jacket';
    if (feelsMin < coat) {
        simp = 'Yes';
        rec = 'it\'s cold out today, you need a coat';
    } else if (feelsMin < jacket) {
        simp = 'Yes';
        rec = 'you should wear a jacket today';
    } else if (feelsMin < sweater) {
        simp = 'No';
        rec = 'you only need a sweater';
    }
    $('title').text('Do I Need a Jacket? ' + simp + '.');
    $('#simpResp').text(simp + '. ');
    $('#longResp').text(rec + '. The coldest it will feel today is ' + feelsMin + ' F.');
}

function search() {
    var query = $('#searchField').val();
    $('#rec').hide();
    $('body').append("");
    return false;
}

function processQuery(data) {
    var entries = data.RESULTS;
    $('#searchResp').empty();
    if (entries.length == 0) {
        $('#searchResp').append('No results.')
    } else {
        $('#searchResp').append('

');
    }
    for (var i = 0; i < entries.length; i++) {
        var entry = entries[i];
        $('#searchResp ul').append('
' + entry.name + '
')
    }
}<|endoftext|>


<commit_before>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs');
const low = require('lowdb');
const lowDbFileAsync = require('lowdb/lib/file-async');
const request = require('request-promise');
const Q = require('q');
const fs = require('fs-extra');
const path = require('path');
const cheerio = require('cheerio');
const sleep = require('system-sleep');
const moment = require('moment');
const Slack = require('slack-node');
const Regex = require('xregexp');
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || '';
const dbFile = path.join(__dirname, 'data/db.json');
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.');
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true });

const scrapperUrlBase = 'http://empregos.maringa.com/';
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end',
];

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig,
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
];

const deferred = Q.defer();
const deferredProcessing = Q.defer();
const deferredSlack = Q.defer();

db.defaults({ jobs: [], settings: {} }).value();

let keywordsProcessed = 0;
let websiteToken = '';
let websiteCookies = [];

request({
  uri: scrapperUrlBase,
  transform: (body, response) => {
    const res = { headers: response.headers, statusCode: response.statusCode, body };

    return res;
  },
}).then((response) => {
  if (response.statusCode !== 200) {
    deferred.reject(new Error('Erro ao obter token e cookies'));
    return false;
  }

  const $ = cheerio.load(response.body);

  websiteCookies = (response.headers['set-cookie'] || []).map((cookie) => {
    const ck = new request.cookie(cookie);

    return ck;
  });

  websiteToken = $('input[name="_token"]').val();

  return websiteToken && websiteCookies.length ? deferred.resolve() : deferred.reject(new Error('Token de busca inválido.'));
}).catch((err) => {
  _log('ERROR');
  _log(err);
  _log('-'.repeat(100));
  process.exit(1);
});

Q.when(deferred.promise).then(() => {
  sleep(1000);

  const cookieJar = request.jar();
  websiteCookies.forEach((cookie) => {
    cookieJar.setCookie(cookie, scrapperUrlBase);
  });

  keywords.forEach((keyword, index) => {
    _log(`Looking for jobs with word "${keyword}`);

    let foundJobs = [];

    request.post(
      scrapperUrlBase,
      {
        jar: cookieJar,
        form: {
          _token: websiteToken,
          text: keyword,
          estado: 'PR',
          cidade: 'Maringá',
          area: '',
          faixa_salarial: '',
        },
      }
    ).then((response) => {
      const $ = cheerio.load(response);

      $('.table-vagas').find('.clickable-row ').each((i, job) => {
        const $job = $(job);

        const title = $job.find('td').eq(1).find('p').eq(0).find('strong').text().trim().replace(/[\r\n]/g, '');
        const company = $job.find('td').eq(1).find('p').eq(1).text().trim().replace(/empresa: /i, '');
        const link = $job.data('href');

        // TODO: checar se existe flag de vaga preenchida no site
        const isFilled = $job.find('td').eq(1).find('p').eq(0).find('.badge-success').length > 0;
        const dateProcessed = Date.now();
        const id = (link.match(/vaga-emprego\/(\d+)\//) || []).pop();

        if (isNaN(id)) {
          return;
        }

        let row = null;
        let date = $job.find('td').eq(2).find('p').eq(0).text().trim();

        if (date.toLowerCase() === 'hoje') {
          date = new Date();
        } else {
          date = new Date(date.toString().split('/').reverse().join('-'));
        }
        date.setHours(0, 0, 0, 0);
        date = date.getTime();

        row = db.get('jobs').find({ id }).value();
        if (row) {
          if (!row.is_filled && isFilled) {
            db.get('jobs').find({ id }).assign({ is_filed: isFilled }).value();
          }

          return;
        }

        foundJobs.push({
          id,
          title,
          company,
          date,
          date_processed: dateProcessed,
          is_filled: isFilled,
          keyword,
          bot_processed: false,
          url: link,
        });
      });

      foundJobs = foundJobs.filter((job) => {
        const test = blacklist.filter((word) => {
          const regex = word.constructor !== RegExp ? new RegExp(`\\b${word}\\b`, 'igm') : word;

          return Regex(regex).test(job.title);
        });

        return test.length === 0;
      });

      _log(`Found ${foundJobs.length} job opportunities...`);
      _log('-'.repeat(100));

      foundJobs.forEach((job) => {
        db.get('jobs').push(job).value();
      });

      keywordsProcessed = index + 1;

      if (keywordsProcessed === keywords.length) {
        setTimeout(() => { deferredProcessing.resolve(); }, 1000);
      }
    }).catch((err) => {
      _log('ERROR');
      _log(err);
      _log('-'.repeat(100));
    })
  });
}, (err) => {
  _log('ERROR');
  _log(err);
  _log('-'.repeat(100));
});

Q.when(deferredProcessing.promise).then(() => {
  let cuttingTime = 1484186400245;

  _log('generating html...');

  try {
    db.set('settings.updated_at', Date.now()).value();

    const outputFile = path.join(__dirname, 'out/jobs.html');
    const html = fs.readFileSync(path.join(__dirname, 'tpl/jobs.ejs'), 'utf8');

    const jobs = db.get('jobs').filter({ is_filled: false }).sortBy('date').reverse().value() || [];
    const result = tpl.render(html, { jobs, moment, updated_at: db.get('settings.updated_at').value() || Date.now() });

    if (!fs.existsSync(path.dirname(outputFile)) && !fs.mkdirsSync(path.dirname(outputFile))) {
      throw new Error('Error creating output directory.');
    }

    fs.writeFileSync(outputFile, result);

//  INTEGRATION WITH DEVPARANA SLACK --------------------------------------------------------------------------------------------
    _log('Done generating HTML file: ', outputFile);
    _log('Cheking for new job entries to send to Slack...')

    const botJobs = db.get('jobs').filter({ bot_processed: false })
      .filter({ is_filled: false }).filter(row => row.date_processed >= cuttingTime)
      .sortBy('date').reverse().value();

    sleep(1000);

    if (SLACK_WEBHOOK && botJobs.length) {
      _log(`Found ${botJobs.length} entries to be posted on slack.`);

      let slack = new Slack();
      slack.setWebhook(SLACK_WEBHOOK);

      botJobs.forEach((item, index) => {
        _log(`Processing item ${(index + 1)}:`, `${item.title} / ${item.company}`);

        slack.webhook({
          attachments: [
            {
              title: `${item.title} / ${item.company}`,
              title_link: item.url,
              text: `Vaga:  ${item.title}\nEmpresa: ${item.company}\nData${moment(item.date).format('DD/MM/YYYY')}`,
              color: '#7CD197',
            },
          ],
          text: `Vaga de trabalho encontrada. Confira!\n\n${item.url}`,
        }, (err, response) => {
          if (err) {
            _log('ERROR: ', err);
            _log('ERROR: ', '-'.repeat(100));
            process.exit(1);
          }
          if (response.statusCode === 200) {
            _log('Done posting item ' + (index + 1));
            db.get('jobs').find({ id: item.id }).assign({ bot_processed: true }).value();
          } else {
            _log('ERROR: ', 'Error processing item ' + (index + 1) + ': ', response.statusCode, response.statusMessage);
          }

          if (index + 1 === botJobs.length) {
            deferredSlack.resolve('DONE');
          }
        });
        sleep(300);
      });
    } else if (!SLACK_WEBHOOK) {
      deferredSlack.reject(new Error('ERROR: Enviroment variable "SLACK_WEBHOOK" is not defined. Aborting slack...'));
    } else {
      deferredSlack.resolve('No new job opening found to send to slack.');
    }
//  /INTEGRATION WITH DEVPARANA SLACK -------------------------------------------------------------------------------------------
    Q.when(deferredSlack.promise).then((msg) => {
      _log(msg);
      _log('-'.repeat(100));
    }, (err) => {
      _log('ERROR');
      _log(err);
      _log('-'.repeat(100));
      process.exit(1);
    });
  } catch (err) {
    _log('ERROR: ', err);
  }
  _log('-'.repeat(100));
}, (err) => {
  _log('ERROR: ', err);
  _log('-'.repeat(100));
  process.exit(1);
});

function _log(...args) {
  console.log(...[].concat([`[${moment().format('DD/MM/YYYY HH:mm:ss')}] =>`], Array.from(args) || []));
}
<commit_msg>improved scrapper error handling<commit_after>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs');
const low = require('lowdb');
const lowDbFileAsync = require('lowdb/lib/file-async');
const request = require('request-promise');
const Q = require('q');
const fs = require('fs-extra');
const path = require('path');
const cheerio = require('cheerio');
const sleep = require('system-sleep');
const moment = require('moment');
const Slack = require('slack-node');
const Regex = require('xregexp');
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || '';
const dbFile = path.join(__dirname, 'data/db.json');
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.');
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true });

const scrapperUrlBase = 'http://empregos.maringa.com/';
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end',
];

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig,
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
];

const deferred = Q.defer();
const deferredProcessing = Q.defer();
const deferredSlack = Q.defer();

db.defaults({ jobs: [], settings: {} }).value();

let keywordsProcessed = 0;
let websiteToken = '';
let websiteCookies = [];

request({
  uri: scrapperUrlBase,
  transform: (body, response) => {
    const res = { headers: response.headers, statusCode: response.statusCode, body };

    return res;
  },
}).then((response) => {
  if (response.statusCode !== 200) {
    deferred.reject(new Error('Erro ao obter token e cookies'));
    return false;
  }

  const $ = cheerio.load(response.body);

  websiteCookies = (response.headers['set-cookie'] || []).map((cookie) => {
    const ck = new request.cookie(cookie);

    return ck;
  });

  websiteToken = $('input[name="_token"]').val();

  return websiteToken && websiteCookies.length ? deferred.resolve() : deferred.reject(new Error('Token de busca inválido.'));
}).catch((err) => {
  _log('ERROR');
  _log(err);
  _log('-'.repeat(100));
  process.exit(1);
});

Q.when(deferred.promise).then(() => {
  sleep(1000);

  const cookieJar = request.jar();
  websiteCookies.forEach((cookie) => {
    cookieJar.setCookie(cookie, scrapperUrlBase);
  });

  keywords.forEach((keyword, index) => {
    _log(`Looking for jobs with word "${keyword}`);

    let foundJobs = [];

    request.post(
      scrapperUrlBase,
      {
        jar: cookieJar,
        form: {
          _token: websiteToken,
          text: keyword,
          estado: 'PR',
          cidade: 'Maringá',
          area: '',
          faixa_salarial: '',
        },
      }
    ).then((response) => {
      const $ = cheerio.load(response);
      const jobRawList = $('.table-vagas').find('.clickable-row ');
      const foundResults = $('h1.text-center.is-primary.h2.my-4').text().trim() || '';
      const totalFoundResults = foundResults ? Number(foundResults.replace(/\D/g, '')) : 0;

      _log(`Found ${totalFoundResults} for the keyword "${keyword}" (raw)`);

      if (!jobRawList.length) {
        if (foundResults) {
          if (totalFoundResults === 0) {
            _log(`No job opportunities was found for keyword "${keyword}"`);
            _log('-'.repeat(100));
          } else {
            _log('ERROR');
            _log(`Error getting jobs for keyword "${keyword}" (total of jobs found: ${totalFoundResults})`);
            _log(`Generated output in /tmp/${keyword}.html`);
            _log('-'.repeat(100));

            const tmp = path.join(__dirname, 'tmp');
            if (!fs.existsSync(tmp)) {
              fs.mkdirSync(tmp);
            }
            fs.writeFileSync(path.join(tmp, `${keyword}.html`), response);
          }
        } else {
          _log('ERROR');
          _log(`Error getting jobs for keyword "${keyword}"`);
          _log(`Generated output in /tmp/${keyword}.html`);
          _log('-'.repeat(100));

          const tmp = path.join(__dirname, 'tmp');
          if (!fs.existsSync(tmp)) {
            fs.mkdirSync(tmp);
          }
          fs.writeFileSync(path.join(tmp, `${keyword}.html`), response);
        }

        return;
      }

      jobRawList.each((i, job) => {
        const $job = $(job);

        const title = $job.find('td').eq(1).find('p').eq(0).find('strong').text().trim().replace(/[\r\n]/g, '');
        const company = $job.find('td').eq(1).find('p').eq(1).text().trim().replace(/empresa: /i, '');
        const link = $job.data('href');

        // TODO: checar se existe flag de vaga preenchida no site
        const isFilled = $job.find('td').eq(1).find('p').eq(0).find('.badge-success').length > 0;
        const dateProcessed = Date.now();
        const id = (link.match(/vaga-emprego\/(\d+)\//) || []).pop();

        if (isNaN(id)) {
          return;
        }

        let row = null;
        let date = $job.find('td').eq(2).find('p').eq(0).text().trim();

        if (date.toLowerCase() === 'hoje') {
          date = new Date();
        } else {
          date = new Date(date.toString().split('/').reverse().join('-'));
        }
        date.setHours(0, 0, 0, 0);
        date = date.getTime();

        row = db.get('jobs').find({ id }).value();
        if (row) {
          if (!row.is_filled && isFilled) {
            db.get('jobs').find({ id }).assign({ is_filed: isFilled }).value();
          }

          return;
        }

        foundJobs.push({
          id,
          title,
          company,
          date,
          date_processed: dateProcessed,
          is_filled: isFilled,
          keyword,
          bot_processed: false,
          url: link,
        });
      });

      foundJobs = foundJobs.filter((job) => {
        const test = blacklist.filter((word) => {
          const regex = word.constructor !== RegExp ? new RegExp(`\\b${word}\\b`, 'igm') : word;

          return Regex(regex).test(job.title);
        });

        return test.length === 0;
      });

      _log(`Found ${foundJobs.length} job opportunities for "${keyword}"...`);
      _log('-'.repeat(100));

      foundJobs.forEach((job) => {
        db.get('jobs').push(job).value();
      });

      keywordsProcessed = index + 1;

      if (keywordsProcessed === keywords.length) {
        setTimeout(() => { deferredProcessing.resolve(); }, 1000);
      }
    }).catch((err) => {
      _log('ERROR');
      _log(err);
      _log('-'.repeat(100));
    })
  });
}, (err) => {
  _log('ERROR');
  _log(err);
  _log('-'.repeat(100));
});

Q.when(deferredProcessing.promise).then(() => {
  let cuttingTime = 1484186400245;

  _log('generating html...');

  try {
    db.set('settings.updated_at', Date.now()).value();

    const outputFile = path.join(__dirname, 'out/jobs.html');
    const html = fs.readFileSync(path.join(__dirname, 'tpl/jobs.ejs'), 'utf8');

    const jobs = db.get('jobs').filter({ is_filled: false }).sortBy('date').reverse().value() || [];
    const result = tpl.render(html, { jobs, moment, updated_at: db.get('settings.updated_at').value() || Date.now() });

    if (!fs.existsSync(path.dirname(outputFile)) && !fs.mkdirsSync(path.dirname(outputFile))) {
      throw new Error('Error creating output directory.');
    }

    fs.writeFileSync(outputFile, result);

//  INTEGRATION WITH DEVPARANA SLACK --------------------------------------------------------------------------------------------
    _log('Done generating HTML file: ', outputFile);
    _log('Cheking for new job entries to send to Slack...')

    const botJobs = db.get('jobs').filter({ bot_processed: false })
      .filter({ is_filled: false }).filter(row => row.date_processed >= cuttingTime)
      .sortBy('date').reverse().value();

    sleep(1000);

    if (SLACK_WEBHOOK && botJobs.length) {
      _log(`Found ${botJobs.length} entries to be posted on slack.`);

      let slack = new Slack();
      slack.setWebhook(SLACK_WEBHOOK);

      botJobs.forEach((item, index) => {
        _log(`Processing item ${(index + 1)}:`, `${item.title} / ${item.company}`);

        slack.webhook({
          attachments: [
            {
              title: `${item.title} / ${item.company}`,
              title_link: item.url,
              text: `Vaga:  ${item.title}\nEmpresa: ${item.company}\nData${moment(item.date).format('DD/MM/YYYY')}`,
              color: '#7CD197',
            },
          ],
          text: `Vaga de trabalho encontrada. Confira!\n\n${item.url}`,
        }, (err, response) => {
          if (err) {
            _log('ERROR: ', err);
            _log('ERROR: ', '-'.repeat(100));
            process.exit(1);
          }
          if (response.statusCode === 200) {
            _log('Done posting item ' + (index + 1));
            db.get('jobs').find({ id: item.id }).assign({ bot_processed: true }).value();
          } else {
            _log('ERROR: ', 'Error processing item ' + (index + 1) + ': ', response.statusCode, response.statusMessage);
          }

          if (index + 1 === botJobs.length) {
            deferredSlack.resolve('DONE');
          }
        });
        sleep(300);
      });
    } else if (!SLACK_WEBHOOK) {
      deferredSlack.reject(new Error('ERROR: Enviroment variable "SLACK_WEBHOOK" is not defined. Aborting slack...'));
    } else {
      deferredSlack.resolve('No new job opening found to send to slack.');
    }
//  /INTEGRATION WITH DEVPARANA SLACK -------------------------------------------------------------------------------------------
    Q.when(deferredSlack.promise).then((msg) => {
      _log(msg);
      _log('-'.repeat(100));
    }, (err) => {
      _log('ERROR');
      _log(err);
      _log('-'.repeat(100));
      process.exit(1);
    });
  } catch (err) {
    _log('ERROR: ', err);
  }
  _log('-'.repeat(100));
}, (err) => {
  _log('ERROR: ', err);
  _log('-'.repeat(100));
  process.exit(1);
});

function _log(...args) {
  console.log(...[].concat([`[${moment().format('DD/MM/YYYY HH:mm:ss')}] =>`], Array.from(args) || []));
}
<|endoftext|>


<commit_before>import * as I from 'immutable';

import { rationalMatrices } from '../arithmetic/types';
const ops = rationalMatrices;

const Edge = I.Record({
  head : undefined,
  tail : undefined,
  shift: undefined
});

Edge.prototype.toString = function toString() {
  return 'Edge('+this.head+', '+this.tail+', '+this.shift+')';
};

Edge.prototype.reverse = function reverse() {
  return new Edge({
    head : this.tail,
    tail : this.head,
    shift: this.shift.map(x => -x)
  });
};

const _isNegative = vec => vec.find(x => ops.sgn(x) != 0) < 0;

Edge.prototype.canonical = function canonical() {
  if (this.tail < this.head || (this.tail == this.head
                                && _isNegative(this.shift)))
    return this.reverse();
  else
    return this;
};

const _makeEdge = function _makeEdge(e) {
  return new Edge({ head: e[0], tail: e[1], shift: e[2] }).canonical();
};

const Graph = I.Record({
  dim  : undefined,
  edges: undefined
});

Graph.prototype.toString = function toString() {
  return 'PGraph('+this.edges+')';
};

export function make(data) {
  const edges = I.Set(data).map(_makeEdge);
  if (edges.size == 0)
    throw new Error('cannot be empty');

  const dim = edges.first().shift.length;
  if (edges.some(e => e.shift.length != dim))
    throw new Error('must have consistent shift dimensions');

  return new Graph({ dim: dim, edges: edges });
};

export function adjacencies(graph) {
  const target = e => ({ v: e.tail, s: e.shift });

  let res = I.Map();
  for (const e of graph.edges) {
    res = res
      .update(e.head, a => (a || I.List()).push(target(e)))
      .update(e.tail, a => (a || I.List()).push(target(e.reverse())));
  }
  return res;
};

export function coordinationSeq(graph, start, dist) {
  const adj  = adjacencies(graph);
  const zero = ops.vector(graph.dim);

  let oldShell = I.Set();
  let thisShell = I.Set([I.fromJS([start, zero])]);
  const res = [1];

  for (const i of I.Range(1, dist+1)) {
    let nextShell = I.Set().asMutable();
    for (const item of thisShell) {
      const [v, s] = item.toJS();
      for (const { v: w, s: t } of adj.get(v)) {
        const key = I.fromJS([w, ops.plus(s, t)]);
        if (!oldShell.contains(key) && !thisShell.contains(key))
          nextShell.add(key);
      }
    }

    res.push(nextShell.size);
    oldShell = thisShell;
    thisShell = nextShell;
  }

  return res;
};

const _componentInOrbitGraph = (graph, start) => {
  const adj = adjacencies(graph);
  const queue = [start]
  const bridges = [];
  const nodeShifts = I.Map([[start, ops.vector(graph.dim)]]).asMutable();

  while (queue.length) {
    const v = queue.shift();
    const av = nodeShifts.get(v);

    for (const {v: w, s: shift} of adj.get(v)) {
      if (nodeShifts.get(w) == undefined) {
        queue.push(w);
        nodeShifts.set(w, ops.minus(av, shift));
      }
      else {
        const aw = nodeShifts.get(w);
        const newShift = ops.plus(shift, ops.minus(aw, av));
        const pivot = newShift.find(x => ops.ne(x, 0));

        if (pivot != undefined && ops.gt(pivot, 0))
          bridges.push({ v, w, s: newShift });
      }
    }
  }

  return {
    nodes: I.Set(nodeShifts.keys()),
    nodeShifts,
    bridges
  };
};

const _isConnectedOrbitGraph = (graph) => {
  const verts = I.List(adjacencies(graph).keySeq());
  const comp = _componentInOrbitGraph(graph, verts.first());

  return comp.nodes.size >= verts.size;
};

const _basis = M => {
  const T = ops.triangulation(M).R;
  return T.slice(0, ops.rank(T));
};

const _componentInCoverGraph = (graph, start) => {
  const { nodes, nodeShifts, bridges } = _componentInOrbitGraph(graph, start);
  const basis = _basis(bridges.map(b => b.s));
  const transform = ops.inverse(basis);
  const old2new = I.Map(I.List(nodes).zip(I.Range(1, nodes.size+1)));

  const newEdges = graph.edges
    .filter(({ head, tail }) =>
            old2new.get(head) != null && old2new.get(tail) != null)
    .map(({ head, tail, shift }) => {
      const [v, w] = [old2new.get(head), old2new.get(tail)];
      const [av, aw] = [nodeShifts.get(head), nodeShifts.get(tail)];
      const t = ops.times(ops.plus(shift, ops.minus(aw, av)), transform);
      return [v, w, t];
    });

  return {
    basis,
    nodes: nodes.toArray(),
    multiplicity: ops.determinant(basis),
    graph: make(newEdges)
  };
};

export function isConnected(graph) {
  const verts = I.List(adjacencies(graph).keySeq());
  const comp = _componentInCoverGraph(graph, verts.first());

  return comp.nodes.length >= verts.size && comp.multiplicity == 1;
};

export function connectedComponents(graph) {
  const verts = I.List(adjacencies(graph).keySeq());
  const seen = I.Set().asMutable();
  const result = [];

  for (const start of verts) {
    if (!seen.contains(start)) {
      const comp = _componentInCoverGraph(graph, start);
      result.push(comp);
      comp.nodes.forEach(v => seen.add(v));
    }
  }

  return result;
};

export function barycentricPlacement(graph) {
  if (!isConnected(graph))
    throw new Error('must have a connected orbit graph');

  const adj   = adjacencies(graph);
  const verts = I.List(adj.keySeq());
  const vIdcs = I.Map(I.Range(0, verts.size).map(i => [verts.get(i), i]));

  const n = verts.size;
  const d = graph.dim;
  let A = ops.matrix(n+1, n);
  let t = ops.matrix(n+1, d);

  verts.forEach((v, i) => {
    adj.get(v).forEach(c => {
      if (c.v != v) {
        const j = vIdcs.get(c.v);
        A[i][j] -= 1;
        A[i][i] += 1;
        t[i] = ops.plus(t[i], c.s);
      }
    });
  });
  A[n][0] = 1;

  const p = ops.solve(A, t);

  return I.Map(I.Range(0, n).map(i => [verts.get(i), p[i]]));
};

export function barycentricPlacementAsFloat(graph) {
  return barycentricPlacement(graph).map(p => p.map(x => ops.toJS(x)));
};

export function isStable(graph, pos=barycentricPlacement(graph)) {
  const verts = I.List(adjacencies(graph).keySeq());
  const seen = I.Set().asMutable();

  for (const v of verts) {
    const p = pos.get(v);
    const key = I.fromJS(ops.repr(p.map(x => ops.mod(x, 1))));
    if (seen.contains(key))
      return false;
    else
      seen.add(key);
  }

  return true;
};

export function isLocallyStable(graph, pos=barycentricPlacement(graph)) {
  const adj = adjacencies(graph);
  const verts = I.List(adj.keySeq());

  for (const v of verts) {
    const seen = I.Set().asMutable();

    for (const w of adj.get(v)) {
      const p = ops.plus(pos.get(w.v), w.s);
      const key = I.fromJS(ops.repr(p));
      if (seen.contains(key))
        return false;
      else
        seen.add(key);
    }
  }

  return true;
};

const _neighborsByEdgeVector = (
  graph, v, adj=adjacencies(graph), pos=barycentricPlacement(graph)
) => {
  const result = I.Map().asMutable();

  for (const { v: w, s } of adj.get(v)) {
    const d = ops.plus(s, ops.minus(pos.get(w), pos.get(v)));
    result.set(I.fromJS(d), { v: w, s })
    if (v == w)
      result.set(I.fromJS(ops.negative(d)), { v, s: ops.negative(s) });
  }

  return result.asImmutable();
};

export function morphism(graph1, graph2, start1, start2, transform) {
  const errors = [];

  if (graph2.dim != graph1.dim)
    errors.push('graphs have different dimensions');
  if (!isLocallyStable(graph1))
    errors.push('first graph is not locally stable');
  if (!isLocallyStable(graph2))
    errors.push('second graph is not locally stable');
  if (!isConnected(graph1))
    errors.push('first graph is not connected');
  if (!isConnected(graph2))
    errors.push('second graph is not connected');
  if (transform != null && ops.dimensions(transform) != graph1.dim)
    errors.push('coordinate transformation has the wrong dimension');

  if (errors.length > 0)
    throw new Error(errors.join('\n'));

  const adj1 = adjacencies(graph1);
  const adj2 = adjacencies(graph2);
  const pos1 = barycentricPlacement(graph1);
  const pos2 = barycentricPlacement(graph2);

  const src2img = I.Map().asMutable();
  const img2src = I.Map().asMutable();
  const queue = [];

  src2img.set(start1, start2);
  img2src.set(start2, start1);
  queue.push(start1);

  while (queue.length) {
    const w1 = queue.shift();
    const w2 = src2img.get(w1);
    const n1 = _neighborsByEdgeVector(graph1, w1, adj1, pos1);
    const n2 = _neighborsByEdgeVector(graph2, w2, adj2, pos2);
  }
};

if (require.main == module) {
  Array.prototype.toString = function() {
    return '[ ' + this.map(x => x.toString()).join(', ') + ' ]';
  };

  const test = function test(g) {
    console.log('g = '+g);
    console.log('  cs  = '+coordinationSeq(g, 1, 10));
    if (isConnected(g)) {
      console.log('  pos = '+barycentricPlacement(g));
      console.log('      = '+barycentricPlacementAsFloat(g));
      console.log('  neighbors of 1: '
                  + JSON.stringify(_neighborsByEdgeVector(g, 1)));
      console.log('  stable: '+isStable(g));
      console.log('  locally stable: '+isLocallyStable(g));
    }

    for (const comp of connectedComponents(g)) {
      console.log('  component:');
      console.log('    nodes = '+comp.nodes);
      console.log('    graph = '+comp.graph);
      console.log('    basis = '+comp.basis);
      console.log('    multiplicity = '+comp.multiplicity);
    }
    console.log();
  };

  test(make([ [ 1, 1, [ 1, 0 ] ],
              [ 1, 1, [ 0, 1 ] ],
              [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 1 ] ],
              [ 1, 3, [ 0, 0 ] ],
              [ 1, 3, [ 1, -1 ] ] ]));

  test(make([ [ 1, 1, [ 1, 0 ] ],
              [ 1, 1, [ 0, 1 ] ],
              [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 1 ] ],
              [ 1, 3, [ 0, 0 ] ],
              [ 1, 3, [ 1, -1 ] ],
              [ 1, 4, [ 0, 0 ] ],
              [ 1, 4, [ 1, -1 ] ] ]));

  test(make([ [ 1, 1, [ -1,  1,  1 ] ],
              [ 1, 1, [  0, -1,  1 ] ],
              [ 1, 1, [  0,  0, -1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 0 ] ],
              [ 1, 2, [ 0, 1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0, 0 ] ],
              [ 1, 2, [ 1, 0, 0 ] ],
              [ 1, 2, [ 0, 1, 0 ] ],
              [ 1, 2, [ 0, 0, 1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0, 0 ] ],
              [ 1, 2, [ 2, 0, 0 ] ],
              [ 1, 2, [ 0, 2, 0 ] ],
              [ 1, 2, [ 0, 0, 2 ] ] ]));

  test(make([ [ 1, 3, [ 0, 0, 0 ] ],
              [ 1, 3, [ 2, 0, 0 ] ],
              [ 1, 3, [ 0, 2, 0 ] ],
              [ 1, 3, [ 0, 0, 2 ] ],
              [ 2, 4, [ 0, 0, 0 ] ],
              [ 2, 4, [ 2, 0, 0 ] ],
              [ 2, 4, [ 0, 2, 0 ] ],
              [ 2, 4, [ 0, 0, 2 ] ] ]));
}
<commit_msg>more work of morphism implementation<commit_after>import * as I from 'immutable';

import { rationalMatrices } from '../arithmetic/types';
const ops = rationalMatrices;

const Edge = I.Record({
  head : undefined,
  tail : undefined,
  shift: undefined
});

Edge.prototype.toString = function toString() {
  return 'Edge('+this.head+', '+this.tail+', '+this.shift+')';
};

Edge.prototype.reverse = function reverse() {
  return new Edge({
    head : this.tail,
    tail : this.head,
    shift: this.shift.map(x => -x)
  });
};

const _isNegative = vec => vec.find(x => ops.sgn(x) != 0) < 0;

Edge.prototype.canonical = function canonical() {
  if (this.tail < this.head || (this.tail == this.head
                                && _isNegative(this.shift)))
    return this.reverse();
  else
    return this;
};

const _makeEdge = function _makeEdge(e) {
  return new Edge({ head: e[0], tail: e[1], shift: e[2] }).canonical();
};

const Graph = I.Record({
  dim  : undefined,
  edges: undefined
});

Graph.prototype.toString = function toString() {
  return 'PGraph('+this.edges+')';
};

const encode = value => I.fromJS(ops.repr(value));
const decode = value => ops.fromRepr(value.toJS());

export function make(data) {
  const edges = I.Set(data).map(_makeEdge);
  if (edges.size == 0)
    throw new Error('cannot be empty');

  const dim = edges.first().shift.length;
  if (edges.some(e => e.shift.length != dim))
    throw new Error('must have consistent shift dimensions');

  return new Graph({ dim: dim, edges: edges });
};

export function adjacencies(graph) {
  const target = e => ({ v: e.tail, s: e.shift });

  let res = I.Map();
  for (const e of graph.edges) {
    res = res
      .update(e.head, a => (a || I.List()).push(target(e)))
      .update(e.tail, a => (a || I.List()).push(target(e.reverse())));
  }
  return res;
};

export function coordinationSeq(graph, start, dist) {
  const adj  = adjacencies(graph);
  const zero = ops.vector(graph.dim);

  let oldShell = I.Set();
  let thisShell = I.Set([I.fromJS([start, zero])]);
  const res = [1];

  for (const i of I.Range(1, dist+1)) {
    let nextShell = I.Set().asMutable();
    for (const item of thisShell) {
      const [v, s] = item.toJS();
      for (const { v: w, s: t } of adj.get(v)) {
        const key = I.fromJS([w, ops.plus(s, t)]);
        if (!oldShell.contains(key) && !thisShell.contains(key))
          nextShell.add(key);
      }
    }

    res.push(nextShell.size);
    oldShell = thisShell;
    thisShell = nextShell;
  }

  return res;
};

const _componentInOrbitGraph = (graph, start) => {
  const adj = adjacencies(graph);
  const queue = [start]
  const bridges = [];
  const nodeShifts = I.Map([[start, ops.vector(graph.dim)]]).asMutable();

  while (queue.length) {
    const v = queue.shift();
    const av = nodeShifts.get(v);

    for (const {v: w, s: shift} of adj.get(v)) {
      if (nodeShifts.get(w) == undefined) {
        queue.push(w);
        nodeShifts.set(w, ops.minus(av, shift));
      }
      else {
        const aw = nodeShifts.get(w);
        const newShift = ops.plus(shift, ops.minus(aw, av));
        const pivot = newShift.find(x => ops.ne(x, 0));

        if (pivot != undefined && ops.gt(pivot, 0))
          bridges.push({ v, w, s: newShift });
      }
    }
  }

  return {
    nodes: I.Set(nodeShifts.keys()),
    nodeShifts,
    bridges
  };
};

const _isConnectedOrbitGraph = (graph) => {
  const verts = I.List(adjacencies(graph).keySeq());
  const comp = _componentInOrbitGraph(graph, verts.first());

  return comp.nodes.size >= verts.size;
};

const _basis = M => {
  const T = ops.triangulation(M).R;
  return T.slice(0, ops.rank(T));
};

const _componentInCoverGraph = (graph, start) => {
  const { nodes, nodeShifts, bridges } = _componentInOrbitGraph(graph, start);
  const basis = _basis(bridges.map(b => b.s));
  const transform = ops.inverse(basis);
  const old2new = I.Map(I.List(nodes).zip(I.Range(1, nodes.size+1)));

  const newEdges = graph.edges
    .filter(({ head, tail }) =>
            old2new.get(head) != null && old2new.get(tail) != null)
    .map(({ head, tail, shift }) => {
      const [v, w] = [old2new.get(head), old2new.get(tail)];
      const [av, aw] = [nodeShifts.get(head), nodeShifts.get(tail)];
      const t = ops.times(ops.plus(shift, ops.minus(aw, av)), transform);
      return [v, w, t];
    });

  return {
    basis,
    nodes: nodes.toArray(),
    multiplicity: ops.determinant(basis),
    graph: make(newEdges)
  };
};

export function isConnected(graph) {
  const verts = I.List(adjacencies(graph).keySeq());
  const comp = _componentInCoverGraph(graph, verts.first());

  return comp.nodes.length >= verts.size && comp.multiplicity == 1;
};

export function connectedComponents(graph) {
  const verts = I.List(adjacencies(graph).keySeq());
  const seen = I.Set().asMutable();
  const result = [];

  for (const start of verts) {
    if (!seen.contains(start)) {
      const comp = _componentInCoverGraph(graph, start);
      result.push(comp);
      comp.nodes.forEach(v => seen.add(v));
    }
  }

  return result;
};

export function barycentricPlacement(graph) {
  if (!isConnected(graph))
    throw new Error('must have a connected orbit graph');

  const adj   = adjacencies(graph);
  const verts = I.List(adj.keySeq());
  const vIdcs = I.Map(I.Range(0, verts.size).map(i => [verts.get(i), i]));

  const n = verts.size;
  const d = graph.dim;
  let A = ops.matrix(n+1, n);
  let t = ops.matrix(n+1, d);

  verts.forEach((v, i) => {
    adj.get(v).forEach(c => {
      if (c.v != v) {
        const j = vIdcs.get(c.v);
        A[i][j] -= 1;
        A[i][i] += 1;
        t[i] = ops.plus(t[i], c.s);
      }
    });
  });
  A[n][0] = 1;

  const p = ops.solve(A, t);

  return I.Map(I.Range(0, n).map(i => [verts.get(i), p[i]]));
};

export function barycentricPlacementAsFloat(graph) {
  return barycentricPlacement(graph).map(p => p.map(x => ops.toJS(x)));
};

export function isStable(graph, pos=barycentricPlacement(graph)) {
  const verts = I.List(adjacencies(graph).keySeq());
  const seen = I.Set().asMutable();

  for (const v of verts) {
    const p = pos.get(v);
    const key = encode(p.map(x => ops.mod(x, 1)));
    if (seen.contains(key))
      return false;
    else
      seen.add(key);
  }

  return true;
};

export function isLocallyStable(graph, pos=barycentricPlacement(graph)) {
  const adj = adjacencies(graph);
  const verts = I.List(adj.keySeq());

  for (const v of verts) {
    const seen = I.Set().asMutable();

    for (const w of adj.get(v)) {
      const p = ops.plus(pos.get(w.v), w.s);
      const key = encode(p);
      if (seen.contains(key))
        return false;
      else
        seen.add(key);
    }
  }

  return true;
};

const _neighborsByEdgeVector = (
  graph, v, adj=adjacencies(graph), pos=barycentricPlacement(graph)
) => {
  const result = I.Map().asMutable();

  for (const { v: w, s } of adj.get(v)) {
    const d = ops.plus(s, ops.minus(pos.get(w), pos.get(v)));
    result.set(encode(d), { v: w, s })
    if (v == w)
      result.set(encode(ops.negative(d)), { v, s: ops.negative(s) });
  }

  return result.asImmutable();
};

export function morphism(graph1, graph2, start1, start2, transform) {
  const errors = [];

  if (graph2.dim != graph1.dim)
    errors.push('graphs have different dimensions');
  if (!isLocallyStable(graph1))
    errors.push('first graph is not locally stable');
  if (!isLocallyStable(graph2))
    errors.push('second graph is not locally stable');
  if (!isConnected(graph1))
    errors.push('first graph is not connected');
  if (!isConnected(graph2))
    errors.push('second graph is not connected');
  if (transform != null && ops.dimension(transform) != graph1.dim)
    errors.push('coordinate transformation has the wrong dimension');

  if (errors.length > 0)
    throw new Error(errors.join('\n'));

  const adj1 = adjacencies(graph1);
  const adj2 = adjacencies(graph2);
  const pos1 = barycentricPlacement(graph1);
  const pos2 = barycentricPlacement(graph2);

  const src2img = I.Map().asMutable();
  const img2src = I.Map().asMutable();
  const queue = [];

  src2img.set(start1, start2);
  img2src.set(start2, start1);
  queue.push(start1);

  while (queue.length) {
    const w1 = queue.shift();
    const w2 = src2img.get(w1);
    const n1 = _neighborsByEdgeVector(graph1, w1, adj1, pos1);
    const n2 = _neighborsByEdgeVector(graph2, w2, adj2, pos2);

    for (const [d1, {v: node1, s: shift1}] of n1) {
      const d2 = encode(ops.times(decode(d1), transform));
      const {v: node2, s: shift2} = n1.get(d2);

      if (node2 == null)
        return null;
    }
  }
};

if (require.main == module) {
  Array.prototype.toString = function() {
    return '[ ' + this.map(x => x.toString()).join(', ') + ' ]';
  };

  const test = function test(g) {
    console.log('g = '+g);
    console.log('  cs  = '+coordinationSeq(g, 1, 10));
    if (isConnected(g)) {
      console.log('  pos = '+barycentricPlacement(g));
      console.log('      = '+barycentricPlacementAsFloat(g));
      //console.log('  neighbors of 1: '+ _neighborsByEdgeVector(g, 1));
      console.log('  stable: '+isStable(g));
      console.log('  locally stable: '+isLocallyStable(g));

      if (isLocallyStable(g)) {
        morphism(g, g, 1, 1, ops.identityMatrix(g.dim));
      }
    }

    for (const comp of connectedComponents(g)) {
      console.log('  component:');
      console.log('    nodes = '+comp.nodes);
      console.log('    graph = '+comp.graph);
      console.log('    basis = '+comp.basis);
      console.log('    multiplicity = '+comp.multiplicity);
    }
    console.log();
  };

  test(make([ [ 1, 1, [ 1, 0 ] ],
              [ 1, 1, [ 0, 1 ] ],
              [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 1 ] ],
              [ 1, 3, [ 0, 0 ] ],
              [ 1, 3, [ 1, -1 ] ] ]));

  test(make([ [ 1, 1, [ 1, 0 ] ],
              [ 1, 1, [ 0, 1 ] ],
              [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 1 ] ],
              [ 1, 3, [ 0, 0 ] ],
              [ 1, 3, [ 1, -1 ] ],
              [ 1, 4, [ 0, 0 ] ],
              [ 1, 4, [ 1, -1 ] ] ]));

  test(make([ [ 1, 1, [ -1,  1,  1 ] ],
              [ 1, 1, [  0, -1,  1 ] ],
              [ 1, 1, [  0,  0, -1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0 ] ],
              [ 1, 2, [ 1, 0 ] ],
              [ 1, 2, [ 0, 1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0, 0 ] ],
              [ 1, 2, [ 1, 0, 0 ] ],
              [ 1, 2, [ 0, 1, 0 ] ],
              [ 1, 2, [ 0, 0, 1 ] ] ]));

  test(make([ [ 1, 2, [ 0, 0, 0 ] ],
              [ 1, 2, [ 2, 0, 0 ] ],
              [ 1, 2, [ 0, 2, 0 ] ],
              [ 1, 2, [ 0, 0, 2 ] ] ]));

  test(make([ [ 1, 3, [ 0, 0, 0 ] ],
              [ 1, 3, [ 2, 0, 0 ] ],
              [ 1, 3, [ 0, 2, 0 ] ],
              [ 1, 3, [ 0, 0, 2 ] ],
              [ 2, 4, [ 0, 0, 0 ] ],
              [ 2, 4, [ 2, 0, 0 ] ],
              [ 2, 4, [ 0, 2, 0 ] ],
              [ 2, 4, [ 0, 0, 2 ] ] ]));
}
<|endoftext|>


<commit_before>var fs = require('fs');
var parse = require('fast-plist').parse;

var musicLibrayXmlFilename = process.argv[2];

if(!musicLibrayXmlFilename){
	console.error("usage: " + process.argv[0] + " " + process.argv[1] + ' <path to "iTunes Music Library.xml">');
	process.exit(1);
}

function escapeQuotes(s){
	return s.replace(/'/, "''");
}

function formatOptional(track, keyName, isString){
	var value = track[keyName];
	if(!value){
		return 'NULL';
	}
	if(isString){
		return `'${escapeQuotes(value)}'`;
	}
	return value;
}

function trackToSql(track){
	var trackTable = "tracks";
	var composer = formatOptional(track, 'Composer', true);
	var artist = formatOptional(track, 'Artist', true);
	return `INSERT INTO ${trackTable}(name, artist, file_size, total_time, year, composer) VALUES ('${escapeQuotes(track.Name)}', '${artist}', ${track["Size"]}, ${track["Total Time"]}, ${track["Year"]}, ${composer});`;
}

function printKeys(obj){
	function formatKey(s){
		return `'\${track["${s}"]}'`
	}
	console.log(Object.keys(obj).map(formatKey).join(', '));
}

function printKinds(tracks){
	var kinds = new Map();

	for(var id in tracks){
		var track = tracks[id];

		if(!kinds.has(track.Kind)){
			kinds.set(track.Kind, 1);
		}
		else{
			kinds.set(track.Kind, kinds.get(track.Kind) + 1);
		}
	}
	console.log(kinds.entries());
}

var plist = parse(fs.readFileSync(musicLibrayXmlFilename, 'utf8'));

var tracks = plist.Tracks;

for(var id in tracks){
	var track = tracks[id];
	if(!track.Kind.match(/\baudio\b/)){
		continue;
	}
	console.log(trackToSql(track));
	// printKeys(track);
	// break;
}

<commit_msg>Fix issue where escape quotes function doesn't escape all quotes in string<commit_after>var fs = require('fs');
var parse = require('fast-plist').parse;

var musicLibrayXmlFilename = process.argv[2];

if(!musicLibrayXmlFilename){
	console.error("usage: " + process.argv[0] + " " + process.argv[1] + ' <path to "iTunes Music Library.xml">');
	process.exit(1);
}

function escapeQuotes(s){
	return s.replace(/'/g, "''");
}

function formatOptional(track, keyName, isString){
	var value = track[keyName];
	if(!value){
		return 'NULL';
	}
	if(isString){
		return `'${escapeQuotes(value)}'`;
	}
	return value;
}

function trackToSql(track){
	var trackTable = "tracks";
	var composer = formatOptional(track, 'Composer', true);
	var artist = formatOptional(track, 'Artist', true);
	return `INSERT INTO ${trackTable}(name, artist, file_size, total_time, year, composer) VALUES ('${escapeQuotes(track.Name)}', '${artist}', ${track["Size"]}, ${track["Total Time"]}, ${track["Year"]}, ${composer});`;
}

function printKeys(obj){
	function formatKey(s){
		return `'\${track["${s}"]}'`
	}
	console.log(Object.keys(obj).map(formatKey).join(', '));
}

function printKinds(tracks){
	var kinds = new Map();

	for(var id in tracks){
		var track = tracks[id];

		if(!kinds.has(track.Kind)){
			kinds.set(track.Kind, 1);
		}
		else{
			kinds.set(track.Kind, kinds.get(track.Kind) + 1);
		}
	}
	console.log(kinds.entries());
}

var plist = parse(fs.readFileSync(musicLibrayXmlFilename, 'utf8'));

var tracks = plist.Tracks;

for(var id in tracks){
	var track = tracks[id];
	if(!track.Kind.match(/\baudio\b/)){
		continue;
	}
	console.log(trackToSql(track));
	// printKeys(track);
	// break;
}

<|endoftext|>


<commit_before>'use strict';

// do npm install marked --save-dev
// do npm install grunt-contrib-jade --save-dev

// copy / overwrite view
// copy additional grunt task config
// alter Gruntfile

var path = require('path');
var fs = require('fs');

module.exports = function (remote, files, wok) {

    return {

        //this is run when the plugin is installed while generating the project
        install: function () {

            files.filter(function (filepath) {
                return filepath.indexOf('template/') === 0;
            }).forEach(function (el) {
                wok.files.push({
                    pathFrom: el,
                    remote: remote,
                    pathTo: el.replace(/^template\//, ''),
                    content: null
                });
            });
        }
    };

};
<commit_msg>install<commit_after>'use strict';

// do npm install marked --save-dev
// do npm install grunt-contrib-jade --save-dev

// copy / overwrite view
// copy additional grunt task config
// alter Gruntfile

var path = require('path');
var fs = require('fs');

module.exports = function (remote, files, wok) {

    return {

        //this is run when the plugin is installed while generating the project
        install: function () {

            files.filter(function (filepath) {
                if (filepath.indexOf('template/') === 0) {
                    return {
                        pathFrom: filepath,
                        pathTo: filepath.replace('/^template\//', '')
                    };
                }
            }).forEach(function (fileItem) {
                remote.copy(fileItem.pathFrom, fileItem.pathTo);
            });
        }
    };

};
<|endoftext|>


<commit_before>const zip = require('lodash.zip');

const arrFlatten = (arr) => [].concat.apply([], arr);

const strReverse = (str) => str.split('').reverse().join('');

// module Dejunk
  // extend self

// All characters on the middle row of a QWERTY keyboard
const MASH_CHARS = 'ASDFGHJKLasdfghjkl;: ';
const letters = arrFlatten([...'abcdefghijklmnopqrstuvwxyz'].map(letter => [`${letter} `, `${letter}${letter}`]));

// All neighboring key pairs on a QWERTY keyboard, except "er" and "re" which
// each make up >1% of bigrams in our "good" sample, plus each letter repeated or with a space
const qwertyBigrams = ['qw', 'we', 'rt', 'ty', 'yu', 'ui', 'op', 'as', 'sd', 'df', 'fg', 'gh', 'hj',
                 'jk', 'kl', 'zx', 'xd', 'cv', 'vb', 'bn', 'nm', 'qa', 'az', 'ws', 'sx', 'ed',
                 'dc', 'rf', 'fv', 'tg', 'gb', 'yh', 'hn', 'uj', 'jm', 'ik', 'ol']

// CURRENT -> trying to make this logic return the same result.
const  MASH_BIGRAMS = new Set(arrFlatten(letters.concat(qwertyBigrams).map(bigram => [bigram, strReverse(bigram)])))

const mashingBigramFrequencies = _mashingBigramFrequencies();
const corpusBigramFrequencies = require('./bigram_frequencies.json');

function isJunk(str, minAlnumChars=3, whitelistRegexes=[], whiteliststrs=[]) {
  if(str && (whiteliststrs.indexOf(str) > -1 || whitelistRegexes.some(rex => rex.test(str)))) {
    return false;
  }

  if (!str || /^\w+$/.test(str) === false) {
    return true; // not alphanumeric
  }

  const normed = normalizeForComparison(str);

  if (tooFewAlphanumericChars(normed, minAlnumChars)) return true; //:too_short
  if (excessiveSingleCharacterRepeats(str, normed)) return true; //:one_char_repeat
  if (startsWithDisallowedPunctuation(str)) return true; //:starts_with_punct
  if (tooManyShortWords(str)) return true; //:too_many_short_words
  if (threePlusCharsRepeatTwice(str)) return true; //:three_chars_repeat_twice
  if (missingVowels(str, normed)) return true; //:missing_vowels
  if (asdfRowAndSuspicious(str)) return true; //:asdf_row

  const ascii_proportion = [...str].filter(c => c.charCodeAt() < 128).length / str.length

  // The bigrams look like the ones you'd get from keyboard mashing
  // (the probability shouldn't be taken too literally, > 0.25 is almost all
  // mashing in practice on our corpus)
  if (str.length > 1 && ascii_proportion > 0.8) {
    if (probabilityOfKeyboardMashing(str) > 0.25)
      return true // :mashing_bigrams
    }
  }

function normalizeForComparison(str) {
  return str.normalize('NFD')
            .replace(/[\u0300-\u036f]/g, "")
            .replace(/[^\w\d]/g, "")
            // gsub(/\p{Mn}+/, ''.freeze). ?? [ruby] Unicode Mark Nonspacing
            .toLowerCase();
}

// Too short (unless we're dealing with a large alphabet with legitimate single-char words)
// NOTE: removed asian alphabet conditionals
function tooFewAlphanumericChars(normed, minAlnumChars) {
  if (normed.length < minAlnumChars) {
      return true
  }
  return false
}

// One character repeated 5 or more times, or 3 or more times and not an
// acronym, roman numeral, or www
function excessiveSingleCharacterRepeats(str, normed) {

    // if there's only one letter in the str
    if ([...new Set(normed)].length == 1) {
      return true
    }

    if (/([^\s])\1\1/i.test(str)) {
      if (/([^0-9])\1\1\1\1/i.test(normed)) {
        return true
      }

      const punct = /[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]/;
      const words = str.split(punct);

      for(let i=0; i < words.length; i++) {
        if (/([^iw0-9])\1\1/i.test(words[i]) && words[i] !== words[i].toUpperCase()) {
          return true;
        }
      }
    }

    return false
}

// Starting punctuation, except ['"#@]
// Ruby Original: str =~ /\A[[:punct:]]/ && str !~ /\A(\p{Pi}|\p{Ps}|['"¿»’]).+/
function startsWithDisallowedPunctuation(str) {
  const startsWithPunct = /^[\s\u2000-\u206F\u2E00-\u2E7F\\'!"$%&()*+,\-.\/:;<=>?\[\]^_`{|}~]/
  return startsWithPunct.test(str)
}

function tooManyShortWords(str) {

  const words = str.split(' ')
  const short = words.filter( (w) => w.length < 3 ).length

  if (short > 2 && short > 0.75 * words.length) {
    return true
  }

  return false
}

// At least 3 characters repeated at least twice in a row (but only on short
// strs, otherwise there are false positives)
function threePlusCharsRepeatTwice(str) {
  // ruby: /(....*)[[:space:][:punct:]]*\1[[:space:][:punct:]]*\1/
  const regexp = /(....*)[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]*\1[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]*\1/

  return (str.length < 80 && regexp.test(str))
}

// Missing vowels (and doesn't look like acronym, and is ASCII so we can tell)
function missing_vowels(str, normed) {

    if (! [...normed].some(c => c.charCodeAt() >= 128) || str == str.toUpperCase() ) {
      if (/[aeiouy]/i.test(normed)) {
        return true
      }
    }

    return false
}

// All characters from the same row of the keyboard is suspicious, but we
// need additional confirmation
function asdfRowAndSuspicious(str) {
    if ([...str].every(c => MASH_CHARS.indexOf(c) > -1)) {
      if (str.length >= 16) return true;
      if (/(...).*\1/.test(str)) return true; // Three-plus characters, repeated
      if(/(..).*\1.*\1/.test(str)) return true; // Two characters, repeated twice
      if(/\b[sdfghjkl]\b/.test(str)) return true;  // Stray lowercase letter
      if(/[^aeiouy]{3}/i.test(str) && (str.length > 5 || str != str.toUpperCase())) return true; // Three consonants in a row, non-acronym
    }

    return false;
}

// The Bayesian probability of a str being keyboard mashing, given the
// probability of each bigram if drawn either from the legit corpus or from
// mashing, and an a priori probability of mashing.
// The probability shouldn't be taken too literally, but it's a useful indicator.
function probabilityOfKeyboardMashing(str, aprioriProbabilityOfMashing=0.1) {
// @incomplete
  const bigrams = _bigrams(str);

  if(!bigrams || bigrams.length == 0) return 0;

  // [todo][ruby] version uses BigDecimal here to use decimal notation of numbers.
  // Not sure if I can just leave it in scientific notation
  const probBigramsGivenMashing = bigrams
    .map(bigram => mashingProbability(bigram))
    .reduce( (x, i) => x * i)

  const probBigramsGivenCorpus = bigrams
    .map(bigram => corpusProbability(bigram))
    .reduce( (x, i) => x * i)

  const numerator = probBigramsGivenMashing * aprioriProbabilityOfMashing;

  return numerator / (numerator + probBigramsGivenCorpus * (1 - aprioriProbabilityOfMashing));
}

function _bigrams(str) {
  str = str.trim()

  if(!str || str.length < 2) { return [] }

  const chars = str.split('');

  return zip(chars, chars.slice(1))
           .filter(x => !!x[0] && !!x[1])
           .map(arr => arr.join('').toLowerCase())
           .map(bigram => bigram.replace(/[\d]/g, '0')) // [ruby]  .gsub(/[0-9]/, '0'.freeze) not sure why it needs immutable '0'
            // map { |bigram| bigram.gsub(/[[:space:]]/, ' '.freeze) } // [ruby]. not sure why replace space
}

function mashingProbability(bigram) {
  const f = mashingBigramFrequencies[bigram];
  if (f) {
    return f
  }
  else if (/[a-z]{2}/i.test(bigram)) {
    // 26**2 = 676, so 1 in 2k seems a reasonable probability for an arbitrary two-letter bigram given mashing
    return 0.0005;
  }
  else {
    // An arbitrary (non-ASCII) bigram with mashing is slightly more probable than with legit strs
    return 1e-6;
  }
}

// This is a guess because we don't have a good corpus, but we assume that
// 50% of mashing bigrams are a neighboring pair on the ASDF row or a duplicate
// and the rest are evenly distributed among other neighboring pairs or char plus-space.
function  _mashingBigramFrequencies() {
    const mBF = {};
    MASH_BIGRAMS.forEach((bigram) => {
      if (bigram[0] == bigram[1] || [...bigram].every(c => c !== ' ' && MASH_CHARS.indexOf(c) > -1)) {
        mBF[bigram] = 0.5 / (16 + 26)
      } else {
        mBF[bigram] = 0.5 / (MASH_BIGRAMS.size - 16 - 26)
      }
    });
    return mBF;
}

  // def corpus_probability(bigram)
  //   corpus_bigram_frequencies[bigram] || 1e-7 # Around the smallest frequency we store for the corpus
  // end

function corpusProbability(bigram) {
  return corpusBigramFrequencies[bigram] || 1e-7 // Around the smallest frequency we store for the corpus
}

/*

    # The bigrams don't look like the bigrams in legitimate strs
    if str.length > 6 && ascii_proportion > 0.8
      corpus_similarity = bigram_similarity_to_corpus(str)

      # The similarity is more accurate for longer strs, and with more ASCII,
      # so increase the value (= lower the threshold) for shorter strs and
      # strs with less ASCII.
      score = corpus_similarity * (1.0/ascii_proportion**2) * (1.0/(1 - Math.exp(-0.1*str.length)))

      if score < 0.03
        return :unlikely_bigrams
      elsif score < 0.08 && str !~ /\A([[:upper:]][[:lower:]]+ )*[[:upper:]][[:lower:]]+\z/
        # The similarity ignores casing, so instead use a higher threshold if
        # the casing looks wrong
        return :unlikely_bigrams
      elsif score < bigram_similarity_to_mashing(str)
        return :mashing_bigrams
      end
    end

    false
 And

  # Cosine similarity between vector of frequencies of bigrams within str,
  # and vector of frequencies of all bigrams within corpus
  def bigram_similarity_to_corpus(str)
    bigrams = bigrams(str)

    freqs = bigrams.
      each_with_object(Hash.new(0)) { |bigram, counts| counts[bigram] += 1 }.
      each_with_object({}) do |(bigram,count), freqs|
        freqs[bigram] = count.to_f / bigrams.length
      end

    numerator = freqs.
      map{ |bigram, freq| corpus_bigram_frequencies[bigram].to_f * freq }.inject(&:+)
    denominator = corpus_bigram_magnitude * ((freqs.values.map{ |v| v**2 }.inject(&:+)) ** 0.5)

    numerator / denominator
  end

  # Cosine similarity between vector of frequencies of bigrams within str,
  # and vector which assumes all bigrams made of neighboring pairs on the keyboard
  # are equally likely, and no others appear
  def bigram_similarity_to_mashing(str)
    bigrams = bigrams(str)

    freqs = bigrams.
      each_with_object(Hash.new(0)) { |bigram, counts| counts[bigram] += 1 }.
      each_with_object({}) do |(bigram,count), freqs|
        freqs[bigram] = count.to_f / bigrams.length
      end

    numerator = freqs.map{ |bigram, freq| freq * mashingBigramFrequencies[bigram].to_f }.inject(&:+)
    denominator = mashing_bigram_magnitude * ((freqs.values.map{ |v| v**2 }.inject(&:+)) ** 0.5)

    numerator / denominator
  end

  # The Bayesian probability of a str being keyboard mashing, given the
  # probability of each bigram if drawn either from the legit corpus or from
  # mashing, and an a priori probability of mashing.
  #
  # The probability shouldn't be taken too literally, but it's a useful
  # indicator.
  def probability_of_keyboard_mashing(str, apriori_probability_of_mashing: 0.1)
    bigrams = bigrams(str)

    return 0 unless bigrams.present?

    prob_bigrams_given_mashing = bigrams.
      map { |bigram| BigDecimal.new(mashing_probability(bigram).to_s) }.
      inject(&:*)

    prob_bigrams_given_corpus = bigrams.
      map { |bigram| BigDecimal.new(corpus_probability(bigram).to_s) }.
      inject(&:*)

    numerator = prob_bigrams_given_mashing * apriori_probability_of_mashing

    numerator / (numerator + prob_bigrams_given_corpus * (1 - apriori_probability_of_mashing))
  end

  private

  def corpus_bigram_magnitude
    @corpus_bigram_magnitude ||= (corpus_bigram_frequencies.values.map{ |v| v**2 }.inject(&:+)) ** 0.5
  end

  def mashing_bigram_magnitude
    @mashing_bigram_magnitude ||= (mashingBigramFrequencies.values.map{ |v| v**2 }.inject(&:+)) ** 0.5
  end
end
*/
<commit_msg>[wip] bigram_similarity_to_corpus<commit_after>const zip = require('lodash.zip');

const arrFlatten = (arr) => [].concat.apply([], arr);

const strReverse = (str) => str.split('').reverse().join('');

// module Dejunk
  // extend self

// All characters on the middle row of a QWERTY keyboard
const MASH_CHARS = 'ASDFGHJKLasdfghjkl;: ';
const letters = arrFlatten([...'abcdefghijklmnopqrstuvwxyz'].map(letter => [`${letter} `, `${letter}${letter}`]));

// All neighboring key pairs on a QWERTY keyboard, except "er" and "re" which
// each make up >1% of bigrams in our "good" sample, plus each letter repeated or with a space
const qwertyBigrams = ['qw', 'we', 'rt', 'ty', 'yu', 'ui', 'op', 'as', 'sd', 'df', 'fg', 'gh', 'hj',
                 'jk', 'kl', 'zx', 'xd', 'cv', 'vb', 'bn', 'nm', 'qa', 'az', 'ws', 'sx', 'ed',
                 'dc', 'rf', 'fv', 'tg', 'gb', 'yh', 'hn', 'uj', 'jm', 'ik', 'ol']

// CURRENT -> trying to make this logic return the same result.
const  MASH_BIGRAMS = new Set(arrFlatten(letters.concat(qwertyBigrams).map(bigram => [bigram, strReverse(bigram)])))

const mashingBigramFrequencies = _mashingBigramFrequencies();
const corpusBigramFrequencies = require('./bigram_frequencies.json');

function isJunk(str, minAlnumChars=3, whitelistRegexes=[], whiteliststrs=[]) {
  if(str && (whiteliststrs.indexOf(str) > -1 || whitelistRegexes.some(rex => rex.test(str)))) {
    return false;
  }

  if (!str || /^\w+$/.test(str) === false) {
    return true; // not alphanumeric
  }

  const normed = normalizeForComparison(str);

  if (tooFewAlphanumericChars(normed, minAlnumChars)) return true; //:too_short
  if (excessiveSingleCharacterRepeats(str, normed)) return true; //:one_char_repeat
  if (startsWithDisallowedPunctuation(str)) return true; //:starts_with_punct
  if (tooManyShortWords(str)) return true; //:too_many_short_words
  if (threePlusCharsRepeatTwice(str)) return true; //:three_chars_repeat_twice
  if (missingVowels(str, normed)) return true; //:missing_vowels
  if (asdfRowAndSuspicious(str)) return true; //:asdf_row

  const ascii_proportion = [...str].filter(c => c.charCodeAt() < 128).length / str.length

  // The bigrams look like the ones you'd get from keyboard mashing (the probability shouldn't be
  // taken too literally, > 0.25 is almost all mashing in practice on our corpus)
  if (str.length > 1 && ascii_proportion > 0.8) {
    if (probabilityOfKeyboardMashing(str) > 0.25){
      return true // :mashing_bigrams
    }
  }

  // The bigrams don't look like the bigrams in legitimate strs
  if (str.length > 6 && ascii_proportion > 0.8) {

      const corpus_similarity = bigram_similarity_to_corpus(str);

/*
      // The similarity is more accurate for longer strs, and with more ASCII,
      // so increase the value (= lower the threshold) for shorter strs and strs with less ASCII.
      score = corpus_similarity * (1.0/ascii_proportion**2) * (1.0/(1 - Math.exp(-0.1*str.length)))

      if (score < 0.03) {
        return true // :unlikely_bigrams
      }
      else if (score < 0.08 && str !~ /\A([[:upper:]][[:lower:]]+ )*[[:upper:]][[:lower:]]+\z/) {
        // The similarity ignores casing, so instead use a higher threshold if the casing looks wrong
        return true //:unlikely_bigrams
      }
      else if (score < bigram_similarity_to_mashing(str)) {
        return true //:mashing_bigrams
      }
*/
  }

  return false
}

function normalizeForComparison(str) {
  return str.normalize('NFD')
            .replace(/[\u0300-\u036f]/g, "")
            .replace(/[^\w\d]/g, "")
            // gsub(/\p{Mn}+/, ''.freeze). ?? [ruby] Unicode Mark Nonspacing
            .toLowerCase();
}

// Too short (unless we're dealing with a large alphabet with legitimate single-char words)
// NOTE: removed asian alphabet conditionals
function tooFewAlphanumericChars(normed, minAlnumChars) {
  if (normed.length < minAlnumChars) {
      return true
  }
  return false
}

// One character repeated 5 or more times, or 3 or more times and not an
// acronym, roman numeral, or www
function excessiveSingleCharacterRepeats(str, normed) {

    // if there's only one letter in the str
    if ([...new Set(normed)].length == 1) {
      return true
    }

    if (/([^\s])\1\1/i.test(str)) {
      if (/([^0-9])\1\1\1\1/i.test(normed)) {
        return true
      }

      const punct = /[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]/;
      const words = str.split(punct);

      for(let i=0; i < words.length; i++) {
        if (/([^iw0-9])\1\1/i.test(words[i]) && words[i] !== words[i].toUpperCase()) {
          return true;
        }
      }
    }

    return false
}

// Starting punctuation, except ['"#@]
// Ruby Original: str =~ /\A[[:punct:]]/ && str !~ /\A(\p{Pi}|\p{Ps}|['"¿»’]).+/
function startsWithDisallowedPunctuation(str) {
  const startsWithPunct = /^[\s\u2000-\u206F\u2E00-\u2E7F\\'!"$%&()*+,\-.\/:;<=>?\[\]^_`{|}~]/
  return startsWithPunct.test(str)
}

function tooManyShortWords(str) {

  const words = str.split(' ')
  const short = words.filter( (w) => w.length < 3 ).length

  if (short > 2 && short > 0.75 * words.length) {
    return true
  }

  return false
}

// At least 3 characters repeated at least twice in a row (but only on short
// strs, otherwise there are false positives)
function threePlusCharsRepeatTwice(str) {
  // ruby: /(....*)[[:space:][:punct:]]*\1[[:space:][:punct:]]*\1/
  const regexp = /(....*)[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]*\1[\s\u2000-\u206F\u2E00-\u2E7F\\'!"#$%&()*+,\-.\/:;<=>?@\[\]^_`{|}~]*\1/

  return (str.length < 80 && regexp.test(str))
}

// Missing vowels (and doesn't look like acronym, and is ASCII so we can tell)
function missing_vowels(str, normed) {

    if (! [...normed].some(c => c.charCodeAt() >= 128) || str == str.toUpperCase() ) {
      if (/[aeiouy]/i.test(normed)) {
        return true
      }
    }

    return false
}

// All characters from the same row of the keyboard is suspicious, but we
// need additional confirmation
function asdfRowAndSuspicious(str) {
    if ([...str].every(c => MASH_CHARS.indexOf(c) > -1)) {
      if (str.length >= 16) return true;
      if (/(...).*\1/.test(str)) return true; // Three-plus characters, repeated
      if(/(..).*\1.*\1/.test(str)) return true; // Two characters, repeated twice
      if(/\b[sdfghjkl]\b/.test(str)) return true;  // Stray lowercase letter
      if(/[^aeiouy]{3}/i.test(str) && (str.length > 5 || str != str.toUpperCase())) return true; // Three consonants in a row, non-acronym
    }

    return false;
}

// The Bayesian probability of a str being keyboard mashing, given the
// probability of each bigram if drawn either from the legit corpus or from
// mashing, and an a priori probability of mashing.
// The probability shouldn't be taken too literally, but it's a useful indicator.
function probabilityOfKeyboardMashing(str, aprioriProbabilityOfMashing=0.1) {
  const bigrams = _bigrams(str);

  if(!bigrams || bigrams.length == 0) return 0;

  // [todo][ruby] version uses BigDecimal here to use decimal notation of numbers.
  // Not sure if I can just leave it in scientific notation
  const probBigramsGivenMashing = bigrams
    .map(bigram => mashingProbability(bigram))
    .reduce( (x, i) => x * i)

  const probBigramsGivenCorpus = bigrams
    .map(bigram => corpusProbability(bigram))
    .reduce( (x, i) => x * i)

  const numerator = probBigramsGivenMashing * aprioriProbabilityOfMashing;

  return numerator / (numerator + probBigramsGivenCorpus * (1 - aprioriProbabilityOfMashing));
}

function _bigrams(str) {
  str = str.trim()

  if(!str || str.length < 2) { return [] }

  const chars = str.split('');

  return zip(chars, chars.slice(1))
           .filter(x => !!x[0] && !!x[1])
           .map(arr => arr.join('').toLowerCase())
           .map(bigram => bigram.replace(/[\d]/g, '0')) // [ruby]  .gsub(/[0-9]/, '0'.freeze) not sure why it needs immutable '0'
            // map { |bigram| bigram.gsub(/[[:space:]]/, ' '.freeze) } // [ruby]. not sure why replace space
}

function mashingProbability(bigram) {
  const f = mashingBigramFrequencies[bigram];
  if (f) {
    return f
  }
  else if (/[a-z]{2}/i.test(bigram)) {
    // 26**2 = 676, so 1 in 2k seems a reasonable probability for an arbitrary two-letter bigram given mashing
    return 0.0005;
  }
  else {
    // An arbitrary (non-ASCII) bigram with mashing is slightly more probable than with legit strs
    return 1e-6;
  }
}

// This is a guess because we don't have a good corpus, but we assume that
// 50% of mashing bigrams are a neighboring pair on the ASDF row or a duplicate
// and the rest are evenly distributed among other neighboring pairs or char plus-space.
function  _mashingBigramFrequencies() {
    const mBF = {};
    MASH_BIGRAMS.forEach((bigram) => {
      if (bigram[0] == bigram[1] || [...bigram].every(c => c !== ' ' && MASH_CHARS.indexOf(c) > -1)) {
        mBF[bigram] = 0.5 / (16 + 26)
      } else {
        mBF[bigram] = 0.5 / (MASH_BIGRAMS.size - 16 - 26)
      }
    });
    return mBF;
}

function corpusProbability(bigram) {
  return corpusBigramFrequencies[bigram] || 1e-7 // Around the smallest frequency we store for the corpus
}

/*
  # Cosine similarity between vector of frequencies of bigrams within str,
  # and vector of frequencies of all bigrams within corpus
  def bigram_similarity_to_corpus(str)
    bigrams = bigrams(str)

    freqs = bigrams.
      each_with_object(Hash.new(0)) { |bigram, counts| counts[bigram] += 1 }.
      each_with_object({}) do |(bigram,count), freqs|
        freqs[bigram] = count.to_f / bigrams.length
      end

    numerator = freqs.
      map{ |bigram, freq| corpus_bigram_frequencies[bigram].to_f * freq }.inject(&:+)
    denominator = corpus_bigram_magnitude * ((freqs.values.map{ |v| v**2 }.inject(&:+)) ** 0.5)

    numerator / denominator
  end
*/

// Cosine similarity between vector of frequencies of bigrams within str,
// and vector of frequencies of all bigrams within corpus
function bigram_similarity_to_corpus(str) {
// @WIP
  const bigrams = _bigrams(str);

  const counts = bigrams.reduce((r, bigram) => {
    r[bigram] = r[bigram] ? r[bigram] + 1 : 1;
    return r;
  }, {});

  const numerator = Object.keys(counts).map(bigram => {
    const freq = counts[bigram] / bigrams.length;
    return (corpusBigramFrequencies[bigram] || 0) * freq
  }).reduce((a, b) => a + b);

  // console.log(freqs);

  // denominator = corpus_bigram_magnitude * ((freqs.values.map{ |v| v**2 }.inject(&:+)) ** 0.5)

  // return numerator / denominator
}

/*

  # Cosine similarity between vector of frequencies of bigrams within str,
  # and vector which assumes all bigrams made of neighboring pairs on the keyboard
  # are equally likely, and no others appear
  def bigram_similarity_to_mashing(str)
    bigrams = bigrams(str)

    freqs = bigrams.
      each_with_object(Hash.new(0)) { |bigram, counts| counts[bigram] += 1 }.
      each_with_object({}) do |(bigram,count), freqs|
        freqs[bigram] = count.to_f / bigrams.length
      end

    numerator = freqs.map{ |bigram, freq| freq * mashingBigramFrequencies[bigram].to_f }.inject(&:+)
    denominator = mashing_bigram_magnitude * ((freqs.values.map{ |v| v**2 }.inject(&:+)) ** 0.5)

    numerator / denominator
  end

  # The Bayesian probability of a str being keyboard mashing, given the
  # probability of each bigram if drawn either from the legit corpus or from
  # mashing, and an a priori probability of mashing.
  #
  # The probability shouldn't be taken too literally, but it's a useful
  # indicator.
  def probability_of_keyboard_mashing(str, apriori_probability_of_mashing: 0.1)
    bigrams = bigrams(str)

    return 0 unless bigrams.present?

    prob_bigrams_given_mashing = bigrams.
      map { |bigram| BigDecimal.new(mashing_probability(bigram).to_s) }.
      inject(&:*)

    prob_bigrams_given_corpus = bigrams.
      map { |bigram| BigDecimal.new(corpus_probability(bigram).to_s) }.
      inject(&:*)

    numerator = prob_bigrams_given_mashing * apriori_probability_of_mashing

    numerator / (numerator + prob_bigrams_given_corpus * (1 - apriori_probability_of_mashing))
  end

  private

  def corpus_bigram_magnitude
    @corpus_bigram_magnitude ||= (corpus_bigram_frequencies.values.map{ |v| v**2 }.inject(&:+)) ** 0.5
  end

  def mashing_bigram_magnitude
    @mashing_bigram_magnitude ||= (mashingBigramFrequencies.values.map{ |v| v**2 }.inject(&:+)) ** 0.5
  end
end
*/

bigram_similarity_to_corpus('hadsjk hk∆j hadsjk sjh7kjé ah-sd');
<|endoftext|>


<commit_before>'use strict';

/**
 * Module dependencies
 */

var fs = require('fs');
var path = require('path');
var minimatch = require('minimatch');

/**
 * Expose `lookup`
 */

module.exports = function lookup(patterns, cwd, opts) {
  patterns = Array.isArray(patterns) ? patterns : [patterns];
  cwd = path.resolve(cwd, '.');

  var files = fs.readdirSync(cwd);
  var len = files.length;
  var i = 0;

  while (i < len) {
    var fp = path.join(cwd, files[i++]);

    for (var j = 0; j < patterns.length; j++) {
      if (minimatch(fp, patterns[j], opts)) {
        return path.resolve(cwd, fp);
      }
    }
  }

  while (cwd !== process.cwd()) {
    cwd = path.resolve(cwd, '..');
    return lookup(patterns, cwd, opts);
  }
  return null;
};
<commit_msg>refactor<commit_after>'use strict';

/**
 * Module dependencies
 */

var fs = require('fs');
var path = require('path');
var normalize = require('normalize-path');
var mm = require('multimatch');

/**
 * Expose `lookup`
 */

module.exports = function lookup(patterns, options) {
  var opts = extend({cwd: process.cwd()}, options);
  patterns = !Array.isArray(patterns)
    ? [patterns]
    : patterns;

  var cwd = opts.cwd;
  var files = fs.readdirSync(cwd);
  var len = files.length;
  var i = 0;

  while (i < len) {
    var fp = path.join(cwd, files[i++]);
    if (path.dirname(fp) === '.') {
      break;
    }

    var match = mm(fp, patterns, opts);
    if (match.length === 0) {
      continue;
    }
    return path.resolve(fp);
  }

  var dir = normalized(process.cwd());
  cwd = normalized(cwd);

  if (dir === cwd) {
    return path.resolve(dir);
  }

  cwd = path.join(cwd, '..');
  if (cwd === '..') {
    return null;
  }

  return lookup(patterns, extend(opts, {cwd: cwd}));
};

/**
 * Basic extend
 *
 * @api private
 */

function extend(a, b) {
  for (var key in b) {
    if (b.hasOwnProperty(key)) {
      a[key] = b[key];
    }
  }
  return a;
}

/**
 * Normalize the paths for comparisons
 *
 * @api private
 */

function normalized(fp) {
  if (/^\w:/.test(fp)) {
    fp = fp.slice(2);
  }

  // strip leading slashes
  fp = fp.replace(/^[\\\/]+/, '');
  return normalize(fp)
    .toLowerCase();
}<|endoftext|>


<commit_before>import * as dynamoDbLib from "./libs/dynamodb-lib";
import { success, failure } from "./libs/response-lib";

export async function main(event, context, callback) {
  const params = {
    TableName: "notes",
    // 'Key' defines the partition key and sort key of the item to be removed
    // - 'userId': Identity Pool identity id of the authenticated user
    // - 'noteId': path parameter
    Key: {
      userId: event.requestContext.identity.cognitoIdentityId,
      noteId: event.pathParameters.id
    }
  };

  try {
    const result = await dynamoDbLib.call("delete", params);
    callback(null, success({ status: true }));
  } catch (e) {
    callback(null, failure({ status: false }));
  }
}
<commit_msg>Returning for async functions<commit_after>import * as dynamoDbLib from "./libs/dynamodb-lib";
import { success, failure } from "./libs/response-lib";

export async function main(event, context) {
  const params = {
    TableName: "notes",
    // 'Key' defines the partition key and sort key of the item to be removed
    // - 'userId': Identity Pool identity id of the authenticated user
    // - 'noteId': path parameter
    Key: {
      userId: event.requestContext.identity.cognitoIdentityId,
      noteId: event.pathParameters.id
    }
  };

  try {
    const result = await dynamoDbLib.call("delete", params);
    return success({ status: true });
  } catch (e) {
    return failure({ status: false });
  }
}
<|endoftext|>


<commit_before>var config = require('./config');
var log = require('./utils/logger')();
var AWSConnector = require('aws-connector');

module.exports = function S3ConnectorConstructor(bucketName) {
    var s3 = this;
    var AWS_AUTH = config.S3.auth;
    var BUCKET_NAME = bucketName || config.S3.bucket;

    AWSConnector.s3.configure(AWS_AUTH);
    s3.connection = AWSConnector.s3.connect();

    // get object
    s3.getObject = function getObject(key, isString, returnOnlyString) {
        isString = isString || true;
        returnOnlyString = returnOnlyString || false;
        var params = {
            Bucket: BUCKET_NAME,
            Key: key
        };
        return s3.connection.getObject(params)
            .then(function decodeObjectToString(object) {
                //if string, return we dont return buffer.
                if (isString) {
                    object.Body = object.Body.toString('utf8');
                }
                if (returnOnlyString) {
                    return object.Body;
                } else {
                    return object;
                }
            })
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in getObject');
                throw e;
            });
    };

    // put object
    s3.putObject = function putObject(key, content, expiresOn){
        if (!key || !content) {
            log.error('Required arguments are missing');
            throw new Error('Required arguments are missing');
        }
        if (!Buffer.isBuffer(content)) {
            content = new Buffer(content);
        }
        var params = {
            Bucket: BUCKET_NAME,
            Key: key,
            Body: content
        };
        if (expiresOn) {
            params.Expires = expiresOn;
        }
        return s3.connection.putObject(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in putObject');
                throw e;
            });
    };

    // delete object
    s3.deleteObject = function deleteObject(key){
        if (!key) {
			log.error('Required arguments are missing');
            throw new Error('Required arguments are missing');
        }
        var params = {
            Bucket: BUCKET_NAME,
            Key: key
        };
        return s3.connection.deleteObject(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in deleteObject');
                throw e;
            });
    };

    // list objects in bucket
    s3.listFilesInBucket = function listFilesInBucket(bucketName) {
        bucketName = bucketName || BUCKET_NAME;
        var params = {
            Bucket: bucketName
        };
        return s3.connection.listObjects(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in listFilesInBucket');
                throw e;
            });
    };

	return s3;
};
<commit_msg>hotfix:)<commit_after>var config = require('./config');
var log = require('./utils/logger')();
var AWSConnector = require('aws-connector');

var s3 = module.exports = function S3ConnectorConstructor(bucketName) {
    var AWS_AUTH = config.S3.auth;
    var BUCKET_NAME = bucketName || config.S3.bucket;

    AWSConnector.s3.configure(AWS_AUTH);
    s3.connection = AWSConnector.s3.connect();

    // get object
    s3.getObject = function getObject(key, isString, returnOnlyString) {
        isString = isString || true;
        returnOnlyString = returnOnlyString || false;
        var params = {
            Bucket: BUCKET_NAME,
            Key: key
        };
        return s3.connection.getObject(params)
            .then(function decodeObjectToString(object) {
                //if string, return we dont return buffer.
                if (isString) {
                    object.Body = object.Body.toString('utf8');
                }
                if (returnOnlyString) {
                    return object.Body;
                } else {
                    return object;
                }
            })
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in getObject');
                throw e;
            });
    };

    // put object
    s3.putObject = function putObject(key, content, expiresOn){
        if (!key || !content) {
            log.error('Required arguments are missing');
            throw new Error('Required arguments are missing');
        }
        if (!Buffer.isBuffer(content)) {
            content = new Buffer(content);
        }
        var params = {
            Bucket: BUCKET_NAME,
            Key: key,
            Body: content
        };
        if (expiresOn) {
            params.Expires = expiresOn;
        }
        return s3.connection.putObject(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in putObject');
                throw e;
            });
    };

    // delete object
    s3.deleteObject = function deleteObject(key){
        if (!key) {
			log.error('Required arguments are missing');
            throw new Error('Required arguments are missing');
        }
        var params = {
            Bucket: BUCKET_NAME,
            Key: key
        };
        return s3.connection.deleteObject(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in deleteObject');
                throw e;
            });
    };

    // list objects in bucket
    s3.listFilesInBucket = function listFilesInBucket(bucketName) {
        bucketName = bucketName || BUCKET_NAME;
        var params = {
            Bucket: bucketName
        };
        return s3.connection.listObjects(params)
            .catch(function handleError(e) {
                log.error({error: e}, 'Error in listFilesInBucket');
                throw e;
            });
    };

	return s3;
};
<|endoftext|>