<commit_before>'use strict';

const apiai = require('apiai');
const express = require('express');
const bodyParser = require('body-parser');
const uuid = require('node-uuid');
const request = require('request');
const JSONbig = require('json-bigint');
const async = require('async');

const REST_PORT = (process.env.PORT || 5000);
const APIAI_ACCESS_TOKEN = process.env.APIAI_ACCESS_TOKEN;
const APIAI_LANG = process.env.APIAI_LANG || 'en';
const FB_VERIFY_TOKEN = process.env.FB_VERIFY_TOKEN;
const FB_PAGE_ACCESS_TOKEN = process.env.FB_PAGE_ACCESS_TOKEN;

const apiAiService = apiai(APIAI_ACCESS_TOKEN, {language: APIAI_LANG, requestSource: "fb"});
const sessionIds = new Map();

function processEvent(event) {
    var sender = event.sender.id.toString();

    if ((event.message && event.message.text) || (event.postback && event.postback.payload)) {
        var text = event.message ? event.message.text : event.postback.payload;
        // Handle a text message from this sender

        if (!sessionIds.has(sender)) {
            sessionIds.set(sender, uuid.v1());
        }

        console.log("Text", text);

        let apiaiRequest = apiAiService.textRequest(text,
            {
                sessionId: sessionIds.get(sender)
            });

        apiaiRequest.on('response', (response) => {
            if (isDefined(response.result)) {
                let responseText = response.result.fulfillment.speech;
                let responseData = response.result.fulfillment.data;
                let action = response.result.action;

                if (isDefined(responseData) && isDefined(responseData.facebook)) {
                    if (!Array.isArray(responseData.facebook)) {
                        try {
                            console.log('Response as formatted message');
                            sendFBMessage(sender, responseData.facebook);
                        } catch (err) {
                            sendFBMessage(sender, {text: err.message});
                        }
                    } else {
                        responseData.facebook.forEach((facebookMessage) => {
                            try {
                                if (facebookMessage.sender_action) {
                                    console.log('Response as sender action');
                                    sendFBSenderAction(sender, facebookMessage.sender_action);
                                }
                                else {
                                    console.log('Response as formatted message');
                                    sendFBMessage(sender, facebookMessage);
                                }
                            } catch (err) {
                                sendFBMessage(sender, {text: err.message});
                            }
                        });
                    }
                } else if (isDefined(responseText)) {
                    console.log('Response as text message');
                    // facebook API limit for text length is 320,
                    // so we split message if needed
                    var splittedText = splitResponse(responseText);

                    async.eachSeries(splittedText, (textPart, callback) => {
                        sendFBMessage(sender, {text: textPart}, callback);
                    });
                }

            }
        });

        apiaiRequest.on('error', (error) => console.error(error));
        apiaiRequest.end();
    }
}

function splitResponse(str) {
    if (str.length <= 320) {
        return [str];
    }

    return chunkString(str, 300);
}

function chunkString(s, len) {
    var curr = len, prev = 0;

    var output = [];

    while (s[curr]) {
        if (s[curr++] == ' ') {
            output.push(s.substring(prev, curr));
            prev = curr;
            curr += len;
        }
        else {
            var currReverse = curr;
            do {
                if (s.substring(currReverse - 1, currReverse) == ' ') {
                    output.push(s.substring(prev, currReverse));
                    prev = currReverse;
                    curr = currReverse + len;
                    break;
                }
                currReverse--;
            } while (currReverse > prev)
        }
    }
    output.push(s.substr(prev));
    return output;
}

function sendFBMessage(sender, messageData, callback) {
    request({
        url: 'https://graph.facebook.com/v2.6/me/messages',
        qs: {access_token: FB_PAGE_ACCESS_TOKEN},
        method: 'POST',
        json: {
            recipient: {id: sender},
            message: messageData
        }
    }, (error, response, body) => {
        if (error) {
            console.log('Error sending message: ', error);
        } else if (response.body.error) {
            console.log('Error: ', response.body.error);
        }

        if (callback) {
            callback();
        }
    });
}

function sendFBSenderAction(sender, action, callback) {
    setTimeout(() => {
        request({
            url: 'https://graph.facebook.com/v2.6/me/messages',
            qs: {access_token: FB_PAGE_ACCESS_TOKEN},
            method: 'POST',
            json: {
                recipient: {id: sender},
                sender_action: action
            }
        }, (error, response, body) => {
            if (error) {
                console.log('Error sending action: ', error);
            } else if (response.body.error) {
                console.log('Error: ', response.body.error);
            }
            if (callback) {
                callback();
            }
        });
    }, 1000);
}

function doSubscribeRequest() {
    request({
            method: 'POST',
            uri: "https://graph.facebook.com/v2.6/me/subscribed_apps?access_token=" + FB_PAGE_ACCESS_TOKEN
        },
        (error, response, body) => {
            if (error) {
                console.error('Error while subscription: ', error);
            } else {
                console.log('Subscription result: ', response.body);
            }
        });
}

function isDefined(obj) {
    if (typeof obj == 'undefined') {
        return false;
    }

    if (!obj) {
        return false;
    }

    return obj != null;
}

const app = express();

app.use(bodyParser.text({type: 'application/json'}));

app.get('/webhook/', (req, res) => {
    if (req.query['hub.verify_token'] == FB_VERIFY_TOKEN) {
        res.send(req.query['hub.challenge']);

        setTimeout(() => {
            doSubscribeRequest();
        }, 3000);
    } else {
        res.send('Error, wrong validation token');
    }
});

app.post('/webhook/', (req, res) => {
    try {
        var data = JSONbig.parse(req.body);

        var messaging_events = data.entry[0].messaging;
        for (var i = 0; i < messaging_events.length; i++) {
            var event = data.entry[0].messaging[i];
            processEvent(event);
        }
        return res.status(200).json({
            status: "ok"
        });
    } catch (err) {
        return res.status(400).json({
            status: "error",
            error: err
        });
    }

});

app.listen(REST_PORT, () => {
    console.log('Rest service ready on port ' + REST_PORT);
});

doSubscribeRequest();
<commit_msg>Changed Facebook protocol support<commit_after>'use strict';

const apiai = require('apiai');
const express = require('express');
const bodyParser = require('body-parser');
const uuid = require('node-uuid');
const request = require('request');
const JSONbig = require('json-bigint');
const async = require('async');

const REST_PORT = (process.env.PORT || 5000);
const APIAI_ACCESS_TOKEN = process.env.APIAI_ACCESS_TOKEN;
const APIAI_LANG = process.env.APIAI_LANG || 'en';
const FB_VERIFY_TOKEN = process.env.FB_VERIFY_TOKEN;
const FB_PAGE_ACCESS_TOKEN = process.env.FB_PAGE_ACCESS_TOKEN;

const apiAiService = apiai(APIAI_ACCESS_TOKEN, {language: APIAI_LANG, requestSource: "fb"});
const sessionIds = new Map();

function processEvent(event) {
    var sender = event.sender.id.toString();

    if ((event.message && event.message.text) || (event.postback && event.postback.payload)) {
        var text = event.message ? event.message.text : event.postback.payload;
        // Handle a text message from this sender

        if (!sessionIds.has(sender)) {
            sessionIds.set(sender, uuid.v1());
        }

        console.log("Text", text);

        let apiaiRequest = apiAiService.textRequest(text,
            {
                sessionId: sessionIds.get(sender)
            });

        apiaiRequest.on('response', (response) => {
            if (isDefined(response.result)) {
                let responseText = response.result.fulfillment.speech;
                let responseData = response.result.fulfillment.data;
                let action = response.result.action;

                if (isDefined(responseData) && isDefined(responseData.facebook)) {
                    if (!Array.isArray(responseData.facebook)) {
                        try {
                            console.log('Response as formatted message');
                            sendFBMessage(sender, responseData.facebook);
                        } catch (err) {
                            sendFBMessage(sender, {text: err.message});
                        }
                    } else {
                        responseData.facebook.forEach((facebookMessage) => {
                            try {
                                if (facebookMessage.sender_action) {
                                    console.log('Response as sender action');
                                    sendFBSenderAction(sender, facebookMessage.sender_action);
                                }
                                else {
                                    console.log('Response as formatted message');
                                    sendFBMessage(sender, facebookMessage);
                                }
                            } catch (err) {
                                sendFBMessage(sender, {text: err.message});
                            }
                        });
                    }
                } else if (isDefined(responseText)) {
                    console.log('Response as text message');
                    // facebook API limit for text length is 320,
                    // so we split message if needed
                    var splittedText = splitResponse(responseText);

                    async.eachSeries(splittedText, (textPart, callback) => {
                        sendFBMessage(sender, {text: textPart}, callback);
                    });
                }

            }
        });

        apiaiRequest.on('error', (error) => console.error(error));
        apiaiRequest.end();
    }
}

function splitResponse(str) {
    if (str.length <= 320) {
        return [str];
    }

    return chunkString(str, 300);
}

function chunkString(s, len) {
    var curr = len, prev = 0;

    var output = [];

    while (s[curr]) {
        if (s[curr++] == ' ') {
            output.push(s.substring(prev, curr));
            prev = curr;
            curr += len;
        }
        else {
            var currReverse = curr;
            do {
                if (s.substring(currReverse - 1, currReverse) == ' ') {
                    output.push(s.substring(prev, currReverse));
                    prev = currReverse;
                    curr = currReverse + len;
                    break;
                }
                currReverse--;
            } while (currReverse > prev)
        }
    }
    output.push(s.substr(prev));
    return output;
}

function sendFBMessage(sender, messageData, callback) {
    request({
        url: 'https://graph.facebook.com/v2.6/me/messages',
        qs: {access_token: FB_PAGE_ACCESS_TOKEN},
        method: 'POST',
        json: {
            recipient: {id: sender},
            message: messageData
        }
    }, (error, response, body) => {
        if (error) {
            console.log('Error sending message: ', error);
        } else if (response.body.error) {
            console.log('Error: ', response.body.error);
        }

        if (callback) {
            callback();
        }
    });
}

function sendFBSenderAction(sender, action, callback) {
    setTimeout(() => {
        request({
            url: 'https://graph.facebook.com/v2.6/me/messages',
            qs: {access_token: FB_PAGE_ACCESS_TOKEN},
            method: 'POST',
            json: {
                recipient: {id: sender},
                sender_action: action
            }
        }, (error, response, body) => {
            if (error) {
                console.log('Error sending action: ', error);
            } else if (response.body.error) {
                console.log('Error: ', response.body.error);
            }
            if (callback) {
                callback();
            }
        });
    }, 1000);
}

function doSubscribeRequest() {
    request({
            method: 'POST',
            uri: "https://graph.facebook.com/v2.6/me/subscribed_apps?access_token=" + FB_PAGE_ACCESS_TOKEN
        },
        (error, response, body) => {
            if (error) {
                console.error('Error while subscription: ', error);
            } else {
                console.log('Subscription result: ', response.body);
            }
        });
}

function isDefined(obj) {
    if (typeof obj == 'undefined') {
        return false;
    }

    if (!obj) {
        return false;
    }

    return obj != null;
}

const app = express();

app.use(bodyParser.text({type: 'application/json'}));

app.get('/webhook/', (req, res) => {
    if (req.query['hub.verify_token'] == FB_VERIFY_TOKEN) {
        res.send(req.query['hub.challenge']);

        setTimeout(() => {
            doSubscribeRequest();
        }, 3000);
    } else {
        res.send('Error, wrong validation token');
    }
});

app.post('/webhook/', (req, res) => {
    try {
        var data = JSONbig.parse(req.body);

        if (data.entry) {
            let entries = data.entry;
            entries.forEach((entry) => {
                let messaging_events = entry.messaging;
                if (messaging_events) {
                    messaging_events.forEach((event) => {
                        if (event.message && !event.message.is_echo ||
                            event.postback && event.postback.payload) {
                            processEvent(event);
                        }
                    });
                }
            });
        }

        return res.status(200).json({
            status: "ok"
        });
    } catch (err) {
        return res.status(400).json({
            status: "error",
            error: err
        });
    }

});

app.listen(REST_PORT, () => {
    console.log('Rest service ready on port ' + REST_PORT);
});

doSubscribeRequest();
<|endoftext|>


<commit_before>import Client from './api/Client';
import Form from './models/form';
import ChoiceList from './models/choice-list';
import ClassificationSet from './models/classification-set';
import Project from './models/project';
import Record from './models/record';
import Photo from './models/photo';
import Video from './models/video';
import {format} from 'util';
import Schema from 'fulcrum-schema/dist/schema';
import Metadata from 'fulcrum-schema/dist/metadata';
import sqldiff from 'sqldiff';
import V2 from 'fulcrum-schema/dist/schemas/postgres-query-v2';
import path from 'path';
import ConcurrentQueue from './concurrent-queue';
import filesize from 'filesize';
import Promise from 'bluebird';
import mkdirp from 'mkdirp';

const {SchemaDiffer, Sqlite} = sqldiff;

require('colors');

// var fs = require('fs');
// var profiler = require('v8-profiler');

function getUserHome() {
  return process.env[(process.platform === 'win32') ? 'USERPROFILE' : 'HOME'];
}

const mediaPath = path.join(getUserHome(), 'Documents', 'fulcrum-media');

mkdirp.sync(mediaPath);

const MEDIA_CONCURRENCY = 5;

const models = {
  Form: Form,
  ChoiceList: ChoiceList,
  ClassificationSet: ClassificationSet,
  Project: Project,
  Record: Record,
  Photo: Photo,
  Video: Video
};

export default class Synchronizer {
  async run(account, dataSource) {
    await Promise.all([ this.syncObject('ChoiceList', 'choice_lists', account),
                        this.syncObject('ClassificationSet', 'classification_sets', account),
                        this.syncObject('Project', 'projects', account) ]);

    await this.syncObject('Form', 'forms', account);

    const forms = await Form.findAll(account.db, {account_id: account.id}, 'name ASC');

    await this.syncVideos(account, null);
    await this.syncPhotos(account, null);

    for (const form of forms) {
      await new Promise((resolve, reject) => {
        form.load(dataSource, resolve);
      });
      await this.syncRecords(account, form);
    }
  }

  async syncObject(objectName, key, account) {
    const results = await Client['get' + objectName + 's'](account);

    const body = results.body;

    const data = JSON.parse(body);

    for (const attributes of data[key]) {
      const object = await models[objectName].findOrCreate(account.db, {resource_id: attributes.id, account_id: account.id});

      let oldForm = null;

      if (object.id) {
        oldForm = {
          id: object.resourceID,
          row_id: object.id,
          name: object.name,
          elements: object.elements
        };
      }

      object.updateFromAPIAttributes(attributes);
      object.accountID = account.id;

      await object.save();

      if (objectName === 'Form') {
        const newForm = Object.assign({row_id: object.id}, {elements: object.elementsJSON});

        await this.updateFormTables(account, oldForm, newForm);
      }
    }

    console.log(account.organizationName.green, 'downloaded ' + key);
  }

  async updateFormTables(account, oldForm, newForm) {
    let oldSchema = null;
    let newSchema = null;

    if (oldForm) {
      oldSchema = new Schema(oldForm, V2, null);
    }

    if (newForm) {
      newSchema = new Schema(newForm, V2, null);
    }

    const differ = new SchemaDiffer(oldSchema, newSchema);

    const meta = new Metadata(differ, {quote: '`', prefix: 'account_' + account.id + '_'});

    const generator = new Sqlite(differ, {afterTransform: meta.build.bind(meta)});

    generator.tablePrefix = 'account_' + account.id + '_';

    const statements = generator.generate();

    await account.db.execute('BEGIN TRANSACTION');

    for (const statement of statements) {
      await account.db.execute(statement);
    }

    await account.db.execute('COMMIT');
  }

  async syncVideos(account, form) {
    return this.syncVideoPage(account, form, 1);
  }

  async syncVideoPage(account, form, page, total) {
    const beginFetchTime = new Date();

    const results = await Client.getVideos(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      // console.log(account.organizationName.green, 'failed'.red, 'to download photos in', form.name.blue);
      console.log(account.organizationName.green, 'failed'.red, 'to download videos');
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    const queue = new ConcurrentQueue(async function (task) {
      const file = path.join(mediaPath, 'videos', task.access_key + '.mp4');

      await Client.download(task.original, file);

      // console.log(format('%s downloaded photo in %s | %s | %s',
      //                    account.organizationName.green,
      //                    form.name.blue,
      //                    task.access_key.cyan,
      //                    filesize(task.file_size).red));
      console.log(format('%s downloaded video | %s | %s',
                         account.organizationName.green,
                         task.access_key.cyan,
                         filesize(task.file_size).red));
    }, MEDIA_CONCURRENCY);

    await db.transaction(async function () {
      for (const attributes of data.videos) {
        const object = await models.Video.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.access_key});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;

        if (attributes.processed) {
          if (!object.isDownloaded) {
            queue.push(attributes, function(err) {
              if (err) {
                console.log('ERROR DOWNLOADING', err);
                throw err;
              }

              object.isDownloaded = true;
              // do we need to await this somehow?
              object.save();
            });
          }
        } else {
          object.isDownloaded = false;
        }

        if (object.isDownloaded == null) {
          object.isDownloaded = false;
        }

        await object.save();
      }
    });

    await queue.drain();

    const totalTime = new Date().getTime() - now.getTime();

    // console.log(format('%s downloaded photos in %s | %s | %s | %s',
    //                    account.organizationName.green,
    //                    form.name.blue,
    //                    format('page %s/%s', page, data.total_pages || 1).yellow,
    //                    (totalFetchTime + 'ms').cyan + ' (api)'.red,
    //                    (totalTime + 'ms').cyan + ' (db)'.red));

    console.log(format('%s downloaded videos | %s | %s | %s',
                       account.organizationName.green,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncVideoPage(account, form, page + 1, data.total_pages);
    }
  }

  async syncPhotos(account, form) {
    return this.syncPhotoPage(account, form, 1);
  }

  async syncPhotoPage(account, form, page, total) {
    const beginFetchTime = new Date();

    const results = await Client.getPhotos(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      // console.log(account.organizationName.green, 'failed'.red, 'to download photos in', form.name.blue);
      console.log(account.organizationName.green, 'failed'.red, 'to download photos');
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    const queue = new ConcurrentQueue(async function (task) {
      const file = path.join(mediaPath, 'photos', task.access_key + '.jpg');

      await Client.download(task.original, file);

      // console.log(format('%s downloaded photo in %s | %s | %s',
      //                    account.organizationName.green,
      //                    form.name.blue,
      //                    task.access_key.cyan,
      //                    filesize(task.file_size).red));
      console.log(format('%s downloaded photo | %s | %s',
                         account.organizationName.green,
                         task.access_key.cyan,
                         filesize(task.file_size).red));
    }, MEDIA_CONCURRENCY);

    await db.transaction(async function () {
      for (const attributes of data.photos) {
        const object = await models.Photo.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.access_key});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;

        if (attributes.processed) {
          if (!object.isDownloaded) {
            queue.push(attributes, function(err) {
              if (err) {
                console.log('ERROR DOWNLOADING', err);
                throw err;
              }

              object.isDownloaded = true;
              // do we need to await this somehow?
              object.save();
            });
          }
        } else {
          object.isDownloaded = false;
        }

        if (object.isDownloaded == null) {
          object.isDownloaded = false;
        }

        await object.save();
      }
    });

    await queue.drain();

    const totalTime = new Date().getTime() - now.getTime();

    // console.log(format('%s downloaded photos in %s | %s | %s | %s',
    //                    account.organizationName.green,
    //                    form.name.blue,
    //                    format('page %s/%s', page, data.total_pages || 1).yellow,
    //                    (totalFetchTime + 'ms').cyan + ' (api)'.red,
    //                    (totalTime + 'ms').cyan + ' (db)'.red));

    console.log(format('%s downloaded photos | %s | %s | %s',
                       account.organizationName.green,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncPhotoPage(account, form, page + 1, data.total_pages);
    }
  }

  async syncRecords(account, form) {
    return this.syncRecordPage(account, form, 1);
  }

  async syncRecordPage(account, form, page, total) {
    // console.log(format('%s downloading records in %s (page %s/%s)',
    //                    account.organizationName.green,
    //                    form.name.blue,
    //                    page,
    //                    total || 1));

    const beginFetchTime = new Date();

    const results = await Client.getRecords(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      console.log(account.organizationName.green, 'failed'.red, 'to download records in', form.name.blue);
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    // profiler.startProfiling('1', true);

    await db.transaction(async function () {
      for (const attributes of data.records) {
        const object = await models.Record.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.id});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;
        object.form = form;
        object.formID = form.id;

        if (attributes.project_id) {
          const project = await account.projectByResourceID(attributes.project_id);

          if (project) {
            object.projectId = project.id;
          }
        }

        await object.save();
      }
    });

    const totalTime = new Date().getTime() - now.getTime();

    // var profile1 = profiler.stopProfiling();

    // console.log(profile1.getHeader());

    // profile1.export(function (error, result) {
    //   if (error) {
    //     throw error;
    //   }
    //   fs.writeFileSync('profile1.cpuprofile', result);
    //   profile1.delete();
    // });

    console.log(format('%s downloaded records in %s | %s | %s | %s',
                       account.organizationName.green,
                       form.name.blue,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncRecordPage(account, form, page + 1, data.total_pages);
    }
  }
}

Synchronizer.instance = new Synchronizer();
<commit_msg>Remove logging<commit_after>import Client from './api/Client';
import Form from './models/form';
import ChoiceList from './models/choice-list';
import ClassificationSet from './models/classification-set';
import Project from './models/project';
import Record from './models/record';
import Photo from './models/photo';
import Video from './models/video';
import {format} from 'util';
import Schema from 'fulcrum-schema/dist/schema';
import Metadata from 'fulcrum-schema/dist/metadata';
import sqldiff from 'sqldiff';
import V2 from 'fulcrum-schema/dist/schemas/postgres-query-v2';
import path from 'path';
import ConcurrentQueue from './concurrent-queue';
import filesize from 'filesize';
import Promise from 'bluebird';
import mkdirp from 'mkdirp';

const {SchemaDiffer, Sqlite} = sqldiff;

require('colors');

// var fs = require('fs');
// var profiler = require('v8-profiler');

function getUserHome() {
  return process.env[(process.platform === 'win32') ? 'USERPROFILE' : 'HOME'];
}

const mediaPath = path.join(getUserHome(), 'Documents', 'fulcrum-media');

mkdirp.sync(mediaPath);

const MEDIA_CONCURRENCY = 5;

const models = {
  Form: Form,
  ChoiceList: ChoiceList,
  ClassificationSet: ClassificationSet,
  Project: Project,
  Record: Record,
  Photo: Photo,
  Video: Video
};

export default class Synchronizer {
  async run(account, dataSource) {
    await Promise.all([ this.syncObject('ChoiceList', 'choice_lists', account),
                        this.syncObject('ClassificationSet', 'classification_sets', account),
                        this.syncObject('Project', 'projects', account) ]);

    await this.syncObject('Form', 'forms', account);

    const forms = await Form.findAll(account.db, {account_id: account.id}, 'name ASC');

    await this.syncVideos(account, null);
    await this.syncPhotos(account, null);

    for (const form of forms) {
      await new Promise((resolve, reject) => {
        form.load(dataSource, resolve);
      });
      await this.syncRecords(account, form);
    }
  }

  async syncObject(objectName, key, account) {
    const results = await Client['get' + objectName + 's'](account);

    const body = results.body;

    const data = JSON.parse(body);

    for (const attributes of data[key]) {
      const object = await models[objectName].findOrCreate(account.db, {resource_id: attributes.id, account_id: account.id});

      let oldForm = null;

      if (object.id) {
        oldForm = {
          id: object.resourceID,
          row_id: object.id,
          name: object.name,
          elements: object.elements
        };
      }

      object.updateFromAPIAttributes(attributes);
      object.accountID = account.id;

      await object.save();

      if (objectName === 'Form') {
        const newForm = Object.assign({row_id: object.id}, {elements: object.elementsJSON});

        await this.updateFormTables(account, oldForm, newForm);
      }
    }

    console.log(account.organizationName.green, 'downloaded ' + key);
  }

  async updateFormTables(account, oldForm, newForm) {
    let oldSchema = null;
    let newSchema = null;

    if (oldForm) {
      oldSchema = new Schema(oldForm, V2, null);
    }

    if (newForm) {
      newSchema = new Schema(newForm, V2, null);
    }

    const differ = new SchemaDiffer(oldSchema, newSchema);

    const meta = new Metadata(differ, {quote: '`', prefix: 'account_' + account.id + '_'});

    const generator = new Sqlite(differ, {afterTransform: meta.build.bind(meta)});

    generator.tablePrefix = 'account_' + account.id + '_';

    const statements = generator.generate();

    await account.db.execute('BEGIN TRANSACTION');

    for (const statement of statements) {
      await account.db.execute(statement);
    }

    await account.db.execute('COMMIT');
  }

  async syncVideos(account, form) {
    return this.syncVideoPage(account, form, 1);
  }

  async syncVideoPage(account, form, page, total) {
    const beginFetchTime = new Date();

    const results = await Client.getVideos(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      console.log(account.organizationName.green, 'failed'.red, 'to download videos');
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    const queue = new ConcurrentQueue(async function (task) {
      const file = path.join(mediaPath, 'videos', task.access_key + '.mp4');

      await Client.download(task.original, file);

      console.log(format('%s downloaded video | %s | %s',
                         account.organizationName.green,
                         task.access_key.cyan,
                         filesize(task.file_size).red));
    }, MEDIA_CONCURRENCY);

    await db.transaction(async function () {
      for (const attributes of data.videos) {
        const object = await models.Video.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.access_key});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;

        if (attributes.processed) {
          if (!object.isDownloaded) {
            queue.push(attributes, function(err) {
              if (err) {
                console.log('ERROR DOWNLOADING', err);
                throw err;
              }

              object.isDownloaded = true;
              // do we need to await this somehow?
              object.save();
            });
          }
        } else {
          object.isDownloaded = false;
        }

        if (object.isDownloaded == null) {
          object.isDownloaded = false;
        }

        await object.save();
      }
    });

    await queue.drain();

    const totalTime = new Date().getTime() - now.getTime();

    console.log(format('%s downloaded videos | %s | %s | %s',
                       account.organizationName.green,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncVideoPage(account, form, page + 1, data.total_pages);
    }
  }

  async syncPhotos(account, form) {
    return this.syncPhotoPage(account, form, 1);
  }

  async syncPhotoPage(account, form, page, total) {
    const beginFetchTime = new Date();

    const results = await Client.getPhotos(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      console.log(account.organizationName.green, 'failed'.red, 'to download photos');
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    const queue = new ConcurrentQueue(async function (task) {
      const file = path.join(mediaPath, 'photos', task.access_key + '.jpg');

      await Client.download(task.original, file);

      console.log(format('%s downloaded photo | %s | %s',
                         account.organizationName.green,
                         task.access_key.cyan,
                         filesize(task.file_size).red));
    }, MEDIA_CONCURRENCY);

    await db.transaction(async function () {
      for (const attributes of data.photos) {
        const object = await models.Photo.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.access_key});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;

        if (attributes.processed) {
          if (!object.isDownloaded) {
            queue.push(attributes, function(err) {
              if (err) {
                console.log('ERROR DOWNLOADING', err);
                throw err;
              }

              object.isDownloaded = true;
              // do we need to await this somehow?
              object.save();
            });
          }
        } else {
          object.isDownloaded = false;
        }

        if (object.isDownloaded == null) {
          object.isDownloaded = false;
        }

        await object.save();
      }
    });

    await queue.drain();

    const totalTime = new Date().getTime() - now.getTime();

    console.log(format('%s downloaded photos | %s | %s | %s',
                       account.organizationName.green,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncPhotoPage(account, form, page + 1, data.total_pages);
    }
  }

  async syncRecords(account, form) {
    return this.syncRecordPage(account, form, 1);
  }

  async syncRecordPage(account, form, page, total) {
    const beginFetchTime = new Date();

    const results = await Client.getRecords(account, form, page);

    const totalFetchTime = new Date().getTime() - beginFetchTime.getTime();

    const body = results.body;

    if (results.statusCode !== 200) {
      console.log(account.organizationName.green, 'failed'.red, 'to download records in', form.name.blue);
      return;
    }

    const data = JSON.parse(body);

    const db = account.db;

    let now = new Date();

    // profiler.startProfiling('1', true);

    await db.transaction(async function () {
      for (const attributes of data.records) {
        const object = await models.Record.findOrCreate(account.db, {account_id: account.id, resource_id: attributes.id});

        object.updateFromAPIAttributes(attributes);
        object.accountID = account.id;
        object.form = form;
        object.formID = form.id;

        if (attributes.project_id) {
          const project = await account.projectByResourceID(attributes.project_id);

          if (project) {
            object.projectId = project.id;
          }
        }

        await object.save();
      }
    });

    const totalTime = new Date().getTime() - now.getTime();

    // var profile1 = profiler.stopProfiling();

    // console.log(profile1.getHeader());

    // profile1.export(function (error, result) {
    //   if (error) {
    //     throw error;
    //   }
    //   fs.writeFileSync('profile1.cpuprofile', result);
    //   profile1.delete();
    // });

    console.log(format('%s downloaded records in %s | %s | %s | %s',
                       account.organizationName.green,
                       form.name.blue,
                       format('page %s/%s', page, data.total_pages || 1).yellow,
                       (totalFetchTime + 'ms').cyan + ' (api)'.red,
                       (totalTime + 'ms').cyan + ' (db)'.red));

    if (data.total_pages > page) {
      await this.syncRecordPage(account, form, page + 1, data.total_pages);
    }
  }
}

Synchronizer.instance = new Synchronizer();
<|endoftext|>


<commit_before>var RolesSchema = new SimpleSchema({
  name: {
    type: String,
    label: 'Name'
  }
});

Meteor.roles.attachSchema(RolesSchema);
<commit_msg>Allow admins to insert<commit_after>var RolesSchema = new SimpleSchema({
  name: {
    type: String,
    label: 'Name'
  }
});

Meteor.roles.attachSchema(RolesSchema);

Meteor.roles.allow({
  "insert": function (userId) {
    // Only Administrators can insert
    return Roles.userIsInRole(userId, ['admin']);
  }
});
<|endoftext|>


<commit_before><commit_msg>(feat) Add db name, username and password<commit_after>import { Sequelize } from 'sequelize';

//Create a database with the name alumnidb
//Serve with the username of root
//No password for now !!
let db = new Sequelize('alumnidb', 'root', '');

export default db;<|endoftext|>


<commit_before>/**
 * Local modules
 */
const Logger = require('./helpers/logger');
const Environment = require('./helpers/env');
const FileSystem = require('./helpers/file');

/**
 * Node modules
 */
const tinify = require('tinify');
const path = require('path');
const util = require('util');
const readdirAsync = util.promisify(require('fs').readdir);
const lstatSync = util.promisify(require('fs').lstat);

/**
 * Application class
 *
 * Receives a given dir (which should have images) and minify all of them
 * by using the TinyPNG API.
 *
 * @class Application
 * @see https://github.com/tinify/tinify-nodejs
 * @author Patrick Ferreira <paatrickferreira@gmail.com>
 */
class Application {
  /**
   * Sets Tinify API key and validate it. Throws an error in case of not valid key.
   *
   * @static
   *
   * @memberof Application
   */
  static setConfig() {
    tinify.key = Environment.getProperty('API_KEY');
    tinify.validate((err) => {
      if (err) throw err;
    });
  }

  /**
   * Main method of the class. Starts all the process by validating the given dir,
   * cloning it and, afterwards, minifying all its files recursively.
   *
   * @static
   * @param {string} dir
   *
   * @memberof Application
   */
  static async run(dir) {
    Application.setConfig();

    if (!path.isAbsolute(dir)) throw new Error('Given path is not an absolute path');

    Logger.info('Starting process...');

    const optimizedDir = await FileSystem.cloneDir(dir);
    Logger.info('Created optimized dir');

    const results = await Application.readDirRecursively(optimizedDir);
    const totalOfMinifiedFiles = results.filter(isFile => isFile).length;

    Logger.info(`${totalOfMinifiedFiles} file(s) minified!`);
  }

  /**
   * Reveives a given dir and checks it recursively in order to find all its files.
   *
   * @static
   * @param {string} optimizedDir
   * @returns {array}
   *
   * @memberof Application
   */
  static async readDirRecursively(optimizedDir) {
    try {
      const contents = await readdirAsync(optimizedDir);

      const results = contents.map(content => Application.addToQueue(optimizedDir, content));

      return await Promise.all(results);
    } catch (err) {
      throw new Error(err);
    }
  }

  /**
   * Checks if given path is another dir or a file and returns it based on the result
   * to a queue (array of promises), which will be executed afterward.
   *
   * @static
   * @param {string} optimizedDir
   * @param {string} content
   * @returns {Promise}
   *
   * @memberof Application
   */
  static async addToQueue(optimizedDir, content) {
    const currentPath = `${optimizedDir}/${content}`;

    const pathStats = await lstatSync(currentPath);

    if (pathStats.isFile()) {
      return Application.minifyFile(currentPath);
    }

    return Application.readDirRecursively(currentPath);
  }

  /**
   * Receives a file, checks if it's an image and minifies it afterward.
   *
   * @static
   * @param {string} file
   * @returns {boolean}
   *
   * @memberof Application
   */
  static async minifyFile(file) {
    try {
      if (!FileSystem.isImage(file)) {
        Logger.error(`Oops! "${file}" is not a valid image.`);
        return false;
      }

      await tinify.fromFile(file).toFile(file);

      Logger.info(`Minified image: ${file}`);
      return true;
    } catch (err) {
      throw new Error(err);
    }
  }
}

module.exports = Application;
<commit_msg>Fixed typo<commit_after>/**
 * Local modules
 */
const Logger = require('./helpers/logger');
const Environment = require('./helpers/env');
const FileSystem = require('./helpers/file');

/**
 * Node modules
 */
const tinify = require('tinify');
const path = require('path');
const util = require('util');
const readdirAsync = util.promisify(require('fs').readdir);
const lstatAsync = util.promisify(require('fs').lstat);

/**
 * Application class
 *
 * Receives a given dir (which should have images) and minify all of them
 * by using the TinyPNG API.
 *
 * @class Application
 * @see https://github.com/tinify/tinify-nodejs
 * @author Patrick Ferreira <paatrickferreira@gmail.com>
 */
class Application {
  /**
   * Sets Tinify API key and validate it. Throws an error in case of not valid key.
   *
   * @static
   *
   * @memberof Application
   */
  static setConfig() {
    tinify.key = Environment.getProperty('API_KEY');
    tinify.validate((err) => {
      if (err) throw err;
    });
  }

  /**
   * Main method of the class. Starts all the process by validating the given dir,
   * cloning it and, afterwards, minifying all its files recursively.
   *
   * @static
   * @param {string} dir
   *
   * @memberof Application
   */
  static async run(dir) {
    Application.setConfig();

    if (!path.isAbsolute(dir)) throw new Error('Given path is not an absolute path');

    Logger.info('Starting process...');

    const optimizedDir = await FileSystem.cloneDir(dir);
    Logger.info('Created optimized dir');

    const results = await Application.readDirRecursively(optimizedDir);
    const totalOfMinifiedFiles = results.filter(isFile => isFile).length;

    Logger.info(`${totalOfMinifiedFiles} file(s) minified!`);
  }

  /**
   * Reveives a given dir and checks it recursively in order to find all its files.
   *
   * @static
   * @param {string} optimizedDir
   * @returns {array}
   *
   * @memberof Application
   */
  static async readDirRecursively(optimizedDir) {
    try {
      const contents = await readdirAsync(optimizedDir);

      const results = contents.map(content => Application.addToQueue(optimizedDir, content));

      return await Promise.all(results);
    } catch (err) {
      throw new Error(err);
    }
  }

  /**
   * Checks if given path is another dir or a file and returns it based on the result
   * to a queue (array of promises), which will be executed afterward.
   *
   * @static
   * @param {string} optimizedDir
   * @param {string} content
   * @returns {Promise}
   *
   * @memberof Application
   */
  static async addToQueue(optimizedDir, content) {
    const currentPath = `${optimizedDir}/${content}`;

    const pathStats = await lstatAsync(currentPath);

    if (pathStats.isFile()) {
      return Application.minifyFile(currentPath);
    }

    return Application.readDirRecursively(currentPath);
  }

  /**
   * Receives a file, checks if it's an image and minifies it afterward.
   *
   * @static
   * @param {string} file
   * @returns {boolean}
   *
   * @memberof Application
   */
  static async minifyFile(file) {
    try {
      if (!FileSystem.isImage(file)) {
        Logger.error(`Oops! "${file}" is not a valid image.`);
        return false;
      }

      await tinify.fromFile(file).toFile(file);

      Logger.info(`Minified image: ${file}`);
      return true;
    } catch (err) {
      throw new Error(err);
    }
  }
}

module.exports = Application;
<|endoftext|>


<commit_before>module.exports = function(grunt) {

	// core files that make up F2
	var CORE_FILES = [
		'sdk/src/F2.js',
		'sdk/src/app_handlers.js',
		'sdk/src/classes.js',
		'sdk/src/constants.js',
		'sdk/src/events.js',
		'sdk/src/rpc.js',
		'sdk/src/ui.js',
		'sdk/src/container.js'
	];

	// Project config
	grunt.initConfig({
		pkg: grunt.file.readJSON('package.json'),
		concat: {
			options: {
				process: { data: '<%= pkg %>' },
				separator: '\n',
				stripBanners: false
			},
			'no-third-party': {
				src: [
					'sdk/src/template/header.js.tmpl',
					CORE_FILES,
					'sdk/src/template/footer.js.tmpl'
				],
				dest: 'sdk/f2.no-third-party.js'
			},
			dist: {
				src: [
					'sdk/src/template/header.js.tmpl',
					'sdk/src/third-party/json2.js',
					'sdk/src/third-party/jquery.js',
					'sdk/src/third-party/bootstrap-modal.js',
					'sdk/src/third-party/jquery.noconflict.js',
					'sdk/src/third-party/eventemitter2.js',
					'sdk/src/third-party/easyXDM/easyXDM.js',
					CORE_FILES,
					'sdk/src/template/footer.js.tmpl'
				],
				dest: 'sdk/f2.debug.js'
			}
		},
		jshint: {
			options: {
				jshintrc: '.jshintrc'
			},
			files: CORE_FILES
		},
		uglify: {
			options: {
				preserveComments: function(node, comment) {
					return /^!/.test(comment.value);
				}
			},
			dist: {
				files: {'sdk/f2.min.js' : ['sdk/f2.debug.js']},
				options: {
					report: 'gzip'
				}
			},
			sourcemap: {
				files: '<%= uglify.dist.files %>',
				options: {
					sourceMap: function(fileName) {
						return fileName.replace(/\.js$/, '.map');
					},
					sourceMapPrefix: 1,
					sourceMappingURL: function(path) {
						return path.replace(grunt.config('sourcemap.options.prefix'), '').replace(/\.js$/, '.map');
					}
				}
			}
		},
		sourcemap: {
			options: {
				src: 'sdk/f2.min.js',
				prefix: 'sdk/'
			}
		}
	});

	// Load plugins
	grunt.loadNpmTasks('grunt-contrib-concat');
	grunt.loadNpmTasks('grunt-contrib-jshint');
	grunt.loadNpmTasks('grunt-contrib-uglify');

	// Register tasks
	grunt.registerTask('fix-sourcemap', 'Fixes the source map file', function() {
		var uglifyOptions = grunt.config('uglify.sourcemap.options'),
			options = grunt.config('sourcemap.options'),
			dest = uglifyOptions.sourceMap(options.src),
			rawMap = grunt.file.read(dest);

		rawMap = rawMap.replace(options.prefix, '');
		grunt.file.write(dest, rawMap);
	});

	grunt.registerTask('js', ['jshint', 'concat', 'uglify:dist', 'sourcemap']);
	grunt.registerTask('sourcemap', ['uglify:sourcemap', 'fix-sourcemap']);

	// the default task
	grunt.registerTask('default', ['js']);
};<commit_msg>added yuidoc<commit_after>module.exports = function(grunt) {

	// core files that make up F2
	var CORE_FILES = [
		'sdk/src/F2.js',
		'sdk/src/app_handlers.js',
		'sdk/src/classes.js',
		'sdk/src/constants.js',
		'sdk/src/events.js',
		'sdk/src/rpc.js',
		'sdk/src/ui.js',
		'sdk/src/container.js'
	];

	var f2Info = require('./build/F2.json');

	// Project config
	grunt.initConfig({
		pkg: grunt.file.readJSON('package.json'),
		concat: {
			options: {
				process: { data: '<%= pkg %>' },
				separator: '\n',
				stripBanners: false
			},
			'no-third-party': {
				src: [
					'sdk/src/template/header.js.tmpl',
					CORE_FILES,
					'sdk/src/template/footer.js.tmpl'
				],
				dest: 'sdk/f2.no-third-party.js'
			},
			dist: {
				src: [
					'sdk/src/template/header.js.tmpl',
					'sdk/src/third-party/json2.js',
					'sdk/src/third-party/jquery.js',
					'sdk/src/third-party/bootstrap-modal.js',
					'sdk/src/third-party/jquery.noconflict.js',
					'sdk/src/third-party/eventemitter2.js',
					'sdk/src/third-party/easyXDM/easyXDM.js',
					CORE_FILES,
					'sdk/src/template/footer.js.tmpl'
				],
				dest: 'sdk/f2.debug.js'
			}
		},
		jshint: {
			options: {
				jshintrc: '.jshintrc'
			},
			files: CORE_FILES
		},
		uglify: {
			options: {
				preserveComments: function(node, comment) {
					return /^!/.test(comment.value);
				}
			},
			dist: {
				files: {'sdk/f2.min.js' : ['sdk/f2.debug.js']},
				options: {
					report: 'gzip'
				}
			},
			sourcemap: {
				files: '<%= uglify.dist.files %>',
				options: {
					sourceMap: function(fileName) {
						return fileName.replace(/\.js$/, '.map');
					},
					sourceMapPrefix: 1,
					sourceMappingURL: function(path) {
						return path.replace(grunt.config('sourcemap.options.prefix'), '').replace(/\.js$/, '.map');
					}
				}
			}
		},
		sourcemap: {
			options: {
				src: 'sdk/f2.min.js',
				prefix: 'sdk/'
			}
		}
	});

	// Load plugins
	grunt.loadNpmTasks('grunt-contrib-concat');
	grunt.loadNpmTasks('grunt-contrib-jshint');
	grunt.loadNpmTasks('grunt-contrib-uglify');

	// Register tasks
	grunt.registerTask('fix-sourcemap', 'Fixes the source map file', function() {
		var uglifyOptions = grunt.config('uglify.sourcemap.options'),
			options = grunt.config('sourcemap.options'),
			dest = uglifyOptions.sourceMap(options.src),
			rawMap = grunt.file.read(dest);

		rawMap = rawMap.replace(options.prefix, '');
		grunt.file.write(dest, rawMap);
	});

	grunt.registerTask('yuidoc', 'Builds the reference documentation with YUIDoc', function() {

		var builder,
			docOptions = {
				quiet: true,
				norecurse: true,
				paths: ['./sdk/src'],
				outdir: './docs/sdk/',
				themedir: './docs/src/sdk-template'
			},
			done = this.async(),
			json,
			readmeMd = grunt.file.read('README.md'),
			Y = require('yuidocjs');

		json = (new Y.YUIDoc(docOptions)).run();
		// massage in some meta information from F2.json
		json.project = {
			docsAssets: '../',
			version: grunt.package.version,
			docsVersion: grunt.package.version,
			docsLastUpdateDateFormatted: f2Info.docs.lastUpdateDateFormatted,
			branch: f2Info.branch
		};
		docOptions = Y.Project.mix(json, docOptions);

		// hasClassMembers
		// ensures that the class has members and isn't just an empty namespace
		Y.Handlebars.registerHelper('hasClassMembers', function() {

			for (var i = 0, len = json.classitems.length; i < len; i++) {
				//console.log(json.classitems[i].class, this.name);
				if (json.classitems[i].class === this.name) {
					return '';
				}
			}

			return 'hide';
		});

		// title tag
		Y.Handlebars.registerHelper('htmlTitle',function () {
			var name  = this.displayName || this.name,
					title = name;

			if (title) {
				title = 'F2 - ' + title;
			} else {
				title = 'F2 - The Open Financial Framework';
			}

			return title;
		});

		// insert readme markdown
		Y.Handlebars.registerHelper('readme', function() {
			return builder.markdown(readmeMd, true);
		});

		builder = new Y.DocBuilder(docOptions, json);
		builder.compile(function() {
			grunt.log.writeln('YUIDoc compilation complete');
			done();
		});
	});

	grunt.registerTask('docs', ['yuidoc']);
	grunt.registerTask('js', ['jshint', 'concat', 'uglify:dist', 'sourcemap']);
	grunt.registerTask('sourcemap', ['uglify:sourcemap', 'fix-sourcemap']);

	// the default task
	grunt.registerTask('default', ['js', 'docs']);
};<|endoftext|>


<commit_before>/**
 * An application tracks Mario Kart results and calculates rankings,
 * and includes a Slack Bot integration.
 */

// SLACK_TOKEN is required for connecting to Slack
if (!process.env.SLACK_TOKEN)
{
	console.log('Error: Please specify SLACK_TOKEN in the environment.');
    process.exit(1);
}
else if (!process.env.MONGODB_URI)
{
	console.log('Error: Please specify MONGODB_URI in the environment.');
    process.exit(1);
}
else if (!process.env.DEPLOYMENT_ENVIRONMENT)
{
	console.log('Error: Please specify DEPLOYMENT_ENVIRONMENT in the environment.');
    process.exit(1);
}

// dependencies
const PORT = process.env.PORT || 80;
var express = require("express"),
	bodyParser = require("body-parser"),
	mongodb = require("mongodb"),
	Botkit = require('botkit'),
	requester = require('request'),
	TextTable = require('text-table'),
	Plotly = require('plotly')("jgroppe", "hOvosfAJF74xOWSPKJh5"),
	fs = require('fs'),
	Q = require('q'),
	dateFormat = require('dateformat');

// create the express app
var app = express();

// tell express where to load static (client-side) files from
app.use(express.static(__dirname + "/www"));

// tell express to parse request bodies as json
app.use(bodyParser.json());

// create a database variable outside of the database connection callback
// to reuse the connection pool.
var db, db_collection_player,
	db_collection_game, db_collection_character;

// connect to the database before starting the application server.
mongodb.MongoClient.connect(process.env.MONGODB_URI, function (err, database) {

	// if there was an error connecting, shutdown the app
	if (err) {
		console.log(err);
		process.exit(1);
	}

	// save database object from the callback for reuse
	db = database;

	console.log("Database connection ready");

	// gather the collections
	//db_collection_players = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_PLAYER');
	//db_collection_games = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_GAME');
	//db_collection_characters = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_CHARACTER');
	db_collection_players = db.collection('HEROKU_PLAYER');
	db_collection_games = db.collection('HEROKU_GAME');
	db_collection_characters = db.collection('HEROKU_CHARACTER');

	// initialize the express app
	var server = app.listen(PORT, function () {
		var port = server.address().port;
    	console.log("App now running on port", port);
	});
});

// GAME RESULTS API ROUTES BELOW

// Generic error handler used by all endpoints
function handleError(res, reason, message, code) {
	console.log("ERROR: " + reason);
	res.status(code || 500).json({"error": message});
}

app.get('/', function (req, res) {
 	res.send('Hello World!');
});

app.post('/', function (req, res) {
	console.log(req);
	res.setHeader('Content-Type', 'application/json');
	var response = {
		response_type: 'in_channel',
    	text: 'It\s-a me, mario!',
	};
 	res.send(JSON.stringify(response));
});

// SLACK BOT INTEGRATION

// create the bot
controller = Botkit.slackbot();
var bot = controller.spawn({
	token: process.env.SLACK_TOKEN
})

// connect to slack
bot.startRTM(function(err,bot,payload) {
	if (err) {
		throw new Error('Could not connect to Slack');
	}
});

// when mario gets invited to a channel
controller.on('channel_joined',
	function(bot, message) {
		console.log(message);

		// reply with mario's excitement
		bot.reply(message, '<!channel>, It\'s a me, Mario!');
	}
);

// when someone suggests a game of kart
controller.hears(
	['(kart)(.*)(\\?)'],
	['direction_mention', 'mention', 'ambient'],
	function(bot, message) {

		// let everyone know
		bot.reply(message, '<!channel>, Let\'s a go!');
	}
);

// list commands
controller.hears(
	['^help$'],
	['direct_mention'],
	function(bot, message) {

		var text = '*Usage:* @mario <command>\n*Commands:*\n```';
		text += '\n1. add character "<character name>" <character icon url>';
		text += '\n2. characters';
		text += '\n3. my character is "<valid character name>"';
		text += '\n4. my name is "<name you want to be called>"';
		text += '\n5. played <#> games, <user1> <score1>, <user2> <score2>[, <user3> <score3>]';
		text += '\n6. bigboard';
		text += '```';
		bot.reply(message, text);
	}
);

// update the plot
var updatePlot = function() {
	var resultsDate = new Date();
 	resultsDate.setDate(resultsDate.getDate() - 30);

	db_collection_players.find().toArray(function (players_error, players) {
		var board = {}, promises = [];

		players.forEach(function (player) {
			var deferred = Q.defer();
			promises.push(deferred.promise);

			// get the games from the last 25 days
			db_collection_games.aggregate([
				{
					$match: {
						scores: {
							$elemMatch: {
								player_id: player._id
							}
						},
						datetime: {
							$gte: resultsDate.getTime()
						}
					}
				},
				{
					$sort: {
						datetime: -1
					}
				},
				{
					$project: {
						games: 1,
						datetime: 1,
						scores: {
							$filter: {
								input: '$scores',
								as: 'score',
								cond: { '$eq': [ '$$score.player_id', player._id ] }
							}
						}
					}
				}
			]
			).toArray(function (games_error, games) {
				if (games_error) {
					deferred.reject(new Error(games_error));
				} else {
					player.games = games;
					deferred.resolve(player);
				}
			});
		});

		Q.allSettled(promises).then(function (results) {
			var graph_data = [];

			// format players game information
			results.forEach(function (result) {
				var player = result.value;
				var entry = {
						x: [],
						y: [],
						type: 'scatter',
						line: { shape: 'spline' },
						mode: 'lines',
						name: player.name
					};

				player.games.forEach(function (game, index) {
					var score = game.scores[0];
					entry.x.push(dateFormat(new Date(game.datetime), 'yyyy-mm-dd H:MM:ss'));
					entry.y.push(score.average);
				});

				graph_data.push(entry);
			})

			var graphOptions = {
				filename: 'mario',
				fileopt: 'overwrite'
			};

			Plotly.plot(graph_data, graphOptions, function (err, msg) {
	    		if (err) {
	    			return console.log(err);
	    		}
			});
		});
	});
};

controller.hears(
	['^chart$'],
	['direct_mention'],
	function(bot, message) {
		bot.reply(message, 'https://plot.ly/~jgroppe/4.embed');
	}
);

// rankings
controller.hears(
	['^bigboard$'],
	['direct_mention'],
	function(bot, message) {

		// get all of the players
		db_collection_players.find().toArray(function (players_error, players) {
			var board = {}, promises = [];

			players.forEach(function (player) {
				var deferred = Q.defer();

				promises.push(deferred.promise);
				player.average_score = 0.0;
				player.total_score = 0.0;
				player.games_played = 0;
				player.rounds_played = 0;

				// get the last 25 games that this player was a part of, and only their score
				db_collection_games.aggregate([
					{
						$match: {
							scores: {
								$elemMatch: {
									player_id: player._id
								}
							}
						}
					},
					{
						$sort: {
							datetime: -1
						}
					},
					{
						$limit: 25
					},
					{
						$project: {
							games: 1,
							scores: {
								$filter: {
									input: '$scores',
									as: 'score',
									cond: { '$eq': [ '$$score.player_id', player._id ] }
								}
							}
						}
					}
				]
				).toArray(function (games_error, games) {
					if (games_error) {
						deferred.reject(new Error(games_error));
					} else {
						player.games = games;
						deferred.resolve(player);
					}
				});
			});

			Q.allSettled(promises).then(function (results) {
				var player_results = [];

				// calculate each players information
				results.forEach(function (result) {
					var player = result.value;

					player.games.forEach(function (game, index) {
						var score = game.scores[0];
						player.games_played += parseInt(game.games);
						player.average_score += parseFloat(score.average);
						player.total_score += parseInt(score.score)
						player.rounds_played++;
					});

					player_results.push(player);
				})

				// sort the players
				players.sort(function (a, b) { return (b.total_score / b.games_played) - (a.total_score / a.games_played) });

				var text = '*BOARD*\n';
				var table = [[ 'Rank', 'Name', 'Average', 'Character']];

				for (var i = 0; i < players.length; ++i)
				{
					var player = players[i];
					table.push([ (i + 1).toString(), player.name, (player.total_score / player.games_played).toFixed(2), player.character]);
				}

				var text_table = TextTable(table, { align: [ 'l', 'c', 'c', 'c' ] });
				bot.reply(message, text + text_table);

				updatePlot();
			});
		});
	}
);

// add character
controller.hears(
	['^(add character\\s)(\".*\"\\s)([^\\s]+)$'],
	['direct_mention'],
	function(bot, message) {

		// extract the character information
		var components = message.text.split(/[""]/);

		// make sure its basically in the right format
		if (components.length != 3)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that wasn\'t formatted right.');
			return;
		}

		db_collection_characters.insert({ name: components[1], image: components[2].replace(/[<> ]/g, '') }, function(error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to save that character.');
				return;
			}
			else
			{
				bot.say({text: 'Ok! ' + components[1] + ' has been saved.', channel: message.channel });
				bot.say({
					text: 'I\'m alive!',
					username: components[1],
					channel: message.channel,
					icon_url: components[2].replace(/[<> ]/g, '')
				});
			}
		});
	}
);

// list characters
controller.hears(
	['^characters$'],
	['direct_mention'],
	function(bot, message) {
		db_collection_characters.find().sort({ name: 1 }).toArray(function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to retrieve the characters.');
				return;
			}
			else
			{
				var text = '*Characters*';
				result.forEach(function(character) {
					text = text + '\n- ' + character.name;
				});
				bot.say({ text: text, channel: message.channel });
			}
		});
	}
);

// choose a character
controller.hears(
	['^my character is \".*\"$'],
	['direct_mention', 'mention', 'ambient'],
	function(bot, message) {
		var character;

		// extract the name of the character
		var name = message.text.split(/[""]/)[1];

		// validate it
		if (!name.length || name.length === 0)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that is not a valid character name.');
			return;
		}

		// see if that character exists
		db_collection_characters.find({ name: name }).toArray(function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to determine if that character exists.');
				return;
			}
			else
			{
				character = result[0];

				db_collection_players.update({ _id: message.user }, { $set: { 'character': character.name }}, { upsert: true }, function (error, result) {
					if (error)
					{
						bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to set that as your character.');
						return;
					}
					else
					{
						// reply with the user's new title
						bot.reply(message, 'Alright, <@' + message.user + '>, your character is ' + name + '!');
					}
				});
			}
		});
	}
);

// choose a name
controller.hears(
	['^my name is \".*\"$'],
	['direct_mention', 'mention', 'ambient'],
	function(bot, message) {
		var character;

		// extract the name
		var name = message.text.split(/[""]/)[1];

		// validate it
		if (!name.length || name.length === 0)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that is not a valid character name.');
			return;
		}

		db_collection_players.update({ _id: message.user }, { $set: { 'name': name }}, { upsert: true }, function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to set that as your name.');
				return;
			}
			else
			{
				// reply with the user's new title
				bot.reply(message, 'Alright, <@' + message.user + '>, your name is ' + name + '!');
			}
		});
	}
);

// game results
controller.hears(
	['^(played \\d+ games)((,\\s(<@\\w+>)\\s([0-9]+))+)$'],
	['direct_mention'],
	function(bot, message) {
		var components = message.text.split(",");

		// get the number of games played
		var game_count = parseInt(components[0].split(" ")[1]);

		var round = {
			games: parseInt(game_count),
			datetime: Date.now(),
			scores: [
			]
		};

		for (var i = 1; i < components.length; ++i) {
			var player_score = {
				player_id: components[i].split(" ")[1].split(/[@>]/)[1],
				score: components[i].split(" ")[2],
				average: (parseInt(components[i].split(" ")[2]) / game_count).toFixed(2)
			};

			round.scores.push(player_score);
		}

		db_collection_games.insert(round, function(error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, I wasn\'t able to save that round.');
				return;
			}
			else
			{
				var players = db_collection_players.find().toArray(function (error, result) {

					var text = 'You played ' + game_count + ' games. The average scores were: ';

					for (var i = 0; i < round.scores.length; ++i)
					{
						var player = result.filter(function(player) { return (player._id === round.scores[i].player_id) })[0];
						text += ('\n' + player.name + ': ' + (round.scores[i].score / game_count).toFixed(2));
					}

					bot.reply(message, text);

					updatePlot();
				});
			}
		});
	}
);
<commit_msg>Removing ranking (bigboard) from JS app<commit_after>/**
 * An application tracks Mario Kart results and calculates rankings,
 * and includes a Slack Bot integration.
 */

// SLACK_TOKEN is required for connecting to Slack
if (!process.env.SLACK_TOKEN)
{
	console.log('Error: Please specify SLACK_TOKEN in the environment.');
    process.exit(1);
}
else if (!process.env.MONGODB_URI)
{
	console.log('Error: Please specify MONGODB_URI in the environment.');
    process.exit(1);
}
else if (!process.env.DEPLOYMENT_ENVIRONMENT)
{
	console.log('Error: Please specify DEPLOYMENT_ENVIRONMENT in the environment.');
    process.exit(1);
}

// dependencies
const PORT = process.env.PORT || 80;
var express = require("express"),
	bodyParser = require("body-parser"),
	mongodb = require("mongodb"),
	Botkit = require('botkit'),
	requester = require('request'),
	TextTable = require('text-table'),
	Plotly = require('plotly')("jgroppe", "hOvosfAJF74xOWSPKJh5"),
	fs = require('fs'),
	Q = require('q'),
	dateFormat = require('dateformat');

// create the express app
var app = express();

// tell express where to load static (client-side) files from
app.use(express.static(__dirname + "/www"));

// tell express to parse request bodies as json
app.use(bodyParser.json());

// create a database variable outside of the database connection callback
// to reuse the connection pool.
var db, db_collection_player,
	db_collection_game, db_collection_character;

// connect to the database before starting the application server.
mongodb.MongoClient.connect(process.env.MONGODB_URI, function (err, database) {

	// if there was an error connecting, shutdown the app
	if (err) {
		console.log(err);
		process.exit(1);
	}

	// save database object from the callback for reuse
	db = database;

	console.log("Database connection ready");

	// gather the collections
	//db_collection_players = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_PLAYER');
	//db_collection_games = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_GAME');
	//db_collection_characters = db.collection(process.env.DEPLOYMENT_ENVIRONMENT + '_CHARACTER');
	db_collection_players = db.collection('HEROKU_PLAYER');
	db_collection_games = db.collection('HEROKU_GAME');
	db_collection_characters = db.collection('HEROKU_CHARACTER');

	// initialize the express app
	var server = app.listen(PORT, function () {
		var port = server.address().port;
    	console.log("App now running on port", port);
	});
});

// GAME RESULTS API ROUTES BELOW

// Generic error handler used by all endpoints
function handleError(res, reason, message, code) {
	console.log("ERROR: " + reason);
	res.status(code || 500).json({"error": message});
}

app.get('/', function (req, res) {
 	res.send('Hello World!');
});

app.post('/', function (req, res) {
	console.log(req);
	res.setHeader('Content-Type', 'application/json');
	var response = {
		response_type: 'in_channel',
    	text: 'It\s-a me, mario!',
	};
 	res.send(JSON.stringify(response));
});

// SLACK BOT INTEGRATION

// create the bot
controller = Botkit.slackbot();
var bot = controller.spawn({
	token: process.env.SLACK_TOKEN
})

// connect to slack
bot.startRTM(function(err,bot,payload) {
	if (err) {
		throw new Error('Could not connect to Slack');
	}
});

// when mario gets invited to a channel
controller.on('channel_joined',
	function(bot, message) {
		console.log(message);

		// reply with mario's excitement
		bot.reply(message, '<!channel>, It\'s a me, Mario!');
	}
);

// when someone suggests a game of kart
controller.hears(
	['(kart)(.*)(\\?)'],
	['direction_mention', 'mention', 'ambient'],
	function(bot, message) {

		// let everyone know
		bot.reply(message, '<!channel>, Let\'s a go!');
	}
);

// list commands
controller.hears(
	['^help$'],
	['direct_mention'],
	function(bot, message) {

		var text = '*Usage:* @mario <command>\n*Commands:*\n```';
		text += '\n1. add character "<character name>" <character icon url>';
		text += '\n2. characters';
		text += '\n3. my character is "<valid character name>"';
		text += '\n4. my name is "<name you want to be called>"';
		text += '\n5. played <#> games, <user1> <score1>, <user2> <score2>[, <user3> <score3>]';
		text += '```';
		bot.reply(message, text);
	}
);

// update the plot
var updatePlot = function() {
	var resultsDate = new Date();
 	resultsDate.setDate(resultsDate.getDate() - 30);

	db_collection_players.find().toArray(function (players_error, players) {
		var board = {}, promises = [];

		players.forEach(function (player) {
			var deferred = Q.defer();
			promises.push(deferred.promise);

			// get the games from the last 25 days
			db_collection_games.aggregate([
				{
					$match: {
						scores: {
							$elemMatch: {
								player_id: player._id
							}
						},
						datetime: {
							$gte: resultsDate.getTime()
						}
					}
				},
				{
					$sort: {
						datetime: -1
					}
				},
				{
					$project: {
						games: 1,
						datetime: 1,
						scores: {
							$filter: {
								input: '$scores',
								as: 'score',
								cond: { '$eq': [ '$$score.player_id', player._id ] }
							}
						}
					}
				}
			]
			).toArray(function (games_error, games) {
				if (games_error) {
					deferred.reject(new Error(games_error));
				} else {
					player.games = games;
					deferred.resolve(player);
				}
			});
		});

		Q.allSettled(promises).then(function (results) {
			var graph_data = [];

			// format players game information
			results.forEach(function (result) {
				var player = result.value;
				var entry = {
						x: [],
						y: [],
						type: 'scatter',
						line: { shape: 'spline' },
						mode: 'lines',
						name: player.name
					};

				player.games.forEach(function (game, index) {
					var score = game.scores[0];
					entry.x.push(dateFormat(new Date(game.datetime), 'yyyy-mm-dd H:MM:ss'));
					entry.y.push(score.average);
				});

				graph_data.push(entry);
			})

			var graphOptions = {
				filename: 'mario',
				fileopt: 'overwrite'
			};

			Plotly.plot(graph_data, graphOptions, function (err, msg) {
	    		if (err) {
	    			return console.log(err);
	    		}
			});
		});
	});
};

controller.hears(
	['^chart$'],
	['direct_mention'],
	function(bot, message) {
		bot.reply(message, 'https://plot.ly/~jgroppe/4.embed');
	}
);

// add character
controller.hears(
	['^(add character\\s)(\".*\"\\s)([^\\s]+)$'],
	['direct_mention'],
	function(bot, message) {

		// extract the character information
		var components = message.text.split(/[""]/);

		// make sure its basically in the right format
		if (components.length != 3)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that wasn\'t formatted right.');
			return;
		}

		db_collection_characters.insert({ name: components[1], image: components[2].replace(/[<> ]/g, '') }, function(error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to save that character.');
				return;
			}
			else
			{
				bot.say({text: 'Ok! ' + components[1] + ' has been saved.', channel: message.channel });
				bot.say({
					text: 'I\'m alive!',
					username: components[1],
					channel: message.channel,
					icon_url: components[2].replace(/[<> ]/g, '')
				});
			}
		});
	}
);

// list characters
controller.hears(
	['^characters$'],
	['direct_mention'],
	function(bot, message) {
		db_collection_characters.find().sort({ name: 1 }).toArray(function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to retrieve the characters.');
				return;
			}
			else
			{
				var text = '*Characters*';
				result.forEach(function(character) {
					text = text + '\n- ' + character.name;
				});
				bot.say({ text: text, channel: message.channel });
			}
		});
	}
);

// choose a character
controller.hears(
	['^my character is \".*\"$'],
	['direct_mention', 'mention', 'ambient'],
	function(bot, message) {
		var character;

		// extract the name of the character
		var name = message.text.split(/[""]/)[1];

		// validate it
		if (!name.length || name.length === 0)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that is not a valid character name.');
			return;
		}

		// see if that character exists
		db_collection_characters.find({ name: name }).toArray(function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to determine if that character exists.');
				return;
			}
			else
			{
				character = result[0];

				db_collection_players.update({ _id: message.user }, { $set: { 'character': character.name }}, { upsert: true }, function (error, result) {
					if (error)
					{
						bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to set that as your character.');
						return;
					}
					else
					{
						// reply with the user's new title
						bot.reply(message, 'Alright, <@' + message.user + '>, your character is ' + name + '!');
					}
				});
			}
		});
	}
);

// choose a name
controller.hears(
	['^my name is \".*\"$'],
	['direct_mention', 'mention', 'ambient'],
	function(bot, message) {
		var character;

		// extract the name
		var name = message.text.split(/[""]/)[1];

		// validate it
		if (!name.length || name.length === 0)
		{
			bot.reply(message, 'Sorry, <@' + message.user + '>, that is not a valid character name.');
			return;
		}

		db_collection_players.update({ _id: message.user }, { $set: { 'name': name }}, { upsert: true }, function (error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, <@' + message.user + '>, I wasn\'t able to set that as your name.');
				return;
			}
			else
			{
				// reply with the user's new title
				bot.reply(message, 'Alright, <@' + message.user + '>, your name is ' + name + '!');
			}
		});
	}
);

// game results
controller.hears(
	['^(played \\d+ games)((,\\s(<@\\w+>)\\s([0-9]+))+)$'],
	['direct_mention'],
	function(bot, message) {
		var components = message.text.split(",");

		// get the number of games played
		var game_count = parseInt(components[0].split(" ")[1]);

		var round = {
			games: parseInt(game_count),
			datetime: Date.now(),
			scores: [
			]
		};

		for (var i = 1; i < components.length; ++i) {
			var player_score = {
				player_id: components[i].split(" ")[1].split(/[@>]/)[1],
				score: components[i].split(" ")[2],
				average: (parseInt(components[i].split(" ")[2]) / game_count).toFixed(2)
			};

			round.scores.push(player_score);
		}

		db_collection_games.insert(round, function(error, result) {
			if (error)
			{
				bot.reply(message, 'Sorry, I wasn\'t able to save that round.');
				return;
			}
			else
			{
				var players = db_collection_players.find().toArray(function (error, result) {

					var text = 'You played ' + game_count + ' games. The average scores were: ';

					for (var i = 0; i < round.scores.length; ++i)
					{
						var player = result.filter(function(player) { return (player._id === round.scores[i].player_id) })[0];
						text += ('\n' + player.name + ': ' + (round.scores[i].score / game_count).toFixed(2));
					}

					bot.reply(message, text);

					updatePlot();
				});
			}
		});
	}
);
<|endoftext|>