<commit_before>#!/usr/bin/env node
//References & Resources
//http://www.spacjer.com/blog/2014/02/10/defining-node-dot-js-task-for-heroku-scheduler/
//http://stackoverflow.com/questions/25659134/insert-error-using-node-js-async-series
var archive = require('./mikeal-githubarchive.js');
var fs = require('fs');
var util = require('util');
var mongodb = require('mongodb');
var MongoClient = mongodb.MongoClient;
var async = require('async');
var assert = require('assert');
var underscore = require('underscore');
var moment = require('moment');
var path = require("path");

GetParse();

//Fetch GitHub Archives, parse  'PushEvent' event notifications and insert into MongoDB
//Sample JSON generated
//{"078d40cc5":{"created_at":"2014-08-23T15:05:37-07:00","full_name":"ssss"},
// "0b0084e21":{"created_at":"2014-08-23T15:05:37-07:00","full_name":yyyyy"}}
function GetParse() {
	var count = 0; //number of entries
	var rows = [];
	var TimeNow = moment().format("YYYY-MM-DD HH:MM:SS");
	var TimeAgo = moment().subtract(2, 'hours').format("YYYY-MM-DD-H");

	//Test 1 - fetch prior URL
	//TimeAgo ="2014-08-28-10"  //10345

	var URL = "http://data.githubarchive.org/" + TimeAgo + ".json.gz";
	console.log (TimeNow + " processing " + URL);
	var a = archive(URL, {gzip:true});

	//Test 2 - read test file
    //var npath = path.join(__dirname, '2014-sample.json')
    //console.log ("Testing sample json ..." + npath);
    //var a = archive.readFile(npath, {gzip:false});

    var com = a.MyParser(function (err, commits) {
    	if (err) return console.log(err);
    	//console.log(JSON.stringify(com.commits));
    	var tmp = JSON.stringify(com.commits);
    	var result = JSON.parse(tmp);
    	//create array
    	for(var k in result) {
    	    //console.log (k);          //print keys
    	    count ++;
    	    rows.push(result[k]);       //create array
            //console.log (result[k]);  //print values
    	}
    	console.log ("## entries: " + count);
    	//Insert into MongoDB
    	console.log("## start insert: "+ moment().format("YYYY-MM-DD HH:MM:SS"));
    	MongoInsert(rows,count);

    //Debug - store to external file
	//var tmp_dir = path.join(process.cwd(), "output/");
	//if (!fs.existsSync(tmp_dir))
	    //fs.mkdirSync(tmp_dir);
	//var p = tmp_dir + 'PushEvent.json';
	//fs.writeFile(p, JSON.stringify(com.commits), function (err) {
		//if (err) return console.log(err);
	//});
    }); //end MyParser

} //end GetParse()

function MongoInsert(rows,count)
{
	var connectURL  = process.env.connectURL;
	var mycollection= process.env.mycollection;
	var db;
	var col;

	async.series([
	  // Connect to DB
	  function(callback) {
		  MongoClient.connect(connectURL,function(error, db2) {
			  if (error) {console.log("db connect error" + error);callback(error,"db connect error"); return;}
			  db = db2;
			  callback(null,"connect success");
		  });
	  },
	  function(callback) {
		  col = db.collection(mycollection);
	      callback(null,"collection success");
	  },
	  function(callback) {
		  //console.log ("insert begin ...");
		  var i = 1;
		  async.whilst(
		    function() { return i <= count },
		    function(callback) {
		    	var mydocument = rows.shift();
		        col.insert(mydocument,function(error,result) {
		            if (error) {
		                console.log("insert error:" + error);
		                callback(error);
		                return;
		            }
		            //console.log ("inserted ...");
		            i++;
		            callback(error);
		        });
		    },
		    function(error) {
		      callback(error,"insert sucess")
		    }
		  );
	  },
	  function (callback){
		  //console.log ("close db");
		  db.close();
		  console.log("## end insert: "+ moment().format("YYYY-MM-DD HH:MM:SS"));
		  callback(null,"connection closed");
	  }
	 ], function(error, results) {
			if (error) { console.log("error"); }
				//console.log(results);
		});
}
<commit_msg>modify moment.format()<commit_after>#!/usr/bin/env node
//References & Resources
//http://www.spacjer.com/blog/2014/02/10/defining-node-dot-js-task-for-heroku-scheduler/
//http://stackoverflow.com/questions/25659134/insert-error-using-node-js-async-series
var archive = require('./mikeal-githubarchive.js');
var fs = require('fs');
var util = require('util');
var mongodb = require('mongodb');
var MongoClient = mongodb.MongoClient;
var async = require('async');
var assert = require('assert');
var underscore = require('underscore');
var moment = require('moment');
var path = require("path");

GetParse();

//Fetch GitHub Archives, parse  'PushEvent' event notifications and insert into MongoDB
//Sample JSON generated
//{"078d40cc5":{"created_at":"2014-08-23T15:05:37-07:00","full_name":"ssss"},
// "0b0084e21":{"created_at":"2014-08-23T15:05:37-07:00","full_name":yyyyy"}}
function GetParse() {
	var count = 0; //number of entries
	var rows = [];
	var TimeNow = moment().format();
	var TimeAgo = moment().subtract(2, 'hours').format("YYYY-MM-DD-H");

	//Test 1 - fetch prior URL
	//TimeAgo ="2014-08-28-10"  //10345

	var URL = "http://data.githubarchive.org/" + TimeAgo + ".json.gz";
	console.log (TimeNow + " processing " + URL);
	var a = archive(URL, {gzip:true});

	//Test 2 - read test file
    //var npath = path.join(__dirname, '2014-sample.json')
    //console.log ("Testing sample json ..." + npath);
    //var a = archive.readFile(npath, {gzip:false});

    var com = a.MyParser(function (err, commits) {
    	if (err) return console.log(err);
    	//console.log(JSON.stringify(com.commits));
    	var tmp = JSON.stringify(com.commits);
    	var result = JSON.parse(tmp);
    	//create array
    	for(var k in result) {
    	    //console.log (k);          //print keys
    	    count ++;
    	    rows.push(result[k]);       //create array
            //console.log (result[k]);  //print values
    	}
    	console.log ("## entries: " + count);
    	//Insert into MongoDB
    	console.log("## start insert: "+ moment().format());
    	MongoInsert(rows,count);

    //Debug - store to external file
	//var tmp_dir = path.join(process.cwd(), "output/");
	//if (!fs.existsSync(tmp_dir))
	    //fs.mkdirSync(tmp_dir);
	//var p = tmp_dir + 'PushEvent.json';
	//fs.writeFile(p, JSON.stringify(com.commits), function (err) {
		//if (err) return console.log(err);
	//});
    }); //end MyParser

} //end GetParse()

function MongoInsert(rows,count)
{
	var connectURL  = process.env.connectURL;
	var mycollection= process.env.mycollection;
	var db;
	var col;

	async.series([
	  // Connect to DB
	  function(callback) {
		  MongoClient.connect(connectURL,function(error, db2) {
			  if (error) {console.log("db connect error" + error);callback(error,"db connect error"); return;}
			  db = db2;
			  callback(null,"connect success");
		  });
	  },
	  function(callback) {
		  col = db.collection(mycollection);
	      callback(null,"collection success");
	  },
	  function(callback) {
		  //console.log ("insert begin ...");
		  var i = 1;
		  async.whilst(
		    function() { return i <= count },
		    function(callback) {
		    	var mydocument = rows.shift();
                        //TODO: Check for unique SHA before insert
		        col.insert(mydocument,function(error,result) {
		            if (error) {
		                console.log("insert error:" + error);
		                callback(error);
		                return;
		            }
		            //console.log ("inserted ...");
		            i++;
		            callback(error);
		        }); //end insert
		    },
		    function(error) {
		      callback(error,"insert sucess")
		    }
		  );
	  },
	  function (callback){
		  //console.log ("close db");
		  db.close();
		  console.log("## end insert: "+ moment().format());
		  callback(null,"connection closed");
	  }
	 ], function(error, results) {
			if (error) { console.log("error"); }
				//console.log(results);
		});
}
<|endoftext|>


<commit_before>import React, {PropTypes, Component} from 'react'
import {connect} from 'react-redux'
import {bindActionCreators} from 'redux'

import _ from 'lodash'

import QueryMaker from '../components/QueryMaker'
import Visualization from '../components/Visualization'
import WriteDataForm from 'src/data_explorer/components/WriteDataForm'
import Header from '../containers/Header'
import ResizeContainer from 'shared/components/ResizeContainer'
import OverlayTechnologies from 'shared/components/OverlayTechnologies'

import {VIS_VIEWS} from 'shared/constants'
import {MINIMUM_HEIGHTS, INITIAL_HEIGHTS} from '../constants'
import {errorThrown} from 'shared/actions/errors'
import {setAutoRefresh} from 'shared/actions/app'
import * as dataExplorerActionCreators from 'src/data_explorer/actions/view'
import {writeLineProtocolAsync} from 'src/data_explorer/actions/view/write'

class DataExplorer extends Component {
  constructor(props) {
    super(props)

    this.state = {
      activeQueryIndex: 0,
      showWriteForm: false,
    }
  }

  handleSetActiveQueryIndex = index => {
    this.setState({activeQueryIndex: index})
  }

  handleDeleteQuery = index => {
    const {queryConfigs, queryConfigActions} = this.props
    const query = queryConfigs[index]
    queryConfigActions.deleteQuery(query.id)
  }

  handleAddQuery = () => {
    const newIndex = this.props.queryConfigs.length
    this.props.queryConfigActions.addQuery()
    this.handleSetActiveQueryIndex(newIndex)
  }

  getActiveQuery = () => {
    const {activeQueryIndex} = this.state
    const {queryConfigs} = this.props
    const activeQuery = queryConfigs[activeQueryIndex]
    const defaultQuery = queryConfigs[0]

    return activeQuery || defaultQuery
  }

  render() {
    const {
      autoRefresh,
      errorThrownAction,
      handleChooseAutoRefresh,
      timeRange,
      setTimeRange,
      queryConfigs,
      queryConfigActions,
      source,
      writeLineProtocol,
    } = this.props

    const {activeQueryIndex, showWriteForm} = this.state
    const selectedDatabase = _.get(
      queryConfigs,
      [`${activeQueryIndex}`, 'database'],
      null
    )

    return (

        {showWriteForm
          ? <OverlayTechnologies>
              <WriteDataForm
                selectedDatabase={selectedDatabase}
                errorThrown={errorThrownAction}
                onClose={() => this.setState({showWriteForm: false})}
                source={source}
                writeLineProtocol={writeLineProtocol}
              />
            </OverlayTechnologies>
          : null}
         this.setState({showWriteForm: true})}
        />
        <ResizeContainer
          containerClass="page-contents"
          minTopHeight={MINIMUM_HEIGHTS.queryMaker}
          minBottomHeight={MINIMUM_HEIGHTS.visualization}
          initialTopHeight={INITIAL_HEIGHTS.queryMaker}
          initialBottomHeight={INITIAL_HEIGHTS.visualization}
        >
          <QueryMaker
            source={source}
            queries={queryConfigs}
            actions={queryConfigActions}
            autoRefresh={autoRefresh}
            timeRange={timeRange}
            setActiveQueryIndex={this.handleSetActiveQueryIndex}
            onDeleteQuery={this.handleDeleteQuery}
            onAddQuery={this.handleAddQuery}
            activeQueryIndex={activeQueryIndex}
            activeQuery={this.getActiveQuery()}
          />
          <Visualization
            isInDataExplorer={true}
            autoRefresh={autoRefresh}
            timeRange={timeRange}
            queryConfigs={queryConfigs}
            activeQueryIndex={activeQueryIndex}
            editQueryStatus={queryConfigActions.editQueryStatus}
            views={VIS_VIEWS}
          />
        </ResizeContainer>

    )
  }
}

const {arrayOf, func, number, shape, string} = PropTypes

DataExplorer.propTypes = {
  source: shape({
    links: shape({
      proxy: string.isRequired,
      self: string.isRequired,
      queries: string.isRequired,
    }).isRequired,
  }).isRequired,
  queryConfigs: arrayOf(shape({})).isRequired,
  queryConfigActions: shape({
    editQueryStatus: func.isRequired,
  }).isRequired,
  autoRefresh: number.isRequired,
  handleChooseAutoRefresh: func.isRequired,
  timeRange: shape({
    upper: string,
    lower: string,
  }).isRequired,
  setTimeRange: func.isRequired,
  dataExplorer: shape({
    queryIDs: arrayOf(string).isRequired,
  }).isRequired,
  writeLineProtocol: func.isRequired,
  errorThrownAction: func.isRequired,
}

DataExplorer.childContextTypes = {
  source: shape({
    links: shape({
      proxy: string.isRequired,
      self: string.isRequired,
    }).isRequired,
  }).isRequired,
}

const mapStateToProps = state => {
  const {
    app: {persisted: {autoRefresh}},
    dataExplorer,
    dataExplorerQueryConfigs: queryConfigs,
    timeRange,
  } = state
  const queryConfigValues = _.values(queryConfigs)

  return {
    autoRefresh,
    dataExplorer,
    queryConfigs: queryConfigValues,
    timeRange,
  }
}

const mapDispatchToProps = dispatch => {
  return {
    handleChooseAutoRefresh: bindActionCreators(setAutoRefresh, dispatch),
    errorThrownAction: bindActionCreators(errorThrown, dispatch),
    setTimeRange: bindActionCreators(
      dataExplorerActionCreators.setTimeRange,
      dispatch
    ),
    writeLineProtocol: bindActionCreators(writeLineProtocolAsync, dispatch),
    queryConfigActions: bindActionCreators(
      dataExplorerActionCreators,
      dispatch
    ),
  }
}

export default connect(mapStateToProps, mapDispatchToProps)(DataExplorer)
<commit_msg>Remove arrow props from DataExplorer<commit_after>import React, {PropTypes, Component} from 'react'
import {connect} from 'react-redux'
import {bindActionCreators} from 'redux'

import _ from 'lodash'

import QueryMaker from '../components/QueryMaker'
import Visualization from '../components/Visualization'
import WriteDataForm from 'src/data_explorer/components/WriteDataForm'
import Header from '../containers/Header'
import ResizeContainer from 'shared/components/ResizeContainer'
import OverlayTechnologies from 'shared/components/OverlayTechnologies'

import {VIS_VIEWS} from 'shared/constants'
import {MINIMUM_HEIGHTS, INITIAL_HEIGHTS} from '../constants'
import {errorThrown} from 'shared/actions/errors'
import {setAutoRefresh} from 'shared/actions/app'
import * as dataExplorerActionCreators from 'src/data_explorer/actions/view'
import {writeLineProtocolAsync} from 'src/data_explorer/actions/view/write'

class DataExplorer extends Component {
  constructor(props) {
    super(props)

    this.state = {
      activeQueryIndex: 0,
      showWriteForm: false,
    }
  }

  handleSetActiveQueryIndex = index => {
    this.setState({activeQueryIndex: index})
  }

  handleDeleteQuery = index => {
    const {queryConfigs, queryConfigActions} = this.props
    const query = queryConfigs[index]
    queryConfigActions.deleteQuery(query.id)
  }

  handleAddQuery = () => {
    const newIndex = this.props.queryConfigs.length
    this.props.queryConfigActions.addQuery()
    this.handleSetActiveQueryIndex(newIndex)
  }

  getActiveQuery = () => {
    const {activeQueryIndex} = this.state
    const {queryConfigs} = this.props
    const activeQuery = queryConfigs[activeQueryIndex]
    const defaultQuery = queryConfigs[0]

    return activeQuery || defaultQuery
  }

  handleCloseWriteData = () => {
    this.setState({showWriteForm: false})
  }

  handleOpenWriteData = () => {
    this.setState({showWriteForm: true})
  }

  render() {
    const {
      autoRefresh,
      errorThrownAction,
      handleChooseAutoRefresh,
      timeRange,
      setTimeRange,
      queryConfigs,
      queryConfigActions,
      source,
      writeLineProtocol,
    } = this.props

    const {activeQueryIndex, showWriteForm} = this.state
    const selectedDatabase = _.get(
      queryConfigs,
      [`${activeQueryIndex}`, 'database'],
      null
    )

    return (

        {showWriteForm
          ? <OverlayTechnologies>
              <WriteDataForm
                source={source}
                errorThrown={errorThrownAction}
                selectedDatabase={selectedDatabase}
                onClose={this.handleCloseWriteData}
                writeLineProtocol={writeLineProtocol}
              />
            </OverlayTechnologies>
          : null}

        <ResizeContainer
          containerClass="page-contents"
          minTopHeight={MINIMUM_HEIGHTS.queryMaker}
          minBottomHeight={MINIMUM_HEIGHTS.visualization}
          initialTopHeight={INITIAL_HEIGHTS.queryMaker}
          initialBottomHeight={INITIAL_HEIGHTS.visualization}
        >
          <QueryMaker
            source={source}
            queries={queryConfigs}
            actions={queryConfigActions}
            autoRefresh={autoRefresh}
            timeRange={timeRange}
            setActiveQueryIndex={this.handleSetActiveQueryIndex}
            onDeleteQuery={this.handleDeleteQuery}
            onAddQuery={this.handleAddQuery}
            activeQueryIndex={activeQueryIndex}
            activeQuery={this.getActiveQuery()}
          />
          <Visualization
            isInDataExplorer={true}
            autoRefresh={autoRefresh}
            timeRange={timeRange}
            queryConfigs={queryConfigs}
            activeQueryIndex={activeQueryIndex}
            editQueryStatus={queryConfigActions.editQueryStatus}
            views={VIS_VIEWS}
          />
        </ResizeContainer>

    )
  }
}

const {arrayOf, func, number, shape, string} = PropTypes

DataExplorer.propTypes = {
  source: shape({
    links: shape({
      proxy: string.isRequired,
      self: string.isRequired,
      queries: string.isRequired,
    }).isRequired,
  }).isRequired,
  queryConfigs: arrayOf(shape({})).isRequired,
  queryConfigActions: shape({
    editQueryStatus: func.isRequired,
  }).isRequired,
  autoRefresh: number.isRequired,
  handleChooseAutoRefresh: func.isRequired,
  timeRange: shape({
    upper: string,
    lower: string,
  }).isRequired,
  setTimeRange: func.isRequired,
  dataExplorer: shape({
    queryIDs: arrayOf(string).isRequired,
  }).isRequired,
  writeLineProtocol: func.isRequired,
  errorThrownAction: func.isRequired,
}

DataExplorer.childContextTypes = {
  source: shape({
    links: shape({
      proxy: string.isRequired,
      self: string.isRequired,
    }).isRequired,
  }).isRequired,
}

const mapStateToProps = state => {
  const {
    app: {persisted: {autoRefresh}},
    dataExplorer,
    dataExplorerQueryConfigs: queryConfigs,
    timeRange,
  } = state
  const queryConfigValues = _.values(queryConfigs)

  return {
    autoRefresh,
    dataExplorer,
    queryConfigs: queryConfigValues,
    timeRange,
  }
}

const mapDispatchToProps = dispatch => {
  return {
    handleChooseAutoRefresh: bindActionCreators(setAutoRefresh, dispatch),
    errorThrownAction: bindActionCreators(errorThrown, dispatch),
    setTimeRange: bindActionCreators(
      dataExplorerActionCreators.setTimeRange,
      dispatch
    ),
    writeLineProtocol: bindActionCreators(writeLineProtocolAsync, dispatch),
    queryConfigActions: bindActionCreators(
      dataExplorerActionCreators,
      dispatch
    ),
  }
}

export default connect(mapStateToProps, mapDispatchToProps)(DataExplorer)
<|endoftext|>


<commit_before>/*    Copyright 2021 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const flowTool = require('../net2/FlowTool');
const log = require('../net2/logger.js')(__filename)
const _ = require('lodash');
const Sensor = require('./Sensor.js').Sensor
const fc = require('../net2/config.js');
const featureName = 'compress_flows'
const Promise = require('bluebird');
const zlib = require('zlib');
const extensionManager = require('./ExtensionManager.js')
const { rclient } = require('../util/redis_manager');
const deflateAsync = Promise.promisify(zlib.deflate);
const sem = require('./SensorEventManager.js').getInstance();
const MAX_MEM = 10 * 1000 * 1000

class FlowCompressionSensor extends Sensor {
  constructor() {
    super()
    this.recentlyTickKey = "compressed:flows:lastest:ts"
    this.step = 60 * 60 // one hour
    this.maxInterval = 24 * 60 * 60 // 24 hours
  }

  async run() {
    sem.once('IPTABLES_READY', async () => {
      await this.build()
      setInterval(async () => {
        await this.build()
      }, 30 * 60 * 1000);
    })
  }

  async checkAndCleanMem() {
    let compressedMem = 0
    let oldestKey
    const compressedFlowsKeys = await rclient.scanResults(this.getKey("*", "*"), 1000)
    for (const key of compressedFlowsKeys) {
      const mem = Number(await rclient.memoryAsync("usage", key) || 0)
      compressedMem += mem
      if (!oldestKey) {
        oldestKey = key
      } else if (await rclient.ttlAsync(key) < await rclient.ttlAsync(oldestKey)) {
        oldestKey = key
      }
    }
    if (compressedMem > MAX_MEM) { // del the oldest key
      await rclient.delAsync(oldestKey)
    }
  }

  async apiRun() {
    extensionManager.onGet("compressedflows", (msg, data) => {
      return this.loadCompressedFlows(data)
    });
  }

  async loadCompressedFlows(options) {
    // options {begin,end}
    let { begin, end } = options
    begin = begin - begin % this.step
    end = end - end % this.step
    log.info(`Load compressed flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
    const compressedFlows = []
    for (let i = 0; i < (end - begin) / this.step; i++) {
      const beginTs = begin + this.step * i
      const endTs = begin + this.step * (i + 1)
      const str = await rclient.getAsync(this.getKey(beginTs, endTs))
      str && compressedFlows.push(str)
    }
    return compressedFlows
  }

  getKey(begin, end) {
    return `compressed:flows:${begin}:${end}`
  }

  async sensorCheck() {
    if (!fc.isFeatureOn(featureName)) {
      log.info(`${featureName} feature is disabled.`)
      return false
    }
    const members = await rclient.smembersAsync("sys:ept:members")
    if (members && members.length > 0) {
      const webEnabled = members.map((m) => {
        try {
          return JSON.parse(m)
        } catch (err) {
          return null
        }
      }).filter((x) => x != null && x.name == 'firewalla_web').length > 0
      if (!webEnabled) {
        log.info(`firewalla_web haven't enabled`)
      }
      return webEnabled
    }
    return false
  }

  async build() {
    try {
      if (! await this.sensorCheck()) return
      const { begin, end } = await this.getBuildingWindow()
      const now = new Date() / 1000
      log.info(`Going to compress flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
      for (let i = 0; i < (end - begin) / this.step; i++) {
        const beginTs = begin + this.step * i
        const endTs = begin + this.step * (i + 1)
        const flows = await this.loadFlows(beginTs, endTs)
        await this.save(beginTs, endTs, flows)
      }
      await rclient.setAsync(this.recentlyTickKey, end)
      log.info(`Compressed flows build complted, cost ${(new Date() / 1000 - now).toFixed(2)}`)
    } catch (e) {
      log.error(`Compress flows error`, e)
    }
  }

  async save(begin, end, flows) { // might save to disk in future
    const base64Str = await this.compress(flows)
    const key = this.getKey(begin, end)
    await rclient.setAsync(key, base64Str)
    await rclient.expireatAsync(key, end + this.maxInterval)
    await this.checkAndCleanMem()
  }

  async getBuildingWindow() {
    const now = new Date() / 1000
    const nowTickTs = now - now % this.step
    let recentlyTickTs = Number(await rclient.getAsync(this.recentlyTickKey) || 0)
    if (nowTickTs - recentlyTickTs > this.maxInterval) {
      recentlyTickTs = nowTickTs - this.maxInterval
    }
    return {
      begin: recentlyTickTs,
      end: nowTickTs
    }
  }

  async loadFlows(begin, end) {
    log.info(`Going to load flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
    let completed = false
    const options = {
      begin: begin,
      end: end,
      audit: true,
      count: 2000,
      asc: true
    }
    let allFlows = []
    const now = new Date() / 1000
    while (!completed) {
      try {
        const flows = await flowTool.prepareRecentFlows({}, JSON.parse(JSON.stringify(options))) || []
        if (flows.length < options.count) {
          completed = true
        } else {
          options.begin = flows[flows.length - 1].ts
        }
        allFlows = allFlows.concat(flows)
      } catch (e) {
        log.error(`Load flows error`, e)
        completed = true
      }
    }
    log.debug(`get ${allFlows.length} flows cost ${(new Date() / 1000 - now).toFixed(2)} seconds`)
    // debug purpose
    log.debug(`there are ${allFlows.reduce((ac, val) => ac + val.count, 0)} zeek logs for these flows`)
    return allFlows
  }

  mergeFlows(flows) {
    if (!flows || flows.length == 0) return [];
    let stash = flows[0];
    const mergedFlows = [stash];
    const compareKeys = ["ltype", "fd", "device", "protocol", "host", "ip", "domain"]
    for (var i = 1; i < flows.length; i++) {
      const flow = flows[i]
      if (_.isEqual(_.pick(stash, compareKeys), _.pick(flow, compareKeys))) {
        stash.count += flow.count
        stash.download += flow.download
        stash.upload += flow.upload
        stash.duration += flow.duration
      } else {
        stash = flow
        mergedFlows.push(stash)
      }
    }
    return mergedFlows
  }
  async compress(flows) {
    const mergedFlows = this.mergeFlows(flows)
    const str = JSON.stringify(mergedFlows)
    const deflateBuffer = await deflateAsync(str)
    const base64Str = deflateBuffer.toString('base64')
    log.debug(`Compress ${mergedFlows.length} flows, raw: ${str.length} deflate: ${deflateBuffer.length} base64:${base64Str.length}`)
    return base64Str
  }
}

module.exports = FlowCompressionSensor;<commit_msg>update<commit_after>/*    Copyright 2021 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const flowTool = require('../net2/FlowTool');
const log = require('../net2/logger.js')(__filename)
const _ = require('lodash');
const Sensor = require('./Sensor.js').Sensor
const fc = require('../net2/config.js');
const featureName = 'compress_flows'
const Promise = require('bluebird');
const zlib = require('zlib');
const extensionManager = require('./ExtensionManager.js')
const { rclient } = require('../util/redis_manager');
const deflateAsync = Promise.promisify(zlib.deflate);
const sem = require('./SensorEventManager.js').getInstance();
const MAX_MEM = 10 * 1000 * 1000

class FlowCompressionSensor extends Sensor {
  constructor() {
    super()
    this.recentlyTickKey = "compressed:flows:lastest:ts"
    this.step = 60 * 60 // one hour
    this.maxInterval = 24 * 60 * 60 // 24 hours
  }

  async run() {
    sem.once('IPTABLES_READY', async () => {
      await this.build()
      setInterval(async () => {
        await this.build()
      }, 30 * 60 * 1000);
    })
  }

  async checkAndCleanMem() {
    let compressedFlowsKeys = await rclient.scanResults(this.getKey("*", "*"), 1000)
    if (compressedFlowsKeys && compressedFlowsKeys.length > 0) {
      compressedFlowsKeys = compressedFlowsKeys.filter(key => key != this.recentlyTickKey).sort((a, b) => {
        const ts1 = a.split(":")[2];
        const ts2 = b.split(":")[2];
        return ts1 > ts2 ? -1 : 2
      })
      let compressedMem = 0
      let delFlag = false
      for (const key of compressedFlowsKeys) {
        if (delFlag) { // delete all earlier keys
          await rclient.delAsync(key);
          continue;
        }
        const mem = Number(await rclient.memoryAsync("usage", key) || 0)
        compressedMem += mem
        if (compressedMem > MAX_MEM) { // accumulate memory size from the latest
          delFlag = true;
          await rclient.delAsync(key);
        }
      }
    }
  }

  async apiRun() {
    extensionManager.onGet("compressedflows", (msg, data) => {
      return this.loadCompressedFlows(data)
    });
  }

  async loadCompressedFlows(options) {
    // options {begin,end}
    let { begin, end } = options
    begin = begin - begin % this.step
    end = end - end % this.step
    log.info(`Load compressed flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
    const compressedFlows = []
    for (let i = 0; i < (end - begin) / this.step; i++) {
      const beginTs = begin + this.step * i
      const endTs = begin + this.step * (i + 1)
      const str = await rclient.getAsync(this.getKey(beginTs, endTs))
      str && compressedFlows.push(str)
    }
    return compressedFlows
  }

  getKey(begin, end) {
    return `compressed:flows:${begin}:${end}`
  }

  async sensorCheck() {
    if (!fc.isFeatureOn(featureName)) {
      log.info(`${featureName} feature is disabled.`)
      return false
    }
    const members = await rclient.smembersAsync("sys:ept:members")
    if (members && members.length > 0) {
      const webEnabled = members.map((m) => {
        try {
          return JSON.parse(m)
        } catch (err) {
          return null
        }
      }).filter((x) => x != null && x.name == 'firewalla_web').length > 0
      if (!webEnabled) {
        log.info(`firewalla_web haven't enabled`)
      }
      return webEnabled
    }
    return false
  }

  async build() {
    try {
      if (! await this.sensorCheck()) return
      const { begin, end } = await this.getBuildingWindow()
      const now = new Date() / 1000
      log.info(`Going to compress flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
      for (let i = 0; i < (end - begin) / this.step; i++) {
        const beginTs = begin + this.step * i
        const endTs = begin + this.step * (i + 1)
        const flows = await this.loadFlows(beginTs, endTs)
        await this.save(beginTs, endTs, flows)
      }
      await rclient.setAsync(this.recentlyTickKey, end)
      log.info(`Compressed flows build complted, cost ${(new Date() / 1000 - now).toFixed(2)}`)
    } catch (e) {
      log.error(`Compress flows error`, e)
    }
  }

  async save(begin, end, flows) { // might save to disk in future
    const base64Str = await this.compress(flows)
    const key = this.getKey(begin, end)
    await rclient.setAsync(key, base64Str)
    await rclient.expireatAsync(key, end + this.maxInterval)
    await this.checkAndCleanMem()
  }

  async getBuildingWindow() {
    const now = new Date() / 1000
    const nowTickTs = now - now % this.step
    let recentlyTickTs = Number(await rclient.getAsync(this.recentlyTickKey) || 0)
    if (nowTickTs - recentlyTickTs > this.maxInterval) {
      recentlyTickTs = nowTickTs - this.maxInterval
    }
    return {
      begin: recentlyTickTs,
      end: nowTickTs
    }
  }

  async loadFlows(begin, end) {
    log.info(`Going to load flows between ${new Date(begin * 1000)} - ${new Date(end * 1000)}`)
    let completed = false
    const options = {
      begin: begin,
      end: end,
      audit: true,
      count: 2000,
      asc: true
    }
    let allFlows = []
    const now = new Date() / 1000
    while (!completed) {
      try {
        const flows = await flowTool.prepareRecentFlows({}, JSON.parse(JSON.stringify(options))) || []
        if (flows.length < options.count) {
          completed = true
        } else {
          options.begin = flows[flows.length - 1].ts
        }
        allFlows = allFlows.concat(flows)
      } catch (e) {
        log.error(`Load flows error`, e)
        completed = true
      }
    }
    log.debug(`get ${allFlows.length} flows cost ${(new Date() / 1000 - now).toFixed(2)} seconds`)
    // debug purpose
    log.debug(`there are ${allFlows.reduce((ac, val) => ac + val.count, 0)} zeek logs for these flows`)
    return allFlows
  }

  mergeFlows(flows) {
    if (!flows || flows.length == 0) return [];
    let stash = flows[0];
    const mergedFlows = [stash];
    const compareKeys = ["ltype", "fd", "device", "protocol", "host", "ip", "domain"]
    for (var i = 1; i < flows.length; i++) {
      const flow = flows[i]
      if (_.isEqual(_.pick(stash, compareKeys), _.pick(flow, compareKeys))) {
        stash.count += flow.count
        stash.download += flow.download
        stash.upload += flow.upload
        stash.duration += flow.duration
      } else {
        stash = flow
        mergedFlows.push(stash)
      }
    }
    return mergedFlows
  }
  async compress(flows) {
    const mergedFlows = this.mergeFlows(flows)
    const str = JSON.stringify(mergedFlows)
    const deflateBuffer = await deflateAsync(str)
    const base64Str = deflateBuffer.toString('base64')
    log.debug(`Compress ${mergedFlows.length} flows, raw: ${str.length} deflate: ${deflateBuffer.length} base64:${base64Str.length}`)
    return base64Str
  }
}

module.exports = FlowCompressionSensor;<|endoftext|>


<commit_before>
$( document ).ready(function() {

            /*GLOBAL STATE VARS */
            mode = "none";
            pauseButtonState = "pause";
            hasCalib = false;
            log = [];
            inBetweenSpeech = 5; //seconds
            timeOut = 3; //seconds
            successCount = 0; //count of # of times in bounds
            loopRun = 0;
            loopSuccessRun = 0;
            wasInBounds = false;
            /*END GLOBAL STATE */

            /* BUTTON LISTENERS */

            $("#calibMode").click(function(){
                if(!hasCalib){
                    var msg = new SpeechSynthesisUtterance('Move your cursor and follow it with your eyes');
                    setTimeout(window.speechSynthesis.speak(msg), 2000);
                    msg = new SpeechSynthesisUtterance('Click in various spots while looking at the cursor');
                    setTimeout(window.speechSynthesis.speak(msg), 10000);
                    msg = new SpeechSynthesisUtterance('Make sure you get points on and around the eyes and face');
                    setTimeout(window.speechSynthesis.speak(msg), 10000);
                }
                mode = "calibration";
                webgazer.showPredictionPoints(true);
                $('#gazedot').css('opacity',0.7);
                hasCalib = true;
            });

            $("#useMode").click(function(){
                if(!hasCalib){
                    alert("Please calibrate first");
                    return false;
                }
                else{
                    if(mode !== "use"){
                        loopRun= 0; //reset, so that speech doesn't go bonkers
                    }
                    mode = "use";
                    webgazer.showPredictionPoints(true);
                    $('#gazedot').css('opacity',0);
                }
            });

            $("#pause").click(function(){
                if(pauseButtonState === "pause"){
                    mode = "pause";
                    pauseButtonState = "resume";
                    webgazer.pause();
                    $("#pause").html('RESUME');
                }
                else if(pauseButtonState === "resume"){
                    mode = "resume";
                    pauseButtonState = "pause";
                    webgazer.resume();
                    $("#pause").html('PAUSE');
                }
            });

            $("#devMode").click(function(){
                if(mode !== "developer" && mode !== "forceOut" && mode !== "forceIn"){
                    mode = "developer";
                }
                else{
                    mode = "use";
                }
                $('#gazedot').css('opacity',0.7);
            });

            $("forceIn").click(function(){
                if(mode !== "devMode" && mode !== "forceOut"){
                    alert("You must be in developer mode to use this feature");
                    return false;
                }
                mode = "forceIn";
            });

            $("forceOut").click(function(){
                if(mode !== "devMode" && mode !== "forceOut"){
                    alert("You must be in developer mode to use this feature");
                    return false;
                }
                mode = "forceOut";
            });
            /*END LISTENERS*/

            function getTransform(el) {
                var results = $(el).css('-webkit-transform').match(/matrix(?:(3d)\(\d+(?:, \d+)*(?:, (\d+))(?:, (\d+))(?:, (\d+)), \d+\)|\(\d+(?:, \d+)*(?:, (\d+))(?:, (\d+))\))/)

                if(!results) return /*[0, 0, 0]*/undefined;
                if(results[1] == '3d') return results.slice(2,5);

                results.push(0);
                return results.slice(5, 8);
            }
            setInterval(updatePosDisplay, 100);
            function updatePosDisplay(){
                var transform = getTransform('#gazedot');
                ++loopRun;
                $('#pos').text(transform);
                console.log(transform);
                console.log(isInBounds(transform));
                //$('#isinbounds').text(isInBounds(transform)) removed
                if(isInBounds(transform) && mode !== "forceOut" || mode === "forceIn"){
                    $('#pos').css('background-color','blue');
                    log.push(true);
                    if(mode !== "calibration"){
                        ++loopSuccessRun;
                            if(loopSuccessRun >= timeOut * 10){
                                $('#buttonbox').css('background-color', 'green');
                                console.log("success");
                                document.getElementById('buttonbox').id = 'buttonboximmune';
                            }
                    }
                    if(loopRun % (10*inBetweenSpeech) === 0|| loopRun === 0||!wasInBounds){
                        if(mode !== "calibration"){
                            var msg = new SpeechSynthesisUtterance('Good');
                            window.speechSynthesis.speak(msg);
                        }
                        wasInBounds = true;
                    }
                }
                else if(!isInBounds(transform) && mode !== "forceIn" || mode === "forceOut"){
                    $('#pos').css('background-color','red');
                    log.push(false);
                    if(loopRun % (10*inBetweenSpeech) === 0|| loopRun === 0||wasInBounds){
                        if(mode !== "calibration"){
                            loopSuccessRun = 0;
                            var msg = new SpeechSynthesisUtterance('Make eye contact');
                            window.speechSynthesis.speak(msg);
                        }
                    }
                    wasInBounds = false;
                }
            }

            function isInBounds(coords){
                var w = window.innerWidth;
                var h = window.innerHeight;
                var topBound = .66115702478*h;
                var bottomBound = .38567493112*h;
                var leftBound = .30769230769*w;
                var rightBound = .69336723881*w;
                var left = coords[0];
                var top = coords[1];
                var x = left;
                var y = h- top;
                console.log('x: '+x);
                console.log('y: '+y);
                console.log('bottom: '+bottomBound);
                console.log('top: '+topBound);
                return x >= leftBound && x <= rightBound && y >= bottomBound && y <= topBound;
            }
});<commit_msg>Dev mode - PROBABLY WON'T WORK<commit_after>
$( document ).ready(function() {

            /*GLOBAL STATE VARS */
            mode = "none";
            pauseButtonState = "pause";
            hasCalib = false;
            log = [];
            inBetweenSpeech = 5; //seconds
            timeOut = 3; //seconds
            successCount = 0; //count of # of times in bounds
            loopRun = 0;
            loopSuccessRun = 0;
            wasInBounds = false;
            /*END GLOBAL STATE */

            /* BUTTON LISTENERS */

            $("#calibMode").click(function(){
                if(!hasCalib){
                    var msg = new SpeechSynthesisUtterance('Move your cursor and follow it with your eyes');
                    setTimeout(window.speechSynthesis.speak(msg), 2000);
                    msg = new SpeechSynthesisUtterance('Click in various spots while looking at the cursor');
                    setTimeout(window.speechSynthesis.speak(msg), 10000);
                    msg = new SpeechSynthesisUtterance('Make sure you get points on and around the eyes and face');
                    setTimeout(window.speechSynthesis.speak(msg), 10000);
                }
                mode = "calibration";
                webgazer.showPredictionPoints(true);
                $('#gazedot').css('opacity',0.7);
                hasCalib = true;
            });

            $("#useMode").click(function(){
                if(!hasCalib){
                    alert("Please calibrate first");
                    return false;
                }
                else{
                    if(mode !== "use"){
                        loopRun= 0; //reset, so that speech doesn't go bonkers
                    }
                    mode = "use";
                    webgazer.showPredictionPoints(true);
                    $('#gazedot').css('opacity',0);
                }
            });

            $("#pause").click(function(){
                if(pauseButtonState === "pause"){
                    mode = "pause";
                    pauseButtonState = "resume";
                    webgazer.pause();
                    $("#pause").html('RESUME');
                }
                else if(pauseButtonState === "resume"){
                    mode = "resume";
                    pauseButtonState = "pause";
                    webgazer.resume();
                    $("#pause").html('PAUSE');
                }
            });

            $("#devMode").click(function(){
                if(mode !== "developer" && mode !== "forceOut" && mode !== "forceIn"){
                    mode = "developer";
                    $('#gazedot').css('opacity',0.7);
                }
                else{
                    mode = "use";
                }

            });

            $("forceIn").click(function(){
                if(mode !== "developer" && mode !== "forceOut"){
                    alert("You must be in developer mode to use this feature");
                    return false;
                }
                mode = "forceIn";
            });

            $("forceOut").click(function(){
                if(mode !== "developer" && mode !== "forceOut"){
                    alert("You must be in developer mode to use this feature");
                    return false;
                }
                mode = "forceOut";
            });
            /*END LISTENERS*/

            function getTransform(el) {
                var results = $(el).css('-webkit-transform').match(/matrix(?:(3d)\(\d+(?:, \d+)*(?:, (\d+))(?:, (\d+))(?:, (\d+)), \d+\)|\(\d+(?:, \d+)*(?:, (\d+))(?:, (\d+))\))/)

                if(!results) return /*[0, 0, 0]*/undefined;
                if(results[1] == '3d') return results.slice(2,5);

                results.push(0);
                return results.slice(5, 8);
            }
            setInterval(updatePosDisplay, 100);
            function updatePosDisplay(){
                var transform = getTransform('#gazedot');
                ++loopRun;
                $('#pos').text(transform);
                console.log(transform);
                console.log(isInBounds(transform));
                //$('#isinbounds').text(isInBounds(transform)) removed
                if(isInBounds(transform) && mode !== "forceOut" || mode === "forceIn"){
                    $('#pos').css('background-color','blue');
                    log.push(true);
                    if(mode !== "calibration"){
                        ++loopSuccessRun;
                            if(loopSuccessRun >= timeOut * 10){
                                $('#buttonbox').css('background-color', 'green');
                                console.log("success");
                                document.getElementById('buttonbox').id = 'buttonboximmune';
                            }
                    }
                    if(loopRun % (10*inBetweenSpeech) === 0|| loopRun === 0||!wasInBounds){
                        if(mode !== "calibration"){
                            var msg = new SpeechSynthesisUtterance('Good');
                            window.speechSynthesis.speak(msg);
                        }
                        wasInBounds = true;
                    }
                }
                else if(!isInBounds(transform) && mode !== "forceIn" || mode === "forceOut"){
                    $('#pos').css('background-color','red');
                    log.push(false);
                    if(loopRun % (10*inBetweenSpeech) === 0|| loopRun === 0||wasInBounds){
                        if(mode !== "calibration"){
                            loopSuccessRun = 0;
                            var msg = new SpeechSynthesisUtterance('Make eye contact');
                            window.speechSynthesis.speak(msg);
                        }
                    }
                    wasInBounds = false;
                }
            }

            function isInBounds(coords){
                var w = window.innerWidth;
                var h = window.innerHeight;
                var topBound = .66115702478*h;
                var bottomBound = .38567493112*h;
                var leftBound = .30769230769*w;
                var rightBound = .69336723881*w;
                var left = coords[0];
                var top = coords[1];
                var x = left;
                var y = h- top;
                console.log('x: '+x);
                console.log('y: '+y);
                console.log('bottom: '+bottomBound);
                console.log('top: '+topBound);
                return x >= leftBound && x <= rightBound && y >= bottomBound && y <= topBound;
            }
});<|endoftext|>


<commit_before>import {
  defaultResolveConfig,
  build,
  start,
  watch,
  runTestcafe,
  merge,
  reset
} from 'resolve-scripts'

import resolveModuleAuth from 'resolve-module-auth'
import resolveModuleUploader from 'resolve-module-uploader'

import appConfig from './config.app'
import devConfig from './config.dev'
import prodConfig from './config.prod'
import cloudConfig from './config.cloud'
import testFunctionalConfig from './config.test-functional'

const launchMode = process.argv[2]

void (async () => {
  const moduleUploader = resolveModuleUploader({
    publicDirs: ['images', 'archives'],
    expireTime: 604800,
    jwtSecret: 'SECRETJWT'
  })

  const moduleAuth = resolveModuleAuth([
    {
      name: 'local-strategy',
      createStrategy: 'auth/create-strategy.js',
      logoutRoute: {
        path: 'logout',
        method: 'POST'
      },
      routes: [
        {
          path: 'register',
          method: 'POST',
          callback: 'auth/route-register-callback.js'
        },
        {
          path: 'login',
          method: 'POST',
          callback: 'auth/route-login-callback.js'
        }
      ]
    }
  ])

  switch (launchMode) {
    case 'dev': {
      const resolveConfig = merge(
        defaultResolveConfig,
        appConfig,
        devConfig,
        moduleAuth,
        moduleUploader
      )

      await reset(resolveConfig, {
        dropEventStore: false,
        dropSnapshots: true,
        dropReadModels: true,
        dropSagas: true
      })

      await watch(resolveConfig)
      break
    }

    case 'build': {
      await build(
        merge(defaultResolveConfig, appConfig, prodConfig, moduleAuth)
      )
      break
    }

    case 'start': {
      await start(
        merge(defaultResolveConfig, appConfig, prodConfig, moduleAuth)
      )
      break
    }

    case 'cloud': {
      await build(
        merge(defaultResolveConfig, appConfig, cloudConfig, moduleAuth)
      )
      break
    }

    case 'test:functional': {
      const resolveConfig = merge(
        defaultResolveConfig,
        appConfig,
        testFunctionalConfig
      )

      await reset(resolveConfig, {
        dropEventStore: true,
        dropSnapshots: true,
        dropReadModels: true,
        dropSagas: true
      })

      await runTestcafe({
        resolveConfig,
        functionalTestsDir: 'test/functional',
        browser: process.argv[3]
      })
      break
    }

    default: {
      throw new Error('Unknown option')
    }
  }
})().catch(error => {
  // eslint-disable-next-line no-console
  console.log(error)
  process.exit(1)
})
<commit_msg>Fix run.js<commit_after>import {
  defaultResolveConfig,
  build,
  start,
  watch,
  runTestcafe,
  merge,
  reset
} from 'resolve-scripts'

import resolveModuleAuth from 'resolve-module-auth'
import resolveModuleUploader from 'resolve-module-uploader'

import appConfig from './config.app'
import devConfig from './config.dev'
import prodConfig from './config.prod'
import cloudConfig from './config.cloud'
import testFunctionalConfig from './config.test-functional'

const launchMode = process.argv[2]

void (async () => {
  const moduleUploader = resolveModuleUploader({
    publicDirs: ['images', 'archives'],
    expireTime: 604800,
    jwtSecret: 'SECRETJWT'
  })

  const moduleAuth = resolveModuleAuth([
    {
      name: 'local-strategy',
      createStrategy: 'auth/create-strategy.js',
      logoutRoute: {
        path: 'logout',
        method: 'POST'
      },
      routes: [
        {
          path: 'register',
          method: 'POST',
          callback: 'auth/route-register-callback.js'
        },
        {
          path: 'login',
          method: 'POST',
          callback: 'auth/route-login-callback.js'
        }
      ]
    }
  ])

  switch (launchMode) {
    case 'dev': {
      const resolveConfig = merge(
        defaultResolveConfig,
        appConfig,
        devConfig,
        moduleAuth,
        moduleUploader
      )

      await reset(resolveConfig, {
        dropEventStore: false,
        dropSnapshots: true,
        dropReadModels: true,
        dropSagas: true
      })

      await watch(resolveConfig)
      break
    }

    case 'build': {
      await build(
        merge(
          defaultResolveConfig,
          appConfig,
          prodConfig,
          moduleAuth,
          moduleUploader
        )
      )
      break
    }

    case 'start': {
      await start(
        merge(
          defaultResolveConfig,
          appConfig,
          prodConfig,
          moduleAuth,
          moduleUploader
        )
      )
      break
    }

    case 'cloud': {
      await build(
        merge(
          defaultResolveConfig,
          appConfig,
          cloudConfig,
          moduleAuth,
          moduleUploader
        )
      )
      break
    }

    case 'test:functional': {
      const resolveConfig = merge(
        defaultResolveConfig,
        appConfig,
        testFunctionalConfig
      )

      await reset(resolveConfig, {
        dropEventStore: true,
        dropSnapshots: true,
        dropReadModels: true,
        dropSagas: true
      })

      await runTestcafe({
        resolveConfig,
        functionalTestsDir: 'test/functional',
        browser: process.argv[3]
      })
      break
    }

    default: {
      throw new Error('Unknown option')
    }
  }
})().catch(error => {
  // eslint-disable-next-line no-console
  console.log(error)
  process.exit(1)
})
<|endoftext|>


<commit_before>/*
 * keys.js: Commands related to working with SSH keys
 *
 * (C) 2010, Nodejitsu Inc.
 *
 */

var fs = require('fs'),
    path = require('path'),
    exec = require('child_process').exec,
    common = require('flatiron').common,
    quill = require('../../quill'),
    authorizedKeys = quill.common.authorizedKeys;

var keys = exports;

keys.usage = [
  '`quill keys *` commands allow you to download and update',
  'SSH keys on target machines',
  '',
  'quill keys authorize-all',
  'quill keys authorize <username>',
  'quill keys upload <public-key> <name>',
  'quill keys hostname'
];

//
// ### function hostname (callback)
// #### @callback {function} Continuation to respond to when complete.
// Displays the hostname for the current machine.
//
keys.hostname = function (callback) {
  exec('hostname', function (err, stdout, stderr) {
    if (err) {
      return callback(err);
    }

    var hostname = stdout.split('\n').filter(Boolean)[0];
    quill.log.info(hostname);
    callback(null, hostname);
  });
};

//
// Usage for `quill keys hostname`.
//
keys.hostname.usage = [
  'Displays the hostname of the current machine',
  '',
  'quill keys hostname',
  'quill hostname'
];

//
// ### function authorize-all (callback)
// #### @callback {function} Continuation to respond to when complete.
// Downloads all known SSH public keys to `~/.ssh/authorized_keys`.
//
keys['authorize-all'] = function (callback) {
  quill.log.info('Downloading SSH keys for all users');
  quill.users.getKeys(function (err, keys) {
    if (err) {
      return callback(err, true, true);
    }

    quill.log.info('Downloaded ' + keys.length + ' SSH keys.');
    quill.log.warn('About to overwrite ' + authorizedKeys.filename.magenta);
    quill.prompt.get(['yesno'], function (err, result) {
      if (/n[o]?/.test(result.yesno)) {
        quill.log.warn('Operation cancelled');
        return callback();
      }

      quill.log.warn('Overwriting ' + authorizedKeys.filename.magenta);
      authorizedKeys.overwrite(keys, callback);
    })
  });
};

//
// Usage for `quill keys authorize-all`.
//
keys['authorize-all'].usage = [
  'Downloads all known SSH public keys to `~/.ssh/authorized_keys`',
  '',
  'quill keys authorize-all'
];

//
// ### function authorize (username, callback)
// #### @username {string} Username to download keys for.
// #### @callback {function} Continuation to respond to when complete.
// Appends the SSH public keys for the specified `username` to `~/.ssh/authorized_keys`.
//
keys.authorize = function (username, callback) {
  if (!username) {
    return callback(new Error('username is required to download keys'));
  }

  quill.users.getKeys(username, function (err, keys) {
    if (err) {
      return callback(err, true, true);
    }

    quill.log.info('Downloaded ' + keys.length + ' SSH keys.');
    quill.log.warn('Appending keys to ' + authorizedKeys.filename.magenta);
    authorizedKeys.append(keys, callback);
  });
};

//
// Usage for `quill keys authorize`.
//
keys.authorize.usage = [
  'Appends the SSH public keys for the specified `username` to `~/.ssh/authorized_keys`',
  '',
  'quill keys authorize <username>'
];

keys.upload = function (key, name, callback) {
  name = name || 'publicKey';
  key = path.resolve(key);

  var username = quill.config.get('username');

  quill.log.info('Reading local key: ' + key.magenta);

  fs.readFile(key, 'utf8', function (err, data) {
    if (err) {
      return callback(err);
    }

    quill.log.info('Uploading ' + name.yellow + ' for ' + username.magenta);
    quill.log.info(key.grey);
    quill.users.addKey(username, name, data, callback);
  });
};<commit_msg>[doc] Added code docs for `quill keys upload`<commit_after>/*
 * keys.js: Commands related to working with SSH keys
 *
 * (C) 2010, Nodejitsu Inc.
 *
 */

var fs = require('fs'),
    path = require('path'),
    exec = require('child_process').exec,
    common = require('flatiron').common,
    quill = require('../../quill'),
    authorizedKeys = quill.common.authorizedKeys;

var keys = exports;

keys.usage = [
  '`quill keys *` commands allow you to download and update',
  'SSH keys on target machines',
  '',
  'quill keys authorize-all',
  'quill keys authorize <username>',
  'quill keys upload <public-key> <name>',
  'quill keys hostname'
];

//
// ### function hostname (callback)
// #### @callback {function} Continuation to respond to when complete.
// Displays the hostname for the current machine.
//
keys.hostname = function (callback) {
  exec('hostname', function (err, stdout, stderr) {
    if (err) {
      return callback(err);
    }

    var hostname = stdout.split('\n').filter(Boolean)[0];
    quill.log.info(hostname);
    callback(null, hostname);
  });
};

//
// Usage for `quill keys hostname`.
//
keys.hostname.usage = [
  'Displays the hostname of the current machine',
  '',
  'quill keys hostname',
  'quill hostname'
];

//
// ### function authorize-all (callback)
// #### @callback {function} Continuation to respond to when complete.
// Downloads all known SSH public keys to `~/.ssh/authorized_keys`.
//
keys['authorize-all'] = function (callback) {
  quill.log.info('Downloading SSH keys for all users');
  quill.users.getKeys(function (err, keys) {
    if (err) {
      return callback(err, true, true);
    }

    quill.log.info('Downloaded ' + keys.length + ' SSH keys.');
    quill.log.warn('About to overwrite ' + authorizedKeys.filename.magenta);
    quill.prompt.get(['yesno'], function (err, result) {
      if (/n[o]?/.test(result.yesno)) {
        quill.log.warn('Operation cancelled');
        return callback();
      }

      quill.log.warn('Overwriting ' + authorizedKeys.filename.magenta);
      authorizedKeys.overwrite(keys, callback);
    })
  });
};

//
// Usage for `quill keys authorize-all`.
//
keys['authorize-all'].usage = [
  'Downloads all known SSH public keys to `~/.ssh/authorized_keys`',
  '',
  'quill keys authorize-all'
];

//
// ### function authorize (username, callback)
// #### @username {string} Username to download keys for.
// #### @callback {function} Continuation to respond to when complete.
// Appends the SSH public keys for the specified `username` to `~/.ssh/authorized_keys`.
//
keys.authorize = function (username, callback) {
  if (!username) {
    return callback(new Error('username is required to download keys'));
  }

  quill.users.getKeys(username, function (err, keys) {
    if (err) {
      return callback(err, true, true);
    }

    quill.log.info('Downloaded ' + keys.length + ' SSH keys.');
    quill.log.warn('Appending keys to ' + authorizedKeys.filename.magenta);
    authorizedKeys.append(keys, callback);
  });
};

//
// Usage for `quill keys authorize`.
//
keys.authorize.usage = [
  'Appends the SSH public keys for the specified `username` to `~/.ssh/authorized_keys`',
  '',
  'quill keys authorize <username>'
];

//
// ### function (key, name, callback)
// #### @key {string} File to upload as public key
// #### @name {string} **Optional** Name of the key to upload (e.g. `my-compy`)
// #### @callback {function} Continuation to respond to when complete.
//
// Uploads the local `key` file for the current user to the `name` in conservatory.
// If no `name` is supplied, then `publicKey` will be used.
//
keys.upload = function (key, name, callback) {
  name = name || 'publicKey';
  key = path.resolve(key);

  var username = quill.config.get('username');

  quill.log.info('Reading local key: ' + key.magenta);

  if (path.extname(key) !== '.pub') {
    quill.log.warn('File is not a SSH public key file.');
  }

  fs.readFile(key, 'utf8', function (err, data) {
    if (err) {
      return callback(err);
    }

    quill.log.info('Uploading ' + name.yellow + ' for ' + username.magenta);
    quill.log.info(key.grey);
    quill.users.addKey(username, name, data, callback);
  });
};

//
// Usage for `quill keys upload`.
//
keys.upload.usage = [
  'Uploads the local `key` file for the current user to the `name` in conservatory.',
  'If no `name` is supplied, then `publicKey` will be used.',
  '',
  'quill keys upload <public-key>',
  'quill keys upload <name>'
];<|endoftext|>


<commit_before>import process from 'process'
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
import {execSync} from 'child_process'
import _ from 'lodash'
import gm from 'gm'
import github from 'octonode'

const BUCKET_S3 = 's3://keybase-app-visdiff'
const BUCKET_HTTP = 'http://keybase-app-visdiff.s3.amazonaws.com'
const MAX_INLINE_IMAGES = 6

const DIFF_NEW = 'new'
const DIFF_REMOVED = 'removed'
const DIFF_CHANGED = 'changed'
const DIFF_SAME = 'same'

const DRY_RUN = !!process.env['VISDIFF_DRY_RUN']

function packageHash () {
  return crypto.createHash('sha1').update(fs.readFileSync('package.json')).digest('hex')
}

function checkout (commit) {
  const origPackageHash = packageHash()
  execSync(`rm -rf node_modules.${origPackageHash} && mv node_modules node_modules.${origPackageHash}`)
  console.log(`Shelved node_modules to node_modules.${origPackageHash}.`)

  execSync(`git checkout -f ${commit}`)
  console.log(`Checked out ${commit}.`)

  const newPackageHash = packageHash()
  if (fs.existsSync(`node_modules.${newPackageHash}`)) {
    console.log(`Reusing existing node_modules.${newPackageHash} directory.`)
    execSync(`mv node_modules.${newPackageHash} node_modules`)
  } else {
    console.log(`Installing dependencies for package.json:${newPackageHash}...`)
    execSync('../packaging/npm_mess.sh', {stdio: 'inherit'})
  }
}

function renderScreenshots (commitRange) {
  for (const commit of commitRange) {
    checkout(commit)
    console.log(`Rendering screenshots of ${commit}`)
    execSync(`mkdir -p screenshots/${commit} && npm run render-screenshots -- screenshots/${commit}`, {stdio: 'inherit'})
  }
}

function compareScreenshots (commitRange, diffDir, callback) {
  const results = {}

  execSync(`mkdir -p screenshots/${diffDir}`)

  const files0 = fs.readdirSync(`screenshots/${commitRange[0]}`)
  const files1 = fs.readdirSync(`screenshots/${commitRange[1]}`)
  const files = _.union(files0, files1)
  function compareNext () {
    const filename = files.pop()
    if (!filename) {
      callback(commitRange, results)
      return
    }

    if (filename.startsWith('.')) {
      return
    }

    const diffPath = `screenshots/${diffDir}/${filename}`

    const oldPath = `screenshots/${commitRange[0]}/${filename}`
    if (!fs.existsSync(oldPath)) {
      results[diffPath] = DIFF_NEW
      compareNext()
      return
    }

    const newPath = `screenshots/${commitRange[1]}/${filename}`
    if (!fs.existsSync(newPath)) {
      results[diffPath] = DIFF_REMOVED
      compareNext()
      return
    }

    const compareOptions = {
      tolerance: 1e-6,  // leave a little wiggle room for antialiasing inconsistencies
      file: diffPath
    }
    gm.compare(oldPath, newPath, compareOptions, (err, isEqual) => {
      if (err) {
        console.log(err)
        process.exit(1)
      }
      results[diffPath] = isEqual ? DIFF_SAME : DIFF_CHANGED
      compareNext()
    })
  }
  compareNext()
}

function processDiff (commitRange, results) {
  const changedResults = []
  const newResults = []
  const removedResults = []
  Object.keys(results).forEach(filePath => {
    const result = results[filePath]
    if (result === DIFF_SAME) {
      // clean up results of identical diffs so they don't get uploaded to S3
      fs.unlinkSync(filePath)
    } else {
      // a difference was detected. include the before and after images, if they exist, in our upload set
      const filenameParts = path.parse(filePath, '.png')
      for (const commit of commitRange) {
        const fromFile = `screenshots/${commit}/${filenameParts.base}`
        if (fs.existsSync(fromFile)) {
          execSync(`cp ${fromFile} ${filenameParts.dir}/${filenameParts.name}-${commit}${filenameParts.ext}`)
        }
      }

      if (result === DIFF_CHANGED) {
        changedResults.push(filenameParts.name)
      } else if (result === DIFF_NEW) {
        newResults.push(filenameParts.name)
      } else if (result === DIFF_REMOVED) {
        removedResults.push(filenameParts.name)
      }
    }
  })

  const commentLines = []
  let imageCount = 0

  newResults.forEach(name => {
    const afterURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[1]}.png`
    const line = ` * Added: **${name}**`
    if (imageCount > MAX_INLINE_IMAGES) {
      commentLines.push(line + ` [(view)](${afterURL})`)
    } else {
      commentLines.push(line + '  ')
      commentLines.push(`   ![${name} rendered](${afterURL})`)
      imageCount++
    }
  })

  changedResults.forEach(name => {
    const diffURL = `${BUCKET_HTTP}/${diffDir}/${name}.png`
    const beforeURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[0]}.png`
    const afterURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[1]}.png`
    const line = ` * Changed: **${name}** [(before)](${beforeURL}) [(after)](${afterURL})`
    if (imageCount > MAX_INLINE_IMAGES) {
      commentLines.push(line + ` [(diff)](${diffURL})`)
    } else {
      commentLines.push(line + '  ')
      commentLines.push(`   ![${name} comparison](${diffURL})`)
      imageCount++
    }
  })

  removedResults.forEach(name => {
    const beforeURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[0]}.png`
    commentLines.push(` * Removed: **${name}** [(view original)](${beforeURL})`)
  })

  if (commentLines.length > 0) {
    commentLines.unshift(':mag_right: These commits introduced some visual changes:')
    const commentBody = commentLines.join('\n')

    if (!DRY_RUN) {
      const s3Env = {
        ...process.env,
        AWS_ACCESS_KEY_ID: process.env['VISDIFF_AWS_ACCESS_KEY_ID'],
        AWS_SECRET_ACCESS_KEY: process.env['VISDIFF_AWS_SECRET_ACCESS_KEY']
      }
      console.log(`Uploading ${diffDir} to ${BUCKET_S3}...`)
      execSync(`s3cmd put --acl-public -r screenshots/${diffDir} ${BUCKET_S3}`, {env: s3Env})
      console.log('Screenshots uploaded.')

      var ghClient = github.client(process.env['VISDIFF_GH_TOKEN'])
      var ghIssue = ghClient.issue('keybase/client', process.env['VISDIFF_PR_ID'])
      ghIssue.createComment({body: commentBody}, (err, res) => {
        if (err) {
          console.log('Failed to post visual diff on GitHub:', err.toString(), err.body)
          process.exit(1)
        }
        console.log('Posted visual diff on GitHub:', res.html_url)
      })
    } else {
      console.log(commentBody)
    }
  } else {
    console.log('No visual changes found as a result of these commits.')
  }
}

if (process.argv.length !== 3) {
  console.log(`Usage: node ${path.basename(process.argv[1])} COMMIT1..COMMIT2`)
  process.exit(1)
}

const commitRange = process.argv[2]
  .split(/\.{2,3}/)  // Travis gives us ranges like START...END
  .map(s => s.substr(0, 12))  // trim the hashes a bit for shorter paths

console.log(`Performing visual diff of ${commitRange[0]}...${commitRange[1]}:`)
const diffDir = `${Date.now()}-${commitRange[0]}-${commitRange[1]}`
renderScreenshots(commitRange)
compareScreenshots(commitRange, diffDir, processDiff)
<commit_msg>visdiff: Include commit range checked in comment<commit_after>import process from 'process'
import fs from 'fs'
import path from 'path'
import crypto from 'crypto'
import {execSync} from 'child_process'
import _ from 'lodash'
import gm from 'gm'
import github from 'octonode'

const BUCKET_S3 = 's3://keybase-app-visdiff'
const BUCKET_HTTP = 'http://keybase-app-visdiff.s3.amazonaws.com'
const MAX_INLINE_IMAGES = 6

const DIFF_NEW = 'new'
const DIFF_REMOVED = 'removed'
const DIFF_CHANGED = 'changed'
const DIFF_SAME = 'same'

const DRY_RUN = !!process.env['VISDIFF_DRY_RUN']

function packageHash () {
  return crypto.createHash('sha1').update(fs.readFileSync('package.json')).digest('hex')
}

function checkout (commit) {
  const origPackageHash = packageHash()
  execSync(`rm -rf node_modules.${origPackageHash} && mv node_modules node_modules.${origPackageHash}`)
  console.log(`Shelved node_modules to node_modules.${origPackageHash}.`)

  execSync(`git checkout -f ${commit}`)
  console.log(`Checked out ${commit}.`)

  const newPackageHash = packageHash()
  if (fs.existsSync(`node_modules.${newPackageHash}`)) {
    console.log(`Reusing existing node_modules.${newPackageHash} directory.`)
    execSync(`mv node_modules.${newPackageHash} node_modules`)
  } else {
    console.log(`Installing dependencies for package.json:${newPackageHash}...`)
    execSync('../packaging/npm_mess.sh', {stdio: 'inherit'})
  }
}

function renderScreenshots (commitRange) {
  for (const commit of commitRange) {
    checkout(commit)
    console.log(`Rendering screenshots of ${commit}`)
    execSync(`mkdir -p screenshots/${commit} && npm run render-screenshots -- screenshots/${commit}`, {stdio: 'inherit'})
  }
}

function compareScreenshots (commitRange, diffDir, callback) {
  const results = {}

  execSync(`mkdir -p screenshots/${diffDir}`)

  const files0 = fs.readdirSync(`screenshots/${commitRange[0]}`)
  const files1 = fs.readdirSync(`screenshots/${commitRange[1]}`)
  const files = _.union(files0, files1)
  function compareNext () {
    const filename = files.pop()
    if (!filename) {
      callback(commitRange, results)
      return
    }

    if (filename.startsWith('.')) {
      return
    }

    const diffPath = `screenshots/${diffDir}/${filename}`

    const oldPath = `screenshots/${commitRange[0]}/${filename}`
    if (!fs.existsSync(oldPath)) {
      results[diffPath] = DIFF_NEW
      compareNext()
      return
    }

    const newPath = `screenshots/${commitRange[1]}/${filename}`
    if (!fs.existsSync(newPath)) {
      results[diffPath] = DIFF_REMOVED
      compareNext()
      return
    }

    const compareOptions = {
      tolerance: 1e-6,  // leave a little wiggle room for antialiasing inconsistencies
      file: diffPath
    }
    gm.compare(oldPath, newPath, compareOptions, (err, isEqual) => {
      if (err) {
        console.log(err)
        process.exit(1)
      }
      results[diffPath] = isEqual ? DIFF_SAME : DIFF_CHANGED
      compareNext()
    })
  }
  compareNext()
}

function processDiff (commitRange, results) {
  const changedResults = []
  const newResults = []
  const removedResults = []
  Object.keys(results).forEach(filePath => {
    const result = results[filePath]
    if (result === DIFF_SAME) {
      // clean up results of identical diffs so they don't get uploaded to S3
      fs.unlinkSync(filePath)
    } else {
      // a difference was detected. include the before and after images, if they exist, in our upload set
      const filenameParts = path.parse(filePath, '.png')
      for (const commit of commitRange) {
        const fromFile = `screenshots/${commit}/${filenameParts.base}`
        if (fs.existsSync(fromFile)) {
          execSync(`cp ${fromFile} ${filenameParts.dir}/${filenameParts.name}-${commit}${filenameParts.ext}`)
        }
      }

      if (result === DIFF_CHANGED) {
        changedResults.push(filenameParts.name)
      } else if (result === DIFF_NEW) {
        newResults.push(filenameParts.name)
      } else if (result === DIFF_REMOVED) {
        removedResults.push(filenameParts.name)
      }
    }
  })

  const commentLines = []
  let imageCount = 0

  newResults.forEach(name => {
    const afterURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[1]}.png`
    const line = ` * Added: **${name}**`
    if (imageCount > MAX_INLINE_IMAGES) {
      commentLines.push(line + ` [(view)](${afterURL})`)
    } else {
      commentLines.push(line + '  ')
      commentLines.push(`   ![${name} rendered](${afterURL})`)
      imageCount++
    }
  })

  changedResults.forEach(name => {
    const diffURL = `${BUCKET_HTTP}/${diffDir}/${name}.png`
    const beforeURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[0]}.png`
    const afterURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[1]}.png`
    const line = ` * Changed: **${name}** [(before)](${beforeURL}) [(after)](${afterURL})`
    if (imageCount > MAX_INLINE_IMAGES) {
      commentLines.push(line + ` [(diff)](${diffURL})`)
    } else {
      commentLines.push(line + '  ')
      commentLines.push(`   ![${name} comparison](${diffURL})`)
      imageCount++
    }
  })

  removedResults.forEach(name => {
    const beforeURL = `${BUCKET_HTTP}/${diffDir}/${name}-${commitRange[0]}.png`
    commentLines.push(` * Removed: **${name}** [(view original)](${beforeURL})`)
  })

  if (commentLines.length > 0) {
    commentLines.unshift(`:mag_right: The commits ${commitRange[0]}...${commitRange[1]} introduced some visual changes:`)
    const commentBody = commentLines.join('\n')

    if (!DRY_RUN) {
      const s3Env = {
        ...process.env,
        AWS_ACCESS_KEY_ID: process.env['VISDIFF_AWS_ACCESS_KEY_ID'],
        AWS_SECRET_ACCESS_KEY: process.env['VISDIFF_AWS_SECRET_ACCESS_KEY']
      }
      console.log(`Uploading ${diffDir} to ${BUCKET_S3}...`)
      execSync(`s3cmd put --acl-public -r screenshots/${diffDir} ${BUCKET_S3}`, {env: s3Env})
      console.log('Screenshots uploaded.')

      var ghClient = github.client(process.env['VISDIFF_GH_TOKEN'])
      var ghIssue = ghClient.issue('keybase/client', process.env['VISDIFF_PR_ID'])
      ghIssue.createComment({body: commentBody}, (err, res) => {
        if (err) {
          console.log('Failed to post visual diff on GitHub:', err.toString(), err.body)
          process.exit(1)
        }
        console.log('Posted visual diff on GitHub:', res.html_url)
      })
    } else {
      console.log(commentBody)
    }
  } else {
    console.log('No visual changes found as a result of these commits.')
  }
}

if (process.argv.length !== 3) {
  console.log(`Usage: node ${path.basename(process.argv[1])} COMMIT1..COMMIT2`)
  process.exit(1)
}

const commitRange = process.argv[2]
  .split(/\.{2,3}/)  // Travis gives us ranges like START...END
  .map(s => s.substr(0, 12))  // trim the hashes a bit for shorter paths

console.log(`Performing visual diff of ${commitRange[0]}...${commitRange[1]}:`)
const diffDir = `${Date.now()}-${commitRange[0]}-${commitRange[1]}`
renderScreenshots(commitRange)
compareScreenshots(commitRange, diffDir, processDiff)
<|endoftext|>


<commit_before>//this is a sample js file that can be served for playback. All js files added
//for playback should be placed in /core/public/js/ and the storyteller server
//will serve these.

//We can use classes here instead of plain functions

async function InitializePlayback()
{
    try {
        const playbackInfo = await Promise.all([
            fetch('/event'),
            fetch('/comment')

        ]);

        const results = await Promise.all([
            playbackInfo[0].json(),
            playbackInfo[1].json()
        ]);

        playbackData.events = results[0];
        playbackData.comments = results[1];

        //displays all comments
        displayAllComments();

        console.log(playbackData.comments);
        playbackData.numEvents = playbackData.events.length;
        AddEventListeners();

        //grab any existing media from the server and display it in the media control modal
        initImageGallery();
    } catch(err) {
        console.log(`Error retrieving data`);
    }

}

function AddEventListeners()
{
    //get the controls
    const stepBackOne = document.getElementById("stepBackOne");
    const stepForwardOne = document.getElementById("stepForwardOne");
    const restartButton = document.getElementById("restartButton");
    const playbackSlider = document.getElementById("playbackSlider");
    const highlightButton = document.getElementById("highlightButton");
    const playPauseButton = document.getElementById("playPauseButton");

    //Get references to the tabs and where the tabs get their content
    const tabsList = document.getElementById("tabsList");
    const tabContent = document.getElementById("tabContent");

    playbackSlider.setAttribute("max", playbackData.numEvents);

    //add event handlers for clicking the buttons
    stepBackOne.addEventListener("click", event => {
        pausePlayback();
        step(-1);
    });

    stepForwardOne.addEventListener("click", event => {
        pausePlayback();
        step(1);
    });

    //The restart will be depreciated soon
    restartButton.addEventListener("click", event => {

        //reset the next event and the slider
        playbackData.nextEventPosition = 0;
        playbackSlider.value = 0;

        //for measuring which approach is faster
        //get the starting timestamp
        const t0 = performance.now();

        editor.setValue("");

        //get the ending timestamp
        const t1 = performance.now();

        //print the duration in ms
        console.log(`Reset took: ${(t1 - t0)} ms`);
    });

    //add event handler to listen for changes to the slider
    playbackSlider.addEventListener("input", event => {
        pausePlayback();
        //DEBUG
        // console.log(`slide: ${playbackSlider.value}`);

        //take the slider value and subtract the next event's position
        step(playbackSlider.value - playbackData.nextEventPosition);
    });

    highlightButton.addEventListener("click", event =>{
        //get the selected code
        //const selection = editor.getSession().getSelection().getRange();
        const selections = editor.getSession().getSelection().getAllRanges();

        //clear any existing highlights
        clearHighlights();

        for (let i = 0; i < selections.length; i++){
            //add the highlight to the selected code
            addHighlight(selections[i].start.row, selections[i].start.column, selections[i].end.row, selections[i].end.column);
        }
    });

    document.querySelector("#addCommentButton").addEventListener("click", event =>{

        var textCommentTextArea = document.querySelector("#textCommentTextArea");

        var commentText = null;
        commentText = textCommentTextArea.value.trim();

        //get all selected ranges from Ace
        //builds an array of ranges
        const ranges = editor.getSession().getSelection().getAllRanges();
        var rangeArray = [];
        for (let i = 0; i < ranges.length; i++){
            var rangeObj = {};
            rangeObj.startRow = ranges[i].start.row
            rangeObj.startColumn = ranges[i].start.column;
            rangeObj.endRow = ranges[i].end.row;
            rangeObj.endColumn = ranges[i].end.column;
            rangeArray.push(rangeObj);
        }

        //get all selected text from Ace
        const selectedText = editor.getSelectedText();

        //get all images associated with this comment
        var commentImages = [];

        //if there was a comment, some selected text, or at least one image
        if (commentText != null || rangeArray.length != 0 || commentImages.length != 0)
        {

            if (playbackData.isPlaying)
            {
                var playButtonClickEvent = new MouseEvent("click",{

                });
                playPauseButton.dispatchEvent(playButtonClickEvent);
            }
            //get the event to playback this comment
            var eventIndex = playbackData.nextEventPosition > 0  ? playbackData.nextEventPosition -1: 0;
            var commentEvent = playbackData.events[eventIndex];

            //create an object that has all of the comment info
            var comment = {
                commentText,
                timestamp: new Date().getTime(),
                displayCommentEvent: commentEvent,
                selectedCodeText: selectedText,
                selectedCodeBlocks: rangeArray,
                imageURLs: [],
                videoURLs: [],
                audioURLs: []
            };

            //determine if any comments already exist for this event
            //if so add the new comment
            //if not create a new array for the comments then add the comments
            if (!playbackData.comments[commentEvent.id]){
                playbackData.comments[commentEvent.id] = [];
            }
            playbackData.comments[commentEvent.id].push(comment);

            //clear out the text area
            textCommentTextArea.value = "";

            sendCommentToServer(comment);

            //display a newly added comment on the current event
            displayAllComments();

            //clear out any images uploaded for this comment
            playbackData.mediaForNewComment = [];

        }

    });

    document.getElementById("handler").addEventListener('mousedown', function (e){
        //add listeners for moving and releasing the drag and disable selection of text
        window.addEventListener('selectstart', disableSelect);
        document.documentElement.addEventListener('mousemove', doDrag, false);
        document.documentElement.addEventListener('mouseup', stopDrag, false);
    });

    //detects key presses
    document.addEventListener('keydown', function(e){
        let keyPressed = e.key;
        let shiftPressed = e.shiftKey;

        if (keyPressed == "ArrowRight"){
            pausePlayback();
            if (shiftPressed){
                //find next event that has a comment
                let targetEvent = -1;
                let commentPositions = Object.keys(playbackData.comments);
                for (let i = playbackData.nextEventPosition; i < playbackData.events.length; i++)
                {

                    for (let j = 0; j < commentPositions.length; j++)
                    {
                        if (playbackData.events[i].id == commentPositions[j])
                        {
                            targetEvent = playbackData.events[i].id;
                            break;
                        }
                    }

                    if (targetEvent != -1)
                        break;
                }

                if (targetEvent < 0)
                {
                    targetEvent = playbackData.events[playbackData.events.length-1].eventSequenceNumber;

                    clearHighlights();
                    step(targetEvent - playbackData.nextEventPosition + 1);
                }
                else{

                    var commentClickEvent = new MouseEvent("click",{

                    });
                    let commentToLoad = document.getElementById(`${targetEvent}-0`);

                    commentToLoad.dispatchEvent(commentClickEvent);
                }

            }
            else{
                step(1);
            }
        }
        else if (keyPressed == "ArrowLeft"){
            pausePlayback();
            if (shiftPressed){
                //find next event that has a comment
                let targetEvent = -1;
                let commentPositions = Object.keys(playbackData.comments);
                for (let i = playbackData.nextEventPosition-2; i >= 0; i--)
                {

                    for (let j = 0; j < commentPositions.length; j++)
                    {
                        if (playbackData.events[i].id == commentPositions[j])
                        {
                            targetEvent = playbackData.events[i].id;
                            break;
                        }
                    }

                    if (targetEvent != -1)
                        break;
                }

                if (targetEvent < 0)
                {
                    targetEvent = 0;

                    clearHighlights();
                    step(targetEvent - playbackData.nextEventPosition + 1);
                }
                else{

                    var commentClickEvent = new MouseEvent("click",{

                    });
                    let commentToLoad = document.getElementById(`${targetEvent}-0`);

                    commentToLoad.dispatchEvent(commentClickEvent);
                }
                //console.log(`currentEvent = ${playbackData.nextEventPosition-1}, targetevent = ${targetEvent}`);

            }
            else{
                step(-1);
            }
        }
    });

    var playPauseInterval = null;
    playPauseButton.addEventListener("click", event =>{
        if (!playbackData.isPlaying && playbackData.nextEventPosition < playbackData.events.length){
            playPauseInterval = setInterval(function() {step(1)}, 300);
            $("i", playPauseButton).toggleClass("fa-play fa-pause");
            playbackData.isPlaying = true;
        }
        else if (playbackData.isPlaying){
            clearInterval(playPauseInterval);
            $("i", playPauseButton).toggleClass("fa-play fa-pause");
            playbackData.isPlaying = false;
        }
    });
}

//send the comment object to the server
async function sendCommentToServer(comment){
    try {
        const fetchConfigData = {
            method: "POST",
            body: JSON.stringify(comment),
            headers: {
                "Content-Type": "application/json"
            }
        };
        const response = await fetch("/comment", fetchConfigData);

        //check the response
        if(response.ok) {
           console.log("Success");
        } else {
            console.log("Error with the response data");
        }

    } catch (error) {

    }
}

function doDrag(e){
    var wrapper = handler.closest('.wrapper');
    var boxA = wrapper.querySelector('.box');

    // Get offset
    var containerOffsetLeft = wrapper.offsetLeft;

    // Get x-coordinate of pointer relative to container
    var pointerRelativeXpos = e.clientX - containerOffsetLeft;

    if (pointerRelativeXpos > screen.width * .1 && pointerRelativeXpos < screen.width * .75) {
        boxA.style.width = e.pageX + 'px';
        boxA.style.flexGrow = 0;
        $('#codePanel').css("width", screen.width - pointerRelativeXpos);
        commentsDiv.style.width = e.pageX + 'px';
    }
}

function stopDrag(event){
    //remove the listeners for dragging movement
    document.documentElement.removeEventListener('mouseup', stopDrag, false);
    document.documentElement.removeEventListener('mousemove', doDrag, false);
    window.removeEventListener('selectstart', disableSelect);
}

//disables mouse selection of text
function disableSelect(event) {
    event.preventDefault();
}<commit_msg>changed button for go to next and previous comment<commit_after>//this is a sample js file that can be served for playback. All js files added
//for playback should be placed in /core/public/js/ and the storyteller server
//will serve these.

//We can use classes here instead of plain functions

async function InitializePlayback()
{
    try {
        const playbackInfo = await Promise.all([
            fetch('/event'),
            fetch('/comment')

        ]);

        const results = await Promise.all([
            playbackInfo[0].json(),
            playbackInfo[1].json()
        ]);

        playbackData.events = results[0];
        playbackData.comments = results[1];

        //displays all comments
        displayAllComments();

        console.log(playbackData.comments);
        playbackData.numEvents = playbackData.events.length;
        AddEventListeners();

        //grab any existing media from the server and display it in the media control modal
        initImageGallery();
    } catch(err) {
        console.log(`Error retrieving data`);
    }

}

function AddEventListeners()
{
    //get the controls
    const stepBackOne = document.getElementById("stepBackOne");
    const stepForwardOne = document.getElementById("stepForwardOne");
    const restartButton = document.getElementById("restartButton");
    const playbackSlider = document.getElementById("playbackSlider");
    const highlightButton = document.getElementById("highlightButton");
    const playPauseButton = document.getElementById("playPauseButton");

    //Get references to the tabs and where the tabs get their content
    const tabsList = document.getElementById("tabsList");
    const tabContent = document.getElementById("tabContent");

    playbackSlider.setAttribute("max", playbackData.numEvents);

    //add event handlers for clicking the buttons
    stepBackOne.addEventListener("click", event => {
        pausePlayback();
        step(-1);
    });

    stepForwardOne.addEventListener("click", event => {
        pausePlayback();
        step(1);
    });

    //The restart will be depreciated soon
    restartButton.addEventListener("click", event => {

        //reset the next event and the slider
        playbackData.nextEventPosition = 0;
        playbackSlider.value = 0;

        //for measuring which approach is faster
        //get the starting timestamp
        const t0 = performance.now();

        editor.setValue("");

        //get the ending timestamp
        const t1 = performance.now();

        //print the duration in ms
        console.log(`Reset took: ${(t1 - t0)} ms`);
    });

    //add event handler to listen for changes to the slider
    playbackSlider.addEventListener("input", event => {
        pausePlayback();
        //DEBUG
        // console.log(`slide: ${playbackSlider.value}`);

        //take the slider value and subtract the next event's position
        step(playbackSlider.value - playbackData.nextEventPosition);
    });

    highlightButton.addEventListener("click", event =>{
        //get the selected code
        //const selection = editor.getSession().getSelection().getRange();
        const selections = editor.getSession().getSelection().getAllRanges();

        //clear any existing highlights
        clearHighlights();

        for (let i = 0; i < selections.length; i++){
            //add the highlight to the selected code
            addHighlight(selections[i].start.row, selections[i].start.column, selections[i].end.row, selections[i].end.column);
        }
    });

    document.querySelector("#addCommentButton").addEventListener("click", event =>{

        var textCommentTextArea = document.querySelector("#textCommentTextArea");

        var commentText = null;
        commentText = textCommentTextArea.value.trim();

        //get all selected ranges from Ace
        //builds an array of ranges
        const ranges = editor.getSession().getSelection().getAllRanges();
        var rangeArray = [];
        for (let i = 0; i < ranges.length; i++){
            var rangeObj = {};
            rangeObj.startRow = ranges[i].start.row
            rangeObj.startColumn = ranges[i].start.column;
            rangeObj.endRow = ranges[i].end.row;
            rangeObj.endColumn = ranges[i].end.column;
            rangeArray.push(rangeObj);
        }

        //get all selected text from Ace
        const selectedText = editor.getSelectedText();

        //get all images associated with this comment
        var commentImages = [];

        //if there was a comment, some selected text, or at least one image
        if (commentText != null || rangeArray.length != 0 || commentImages.length != 0)
        {

            if (playbackData.isPlaying)
            {
                var playButtonClickEvent = new MouseEvent("click",{

                });
                playPauseButton.dispatchEvent(playButtonClickEvent);
            }
            //get the event to playback this comment
            var eventIndex = playbackData.nextEventPosition > 0  ? playbackData.nextEventPosition -1: 0;
            var commentEvent = playbackData.events[eventIndex];

            //create an object that has all of the comment info
            var comment = {
                commentText,
                timestamp: new Date().getTime(),
                displayCommentEvent: commentEvent,
                selectedCodeText: selectedText,
                selectedCodeBlocks: rangeArray,
                imageURLs: [],
                videoURLs: [],
                audioURLs: []
            };

            //determine if any comments already exist for this event
            //if so add the new comment
            //if not create a new array for the comments then add the comments
            if (!playbackData.comments[commentEvent.id]){
                playbackData.comments[commentEvent.id] = [];
            }
            playbackData.comments[commentEvent.id].push(comment);

            //clear out the text area
            textCommentTextArea.value = "";

            sendCommentToServer(comment);

            //display a newly added comment on the current event
            displayAllComments();

            //clear out any images uploaded for this comment
            playbackData.mediaForNewComment = [];

        }

    });

    document.getElementById("handler").addEventListener('mousedown', function (e){
        //add listeners for moving and releasing the drag and disable selection of text
        window.addEventListener('selectstart', disableSelect);
        document.documentElement.addEventListener('mousemove', doDrag, false);
        document.documentElement.addEventListener('mouseup', stopDrag, false);
    });

    //detects key presses
    document.addEventListener('keydown', function(e){
        let keyPressed = e.key;
        let shiftPressed = e.shiftKey;

        if (keyPressed == "ArrowRight"){
            pausePlayback();
            if (!shiftPressed)
            {
                step(1);
            }
        }
        else if (keyPressed == "ArrowLeft"){
            pausePlayback();
            if (!shiftPressed)
            {
                step(-1);
            }
        }
        else if (keyPressed == ">")
        {
            //find next event that has a comment
            let targetEvent = -1;
            let commentPositions = Object.keys(playbackData.comments);
            for (let i = playbackData.nextEventPosition; i < playbackData.events.length; i++)
            {

                for (let j = 0; j < commentPositions.length; j++)
                {
                    if (playbackData.events[i].id == commentPositions[j])
                    {
                        targetEvent = playbackData.events[i].id;
                        break;
                    }
                }

                if (targetEvent != -1)
                    break;
            }

            if (targetEvent < 0)
            {
                targetEvent = playbackData.events[playbackData.events.length-1].eventSequenceNumber;

                clearHighlights();
                step(targetEvent - playbackData.nextEventPosition + 1);
            }
            else{

                var commentClickEvent = new MouseEvent("click",{

                });
                let commentToLoad = document.getElementById(`${targetEvent}-0`);

                commentToLoad.dispatchEvent(commentClickEvent);
            }
        }
        else if (keyPressed == "<")
        {
            //find next event that has a comment
            let targetEvent = -1;
            let commentPositions = Object.keys(playbackData.comments);
            for (let i = playbackData.nextEventPosition-2; i >= 0; i--)
            {

                for (let j = 0; j < commentPositions.length; j++)
                {
                    if (playbackData.events[i].id == commentPositions[j])
                    {
                        targetEvent = playbackData.events[i].id;
                        break;
                    }
                }

                if (targetEvent != -1)
                    break;
            }

            if (targetEvent < 0)
            {
                targetEvent = 0;

                clearHighlights();
                step(targetEvent - playbackData.nextEventPosition + 1);
            }
            else{

                var commentClickEvent = new MouseEvent("click",{

                });
                let commentToLoad = document.getElementById(`${targetEvent}-0`);

                commentToLoad.dispatchEvent(commentClickEvent);
            }
            //console.log(`currentEvent = ${playbackData.nextEventPosition-1}, targetevent = ${targetEvent}`);
        }
    });

    var playPauseInterval = null;
    playPauseButton.addEventListener("click", event =>{
        if (!playbackData.isPlaying && playbackData.nextEventPosition < playbackData.events.length){
            playPauseInterval = setInterval(function() {step(1)}, 300);
            $("i", playPauseButton).toggleClass("fa-play fa-pause");
            playbackData.isPlaying = true;
        }
        else if (playbackData.isPlaying){
            clearInterval(playPauseInterval);
            $("i", playPauseButton).toggleClass("fa-play fa-pause");
            playbackData.isPlaying = false;
        }
    });
}

//send the comment object to the server
async function sendCommentToServer(comment){
    try {
        const fetchConfigData = {
            method: "POST",
            body: JSON.stringify(comment),
            headers: {
                "Content-Type": "application/json"
            }
        };
        const response = await fetch("/comment", fetchConfigData);

        //check the response
        if(response.ok) {
           console.log("Success");
        } else {
            console.log("Error with the response data");
        }

    } catch (error) {

    }
}

function doDrag(e){
    var wrapper = handler.closest('.wrapper');
    var boxA = wrapper.querySelector('.box');

    // Get offset
    var containerOffsetLeft = wrapper.offsetLeft;

    // Get x-coordinate of pointer relative to container
    var pointerRelativeXpos = e.clientX - containerOffsetLeft;

    if (pointerRelativeXpos > screen.width * .1 && pointerRelativeXpos < screen.width * .75) {
        boxA.style.width = e.pageX + 'px';
        boxA.style.flexGrow = 0;
        $('#codePanel').css("width", screen.width - pointerRelativeXpos);
        commentsDiv.style.width = e.pageX + 'px';
    }
}

function stopDrag(event){
    //remove the listeners for dragging movement
    document.documentElement.removeEventListener('mouseup', stopDrag, false);
    document.documentElement.removeEventListener('mousemove', doDrag, false);
    window.removeEventListener('selectstart', disableSelect);
}

//disables mouse selection of text
function disableSelect(event) {
    event.preventDefault();
}<|endoftext|>