<commit_before>const _ = require('lodash');
const Promise = require('bluebird');
const debug = require('debug')('lib:parse');
const moment = require('moment');
const nconf = require('nconf');
const xpath = require('xpath');
const xmldom = require('xmldom');
const JSDOM = require('jsdom').JSDOM;

const mongo = require('./mongo');
const sequence = require('../parsers/components/utils/sequence');
const summarize = require('../parsers/components/utils/summarize');
const echoes = require('./echoes');
const utils = require('./utils');

function checkMetadata(impression, repeat) {
    if(_.isUndefined(impression.id))
        throw new Error("impression missing");

    return mongo
        .readOne(nconf.get('schema').metadata, { id: impression.id })
        .then(function(i) {
            if( _.get(i, 'id') === impression.id && !repeat) {
                debug("metadata [%s] already exists: skipping", i.id);
                return null;
            }

            if( _.get(i, 'id') === impression.id && repeat) {
                debug("metadata [%s] exists, but repeat is requested", i.id);
                return mongo
                    .remove(nconf.get('schema').metadata, { id: impression.id })
                    .return(impression);
            }

            /* else if _.isUndefined(i) is returned the impression */
            return impression;
        });
}

function initialize(impression) {
    const xmlerr = { warning: 0, error: 0, fatal: 0 };
    var domOptions = {
          errorHandler:{
              warning: function() { xmlerr.warning += 1; },
              error: function() { xmlerr.error += 1; },
              fatalError: function() { xmlerr.fatal += 1;  },
          }
    };
    /* this is the envelope with get appended the metadata from the various parsers */
    return {
        impression,
        xmlerr,
        xpath,
        jsdom: new JSDOM(impression.html).window.document,
        dom: new xmldom.DOMParser(domOptions).parseFromString(impression.html),
    };
}

function finalize(envelope) {
    if(!_.size(envelope))
        return { metadata: [], summary: [], errors: [] };

    const errors = _.reduce(envelope, function(memo, e) {
        if(!_.size(e.errors))
            return memo;
        var r = {
            errors: e.errors,
            when:  new Date(),
            id: e.id,
            timelineId: e.timelineId
        }
        memo.push(r);
        return memo;
    }, []);

    const removef = ['dom', 'jsdom', 'xpath', 'xmlerr', 'impression', 'errors'];
    const impressionFields =
        ['id', 'timelineId', 'userId', 'impressionOrder', 'impressionTime'];

    return {
        metadata: _.map(envelope, function(e) {
            return _.extend(
                _.pick(e.impression, impressionFields),
                _.omit(e, removef),
            );
        }),
        summary: _.map(envelope, summarize),
        errors
    };
}

function logSummary(blobs) {
    _.each(blobs.summary, function(e) {
        echoes.echo(
            _.extend({'index': 'parserv' },
            _.pick(e, ['errors', 'type', 'publicationTime', 'postId',
                       'permaLink', 'author', 'textlength', 'impressionTime',
                       'impressionOrder', 'pseudo', 'timeline', 'regexp' ])
            )
        );
    });
    if(nconf.get('fulldump'))
        _.each(blobs.metadata, function(o) {
            console.log(JSON.stringify(o, undefined, 2));
        });
}

function mark(blobs) {
    return Promise.map(blobs.metadata, function(e) {
        return mongo
            .readOne(nconf.get('schema').htmls, { id: e.id })
            .then(function(existing) {
                existing.processed = true;
                return mongo
                    .updateOne(nconf.get('schema').htmls, { _id: existing._id }, existing);
            });
    }, { concurrency: 1});
}

function save(blobs) {
    let chain = [];
    debug("Received %d metadata and %d errors", _.size(blobs.metadata), _.size(blobs.errors));

    if(_.size(blobs.metadata)) {
        chain.push(
            mongo.writeMany(nconf.get('schema').metadata, blobs.metadata)  );
        chain.push(
            mongo.writeMany(nconf.get('schema').summary, blobs.summary)  );
    }

    if(_.size(blobs.errors))
        chain.push(
            mongo.writeMany(nconf.get('schema').errors, blobs.errors)  );

    return Promise
        .map(chain, { concurrency: 1 })
        .return({
            metadata: _.size(blobs.metadata),
            errors: _.size(blobs.errors)
        });
}

function postIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .linkedtime.postId, returns without
     *   creating postCount
     * - postCount { personal, global } gets created always */
    if(!e.linkedtime || !e.linkedtime.postId) {
        debug("postIdCount: postId not present, the nature is [%s]", e.nature);
        return e;
    }

    e.postCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.linkedtime.postId } }),
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.linkedtime.postId }, userId: e.userId })
    ])
    .then(function(results) {
        e.postCount.global = results[0];
        e.postCount.personal = results[1];
        return e;
    });
};

function semanticIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .texts or zero-length texts, it returns
     * - semanticId gets created always
     * - semanticCount { personal, global } gets created always
     * - semantic: true, gets created when the semanticCount.global is zero  */
    if(!_.size(e.texts))
        return e;

    if(!( _.reduce(e.texts, function(memo, o) { return memo || !!_.size(o.text) }, false) )) {
        debug("check to be sure: the texts should be empty");
        debug("%s", JSON.stringify(e.texts));
        return e;
    }

    e.semanticId = utils.hashList(e.texts);
    e.semanticCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId }),
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId, userId: e.userId })
    ])
    .then(function(results) {
        e.semanticCount.global = results[0];
        e.semanticCount.personal = results[1];
        if(!e.semanticCount.global)
            e.semantic = true;
        return e;
    });
};

function mergeHTMLImpression(html) {
    return mongo
        .readOne(nconf.get('schema').impressions, { id: html.impressionId })
        .then(function(impression) {
            _.unset(impression, 'id');
            _.unset(impression, 'htmlId');
            return _.merge(html, impression);
        });
}

function parseHTML(htmlfilter, repeat) {
    return mongo
        .read(nconf.get('schema').htmls, htmlfilter)
        .map(mergeHTMLImpression, { concurrency: 1 })
        .then(_.compact)
        .then(function(impressions) {
            return _.orderBy(impressions, { impressionOrder: -1 });
        })
        .tap(function(impressions) {
            if(_.size(impressions)) {
                const firstT = moment(_.first(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const lastT = moment(_.last(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const humanized = moment
                    .duration( _.last(impressions).impressionTime - _.first(impressions).impressionTime )
                    .humanize();
                debug("Processing %d impressions: %s [%s] %s %s [%s] %s",
                    /*                                    ^^^^^^^^^^^^^ conditionals */
                    _.size(impressions),
                    _.first(impressions).impressionOrder, firstT,
                    _.size(impressions) > 1 ? "-" : "",
                    _.size(impressions) > 1 ? _.last(impressions).impressionOrder : "",
                    _.size(impressions) > 1 ? "[" + lastT + "]" : "",
                    _.size(impressions) > 1 ? "(" + humanized + ")" : ""
                );
            }
        })
        .map(function(e) {
            return checkMetadata(e, repeat);
        }, { concurrency: 1 })
        .then(_.compact)
        .map(initialize)
        .map(sequence, { concurrency: 1 })
        /* this is the function processing the parsers
         * it is call of every .id which should be analyzed */
        .catch(function(error) {
            debug("[E] Unmanaged error in parser sequence: %s", error.message);
            console.log(error.stack);
            return null;
        })
        .tap(function(metadata) {
            debug("⁂  completed %d metadata", _.size(metadata));
            if(!_.isUndefined(nconf.get('verbose')))
                debug("⁂  %s", JSON.stringify(metadata, undefined, 2));
        })
        .map(postIdCount)
        .map(semanticIdCount)
        .then(finalize)
        .tap(logSummary)
        .tap(mark)
        .then(save)
        .catch(function(error) {
            debug("[error after parsing] %s", error.message);
            console.log(error.stack);
            return null;
        });
}

module.exports = {
    checkMetadata: checkMetadata,
    initialize: initialize,
    mergeHTMLImpression: mergeHTMLImpression,
    finalize: finalize,
    logSummary: logSummary,
    save: save,
    postIdCount: postIdCount,
    semanticIdCount: semanticIdCount,
    mark: mark,
    parseHTML: parseHTML,
};
<commit_msg>review sequence for semantic and post count<commit_after>const _ = require('lodash');
const Promise = require('bluebird');
const debug = require('debug')('lib:parse');
const moment = require('moment');
const nconf = require('nconf');
const xpath = require('xpath');
const xmldom = require('xmldom');
const JSDOM = require('jsdom').JSDOM;

const mongo = require('./mongo');
const sequence = require('../parsers/components/utils/sequence');
const summarize = require('../parsers/components/utils/summarize');
const echoes = require('./echoes');
const utils = require('./utils');

function checkMetadata(impression, repeat) {
    if(_.isUndefined(impression.id))
        throw new Error("impression missing");

    return mongo
        .readOne(nconf.get('schema').metadata, { id: impression.id })
        .then(function(i) {
            if( _.get(i, 'id') === impression.id && !repeat) {
                debug("metadata [%s] already exists: skipping", i.id);
                return null;
            }

            if( _.get(i, 'id') === impression.id && repeat) {
                debug("metadata [%s] exists, but repeat is requested", i.id);
                return mongo
                    .remove(nconf.get('schema').metadata, { id: impression.id })
                    .return(impression);
            }

            /* else if _.isUndefined(i) is returned the impression */
            return impression;
        });
}

function initialize(impression) {
    const xmlerr = { warning: 0, error: 0, fatal: 0 };
    var domOptions = {
          errorHandler:{
              warning: function() { xmlerr.warning += 1; },
              error: function() { xmlerr.error += 1; },
              fatalError: function() { xmlerr.fatal += 1;  },
          }
    };
    /* this is the envelope with get appended the metadata from the various parsers */
    return {
        impression,
        xmlerr,
        xpath,
        jsdom: new JSDOM(impression.html).window.document,
        dom: new xmldom.DOMParser(domOptions).parseFromString(impression.html),
    };
}

function finalize(envelope) {
    if(!_.size(envelope))
        return { metadata: [], summary: [], errors: [] };

    const errors = _.reduce(envelope, function(memo, e) {
        if(!_.size(e.errors))
            return memo;
        var r = {
            errors: e.errors,
            when:  new Date(),
            id: e.id,
            timelineId: e.timelineId
        }
        memo.push(r);
        return memo;
    }, []);

    const removef = ['dom', 'jsdom', 'xpath', 'xmlerr', 'impression', 'errors'];
    const impressionFields =
        ['id', 'timelineId', 'userId', 'impressionOrder', 'impressionTime'];

    return {
        metadata: _.map(envelope, function(e) {
            return _.extend(
                _.pick(e.impression, impressionFields),
                _.omit(e, removef),
            );
        }),
        summary: _.map(envelope, summarize),
        errors
    };
}

function logSummary(blobs) {
    _.each(blobs.summary, function(e) {
        echoes.echo(
            _.extend({'index': 'parserv' },
            _.pick(e, ['errors', 'type', 'publicationTime', 'postId',
                       'permaLink', 'author', 'textlength', 'impressionTime',
                       'impressionOrder', 'pseudo', 'timeline', 'regexp' ])
            )
        );
    });
    if(nconf.get('fulldump'))
        _.each(blobs.metadata, function(o) {
            console.log(JSON.stringify(o, undefined, 2));
        });
}

function mark(blobs) {
    return Promise.map(blobs.metadata, function(e) {
        return mongo
            .readOne(nconf.get('schema').htmls, { id: e.id })
            .then(function(existing) {
                existing.processed = true;
                return mongo
                    .updateOne(nconf.get('schema').htmls, { _id: existing._id }, existing);
            });
    }, { concurrency: 1});
}

function save(blobs) {
    let chain = [];
    debug("Received %d metadata and %d errors", _.size(blobs.metadata), _.size(blobs.errors));

    if(_.size(blobs.metadata)) {
        debugger;
        chain.push(
            mongo.writeMany(nconf.get('schema').metadata, blobs.metadata)  );
        chain.push(
            mongo.writeMany(nconf.get('schema').summary, blobs.summary)  );
    }

    if(_.size(blobs.errors))
        chain.push(
            mongo.writeMany(nconf.get('schema').errors, blobs.errors)  );

    return Promise
        .all(chain)
        .return({
            metadata: _.size(blobs.metadata),
            errors: _.size(blobs.errors)
        });
}

function postIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .linkedtime.postId, returns without
     *   creating postCount
     * - postCount { personal, global } gets created always */
    if(!e.postId) {
        debug("postIdCount: postId not present, the nature is [%s]", e.nature);
        return e;
    }

    e.postCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId } }),
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId }, userId: e.userId })
    ])
    .then(function(results) {
        e.postCount.global = results[0];
        e.postCount.personal = results[1];
        return e;
    });
};

function semanticIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .texts or zero-length texts, it returns
     * - semanticId gets created always
     * - semanticCount { personal, global } gets created always
     * - semantic: true, gets created when the semanticCount.global is zero  */

    if(!e.fullTextSize)
        return e;

    e.semanticId = utils.hash({ text: e.fullText });
    e.semanticCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId }),
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId, userId: e.userId })
    ])
    .then(function(results) {
        e.semanticCount.global = results[0];
        e.semanticCount.personal = results[1];
        if(!e.semanticCount.global)
            e.semantic = true;
        return e;
    });
};

function mergeHTMLImpression(html) {
    return mongo
        .readOne(nconf.get('schema').impressions, { id: html.impressionId })
        .then(function(impression) {
            _.unset(impression, 'id');
            _.unset(impression, 'htmlId');
            return _.merge(html, impression);
        });
}

function parseHTML(htmlfilter, repeat) {
    return mongo
        .read(nconf.get('schema').htmls, htmlfilter)
        .map(mergeHTMLImpression, { concurrency: 1 })
        .then(_.compact)
        .then(function(impressions) {
            return _.orderBy(impressions, { impressionOrder: -1 });
        })
        .tap(function(impressions) {
            if(_.size(impressions)) {
                const firstT = moment(_.first(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const lastT = moment(_.last(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const humanized = moment
                    .duration( _.last(impressions).impressionTime - _.first(impressions).impressionTime )
                    .humanize();
                debug("Processing %d impressions: %s [%s] %s %s [%s] %s",
                    /*                                    ^^^^^^^^^^^^^ conditionals */
                    _.size(impressions),
                    _.first(impressions).impressionOrder, firstT,
                    _.size(impressions) > 1 ? "-" : "",
                    _.size(impressions) > 1 ? _.last(impressions).impressionOrder : "",
                    _.size(impressions) > 1 ? "[" + lastT + "]" : "",
                    _.size(impressions) > 1 ? "(" + humanized + ")" : ""
                );
            }
        })
        .map(function(e) {
            return checkMetadata(e, repeat);
        }, { concurrency: 1 })
        .then(_.compact)
        .map(initialize)
        .map(sequence, { concurrency: 1 })
        /* this is the function processing the parsers
         * it is call of every .id which should be analyzed */
        .catch(function(error) {
            debug("[E] Unmanaged error in parser sequence: %s", error.message);
            console.log(error.stack);
            return null;
        })
        .tap(function(metadata) {
            debug("⁂  completed %d metadata", _.size(metadata));
            if(!_.isUndefined(nconf.get('verbose')))
                debug("⁂  %s", JSON.stringify(metadata, undefined, 2));
        })
        .map(postIdCount)
        .map(semanticIdCount)
        .then(finalize)
        .tap(logSummary)
        .tap(mark)
        .then(save)
        .catch(function(error) {
            debug("[error after parsing] %s", error.message);
            console.log(error.stack);
            return null;
        });
}

module.exports = {
    checkMetadata: checkMetadata,
    initialize: initialize,
    mergeHTMLImpression: mergeHTMLImpression,
    finalize: finalize,
    logSummary: logSummary,
    save: save,
    postIdCount: postIdCount,
    semanticIdCount: semanticIdCount,
    mark: mark,
    parseHTML: parseHTML,
};
<|endoftext|>


<commit_before>'use strict';

/**
 * Attempts to recursively convert object properties to dates.
 * @param  {Object}  obj  - Object to iterate over
 * @return {Object}  Returns new object (shallow copy).
*/
function parse(obj) {
  var result = {},
      key,
      value;

  for (key in obj) {
    if (obj.hasOwnProperty(key)) {
      value = obj[key];

      if (typeof value === 'string') {
        if (value === 'true') {
          result[key] = true;
        }
        else if (value === 'false') {
          result[key] = false;
        }
        else {
          result[key] = value;
        }
      }
      else if (value.constructor === Object) {
        result[key] = parse(value);
      }
      else if (Array.isArray(value)) {
        value.forEach(function(item, itemKey) {
          value[itemKey] = parse(item);
        });
        result[key] = value;
      }
      else {
        result[key] = value;
      }
    }
  }

  return result;
}

module.exports = parse;
<commit_msg>:fire: rewrite to be more clear and allow function reuse<commit_after>'use strict';
/* jshint latedef: nofunc */
/* NOTE: this is because fo the recursive nature of the functions here
    we must be able to call parseObject from inside parseValue,
    if the value is an object
*/

/**
 * Convert string to boolean.
 * @param  {String}  string  - String to convert
 * @return {?}  Returns the results of the conversion.
*/
function parseBoolFromString(string) {
  if (string === 'true') {
    return true;
  }
  else if (string === 'false') {
    return false;
  }
  else {
    return string;
  }
}

/**
 * Recursively test values for conversion.
 * @param  {?}  value  - String to convert
 * @return {?}  Returns the results of the conversion.
*/
function parseValue(value) {
  if (typeof value === 'string') {
    return parseBoolFromString(value);
  }
  else if (value.constructor === Object) {
    return parseObject(value);
  }
  else if (Array.isArray(value)) {
    var array = [];
    value.forEach(function(item, itemKey) {
      array[itemKey] = parseValue(item);
    });
    return array;
  }
  else {
    return value;
  }
}

/**
 * Recursively convert object strings to boolean.
 * @param  {Object}  obj  - Object to iterate over
 * @return {Object}  Returns new object (shallow copy).
*/
function parseObject(obj) {
  var result = {},
      key,
      value;

  for (key in obj) {
    if (obj.hasOwnProperty(key)) {
      value = obj[key];
      result[key] = parseValue(value);
    }
  }

  return result;
}

module.exports = parseObject;
<|endoftext|>


<commit_before>const _ = require('lodash');
const Promise = require('bluebird');
const debug = require('debug')('lib:parse');
const moment = require('moment');
const nconf = require('nconf');
const xpath = require('xpath');
const xmldom = require('xmldom');
const JSDOM = require('jsdom').JSDOM;

const mongo = require('./mongo');
const sequence = require('../parsers/components/utils/sequence');
const summarize = require('../parsers/components/utils/summarize');
const parserDebug = require('../parsers/components/utils/parserDebug');
const echoes = require('./echoes');
const utils = require('./utils');
const aggregated = require('./aggregated');
const glue = require('./glue');

function checkMetadata(impression, repeat) {
    if(_.isUndefined(impression.id))
        throw new Error("impression missing");

    return mongo
        .readOne(nconf.get('schema').metadata, { id: impression.id })
        .then(function(i) {
            if( _.get(i, 'id') === impression.id && !repeat) {
                debug("metadata [%s] already exists: skipping", i.id);
                return null;
            }

            if( _.get(i, 'id') === impression.id && repeat) {
                debug("metadata [%s] exists, but repeat is requested", i.id);
                return Promise.all([
                    mongo.remove(nconf.get('schema').metadata, { id: impression.id }),
                    mongo.remove(nconf.get('schema').summary, { id: impression.id })
                ])
                .return(impression);
            }

            /* else if _.isUndefined(i) is returned the impression */
            return impression;
        });
}

function initialize(impression) {
    const xmlerr = { warning: 0, error: 0, fatal: 0 };
    var domOptions = {
          errorHandler:{
              warning: function() { xmlerr.warning += 1; },
              error: function() { xmlerr.error += 1; },
              fatalError: function() { xmlerr.fatal += 1;  },
          }
    };
    /* this is the envelope with get appended the metadata from the various parsers */
    return {
        impression,
        xmlerr,
        xpath,
        jsdom: new JSDOM(impression.html).window.document,
        dom: new xmldom.DOMParser(domOptions).parseFromString(impression.html),
    };
}

function finalize(envelopes) {
    if(!_.size(envelopes))
        return { metadata: [], summary: [], statistics: [], parsers: [], errors: [] };

    /* here is build the new complex object which travels in the pipeline,
     * metadata & parsers are the internal content, not meant to be public */

    const errors = _.reduce(envelopes, function(memo, e) {
        if(!_.size(e.errors))
            return memo;
        var r = {
            errors: e.errors,
            when:  new Date(),
            id: e.impression.id,
            timelineId: e.impression.timelineId
        }
        memo.push(r);
        return memo;
    }, []);

    const removef = ['dom', 'jsdom', 'xpath', 'xmlerr', 'impression', 'errors'];
    const impressionFields =
        ['id', 'timelineId', 'userId', 'impressionOrder', 'impressionTime'];

    const metadata = _.map(envelopes, function(e) {
        return _.extend(
            _.pick(e.impression, impressionFields),
            _.omit(e, removef),
            { when: new Date() }
        );
    });
    const parsers = _.map(metadata, parserDebug);

    /* summary is the format used for user-facing API */
    const summary = _.map(metadata, summarize);
    /* statistics are the public information on how fbtrex is doing */
    const statistics = _.map(summary, aggregated.computeStats);

    return {
        metadata,
        parsers,
        summary,
        statistics,
        errors
    };
}

function logSummary(blobs) {
    _.each(blobs.summary, function(e) {
        echoes.echo(
            _.extend({'index': 'parserv' },
            _.pick(e, ['errors', 'type', 'publicationTime', 'postId',
                       'permaLink', 'author', 'textlength', 'impressionTime',
                       'impressionOrder', 'pseudo', 'timeline', 'regexp' ])
            )
        );
    });

    /* the `fulldump` is set by executers or by `parsers/precise.js` */
    if(nconf.get('fulldump'))
        _.times(_.size(blobs.metadata), function(o, i) {
            let s = _.nth(blobs.summary, i);
            console.log(JSON.stringify(s, undefined, 1));
            console.log("\x1b[36m");
            let m = _.nth(blobs.metadata, i);
            console.log(JSON.stringify(m, undefined, 1));
        });

    /* this is dumped even without fulldump */
    const E = "\x1b[47m\x1b[31m";
    _.each(blobs.errors, function(o) {
        console.log(E, JSON.stringify(o, undefined, 2));
    });
}

function mark(blobs) {
    return Promise.map(blobs.metadata, function(e) {
        return mongo
            .readOne(nconf.get('schema').htmls, { id: e.id })
            .then(function(existing) {
                existing.processed = true;
                return mongo
                    .updateOne(nconf.get('schema').htmls, { _id: existing._id }, existing);
            });
    }, { concurrency: 1});
}

function save(blobs) {
    let chain = [];
    debug("Received %d metadata and %d errors", _.size(blobs.metadata), _.size(blobs.errors));

    if(_.size(blobs.metadata))
        chain.push(
            mongo.writeMany(nconf.get('schema').metadata, blobs.metadata)  );

    if(_.size(blobs.metadata) && !nconf.get('onlymetadata') ) {
        chain.push(
            mongo.writeMany(nconf.get('schema').summary, blobs.summary)  );
        chain.push(
            aggregated.updateHourly(blobs.statistics)   );
        /*
         - still unclear how and what we should records
        chain.push(
            mongo.writeMany(nconf.get('schema').parsers, blobs.parsers)   );
         */
    }

    if(_.size(blobs.errors) && !nconf.get('onlymetadata') )
        chain.push(
            mongo.writeMany(nconf.get('schema').errors, blobs.errors)  );

    return Promise
        .all(chain)
        .return({
            metadata: _.size(blobs.metadata),
            errors: _.size(blobs.errors)
        })
        .catch(function(error) {
            console.log("error in saving:", error.message);
            return null;
        });
}

function postIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * - postId is build at the end of `sequence`
     * - postCount { personal, global } gets created always */

    // this is slowing too much the parsing, I'm going to removed it
    // --- sadClaudio

    if(!e.postId)
        return e;

    e.postCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId } }),
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId }, userId: e.userId })
    ])
    .then(function(results) {
        e.postCount.global = results[0];
        e.postCount.personal = results[1];
        return e;
    });
};

function semanticIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .texts or zero-length texts, it returns
     * - semanticId gets created always
     * - semanticCount { personal, global } gets created always
     * - semantic: true, gets created when the semanticCount.global is zero  */

    if(!e.fullTextSize)
        return e;

    e.semanticId = utils.hash({ text: e.fullText });
    e.semanticCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId }),
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId, userId: e.userId })
    ])
    .then(function(results) {
        e.semanticCount.global = results[0];
        e.semanticCount.personal = results[1];

        /* only the first metadata with a due semanticId is marked with `semantic`,
         * this will be processed later by lib/semantic.js */
        if(!e.semanticCount.global)
            e.semantic = true;
        return e;
    });
};

function mergeHTMLImpression(html) {
    return mongo
        .readOne(nconf.get('schema').impressions, { id: html.impressionId })
        .then(function(impression) {
            _.unset(impression, 'id');
            _.unset(impression, 'htmlId');
            return _.merge(html, impression);
        });
}

function parseHTML(htmlfilter, repeat) {
    return mongo
        .read(nconf.get('schema').htmls, htmlfilter)
        .then(function(found) {
            if(_.size(found) > 0)
                return found;
            if(nconf.get('retrive') != true)
                return [];
            return glue.retrive(htmlfilter)
                .then(glue.writers)
                .tap(function(x) {
                    if(x && x[2] && htmlfilter.id == x[2][0].id)
                        debug("Successfully retrived remote content");
                    else
                        debug("Failure in retriving remote content!");
                })
                .then(function() {
                    return mongo
                        .read(nconf.get('schema').htmls, htmlfilter);
                });
        })
        .catch(function(error) {
            debug("Managed error in retrieving content: %s", error);
            // console.error(error.stack);
            return [];
        })
        .map(mergeHTMLImpression, { concurrency: 1 })
        .then(_.compact)
        .then(function(impressions) {
            return _.orderBy(impressions, { impressionOrder: -1 });
        })
        .tap(function(impressions) {
            if(_.size(impressions)) {
                const firstT = moment(_.first(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const lastT = moment(_.last(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const humanized = moment
                    .duration( _.last(impressions).impressionTime - _.first(impressions).impressionTime )
                    .humanize();
                debug("Processing %d impressionsOrder %s [%s] %s %s [%s] %s",
                    /*                                    ^^^^^^^^^^^^^ conditionals */
                    _.size(impressions),
                    _.first(impressions).impressionOrder, firstT,
                    _.size(impressions) > 1 ? "-" : "",
                    _.size(impressions) > 1 ? _.last(impressions).impressionOrder : "",
                    _.size(impressions) > 1 ? "[" + lastT + "]" : "",
                    _.size(impressions) > 1 ? "(" + humanized + ")" : ""
                );
            }
        })
        .map(function(e) {
            return checkMetadata(e, repeat);
        }, { concurrency: 1 })
        .then(_.compact)
        .map(initialize)
        .map(sequence, { concurrency: 1 })
        /* this is the function processing the parsers
         * it is call of every .id which should be analyzed */
        .catch(function(error) {
            debug("[E] Unmanaged error in parser sequence: %s", error.message);
            console.log(error.stack);
            return null;
        })
        .then(function(metadata) {
            if(!metadata)
                return [];

            debug("⁂  completed %d metadata", _.size(metadata));
            if(!_.isUndefined(nconf.get('verbose')))
                debug("⁂  %s", JSON.stringify(metadata, undefined, 2));

            return metadata;
        })
        // .map(postIdCount)
        .map(semanticIdCount)
        .then(finalize)
        .tap(logSummary)
        .tap(mark)
        .then(save)
        .catch(function(error) {
            console.log("[error after parsing]", error.message);
            console.log(error.stack);
            return null;
        });
}

module.exports = {
    checkMetadata: checkMetadata,
    initialize: initialize,
    mergeHTMLImpression: mergeHTMLImpression,
    finalize: finalize,
    logSummary: logSummary,
    save: save,
    postIdCount: postIdCount,
    semanticIdCount: semanticIdCount,
    mark: mark,
    parseHTML: parseHTML,
};
<commit_msg>removed also semanticIdCount for performance reasons and because was not really used<commit_after>const _ = require('lodash');
const Promise = require('bluebird');
const debug = require('debug')('lib:parse');
const moment = require('moment');
const nconf = require('nconf');
const xpath = require('xpath');
const xmldom = require('xmldom');
const JSDOM = require('jsdom').JSDOM;

const mongo = require('./mongo');
const sequence = require('../parsers/components/utils/sequence');
const summarize = require('../parsers/components/utils/summarize');
const parserDebug = require('../parsers/components/utils/parserDebug');
const echoes = require('./echoes');
const utils = require('./utils');
const aggregated = require('./aggregated');
const glue = require('./glue');

function checkMetadata(impression, repeat) {
    if(_.isUndefined(impression.id))
        throw new Error("impression missing");

    return mongo
        .readOne(nconf.get('schema').metadata, { id: impression.id })
        .then(function(i) {
            if( _.get(i, 'id') === impression.id && !repeat) {
                debug("metadata [%s] already exists: skipping", i.id);
                return null;
            }

            if( _.get(i, 'id') === impression.id && repeat) {
                debug("metadata [%s] exists, but repeat is requested", i.id);
                return Promise.all([
                    mongo.remove(nconf.get('schema').metadata, { id: impression.id }),
                    mongo.remove(nconf.get('schema').summary, { id: impression.id })
                ])
                .return(impression);
            }

            /* else if _.isUndefined(i) is returned the impression */
            return impression;
        });
}

function initialize(impression) {
    const xmlerr = { warning: 0, error: 0, fatal: 0 };
    var domOptions = {
          errorHandler:{
              warning: function() { xmlerr.warning += 1; },
              error: function() { xmlerr.error += 1; },
              fatalError: function() { xmlerr.fatal += 1;  },
          }
    };
    /* this is the envelope with get appended the metadata from the various parsers */
    return {
        impression,
        xmlerr,
        xpath,
        jsdom: new JSDOM(impression.html).window.document,
        dom: new xmldom.DOMParser(domOptions).parseFromString(impression.html),
    };
}

function finalize(envelopes) {
    if(!_.size(envelopes))
        return { metadata: [], summary: [], statistics: [], parsers: [], errors: [] };

    /* here is build the new complex object which travels in the pipeline,
     * metadata & parsers are the internal content, not meant to be public */

    const errors = _.reduce(envelopes, function(memo, e) {
        if(!_.size(e.errors))
            return memo;
        var r = {
            errors: e.errors,
            when:  new Date(),
            id: e.impression.id,
            timelineId: e.impression.timelineId
        }
        memo.push(r);
        return memo;
    }, []);

    const removef = ['dom', 'jsdom', 'xpath', 'xmlerr', 'impression', 'errors'];
    const impressionFields =
        ['id', 'timelineId', 'userId', 'impressionOrder', 'impressionTime'];

    const metadata = _.map(envelopes, function(e) {
        return _.extend(
            _.pick(e.impression, impressionFields),
            _.omit(e, removef),
            { when: new Date() }
        );
    });
    const parsers = _.map(metadata, parserDebug);

    /* summary is the format used for user-facing API */
    const summary = _.map(metadata, summarize);
    /* statistics are the public information on how fbtrex is doing */
    const statistics = _.map(summary, aggregated.computeStats);

    return {
        metadata,
        parsers,
        summary,
        statistics,
        errors
    };
}

function logSummary(blobs) {
    _.each(blobs.summary, function(e) {
        echoes.echo(
            _.extend({'index': 'parserv' },
            _.pick(e, ['errors', 'type', 'publicationTime', 'postId',
                       'permaLink', 'author', 'textlength', 'impressionTime',
                       'impressionOrder', 'pseudo', 'timeline', 'regexp' ])
            )
        );
    });

    /* the `fulldump` is set by executers or by `parsers/precise.js` */
    if(nconf.get('fulldump'))
        _.times(_.size(blobs.metadata), function(o, i) {
            let s = _.nth(blobs.summary, i);
            console.log(JSON.stringify(s, undefined, 1));
            console.log("\x1b[36m");
            let m = _.nth(blobs.metadata, i);
            console.log(JSON.stringify(m, undefined, 1));
        });

    /* this is dumped even without fulldump */
    const E = "\x1b[47m\x1b[31m";
    _.each(blobs.errors, function(o) {
        console.log(E, JSON.stringify(o, undefined, 2));
    });
}

function mark(blobs) {
    return Promise.map(blobs.metadata, function(e) {
        return mongo
            .readOne(nconf.get('schema').htmls, { id: e.id })
            .then(function(existing) {
                existing.processed = true;
                return mongo
                    .updateOne(nconf.get('schema').htmls, { _id: existing._id }, existing);
            });
    }, { concurrency: 1});
}

function save(blobs) {
    let chain = [];
    debug("Received %d metadata and %d errors", _.size(blobs.metadata), _.size(blobs.errors));

    if(_.size(blobs.metadata))
        chain.push(
            mongo.writeMany(nconf.get('schema').metadata, blobs.metadata)  );

    if(_.size(blobs.metadata) && !nconf.get('onlymetadata') ) {
        chain.push(
            mongo.writeMany(nconf.get('schema').summary, blobs.summary)  );
        chain.push(
            aggregated.updateHourly(blobs.statistics)   );
        /*
         - still unclear how and what we should records
        chain.push(
            mongo.writeMany(nconf.get('schema').parsers, blobs.parsers)   );
         */
    }

    if(_.size(blobs.errors) && !nconf.get('onlymetadata') )
        chain.push(
            mongo.writeMany(nconf.get('schema').errors, blobs.errors)  );

    return Promise
        .all(chain)
        .return({
            metadata: _.size(blobs.metadata),
            errors: _.size(blobs.errors)
        })
        .catch(function(error) {
            console.log("error in saving:", error.message);
            return null;
        });
}

function postIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * - postId is build at the end of `sequence`
     * - postCount { personal, global } gets created always */

    // this is slowing too much the parsing, I'm going to removed it
    // --- sadClaudio

    if(!e.postId)
        return e;

    e.postCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId } }),
        mongo.count(nconf.get('schema').metadata,
            { linkedtime: { postId: e.postId }, userId: e.userId })
    ])
    .then(function(results) {
        e.postCount.global = results[0];
        e.postCount.personal = results[1];
        return e;
    });
};

function semanticIdCount(e) {
    /* this function creates certain conditional fields in metadata:
     * * if there is no .texts or zero-length texts, it returns
     * - semanticId gets created always
     * - semanticCount { personal, global } gets created always
     * - semantic: true, gets created when the semanticCount.global is zero  */

    if(!e.fullTextSize)
        return e;

    e.semanticId = utils.hash({ text: e.fullText });
    e.semanticCount = { personal: null, global: null };

    return Promise.all([
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId }),
        mongo.count(nconf.get('schema').metadata, { semanticId: e.semanticId, userId: e.userId })
    ])
    .then(function(results) {
        e.semanticCount.global = results[0];
        e.semanticCount.personal = results[1];

        /* only the first metadata with a due semanticId is marked with `semantic`,
         * this will be processed later by lib/semantic.js */
        if(!e.semanticCount.global)
            e.semantic = true;
        return e;
    });
};

function mergeHTMLImpression(html) {
    return mongo
        .readOne(nconf.get('schema').impressions, { id: html.impressionId })
        .then(function(impression) {
            _.unset(impression, 'id');
            _.unset(impression, 'htmlId');
            return _.merge(html, impression);
        });
}

function parseHTML(htmlfilter, repeat) {
    return mongo
        .read(nconf.get('schema').htmls, htmlfilter)
        .then(function(found) {
            if(_.size(found) > 0)
                return found;
            if(nconf.get('retrive') != true)
                return [];
            return glue.retrive(htmlfilter)
                .then(glue.writers)
                .tap(function(x) {
                    if(x && x[2] && htmlfilter.id == x[2][0].id)
                        debug("Successfully retrived remote content");
                    else
                        debug("Failure in retriving remote content!");
                })
                .then(function() {
                    return mongo
                        .read(nconf.get('schema').htmls, htmlfilter);
                });
        })
        .catch(function(error) {
            debug("Managed error in retrieving content: %s", error);
            // console.error(error.stack);
            return [];
        })
        .map(mergeHTMLImpression, { concurrency: 1 })
        .then(_.compact)
        .then(function(impressions) {
            return _.orderBy(impressions, { impressionOrder: -1 });
        })
        .tap(function(impressions) {
            if(_.size(impressions)) {
                const firstT = moment(_.first(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const lastT = moment(_.last(impressions).impressionTime).format("DD/MMM/YY HH:mm");
                const humanized = moment
                    .duration( _.last(impressions).impressionTime - _.first(impressions).impressionTime )
                    .humanize();
                debug("Processing %d impressionsOrder %s [%s] %s %s [%s] %s",
                    /*                                    ^^^^^^^^^^^^^ conditionals */
                    _.size(impressions),
                    _.first(impressions).impressionOrder, firstT,
                    _.size(impressions) > 1 ? "-" : "",
                    _.size(impressions) > 1 ? _.last(impressions).impressionOrder : "",
                    _.size(impressions) > 1 ? "[" + lastT + "]" : "",
                    _.size(impressions) > 1 ? "(" + humanized + ")" : ""
                );
            }
        })
        .map(function(e) {
            return checkMetadata(e, repeat);
        }, { concurrency: 1 })
        .then(_.compact)
        .map(initialize)
        .map(sequence, { concurrency: 1 })
        /* this is the function processing the parsers
         * it is call of every .id which should be analyzed */
        .catch(function(error) {
            debug("[E] Unmanaged error in parser sequence: %s", error.message);
            console.log(error.stack);
            return null;
        })
        .then(function(metadata) {
            if(!metadata)
                return [];

            debug("⁂  completed %d metadata", _.size(metadata));
            if(!_.isUndefined(nconf.get('verbose')))
                debug("⁂  %s", JSON.stringify(metadata, undefined, 2));

            return metadata;
        })
        //.map(postIdCount)
        //.map(semanticIdCount)
        .then(finalize)
        .tap(logSummary)
        .tap(mark)
        .then(save)
        .catch(function(error) {
            console.log("[error after parsing]", error.message);
            console.log(error.stack);
            return null;
        });
}

module.exports = {
    checkMetadata: checkMetadata,
    initialize: initialize,
    mergeHTMLImpression: mergeHTMLImpression,
    finalize: finalize,
    logSummary: logSummary,
    save: save,
    postIdCount: postIdCount,
    semanticIdCount: semanticIdCount,
    mark: mark,
    parseHTML: parseHTML,
};
<|endoftext|>


<commit_before>const http = require('http');
const https = require('https');
const urlLib = require('url');

function parse(url, method, json, params) {
	return new Parsed(url, method, json, params)
}

class Parsed {
	constructor(url, method, json, params = {}) {
		this.url = url
		this.method = method
		this.json = json
		this.params = params
	}

	getRawResponse(callback)
	{
		const options = urlLib.parse(this.url);
		options.method = this.method;
		options.headers = {};
		if (this.json)
		{
			if (typeof this.json == 'object')
			{
				this.params.body = JSON.stringify(this.json);
				options.headers = {
					'Content-Type': 'application/json',
				};
			}
			else
			{
				this.params.body = String(this.json);
				options.headers = {
					'Content-Type': 'text/plain',
				};
			}
			options.headers['Content-Length'] = Buffer.byteLength(this.params.body)
		}
		options.headers['user-agent'] = 'node.js basic-request bot';
		for (const key in this.params.headers)
		{
			options.headers[key] = this.params.headers[key];
		}
		options.agent = this.params.agent || null;
		let protocol = http;
		if (options.protocol == 'https:')
		{
			protocol = https;
		}
		const request = protocol.request(options, response => {
			this.status = response.statusCode
			if (this.status == 301 || this.status == 302)
			{
				// follow redirection
				this.url = response.headers.location;
				request.abort();
				return this.getRawResponse(callback)
			}
			if (this.status != 200 && this.status != 204)
			{
				request.abort();
				return callback('Invalid status code ' + this.status, {statusCode: this.status});
			}
			if (this.params.timeout)
			{
				response.setTimeout(this.params.timeout, () => {
					return callback('Timeout while reading response', {responseTimeout: true});
				});
			}
			return callback(null, response);
		});
		request.setNoDelay();
		if (this.params.timeout)
		{
			request.setTimeout(this.params.timeout, () => {
				return callback('Timeout while sending request', {requestTimeout: true});
			});
		}
		request.on('error', error => {
			return callback('Error sending request: ' + error, {sendingRequest: true});
		});
		request.on('aborted', () => {
			return callback('Request aborted', {requestAborted: true});
		});
		if (this.params.body)
		{
			request.write(this.params.body, 'utf8');
		}
		request.end();
	}

	parseBody(response, callback) {
		let finished = false;
		this.chunks = [];
		this.headers = response.headers
		response.on('data', chunk => {
			this.chunks.push(chunk);
		});
		response.on('error', error => {
			if (finished) return;
			finished = true;
			return callback('Error reading response: ' + error, {readingResponse: true});
		});
		response.on('aborted', () => {
			if (finished) return;
			finished = true;
			return callback('Response aborted', {responseAborted: true});
		});
		response.on('end', () => {
			if (finished) return;
			finished = true;
			this.finishParsing(callback)
		});
		response.on('close', () => {
			if (finished) return;
			finished = true;
			this.finishParsing(callback)
		});
	}

	finishParsing(callback) {
		this.buffer = Buffer.concat(this.chunks)
		const contentType = this.headers['content-type']
		if (contentType && contentType.includes('application/json'))
		{
			try {
				this.body = JSON.parse(this.buffer)
			} catch(error) {
				return callback(error, String(this.buffer))
			}
		} else {
			this.body = String(this.buffer)
		}
		callback(null, this.getResult())
	}

	getResult()
	{
		if (this.params.buffer) return this.buffer;
		return this.body;
	}
}

module.exports = {parse}

<commit_msg>Bug fix: do not modify incoming params.<commit_after>const http = require('http');
const https = require('https');
const urlLib = require('url');

function parse(url, method, json, params) {
	return new Parsed(url, method, json, params)
}

class Parsed {
	constructor(url, method, json, params = {}) {
		this.url = url
		this.method = method
		this.json = json
		this.params = params
	}

	getRawResponse(callback)
	{
		const options = urlLib.parse(this.url);
		options.method = this.method;
		options.headers = {};
		if (this.json)
		{
			if (typeof this.json == 'object')
			{
				this.bodyToSend = JSON.stringify(this.json);
				options.headers = {
					'Content-Type': 'application/json',
				};
			}
			else
			{
				this.bodyToSend = String(this.json);
				options.headers = {
					'Content-Type': 'text/plain',
				};
			}
			options.headers['Content-Length'] = Buffer.byteLength(this.bodyToSend)
		}
		options.headers['user-agent'] = 'node.js basic-request bot';
		for (const key in this.params.headers)
		{
			options.headers[key] = this.params.headers[key];
		}
		options.agent = this.params.agent || null;
		let protocol = http;
		if (options.protocol == 'https:')
		{
			protocol = https;
		}
		const request = protocol.request(options, response => {
			this.status = response.statusCode
			if (this.status == 301 || this.status == 302)
			{
				// follow redirection
				this.url = response.headers.location;
				request.abort();
				return this.getRawResponse(callback)
			}
			if (this.status != 200 && this.status != 204)
			{
				request.abort();
				return callback('Invalid status code ' + this.status, {statusCode: this.status});
			}
			if (this.params.timeout)
			{
				response.setTimeout(this.params.timeout, () => {
					return callback('Timeout while reading response', {responseTimeout: true});
				});
			}
			return callback(null, response);
		});
		request.setNoDelay();
		if (this.params.timeout)
		{
			request.setTimeout(this.params.timeout, () => {
				return callback('Timeout while sending request', {requestTimeout: true});
			});
		}
		request.on('error', error => {
			return callback('Error sending request: ' + error, {sendingRequest: true});
		});
		request.on('aborted', () => {
			return callback('Request aborted', {requestAborted: true});
		});
		if (this.bodyToSend)
		{
			request.write(this.bodyToSend, 'utf8');
		}
		request.end();
	}

	parseBody(response, callback) {
		let finished = false;
		this.chunks = [];
		this.headers = response.headers
		response.on('data', chunk => {
			this.chunks.push(chunk);
		});
		response.on('error', error => {
			if (finished) return;
			finished = true;
			return callback('Error reading response: ' + error, {readingResponse: true});
		});
		response.on('aborted', () => {
			if (finished) return;
			finished = true;
			return callback('Response aborted', {responseAborted: true});
		});
		response.on('end', () => {
			if (finished) return;
			finished = true;
			this.finishParsing(callback)
		});
		response.on('close', () => {
			if (finished) return;
			finished = true;
			this.finishParsing(callback)
		});
	}

	finishParsing(callback) {
		this.buffer = Buffer.concat(this.chunks)
		const contentType = this.headers['content-type']
		if (contentType && contentType.includes('application/json'))
		{
			try {
				this.body = JSON.parse(this.buffer)
			} catch(error) {
				return callback(error, String(this.buffer))
			}
		} else {
			this.body = String(this.buffer)
		}
		callback(null, this.getResult())
	}

	getResult()
	{
		if (this.params.buffer) return this.buffer;
		return this.body;
	}
}

module.exports = {parse}

<|endoftext|>


<commit_before>/**
 * # PARSE
 *
 * Copyright(c) 2014 Stefano Balietti
 * MIT Licensed
 *
 * Collection of static functions related to parsing strings
 */
(function(JSUS) {

    function PARSE() {}

    /**
     * ## PARSE.stringify_prefix
     *
     * Prefix used by PARSE.stringify and PARSE.parse
     * to decode strings with special meaning
     *
     * @see PARSE.stringify
     * @see PARSE.parse
     */
    PARSE.stringify_prefix = '!?_';

    PARSE.marker_func = PARSE.stringify_prefix + 'function';
    PARSE.marker_null = PARSE.stringify_prefix + 'null';
    PARSE.marker_und = PARSE.stringify_prefix + 'undefined';
    PARSE.marker_nan = PARSE.stringify_prefix + 'NaN';
    PARSE.marker_inf = PARSE.stringify_prefix + 'Infinity';
    PARSE.marker_minus_inf = PARSE.stringify_prefix + '-Infinity';

    /**
     * ## PARSE.getQueryString
     *
     * Parses current querystring and returns the requested variable.
     *
     * If no variable name is specified, returns the full query string.
     * If requested variable is not found returns false.
     *
     * @param {string} name Optional. If set, returns only the value
     *   associated with this variable
     * @param {string} referer Optional. If set, searches this string
     *
     * @return {string|boolean} The querystring, or a part of it, or FALSE
     *
     * Kudos:
     * @see http://stackoverflow.com/q/901115/3347292
     */
    PARSE.getQueryString = function(name, referer) {
        var regex;
        if (referer && 'string' !== typeof referer) {
            throw new TypeError('JSUS.getQueryString: referer must be string ' +
                                'or undefined.');
        }
        referer = referer || window.location.search;
        if ('undefined' === typeof name) return referer;
        name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
        regex = new RegExp("[\\?&]" + name + "=([^&#]*)");
        results = regex.exec(referer);
        return results === null ? false :
            decodeURIComponent(results[1].replace(/\+/g, " "));
    };

    /**
     * ## PARSE.tokenize
     *
     * Splits a string in tokens that users can specified as input parameter.
     * Additional options can be specified with the modifiers parameter
     *
     * - limit: An integer that specifies the number of split items
     *     after the split limit will not be included in the array
     *
     * @param {string} str The string to split
     * @param {array} separators Array containing the separators words
     * @param {object} modifiers Optional. Configuration options
     *   for the tokenizing
     *
     * @return {array} Tokens in which the string was split
     */
    PARSE.tokenize = function(str, separators, modifiers) {
        var pattern, regex;
        if (!str) return;
        if (!separators || !separators.length) return [str];
        modifiers = modifiers || {};

        pattern = '[';

        JSUS.each(separators, function(s) {
            if (s === ' ') s = '\\s';

            pattern += s;
        });

        pattern += ']+';

        regex = new RegExp(pattern);
        return str.split(regex, modifiers.limit);
    };

    /**
     * ## PARSE.stringify
     *
     * Stringifies objects, functions, primitive, undefined or null values
     *
     * Makes uses `JSON.stringify` with a special reviver function, that
     * strinfifies also functions, undefined, and null values.
     *
     * A special prefix is prepended to avoid name collisions.
     *
     * @param {mixed} o The value to stringify
     * @param {number} spaces Optional the number of indentation spaces.
     *   Defaults, 0
     *
     * @return {string} The stringified result
     *
     * @see JSON.stringify
     * @see PARSE.stringify_prefix
     */
    PARSE.stringify = function(o, spaces) {
        return JSON.stringify(o, function(key, value) {
            var type = typeof value;
            if ('function' === type) {
                return PARSE.stringify_prefix + value.toString();
            }

            if ('undefined' === type) return PARSE.marker_und;
            if (value === null) return PARSE.marker_null;
            if ('number' === type && isNaN(value)) return PARSE.marker_nan;
            if (value === Number.POSITIVE_INFINITY) return PARSE.marker_inf;
            if (value === Number.NEGATIVE_INFINITY) {
                return PARSE.marker_minus_inf;
            }

            return value;

        }, spaces);
    };

    /**
     * ## PARSE.stringifyAll
     *
     * Copies all the properties of the prototype before stringifying
     *
     * Notice: The original object is modified!
     *
     * @param {mixed} o The value to stringify
     * @param {number} spaces Optional the number of indentation spaces.
     *   Defaults, 0
     *
     * @return {string} The stringified result
     *
     * @see PARSE.stringify
     */
    PARSE.stringifyAll = function(o, spaces) {
        for (var i in o) {
            if (!o.hasOwnProperty(i)) {
                if ('object' === typeof o[i]) {
                    o[i] = PARSE.stringifyAll(o[i]);
                }
                else {
                    o[i] = o[i];
                }
            }
        }
        return PARSE.stringify(o);
    };

    /**
     * ## PARSE.parse
     *
     * Decodes strings in objects and other values
     *
     * Uses `JSON.parse` and then looks  for special strings
     * encoded by `PARSE.stringify`
     *
     * @param {string} str The string to decode
     * @return {mixed} The decoded value
     *
     * @see JSON.parse
     * @see PARSE.stringify_prefix
     */
    PARSE.parse = function(str) {

        var len_prefix = PARSE.stringify_prefix.length,
            len_func = PARSE.marker_func.length,
            len_null = PARSE.marker_null.length,
            len_und = PARSE.marker_und.length,
            len_nan = PARSE.marker_nan.length,
            len_inf = PARSE.marker_inf.length,
            len_minus_inf = PARSE.marker_minus_inf.length;

        var o = JSON.parse(str);
        return walker(o);

        function walker(o) {
            if ('object' !== typeof o) return reviver(o);

            for (var i in o) {
                if (o.hasOwnProperty(i)) {
                    if ('object' === typeof o[i]) {
                        walker(o[i]);
                    }
                    else {
                        o[i] = reviver(o[i]);
                    }
                }
            }

            return o;
        }

        function reviver(value) {
            var type = typeof value;

            if (type === 'string') {
                if (value.substring(0, len_prefix) !== PARSE.stringify_prefix) {
                    return value;
                }
                else if (value.substring(0, len_func) === PARSE.marker_func) {
                    return JSUS.eval(value.substring(len_prefix));
                }
                else if (value.substring(0, len_null) === PARSE.marker_null) {
                    return null;
                }
                else if (value.substring(0, len_und) === PARSE.marker_und) {
                    return undefined;
                }

                else if (value.substring(0, len_nan) === PARSE.marker_nan) {
                    return NaN;
                }
                else if (value.substring(0, len_inf) === PARSE.marker_inf) {
                    return Infinity;
                }
                else if (value.substring(0, len_minus_inf) ===
                         PARSE.marker_minus_inf) {

                    return -Infinity;
                }

            }
            return value;
        }
    };
    // available can be an array, a string or a object {next(),finished()}
    PARSE.range = function(description, available) {
        var alternatives, i, j;
        var conditions, condition;

        if ('undefined' === typeof available) {
            throw new Error();
        }

        // validity checking.... TODO
        expr = description;

        // & -> && and | -> ||.
        expr = expr.replace(/([^&])&([^&])/g,"$1&&$2");
        expr = expr.replace(/([^|])\|([^|])/g,"$1||$2");

        // * -> true.
        expr = expr.replace('*', 'true');

        if (J.isArray(available)) {
            begin = Math.min.apply(null, available);
            end = Math.max.apply(null, available);
        }
        // If the availble points are also only given implicitly, compute set
        // of available numbers by first guessing a bound.
        else if ('string' === typeof available) {
            numbers = expr.match(/([+-]?(\d?)/g);
            lowerBound = Math.min.apply(null, numbers);
            upperBound = Math.max.apply(null, numbers);

            available = range(available, {
                begin: lowerBound,
                end: upperBound,
                value: lowerBound,
                next: function() {
                    return this.value++;
                },
                finished: function() {
                    return this.value < this.end;
                }
            });
        }
        else {
            begin = available.begin;
            end = available.end;
        }

        // end -> maximal available value.
        expr = expr.replace('end', end);

        // begin -> minimal available value.
        expr = expr.replace('begin', begin);

        // n -> (x == n).
        expr = expr.replace(/[-+]?\d+/g, "(x == $&)");

        // n..m -> (x >= n && x <= m).
        expr = expr.replace(/([-+]?\d+)\.+([-+]?\d+)/g),
            "(x >= $1 && x <= $2)");

        // <n, <=n, >n, >=n -> (x < n), (x <= n), (x > n), (x >= n)
        expr = expr.replace(/[<>]=? *[-+]?\d+/g, "(x $&)");

        // (n,m),(n,m],[n,m),[n,m] -> (x > n && x < m), ..., (x >= n && x <= m).
        expr = expr.replace(/([(\[]) *([-+]?\d+) *, *([-+]?\d+) *([\])])/g,
            function (match, p1, p2, p3, p4) {
                return "(x >" + (p1 == "(" ? " " : "= ") + p2 + " && x <" +
                    (p4 == ")" ? " " : "= ") + p3 + ")";
            }
        );

        // a, b -> (a) || (b)
        splitOnComma = expr.spilt(',');
        expr = "(";
        for (i in splitOnComma) {
            if (splitOnComma.hasOwnProperty(i)) {
                expr += splitOnComma[i] + ") || (";
            }
        }
        expr = expr.slice(0, -5);

        // validation TODO

        if (J.isArray(available)) {
            for (i in available) {
                if (available.hasOwnProperty(i)) {
                    x = available[i];
                    if (eval(expr)) {
                        solution.push(x);
                    }
                }
            }
        else {
            while (!available.finished()) {
                x = available.next();
                if (eval(expr)) {
                    solution.push(x);
                }
            }
        }
    };

    JSUS.extend(PARSE);

})('undefined' !== typeof JSUS ? JSUS : module.parent.exports.JSUS);
<commit_msg>fixing parser;<commit_after>/**
 * # PARSE
 *
 * Copyright(c) 2014 Stefano Balietti
 * MIT Licensed
 *
 * Collection of static functions related to parsing strings
 */
(function(JSUS) {

    function PARSE() {}

    /**
     * ## PARSE.stringify_prefix
     *
     * Prefix used by PARSE.stringify and PARSE.parse
     * to decode strings with special meaning
     *
     * @see PARSE.stringify
     * @see PARSE.parse
     */
    PARSE.stringify_prefix = '!?_';

    PARSE.marker_func = PARSE.stringify_prefix + 'function';
    PARSE.marker_null = PARSE.stringify_prefix + 'null';
    PARSE.marker_und = PARSE.stringify_prefix + 'undefined';
    PARSE.marker_nan = PARSE.stringify_prefix + 'NaN';
    PARSE.marker_inf = PARSE.stringify_prefix + 'Infinity';
    PARSE.marker_minus_inf = PARSE.stringify_prefix + '-Infinity';

    /**
     * ## PARSE.getQueryString
     *
     * Parses current querystring and returns the requested variable.
     *
     * If no variable name is specified, returns the full query string.
     * If requested variable is not found returns false.
     *
     * @param {string} name Optional. If set, returns only the value
     *   associated with this variable
     * @param {string} referer Optional. If set, searches this string
     *
     * @return {string|boolean} The querystring, or a part of it, or FALSE
     *
     * Kudos:
     * @see http://stackoverflow.com/q/901115/3347292
     */
    PARSE.getQueryString = function(name, referer) {
        var regex;
        if (referer && 'string' !== typeof referer) {
            throw new TypeError('JSUS.getQueryString: referer must be string ' +
                                'or undefined.');
        }
        referer = referer || window.location.search;
        if ('undefined' === typeof name) return referer;
        name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
        regex = new RegExp("[\\?&]" + name + "=([^&#]*)");
        results = regex.exec(referer);
        return results === null ? false :
            decodeURIComponent(results[1].replace(/\+/g, " "));
    };

    /**
     * ## PARSE.tokenize
     *
     * Splits a string in tokens that users can specified as input parameter.
     * Additional options can be specified with the modifiers parameter
     *
     * - limit: An integer that specifies the number of split items
     *     after the split limit will not be included in the array
     *
     * @param {string} str The string to split
     * @param {array} separators Array containing the separators words
     * @param {object} modifiers Optional. Configuration options
     *   for the tokenizing
     *
     * @return {array} Tokens in which the string was split
     */
    PARSE.tokenize = function(str, separators, modifiers) {
        var pattern, regex;
        if (!str) return;
        if (!separators || !separators.length) return [str];
        modifiers = modifiers || {};

        pattern = '[';

        JSUS.each(separators, function(s) {
            if (s === ' ') s = '\\s';

            pattern += s;
        });

        pattern += ']+';

        regex = new RegExp(pattern);
        return str.split(regex, modifiers.limit);
    };

    /**
     * ## PARSE.stringify
     *
     * Stringifies objects, functions, primitive, undefined or null values
     *
     * Makes uses `JSON.stringify` with a special reviver function, that
     * strinfifies also functions, undefined, and null values.
     *
     * A special prefix is prepended to avoid name collisions.
     *
     * @param {mixed} o The value to stringify
     * @param {number} spaces Optional the number of indentation spaces.
     *   Defaults, 0
     *
     * @return {string} The stringified result
     *
     * @see JSON.stringify
     * @see PARSE.stringify_prefix
     */
    PARSE.stringify = function(o, spaces) {
        return JSON.stringify(o, function(key, value) {
            var type = typeof value;
            if ('function' === type) {
                return PARSE.stringify_prefix + value.toString();
            }

            if ('undefined' === type) return PARSE.marker_und;
            if (value === null) return PARSE.marker_null;
            if ('number' === type && isNaN(value)) return PARSE.marker_nan;
            if (value === Number.POSITIVE_INFINITY) return PARSE.marker_inf;
            if (value === Number.NEGATIVE_INFINITY) {
                return PARSE.marker_minus_inf;
            }

            return value;

        }, spaces);
    };

    /**
     * ## PARSE.stringifyAll
     *
     * Copies all the properties of the prototype before stringifying
     *
     * Notice: The original object is modified!
     *
     * @param {mixed} o The value to stringify
     * @param {number} spaces Optional the number of indentation spaces.
     *   Defaults, 0
     *
     * @return {string} The stringified result
     *
     * @see PARSE.stringify
     */
    PARSE.stringifyAll = function(o, spaces) {
        for (var i in o) {
            if (!o.hasOwnProperty(i)) {
                if ('object' === typeof o[i]) {
                    o[i] = PARSE.stringifyAll(o[i]);
                }
                else {
                    o[i] = o[i];
                }
            }
        }
        return PARSE.stringify(o);
    };

    /**
     * ## PARSE.parse
     *
     * Decodes strings in objects and other values
     *
     * Uses `JSON.parse` and then looks  for special strings
     * encoded by `PARSE.stringify`
     *
     * @param {string} str The string to decode
     * @return {mixed} The decoded value
     *
     * @see JSON.parse
     * @see PARSE.stringify_prefix
     */
    PARSE.parse = function(str) {

        var len_prefix = PARSE.stringify_prefix.length,
            len_func = PARSE.marker_func.length,
            len_null = PARSE.marker_null.length,
            len_und = PARSE.marker_und.length,
            len_nan = PARSE.marker_nan.length,
            len_inf = PARSE.marker_inf.length,
            len_minus_inf = PARSE.marker_minus_inf.length;

        var o = JSON.parse(str);
        return walker(o);

        function walker(o) {
            if ('object' !== typeof o) return reviver(o);

            for (var i in o) {
                if (o.hasOwnProperty(i)) {
                    if ('object' === typeof o[i]) {
                        walker(o[i]);
                    }
                    else {
                        o[i] = reviver(o[i]);
                    }
                }
            }

            return o;
        }

        function reviver(value) {
            var type = typeof value;

            if (type === 'string') {
                if (value.substring(0, len_prefix) !== PARSE.stringify_prefix) {
                    return value;
                }
                else if (value.substring(0, len_func) === PARSE.marker_func) {
                    return JSUS.eval(value.substring(len_prefix));
                }
                else if (value.substring(0, len_null) === PARSE.marker_null) {
                    return null;
                }
                else if (value.substring(0, len_und) === PARSE.marker_und) {
                    return undefined;
                }

                else if (value.substring(0, len_nan) === PARSE.marker_nan) {
                    return NaN;
                }
                else if (value.substring(0, len_inf) === PARSE.marker_inf) {
                    return Infinity;
                }
                else if (value.substring(0, len_minus_inf) ===
                         PARSE.marker_minus_inf) {

                    return -Infinity;
                }

            }
            return value;
        }
    };
    // available can be an array, a string or a object.
    PARSE.range = function(expr, available) {
        var i;
        var solution = [];
        var numberRegExp;
        var begin, end, lowerBound, upperBound;

        if ("undefined" === typeof expr) {
            return [];
        }

        if ("undefined" === typeof available) {
            throw new Error();
        }
        if (!J.isArray(available)) {
            if ("string" !== typeof available) {
                if ("function" !== typeof available.next ||
                    "function" !== typeof available.isFinished ||
                    "number"   !== typeof available.begin ||
                    "number"   !== typeof available.end
                )
                throw new Error();
            }
        }

        // & -> && and | -> ||.
        expr = expr.replace(/([^&])&([^&])/g, "$1&&$2");
        expr = expr.replace(/([^|])\|([^|])/g, "$1||$2");

        if (J.isArray(available)) {
            begin = Math.min.apply(null, available);
            end = Math.max.apply(null, available);
        }
        // If the availble points are also only given implicitly, compute set
        // of available numbers by first guessing a bound.
        else if ("string" === typeof available) {
            numbers = available.match(/([-+]?\d+)/g);
            lowerBound = Math.min.apply(null, numbers);
            upperBound = Math.max.apply(null, numbers);

            available = range(available, {
                begin: lowerBound,
                end: upperBound,
                value: lowerBound,
                next: function() {
                    return this.value++;
                },
                isFinished: function() {
                    return this.value > this.end;
                }
            });
        }
        else {
            begin = available.begin;
            end = available.end;
        }

        // end -> maximal available value.
        expr = expr.replace("end", end);

        // begin -> minimal available value.
        expr = expr.replace("begin", begin);

        // n -> (x == n).
        expr = expr.replace(/([-+]?\d+)/g, "(x == $1)");

        // n has already been replaced by (x == n) so match for that.
        // <n, <=n, >n, >=n -> (x < n), (x <= n), (x > n), (x >= n)
        expr = expr.replace(/([<>]=?) *\(x == ([-+]?\d+)\)/g, "(x $1 $2)");

        // n..m -> (x >= n && x <= m).
        // n has already been replaced by (x == n) so match for that.
        expr = expr.replace(/\(x == ([-+]?\d+)\)\.+\(x == ([-+]?\d+)\)/g,
                "(x >= $1 && x <= $2)");

        // (n,m), ... ,[n,m] -> (x > n && x < m), ... , (x >= n && x <= m).
        expr = expr.replace(
            /([(\[]) *\(x == ([-+]?\d+)\) *, *\(x == ([-+]?\d+)\) *([\])])/g,
                function (match, p1, p2, p3, p4) {
                    return "(x >" + (p1 == '(' ? ' ' : "= ") + p2 + " && x <" +
                        (p4 == ')' ? ' ' : "= ") + p3 + ')';
            }
        );

        // a, b -> (a) || (b)
        expr = '(' + expr.replace(/,/g, ") || (") + ')';

        // * -> true.
        expr = expr.replace('*', 1);

        // Validating the expression before eval"ing it.
        invalidChars = /[^ \(\)&|<>\+=\*\/\-\dx]/g;
        // Only & | ! may be before an opening bracket.
        invalidBeforeOpeningBracket = /[^ &!|\=\-\*\/\(] *\(/g;
        // Only the no word may be spelled.
        invalidWord = /x[^ <>=\)]/g;

        if (expr.match(invalidChars)) {
            throw new Error();
        }
        if (expr.match(invalidBeforeOpeningBracket)) {
            throw new Error();
        }
        if (expr.match(invalidWord)) {
            throw new Error();
        }

        if (J.isArray(available)) {
            for (i in available) {
                if (available.hasOwnProperty(i)) {
                    x = parseInt(available[i]);
                    if (eval(expr)) {
                        solution.push(x);
                    }
                }
            }
        }
        else {
            while (!available.isFinished()) {
                x = parseInt(available.next());
                if (eval(expr)) {
                    solution.push(x);
                }
            }
        }
        return solution;
    };

    JSUS.extend(PARSE);

})('undefined' !== typeof JSUS ? JSUS : module.parent.exports.JSUS);
<|endoftext|>


<commit_before>/*!*/
// Copyright 2012 Splunk, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

(function() {
    "use strict";

    var root = exports || this;

    // A list of the Splunk REST API endpoint paths
    root.Paths = {
        apps: "/services/apps/local",
        capabilities: "authorization/capabilities",
        configurations: "configs",
        deploymentClient: "deployment/client",
        deploymentServers: "deployment/server",
        deploymentServerClasses: "deployment/serverclass",
        deploymentTenants: "deployment/tenants",
        eventTypes: "saved/eventtypes",
        indexes: "data/indexes",
        info: "/services/server/info",
        inputs: null,
        jobs: "search/jobs",
        licenseGroups: "licenser/groups",
        licenseMessages: "licenser/messages",
        licensePools: "licenser/pools",
        licenseSlaves: "licenser/slaves",
        licenseStacks: "licenser/stacks",
        licenses: "licenser/licenses",
        loggers: "server/logger",
        login: "/services/auth/login",
        messages: "messages",
        passwords: "admin/passwords",
        parser: "search/parser",
        properties: "properties",
        roles: "authorization/roles",
        savedSearches: "saved/searches",
        settings: "server/settings",
        users: "/services/authentication/users",
        typeahead: "search/typeahead",
        views: "data/ui/views",

        currentUser: "/services/authentication/current-context",
        submitEvent: "receivers/simple"
    };
})();
<commit_msg>Hard code receivers/simple to /services<commit_after>/*!*/
// Copyright 2012 Splunk, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

(function() {
    "use strict";

    var root = exports || this;

    // A list of the Splunk REST API endpoint paths
    root.Paths = {
        apps: "/services/apps/local",
        capabilities: "authorization/capabilities",
        configurations: "configs",
        deploymentClient: "deployment/client",
        deploymentServers: "deployment/server",
        deploymentServerClasses: "deployment/serverclass",
        deploymentTenants: "deployment/tenants",
        eventTypes: "saved/eventtypes",
        indexes: "data/indexes",
        info: "/services/server/info",
        inputs: null,
        jobs: "search/jobs",
        licenseGroups: "licenser/groups",
        licenseMessages: "licenser/messages",
        licensePools: "licenser/pools",
        licenseSlaves: "licenser/slaves",
        licenseStacks: "licenser/stacks",
        licenses: "licenser/licenses",
        loggers: "server/logger",
        login: "/services/auth/login",
        messages: "messages",
        passwords: "admin/passwords",
        parser: "search/parser",
        properties: "properties",
        roles: "authorization/roles",
        savedSearches: "saved/searches",
        settings: "server/settings",
        users: "/services/authentication/users",
        typeahead: "search/typeahead",
        views: "data/ui/views",

        currentUser: "/services/authentication/current-context",
        submitEvent: "/services/receivers/simple"
    };
})();
<|endoftext|>


<commit_before>'use strict';

var ProjectListPage = function () {
  browser.get('/projectlist');
};

ProjectListPage.prototype = Object.create({}, {
  searchInputField: {
    get: function () {
      return element(by.model('search'));
    }
  },
  addProjectButton: {
    get: function () {
      return element(by.id('add-project'));
    }
  },
  lastProject: {
    get: function () {
      return element.all(by.className('project')).last();
    }
  },
  openLastProject: {
    value: function () {
      this.lastProject.element(by.css('.fa.fa-edit')).click();
      browser.wait(function () {
        return browser.getCurrentUrl().then(function (url) {
          return url.indexOf('/project/') > -1;
        });
      }, 500);
    }
  },
  addProject: {
    value: function () {
      var page = this;
      page.addProjectButton.click();
      browser.wait(function () {
    	  return browser.executeScript('return !!window.angular');
        }, 1000);
    }
  },
  addProjectAndReturnToProjectList: {
    value: function () {
      var page = this;
      page.addProjectButton.click().then(function () {
        browser.get('/projectlist');
      });
    }
  },
  projectCount: {
    get: function () {
      return element(by.id('project-count')).getText().then(function (text) {
        return parseInt(text);
      });
    }
  },
  filteredProjectCount: {
    get: function () {
      return element.all(by.className('project')).count();
    }
  },
  removeLastProject: {
    value: function () {
      return element.all(by.css('.fa.fa-times')).last().click();
    }
  },
  searchProject: {
    value: function (searchKeyword) {
      this.searchInputField.clear();
      this.searchInputField.sendKeys(searchKeyword);
    }
  }
});

module.exports = ProjectListPage;<commit_msg>HIRE-177: simplified wait statement<commit_after>'use strict';

var ProjectListPage = function () {
  browser.get('/projectlist');
};

ProjectListPage.prototype = Object.create({}, {
  searchInputField: {
    get: function () {
      return element(by.model('search'));
    }
  },
  addProjectButton: {
    get: function () {
      return element(by.id('add-project'));
    }
  },
  lastProject: {
    get: function () {
      return element.all(by.className('project')).last();
    }
  },
  openLastProject: {
    value: function () {
      this.lastProject.element(by.css('.fa.fa-edit')).click();
      browser.wait(function () {
        return browser.getCurrentUrl().then(function (url) {
          return url.indexOf('/project/') > -1;
        });
      }, 500);
    }
  },
  addProject: {
    value: function () {
      var page = this;
      page.addProjectButton.click();
      browser.waitForAngular();
    }
  },
  addProjectAndReturnToProjectList: {
    value: function () {
      var page = this;
      page.addProjectButton.click().then(function () {
        browser.get('/projectlist');
      });
    }
  },
  projectCount: {
    get: function () {
      return element(by.id('project-count')).getText().then(function (text) {
        return parseInt(text);
      });
    }
  },
  filteredProjectCount: {
    get: function () {
      return element.all(by.className('project')).count();
    }
  },
  removeLastProject: {
    value: function () {
      return element.all(by.css('.fa.fa-times')).last().click();
    }
  },
  searchProject: {
    value: function (searchKeyword) {
      this.searchInputField.clear();
      this.searchInputField.sendKeys(searchKeyword);
    }
  }
});

module.exports = ProjectListPage;<|endoftext|>


<commit_before>/**
 * print - return a string representation of the nodes.
 */

const isNeg = (node) => {
    return node.type === 'Apply' && node.op === 'neg';
}

const isAdd = (node) => {
    return node.type === 'Apply' && node.op === 'add';
}

const isMul = (node) => {
    return node.type === 'Apply' && node.op === 'mul';
}

const isDiv = (node) => {
    return node.type === 'Apply' && node.op === 'div';
}

const relationIdentifierMap = {
    'eq': '=',
    'lt': '<',
    'le': '<=',
    'gt': '>',
    'ge': '>=',
    'ne': '!=',
}

const printApply = (node, parent) => {
    const {op, args} = node;

    if (op === 'add') {
        let result = print(args[0], node);
        for (let i = 1; i < args.length; i++) {
            const arg = args[i];
            if (isNeg(arg) && arg.wasMinus) {
                result += ` - ${print(arg.args[0], node)}`;
            } else {
                result += ` + ${print(arg, node)}`;
            }
        }
        return parent ? `(${result})` : result;
    } else if (op === 'mul') {
        if (node.implicit) {
            return args.map(arg => print(arg, node)).join(` `);
        } else {
            return args.map(arg => print(arg, node)).join(` * `);
        }
    } else if (op === 'div') {
        let result = '';
        if (isMul(args[0])) {
            result += `(${print(args[0], node)})`;
        } else {
            result += print(args[0], node);
        }
        result += ' / ';
        if (isMul(args[1]) || isDiv(args[1])) {
            result += `(${print(args[1], node)})`;
        } else {
            result += print(args[1], node);
        }
        return result;
    } else if (op === 'pow') {
        const exponent = node.args[1];
        let bool = false;
        if (exponent.args) {
            if (isMul(exponent) || isDiv(exponent)) {
                bool = true;
            }
        }
        if (bool) {
            return `${print(args[0], node)}^(${print(args[1], node)})`;
        } else {
            return `${print(args[0], node)}^${print(args[1], node)}`;
        }
    } else if (op === 'neg') {
        return `-${print(args[0], node)}`;
    } else if (op === 'pos') {
        return `+${print(args[0], node)}`;
    } else if (op === 'pn') {
        throw new Error(`we don't handle 'pn' operations yet`);
    } else if (op === 'np') {
        throw new Error(`we don't handle 'np' operations yet`);
    } else if (op === 'fact') {
        throw new Error(`we don't handle 'fact' operations yet`);
    } else if (op in relationIdentifierMap) {
        const symbol = relationIdentifierMap[op];
        return args.map(arg => print(arg, node)).join(` ${symbol} `);
    } else {
        return `${op}(${args.map(arg => print(arg, node)).join(', ')})`;
    }
}

export default function print(node, parent = null) {
    switch (node.type) {
        // regular non-leaf nodes
        case 'Apply':
            return printApply(node, parent);

        // irregular non-leaf nodes
        case 'Parentheses':
            return `(${print(node.body, node)})`;

        // leaf nodes
        case 'Identifier':
            return node.name;
        case 'Number':
            return node.value;

        default:
            console.log(node);  // eslint-disable-line no-console
            throw new Error('unrecognized node');
    }
}
<commit_msg>remove unnecessary logic from print<commit_after>/**
 * print - return a string representation of the nodes.
 */

const isNeg = (node) => {
    return node.type === 'Apply' && node.op === 'neg';
}

const isAdd = (node) => {
    return node.type === 'Apply' && node.op === 'add';
}

const isMul = (node) => {
    return node.type === 'Apply' && node.op === 'mul';
}

const isDiv = (node) => {
    return node.type === 'Apply' && node.op === 'div';
}

const relationIdentifierMap = {
    'eq': '=',
    'lt': '<',
    'le': '<=',
    'gt': '>',
    'ge': '>=',
    'ne': '!=',
}

const printApply = (node, parent) => {
    const {op, args} = node;

    if (op === 'add') {
        let result = print(args[0], node);
        for (let i = 1; i < args.length; i++) {
            const arg = args[i];
            if (isNeg(arg) && arg.wasMinus) {
                result += ` - ${print(arg.args[0], node)}`;
            } else {
                result += ` + ${print(arg, node)}`;
            }
        }
        return parent ? `(${result})` : result;
    } else if (op === 'mul') {
        if (node.implicit) {
            return args.map(arg => print(arg, node)).join(` `);
        } else {
            return args.map(arg => print(arg, node)).join(` * `);
        }
    } else if (op === 'div') {
        let result = '';
        if (isMul(args[0])) {
            result += `(${print(args[0], node)})`;
        } else {
            result += print(args[0], node);
        }
        result += ' / ';
        if (isMul(args[1]) || isDiv(args[1])) {
            result += `(${print(args[1], node)})`;
        } else {
            result += print(args[1], node);
        }
        return result;
    } else if (op === 'pow') {
        const [base, exp] = node.args;
        if (isMul(exp) || isDiv(exp)) {
            return `${print(base, node)}^(${print(exp, node)})`;
        } else {
            return `${print(base, node)}^${print(exp, node)}`;
        }
    } else if (op === 'neg') {
        return `-${print(args[0], node)}`;
    } else if (op === 'pos') {
        return `+${print(args[0], node)}`;
    } else if (op === 'pn') {
        throw new Error(`we don't handle 'pn' operations yet`);
    } else if (op === 'np') {
        throw new Error(`we don't handle 'np' operations yet`);
    } else if (op === 'fact') {
        throw new Error(`we don't handle 'fact' operations yet`);
    } else if (op in relationIdentifierMap) {
        const symbol = relationIdentifierMap[op];
        return args.map(arg => print(arg, node)).join(` ${symbol} `);
    } else {
        return `${op}(${args.map(arg => print(arg, node)).join(', ')})`;
    }
}

export default function print(node, parent = null) {
    switch (node.type) {
        // regular non-leaf nodes
        case 'Apply':
            return printApply(node, parent);

        // irregular non-leaf nodes
        case 'Parentheses':
            return `(${print(node.body, node)})`;

        // leaf nodes
        case 'Identifier':
            return node.name;
        case 'Number':
            return node.value;

        default:
            console.log(node);  // eslint-disable-line no-console
            throw new Error('unrecognized node');
    }
}
<|endoftext|>


<commit_before>'use strict'

const HELPERS = require('./helpers')

exports.import = function (data) {
  return `-h ${data.host} --db ${data.db} ${HELPERS.generateAuth(data)} ${HELPERS.generateType(data)}`
}

exports.export = function (data) {
  return `-h ${data.host} --db ${data.db} ${HELPERS.generateAuth(data)} -c ${data.collection} --jsonArray -o ${data.collection}.json`
}
<commit_msg>Refactor the import query template<commit_after>'use strict'

const HELPERS = require('./helpers')

exports.import = function (data) {
  return `-h ${data.host} --db ${data.db} ${HELPERS.generateAuth(data)} ${HELPERS.generateType(data.file)} ${HELPERS.generateUpsertFields(data.upsertFields)} --file ${data.file}`
}

exports.export = function (data) {
  return `-h ${data.host} --db ${data.db} ${HELPERS.generateAuth(data)} -c ${data.collection} --jsonArray -o ${data.collection}.json`
}
<|endoftext|>