<commit_before>"use strict";

/**
 * Page revision API module
 *
 * Main tasks:
 * - keep track of titles, and provide listings for them
 * - keep track of MediaWiki revisions and their metadata
 * - translate MediaWiki revisions into timeuuid ranges for property queries
 * - detect edit conflicts
 */

var rbUtil = require('../lib/rbUtil.js');
var URI = require('swagger-router').URI;

// TODO: move to module
var fs = require('fs');
var yaml = require('js-yaml');
var spec = yaml.safeLoad(fs.readFileSync(__dirname + '/page_revisions.yaml'));

// Title Revision Service
function PRS (options) {
    this.options = options;
    this.log = options.log || function(){};
}

PRS.prototype.tableName = 'title_revisions';
PRS.prototype.tableURI = function(domain) {
    return new URI([domain,'sys','table',this.tableName,'']);
};

// Get the schema for the revision table
PRS.prototype.getTableSchema = function () {
    return {
        table: this.tableName,
        attributes: {
            // listing: /titles.rev/Barack_Obama/master/
            // @specific time: /titles.rev/Barack_Obama?ts=20140312T20:22:33.3Z
            title: 'string',
            rev: 'int',             // MediaWiki oldid
            latest_rev: 'int',      // Latest MediaWiki revision
            tid: 'timeuuid',
            // revision deletion or suppression, can be:
            // - sha1hidden, commenthidden, texthidden
            restrictions: 'set<string>',
            // Revision tags. Examples:
            // - minor revision
            tags: 'set<string>',
            // Page renames. null, to:destination or from:source
            // Followed for linear history, possibly useful for branches / drafts
            renames: 'set<string>',
            nextrev_tid: 'timeuuid',// tid of next revision, or null
            latest_tid: 'timeuuid', // static, CAS synchronization point
            // revision metadata in individual attributes for ease of indexing
            user_id: 'int',         // stable for contributions etc
            user_text: 'string',
            comment: 'string'
        },
        index: [
            { attribute: 'title', type: 'hash' },
            { attribute: 'rev', type: 'range', order: 'desc' },
            { attribute: 'latest_rev', type: 'static' },
            { attribute: 'tid', type: 'range', order: 'desc' }
        ],
        secondaryIndexes: {
            by_rev: [
                { attribute: 'rev', type: 'hash' },
                { attribute: 'tid', type: 'range', order: 'desc' },
                { attribute: 'title', type: 'range', order: 'asc' },
                { attribute: 'restrictions', type: 'proj' }
            ]
        }
    };
};

/**
 * Checks the revision info returned from the storage
 * for restrictions, and if there are any, raises an error
 *
 * @param res Object the result as returned from storage
 * @return true
 * @throws rbUtil.httpError if access to the revision should be denied
 */
PRS.prototype._checkRevReturn = function(res) {
    var item = res.body.items.length && res.body.items[0];
    // if there are any restrictions imposed on this
    // revision, forbid its retrieval, cf.
    // https://phabricator.wikimedia.org/T76165#1030962
    if (item && Array.isArray(item.restrictions) && item.restrictions.length > 0) {
        // there are some restrictions, deny access to the revision
        throw new rbUtil.HTTPError({
            status: 403,
            body: {
                type: 'access_denied#revision',
                title: 'Access to resource denied',
                description: 'Access is restricted for revision ' + item.rev,
                restrictions: item.restrictions
            }
        });
    }
    return true;
}

// /page/
PRS.prototype.listTitles = function(restbase, req, options) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            proj: ['title'],
            distinct: true
        }
    };

    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.title;
            });
        }
        return res;
    });
};

PRS.prototype.fetchAndStoreMWRevision = function (restbase, req) {
    var self = this;
    var rp = req.params;
    // Try to resolve MW oldids to tids
    var apiReq = {
        uri: new URI([rp.domain,'sys','action','query']),
        body: {
            format: 'json',
            action: 'query',
            prop: 'revisions',
            continue: '',
            rvprop: 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|tags'
        }
    };
    if (/^[0-9]+$/.test(rp.revision)) {
        apiReq.body.revids = rp.revision;
    } else {
        apiReq.body.titles = rp.title;
    }
    return restbase.post(apiReq)
    .then(function(apiRes) {
        var items = apiRes.body.items;
        if (!items.length || !items[0].revisions) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiResponse: apiRes
                }
            });
        }
        // the response item
        var dataResp = apiRes.body.items[0];
        // the revision info
        var apiRev = dataResp.revisions[0];
        // are there any restrictions set?
        var restrictions = Object.keys(apiRev).filter(function(key) { return /hidden$/.test(key) });
        // the tid to store this info under
        var tid = rbUtil.tidFromDate(apiRev.timestamp);
        return restbase.put({ // Save / update the revision entry
            uri: self.tableURI(rp.domain),
            body: {
                table: self.tableName,
                attributes: {
                    // FIXME: if a title has been given, check it
                    // matches the one returned by the MW API
                    // cf. https://phabricator.wikimedia.org/T87393
                    title: dataResp.title,
                    rev: parseInt(apiRev.revid),
                    tid: tid,
                    user_id: apiRev.userid,
                    user_text: apiRev.user,
                    comment: apiRev.comment,
                    tags: apiRev.tags,
                    restrictions: restrictions
                }
            }
        })
        .then(function() {
            // if there are any restrictions imposed on this
            // revision, forbid its retrieval, cf.
            // https://phabricator.wikimedia.org/T76165#1030962
            if (restrictions && restrictions.length > 0) {
                throw new rbUtil.HTTPError({
                    status: 403,
                    body: {
                        type: 'access_denied#revision',
                        title: 'Access to resource denied',
                        description: 'Access is restricted for revision ' + apiRev.revid,
                        restrictions: restrictions
                    }
                });
            }
            // no restrictions, continue
            rp.revision = apiRev.revid + '';
            rp.title = dataResp.title;
            return self.getTitleRevision(restbase, req);
        });
    }).catch(function(e) {
        // if a bad revision is supplied, the action module
        // returns a 500 with the 'Missing query pages' message
        // so catch that and turn it into a 404 in our case
        if(e.status === 500 && /^Missing query pages/.test(e.description)) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiRequest: apiReq
                }
            });
        }
        throw e;
    });
};

PRS.prototype.getTitleRevision = function(restbase, req) {
    var self = this;
    var rp = req.params;
    var revisionRequest;
    if (/^[0-9]+$/.test(rp.revision)) {
        // Check the local db
        revisionRequest = restbase.get({
            uri: this.tableURI(rp.domain),
            body: {
                table: this.tableName,
                attributes: {
                    title: rp.title,
                    rev: parseInt(rp.revision)
                },
                limit: 1
            }
        })
        .catch(function(e) {
            if (e.status !== 404) {
                throw e;
            }
            return self.fetchAndStoreMWRevision(restbase, req);
        });
    } else if (rp.revision === 'latest') {
        revisionRequest = self.fetchAndStoreMWRevision(restbase, req);
    } else {
        throw new Error("Invalid revision: " + rp.revision);
    }
    return revisionRequest
    .then(function(res) {
        // check if the revision has any restrictions
        self._checkRevReturn(res);
        if (!res.headers) {
            res.headers = {};
        }
        res.headers.etag = res.body.items[0].tid;
        return res;
    });
    // TODO: handle other revision formats (tid)
};

PRS.prototype.listTitleRevisions = function(restbase, req) {
    var rp = req.params;
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                title: rp.title
            },
            proj: ['rev']
        }
    })
    .then(function(res) {
        // Flatten to an array of revisions rather than an array of objects
        res.body.items = res.body.items.map(function(row) {
            return row.rev;
        });
        return res;
    });
};

// /rev/
PRS.prototype.listRevisions = function(restbase, req) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            index: 'by_rev',
            proj: ['rev'],
            distinct: true
        }
    };
    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.rev;
            });
        }
        return res;
    });
};

PRS.prototype.getRevision = function(restbase, req) {
    var rp = req.params;
    var self = this;
    // sanity check
    if (!/^[0-9]+$/.test(rp.revision)) {
        throw new rbUtil.HTTPError({
            status: 400,
            body: {
                type: 'invalidRevision',
                description: 'Invalid revision specified.'
            }
        });
    }
    if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
        // ask the MW API directly and
        // store and return its result
        return this.fetchAndStoreMWRevision(restbase, req);
    }
    // check the storage, and, if no match is found
    // ask the MW API about the revision
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            index: 'by_rev',
            attributes: {
                rev: parseInt(rp.revision)
            },
            limit: 1
        }
    })
    .then(function(res) {
        // check the return
        self._checkRevReturn(res);
        // and get the revision info for the
        // page now that we have the title
        rp.title = res.body.items[0].title;
        return self.getTitleRevision(restbase, req);
    })
    .catch(function(e) {
        if (e.status !== 404) {
            throw e;
        }
        return self.fetchAndStoreMWRevision(restbase, req);
    });
};

module.exports = function(options) {
    var prs = new PRS(options);
    // XXX: add docs
    return {
        spec: spec,
        operations: {
            listTitles: prs.listTitles.bind(prs),
            listTitleRevisions: prs.listTitleRevisions.bind(prs),
            getTitleRevision: prs.getTitleRevision.bind(prs),
            //getTitleRevisionId: prs.getTitleRevisionId.bind(prs)
            listRevisions: prs.listRevisions.bind(prs),
            getRevision: prs.getRevision.bind(prs)
        },
        resources: [
            {
                // Revision table
                uri: '/{domain}/sys/table/' + prs.tableName,
                body: prs.getTableSchema()
            }
        ]
    };
};
<commit_msg>Add the 'namespace' field and fill it with info from MW API<commit_after>"use strict";

/**
 * Page revision API module
 *
 * Main tasks:
 * - keep track of titles, and provide listings for them
 * - keep track of MediaWiki revisions and their metadata
 * - translate MediaWiki revisions into timeuuid ranges for property queries
 * - detect edit conflicts
 */

var rbUtil = require('../lib/rbUtil.js');
var URI = require('swagger-router').URI;

// TODO: move to module
var fs = require('fs');
var yaml = require('js-yaml');
var spec = yaml.safeLoad(fs.readFileSync(__dirname + '/page_revisions.yaml'));

// Title Revision Service
function PRS (options) {
    this.options = options;
    this.log = options.log || function(){};
}

PRS.prototype.tableName = 'title_revisions';
PRS.prototype.tableURI = function(domain) {
    return new URI([domain,'sys','table',this.tableName,'']);
};

// Get the schema for the revision table
PRS.prototype.getTableSchema = function () {
    return {
        table: this.tableName,
        attributes: {
            // listing: /titles.rev/Barack_Obama/master/
            // @specific time: /titles.rev/Barack_Obama?ts=20140312T20:22:33.3Z
            title: 'string',
            rev: 'int',             // MediaWiki oldid
            latest_rev: 'int',      // Latest MediaWiki revision
            tid: 'timeuuid',
            namespace: 'int',       // the namespace ID of the page
            // revision deletion or suppression, can be:
            // - sha1hidden, commenthidden, texthidden
            restrictions: 'set<string>',
            // Revision tags. Examples:
            // - minor revision
            tags: 'set<string>',
            // Page renames. null, to:destination or from:source
            // Followed for linear history, possibly useful for branches / drafts
            renames: 'set<string>',
            nextrev_tid: 'timeuuid',// tid of next revision, or null
            latest_tid: 'timeuuid', // static, CAS synchronization point
            // revision metadata in individual attributes for ease of indexing
            user_id: 'int',         // stable for contributions etc
            user_text: 'string',
            comment: 'string'
        },
        index: [
            { attribute: 'title', type: 'hash' },
            { attribute: 'rev', type: 'range', order: 'desc' },
            { attribute: 'latest_rev', type: 'static' },
            { attribute: 'tid', type: 'range', order: 'desc' }
        ],
        secondaryIndexes: {
            by_rev: [
                { attribute: 'rev', type: 'hash' },
                { attribute: 'tid', type: 'range', order: 'desc' },
                { attribute: 'title', type: 'range', order: 'asc' },
                { attribute: 'restrictions', type: 'proj' }
            ]
        }
    };
};

/**
 * Checks the revision info returned from the storage
 * for restrictions, and if there are any, raises an error
 *
 * @param res Object the result as returned from storage
 * @return true
 * @throws rbUtil.httpError if access to the revision should be denied
 */
PRS.prototype._checkRevReturn = function(res) {
    var item = res.body.items.length && res.body.items[0];
    // if there are any restrictions imposed on this
    // revision, forbid its retrieval, cf.
    // https://phabricator.wikimedia.org/T76165#1030962
    if (item && Array.isArray(item.restrictions) && item.restrictions.length > 0) {
        // there are some restrictions, deny access to the revision
        throw new rbUtil.HTTPError({
            status: 403,
            body: {
                type: 'access_denied#revision',
                title: 'Access to resource denied',
                description: 'Access is restricted for revision ' + item.rev,
                restrictions: item.restrictions
            }
        });
    }
    return true;
}

// /page/
PRS.prototype.listTitles = function(restbase, req, options) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            proj: ['title'],
            distinct: true
        }
    };

    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.title;
            });
        }
        return res;
    });
};

PRS.prototype.fetchAndStoreMWRevision = function (restbase, req) {
    var self = this;
    var rp = req.params;
    // Try to resolve MW oldids to tids
    var apiReq = {
        uri: new URI([rp.domain,'sys','action','query']),
        body: {
            format: 'json',
            action: 'query',
            prop: 'revisions',
            continue: '',
            rvprop: 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|tags'
        }
    };
    if (/^[0-9]+$/.test(rp.revision)) {
        apiReq.body.revids = rp.revision;
    } else {
        apiReq.body.titles = rp.title;
    }
    return restbase.post(apiReq)
    .then(function(apiRes) {
        var items = apiRes.body.items;
        if (!items.length || !items[0].revisions) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiResponse: apiRes
                }
            });
        }
        // the response item
        var dataResp = apiRes.body.items[0];
        // the revision info
        var apiRev = dataResp.revisions[0];
        // are there any restrictions set?
        var restrictions = Object.keys(apiRev).filter(function(key) { return /hidden$/.test(key) });
        // the tid to store this info under
        var tid = rbUtil.tidFromDate(apiRev.timestamp);
        return restbase.put({ // Save / update the revision entry
            uri: self.tableURI(rp.domain),
            body: {
                table: self.tableName,
                attributes: {
                    // FIXME: if a title has been given, check it
                    // matches the one returned by the MW API
                    // cf. https://phabricator.wikimedia.org/T87393
                    title: dataResp.title,
                    rev: parseInt(apiRev.revid),
                    tid: tid,
                    namespace: parseInt(dataResp.ns),
                    user_id: apiRev.userid,
                    user_text: apiRev.user,
                    comment: apiRev.comment,
                    tags: apiRev.tags,
                    restrictions: restrictions
                }
            }
        })
        .then(function() {
            // if there are any restrictions imposed on this
            // revision, forbid its retrieval, cf.
            // https://phabricator.wikimedia.org/T76165#1030962
            if (restrictions && restrictions.length > 0) {
                throw new rbUtil.HTTPError({
                    status: 403,
                    body: {
                        type: 'access_denied#revision',
                        title: 'Access to resource denied',
                        description: 'Access is restricted for revision ' + apiRev.revid,
                        restrictions: restrictions
                    }
                });
            }
            // no restrictions, continue
            rp.revision = apiRev.revid + '';
            rp.title = dataResp.title;
            return self.getTitleRevision(restbase, req);
        });
    }).catch(function(e) {
        // if a bad revision is supplied, the action module
        // returns a 500 with the 'Missing query pages' message
        // so catch that and turn it into a 404 in our case
        if(e.status === 500 && /^Missing query pages/.test(e.description)) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiRequest: apiReq
                }
            });
        }
        throw e;
    });
};

PRS.prototype.getTitleRevision = function(restbase, req) {
    var self = this;
    var rp = req.params;
    var revisionRequest;
    if (/^[0-9]+$/.test(rp.revision)) {
        // Check the local db
        revisionRequest = restbase.get({
            uri: this.tableURI(rp.domain),
            body: {
                table: this.tableName,
                attributes: {
                    title: rp.title,
                    rev: parseInt(rp.revision)
                },
                limit: 1
            }
        })
        .catch(function(e) {
            if (e.status !== 404) {
                throw e;
            }
            return self.fetchAndStoreMWRevision(restbase, req);
        });
    } else if (rp.revision === 'latest') {
        revisionRequest = self.fetchAndStoreMWRevision(restbase, req);
    } else {
        throw new Error("Invalid revision: " + rp.revision);
    }
    return revisionRequest
    .then(function(res) {
        // check if the revision has any restrictions
        self._checkRevReturn(res);
        if (!res.headers) {
            res.headers = {};
        }
        res.headers.etag = res.body.items[0].tid;
        return res;
    });
    // TODO: handle other revision formats (tid)
};

PRS.prototype.listTitleRevisions = function(restbase, req) {
    var rp = req.params;
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                title: rp.title
            },
            proj: ['rev']
        }
    })
    .then(function(res) {
        // Flatten to an array of revisions rather than an array of objects
        res.body.items = res.body.items.map(function(row) {
            return row.rev;
        });
        return res;
    });
};

// /rev/
PRS.prototype.listRevisions = function(restbase, req) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            index: 'by_rev',
            proj: ['rev'],
            distinct: true
        }
    };
    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.rev;
            });
        }
        return res;
    });
};

PRS.prototype.getRevision = function(restbase, req) {
    var rp = req.params;
    var self = this;
    // sanity check
    if (!/^[0-9]+$/.test(rp.revision)) {
        throw new rbUtil.HTTPError({
            status: 400,
            body: {
                type: 'invalidRevision',
                description: 'Invalid revision specified.'
            }
        });
    }
    if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
        // ask the MW API directly and
        // store and return its result
        return this.fetchAndStoreMWRevision(restbase, req);
    }
    // check the storage, and, if no match is found
    // ask the MW API about the revision
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            index: 'by_rev',
            attributes: {
                rev: parseInt(rp.revision)
            },
            limit: 1
        }
    })
    .then(function(res) {
        // check the return
        self._checkRevReturn(res);
        // and get the revision info for the
        // page now that we have the title
        rp.title = res.body.items[0].title;
        return self.getTitleRevision(restbase, req);
    })
    .catch(function(e) {
        if (e.status !== 404) {
            throw e;
        }
        return self.fetchAndStoreMWRevision(restbase, req);
    });
};

module.exports = function(options) {
    var prs = new PRS(options);
    // XXX: add docs
    return {
        spec: spec,
        operations: {
            listTitles: prs.listTitles.bind(prs),
            listTitleRevisions: prs.listTitleRevisions.bind(prs),
            getTitleRevision: prs.getTitleRevision.bind(prs),
            //getTitleRevisionId: prs.getTitleRevisionId.bind(prs)
            listRevisions: prs.listRevisions.bind(prs),
            getRevision: prs.getRevision.bind(prs)
        },
        resources: [
            {
                // Revision table
                uri: '/{domain}/sys/table/' + prs.tableName,
                body: prs.getTableSchema()
            }
        ]
    };
};
<|endoftext|>


<commit_before>'use strict'
const url = require('url')
const _ = require('lodash')
const routes = require('./routes')

module.exports = {
  init: (app) => {
    const passport = app.services.PassportService.passport
    app.config.web.middlewares.passportInit = passport.initialize()
    app.config.web.middlewares.passportSession = passport.session()

    /**
     * Connect a third-party profile to a local user
     *
     * This is where most of the magic happens when a user is authenticating with a
     * third-party provider. What it does, is the following:
     *
     *   1. Given a provider and an identifier, find a matching Passport.
     *   2. From here, the logic branches into two paths.
     *
     *     - A user is not currently logged in:
     *       1. If a Passport wasn't found, create a new user as well as a new
     *          Passport that will be assigned to the user.
     *       2. If a Passport was found, get the user associated with the passport.
     *
     *     - A user is currently logged in:
     *       1. If a Passport wasn't found, create a new Passport and associate it
     *          with the already logged in user (ie. "Connect")
     *       2. If a Passport was found, nothing needs to happen.
     *
     * As you can see, this function handles both "authentication" and "authori-
     * zation" at the same time. This is due to the fact that we pass in
     * `passReqToCallback: true` when loading the strategies, allowing us to look
     * for an existing session in the request and taking action based on that.
     *
     * For more information on auth(entication|rization) in Passport.js, check out:
     * http://passportjs.org/guide/authenticate/
     * http://passportjs.org/guide/authorize/
     *
     * @param {Object}   req
     * @param {Object}   query
     * @param {Object}   profile
     * @param {Function} next
     */
    passport.connect = function (req, query, profile, next) {
      const user = {}

      // Get the authentication provider from the query.
      query.provider = req.params.provider

      // Use profile.provider or fallback to the query.provider if it is undefined
      // as is the case for OpenID, for example
      const provider = profile.provider || query.provider

      // If the provider cannot be identified we cannot match it to a passport so
      // throw an error and let whoever's next in line take care of it.
      if (!provider) {
        return next(new Error('No authentication provider was identified.'))
      }

      // If the profile object contains a list of emails, grab the first one and
      // add it to the user.
      if (profile.hasOwnProperty('emails')) {
        user.email = profile.emails[0].value
      }
      // If the profile object contains a username, add it to the user.
      if (profile.hasOwnProperty('username')) {
        user.username = profile.username
      }

      // If neither an email or a username was available in the profile, we don't
      // have a way of identifying the user in the future. Throw an error and let
      // whoever's next in the line take care of it.
      if (!user.username && !user.email) {
        return next(new Error('Neither a username nor email was available'))
      }

      app.services.FootprintService.find('passport', {
        provider: provider,
        identifier: query.identifier.toString()
      }, {findOne: true})
        .then(passport => {
          if (!req.user) {
            // Scenario: A new user is attempting to sign up using a third-party
            //           authentication provider.
            // Action:   Create a new user and assign them a passport.
            if (!passport) {
              // Merge the profile received from the third-party authentication
              // provider with the user.
              app.config.passport.mergeThirdPartyProfile(user, profile).then(mergedProfile =>
                app.services.FootprintService.create('user', mergedProfile).then(user => {
                  query.tokens = JSON.stringify(query.tokens)
                  app.services.FootprintService.createAssociation('user', user.id, 'passports', query)
                    .then(passport => next(null, user))
                    .catch(next)
                }).catch(next)
              ).catch(next)
            }
            // Scenario: An existing user is trying to log in using an already
            //           connected passport.
            // Action:   Get the user associated with the passport.
            else {
              // If the tokens have changed since the last session, update them
              if (query.hasOwnProperty('tokens') && query.tokens !== passport.tokens) {
                passport.tokens = JSON.stringify(query.tokens)
              }

              // Save any update to the Passport and read the associated user instance
              return Promise.all([
                app.services.FootprintService.findAssociation('Passport', passport.id, 'User'),
                app.services.FootprintService.update('Passport', passport.id, passport.toJSON())
              ])
                .then(results => {
                  const userInstance = results[0] // Not so usefull, just for keeping namings clear
                  next(null, userInstance)
                })
                .catch(next)
            }
          }
          else {
            // Scenario: A user is currently logged in and trying to connect a new
            //           passport.
            // Action:   Create and assign a new passport to the user.
            if (!passport) {
              if (query.hasOwnProperty('tokens') && typeof(query.tokens) === 'object') {
                query.tokens = JSON.stringify(query.tokens)
              }

              app.services.FootprintService.createAssociation('user', req.user.id, 'passports', query)
                .then(passport => next(null, req.user))
                .catch(next)
            }
            // Scenario: The user is a nutjob or spammed the back-button.
            // Action:   Simply pass along the already established session.
            else {
              // If the user currently logged in is different than the user associated
              // with the third-party authentication provider, throw an error.
              if (passport.UserId != req.user.id) {
                return next(new Error('Third party account already linked to another user'))
              }
              else {
                next(null, req.user)
              }
            }
          }
        }).catch(next)
    }

    passport.serializeUser((user, next) => {
      next(null, user.id)
    })

    passport.deserializeUser((id, next) => {
      app.services.FootprintService.find('user', id, {}).then(user => {
        next(null, user)
      }).catch(err => {
        next(err)
      })
    })

  },
  loadStrategies: (app) => {
    const passport = app.services.PassportService.passport
    _.each(app.config.passport.strategies, (strategiesOptions, name) => {
        const Strategy = strategiesOptions.strategy
        const options = _.extend({passReqToCallback: true}, strategiesOptions.options || {})
        if (name === 'local') {
          passport.use(new Strategy(options, app.services.PassportService.protocols.local(app)))
        }
        else if (name === 'jwt') {
          passport.use(new Strategy(options, app.services.PassportService.protocols[name]))
        }
        else {
          const protocol = strategiesOptions.protocol
          let callback = strategiesOptions.callback

          if (!callback) {
            callback = 'auth/' + name + '/callback'
          }
          const serverProtocol = app.config.web.ssl ? 'https' : 'http'
          const baseUrl = serverProtocol + '://' + (app.config.web.host || 'localhost') + ':' + app.config.web.port

          switch (protocol) {
            case 'oauth':
            case 'oauth2':
              options.callbackURL = url.resolve(baseUrl, callback)
              break
            case 'openid':
              options.returnURL = url.resolve(baseUrl, callback)
              options.realm = baseUrl
              options.profile = true
              break
          }

          // Merge the default options with any options defined in the config. All
          // defaults can be overriden, but I don't see a reason why you'd want to
          // do that.
          _.extend(options, strategiesOptions.options)

          passport.use(new Strategy(options, app.services.PassportService.protocols[protocol](app)))
        }
      }
    )
  },
  addRoutes: app => {
    const prefix = _.get(app.config, 'passport.prefix') || _.get(app.config, 'footprints.prefix')
    const routerUtil = app.packs.router.util
    if (prefix) {
      routes.forEach(route => {
        route.path = prefix + route.path
      })
    }
    app.config.routes = app.config.routes.concat(routes)
  }

}
<commit_msg>eslint fix<commit_after>'use strict'
const url = require('url')
const _ = require('lodash')
const routes = require('./routes')

module.exports = {
  init: (app) => {
    const passport = app.services.PassportService.passport
    app.config.web.middlewares.passportInit = passport.initialize()
    app.config.web.middlewares.passportSession = passport.session()

    /**
     * Connect a third-party profile to a local user
     *
     * This is where most of the magic happens when a user is authenticating with a
     * third-party provider. What it does, is the following:
     *
     *   1. Given a provider and an identifier, find a matching Passport.
     *   2. From here, the logic branches into two paths.
     *
     *     - A user is not currently logged in:
     *       1. If a Passport wasn't found, create a new user as well as a new
     *          Passport that will be assigned to the user.
     *       2. If a Passport was found, get the user associated with the passport.
     *
     *     - A user is currently logged in:
     *       1. If a Passport wasn't found, create a new Passport and associate it
     *          with the already logged in user (ie. "Connect")
     *       2. If a Passport was found, nothing needs to happen.
     *
     * As you can see, this function handles both "authentication" and "authori-
     * zation" at the same time. This is due to the fact that we pass in
     * `passReqToCallback: true` when loading the strategies, allowing us to look
     * for an existing session in the request and taking action based on that.
     *
     * For more information on auth(entication|rization) in Passport.js, check out:
     * http://passportjs.org/guide/authenticate/
     * http://passportjs.org/guide/authorize/
     *
     * @param {Object}   req
     * @param {Object}   query
     * @param {Object}   profile
     * @param {Function} next
     */
    passport.connect = function (req, query, profile, next) {
      const user = {}

      // Get the authentication provider from the query.
      query.provider = req.params.provider

      // Use profile.provider or fallback to the query.provider if it is undefined
      // as is the case for OpenID, for example
      const provider = profile.provider || query.provider

      // If the provider cannot be identified we cannot match it to a passport so
      // throw an error and let whoever's next in line take care of it.
      if (!provider) {
        return next(new Error('No authentication provider was identified.'))
      }

      // If the profile object contains a list of emails, grab the first one and
      // add it to the user.
      if (profile.hasOwnProperty('emails')) {
        user.email = profile.emails[0].value
      }
      // If the profile object contains a username, add it to the user.
      if (profile.hasOwnProperty('username')) {
        user.username = profile.username
      }

      // If neither an email or a username was available in the profile, we don't
      // have a way of identifying the user in the future. Throw an error and let
      // whoever's next in the line take care of it.
      if (!user.username && !user.email) {
        return next(new Error('Neither a username nor email was available'))
      }

      app.services.FootprintService.find('passport', {
        provider: provider,
        identifier: query.identifier.toString()
      }, {findOne: true})
        .then(passport => {
          if (!req.user) {
            // Scenario: A new user is attempting to sign up using a third-party
            //           authentication provider.
            // Action:   Create a new user and assign them a passport.
            if (!passport) {
              // Merge the profile received from the third-party authentication
              // provider with the user.
              app.config.passport.mergeThirdPartyProfile(user, profile).then(mergedProfile =>
                app.services.FootprintService.create('user', mergedProfile).then(user => {
                  query.tokens = JSON.stringify(query.tokens)
                  app.services.FootprintService.createAssociation('user', user.id, 'passports', query)
                    .then(passport => next(null, user))
                    .catch(next)
                }).catch(next)
              ).catch(next)
            }
            // Scenario: An existing user is trying to log in using an already
            //           connected passport.
            // Action:   Get the user associated with the passport.
            else {
              // If the tokens have changed since the last session, update them
              if (query.hasOwnProperty('tokens') && query.tokens !== passport.tokens) {
                passport.tokens = JSON.stringify(query.tokens)
              }

              // Save any update to the Passport and read the associated user instance
              return Promise.all([
                app.services.FootprintService.findAssociation('Passport', passport.id, 'User'),
                app.services.FootprintService.update('Passport', passport.id, passport.toJSON())
              ])
                .then(results => {
                  const userInstance = results[0] // Not so usefull, just for keeping namings clear
                  next(null, userInstance)
                })
                .catch(next)
            }
          }
          else {
            // Scenario: A user is currently logged in and trying to connect a new
            //           passport.
            // Action:   Create and assign a new passport to the user.
            if (!passport) {
              if (query.hasOwnProperty('tokens') && typeof(query.tokens) === 'object') {
                query.tokens = JSON.stringify(query.tokens)
              }

              app.services.FootprintService.createAssociation('user', req.user.id, 'passports', query)
                .then(passport => next(null, req.user))
                .catch(next)
            }
            // Scenario: The user is a nutjob or spammed the back-button.
            // Action:   Simply pass along the already established session.
            else {
              // If the user currently logged in is different than the user associated
              // with the third-party authentication provider, throw an error.
              if (passport.UserId != req.user.id) {
                return next(new Error('Third party account already linked to another user'))
              }
              else {
                next(null, req.user)
              }
            }
          }
        }).catch(next)
    }

    passport.serializeUser((user, next) => {
      next(null, user.id)
    })

    passport.deserializeUser((id, next) => {
      app.services.FootprintService.find('user', id, {}).then(user => {
        next(null, user)
      }).catch(err => {
        next(err)
      })
    })

  },
  loadStrategies: (app) => {
    const passport = app.services.PassportService.passport
    _.each(app.config.passport.strategies, (strategiesOptions, name) => {
        const Strategy = strategiesOptions.strategy
        const options = _.extend({passReqToCallback: true}, strategiesOptions.options || {})
        if (name === 'local') {
          passport.use(new Strategy(options, app.services.PassportService.protocols.local(app)))
        }
        else if (name === 'jwt') {
          passport.use(new Strategy(options, app.services.PassportService.protocols[name]))
        }
        else {
          const protocol = strategiesOptions.protocol
          let callback = strategiesOptions.callback

          if (!callback) {
            callback = 'auth/' + name + '/callback'
          }
          const serverProtocol = app.config.web.ssl ? 'https' : 'http'
          const baseUrl = serverProtocol + '://' + (app.config.web.host || 'localhost') + ':' + app.config.web.port

          switch (protocol) {
            case 'oauth':
            case 'oauth2':
              options.callbackURL = url.resolve(baseUrl, callback)
              break
            case 'openid':
              options.returnURL = url.resolve(baseUrl, callback)
              options.realm = baseUrl
              options.profile = true
              break
          }

          // Merge the default options with any options defined in the config. All
          // defaults can be overriden, but I don't see a reason why you'd want to
          // do that.
          _.extend(options, strategiesOptions.options)

          passport.use(new Strategy(options, app.services.PassportService.protocols[protocol](app)))
        }
      }
    )
  },
  addRoutes: app => {
    const prefix = _.get(app.config, 'passport.prefix') || _.get(app.config, 'footprints.prefix')

    if (prefix) {
      routes.forEach(route => {
        route.path = prefix + route.path
      })
    }
    app.config.routes = app.config.routes.concat(routes)
  }

}
<|endoftext|>


<commit_before>//import * as TensorTools from "../util/TensorTools.js";

export { Base };

/**
 * A test set.
 */
class Base {

  /**
   * @param {number} height      image height
   * @param {number} width       image width
   * @param {number} depth       image channel count
   */
  constructor( height, width, depth ) {

    this.height = height;
    this.width = width;
    this.depth = depth;

    this.valueCount = height * width * depth;

    //this.concatenatedShape = [ height, width, depth ];

    // About half, but the difference should be divisable by 2.
    this.targetSize = [ Math.floor( height / 20 ) * 10, Math.floor( width / 20 ) * 10 ];

    // The filter size for achieving target size in one step.
    let filterHeight_OneStep =      this.targetSize[ 0 ] + 1;
    let filterWidth_OneStep =       this.targetSize[ 1 ] + 1;
    this.filterHeightWidth_OneStep = [ filterHeight_OneStep, filterWidth_OneStep ];

    // How many steps if achieving target size in many 3x3 depthwise convolution.
    let sizeReducedPerStepBy3x3 = 2;
    let this.stepsBy3x3MultiSteps = ( this.height - this.targetSize[ 0 ] ) / sizeReducedPerStepBy3x3;

    this.dataTensor3d = tf.tidy( () => {
      let dataTensor1d = tf.linspace( 0, this.valueCount - 1, this.valueCount );
      let dataTensor3d = dataTensor1d.reshape( [ height, width, depth ] );
      return dataTensor3d;
    });

    let inChannels =        depth;
    let channelMultiplier = 1;

    this.depthwiseConvFilters_OneStep = tf.tidy( () => {
      let filtersShape = [ filterHeight_OneStep, filterWidth_OneStep, inChannels, channelMultiplier ];
      let filtersValueCount = tf.util.sizeFromShape( filtersShape );

      let filtersTensor1d = tf.range( 0, filtersValueCount, 1 );
      let filtersTensor4d = filtersTensor1d.reshape( filtersShape );
      return filtersTensor4d;
    });

    this.depthwiseConv3x3Filters = tf.tidy( () => {
      let filtersShape = [ 3, 3, inChannels, channelMultiplier ];
      let filtersValueCount = tf.util.sizeFromShape( filtersShape );

      let filtersTensor1d = tf.range( 0, filtersValueCount, 1 );
      let filtersTensor4d = filtersTensor1d.reshape( filtersShape );
      return filtersTensor4d;
    });
  }

  disposeTensors() {
    if ( this.dataTensor3d ) {
      tf.dispose( this.dataTensor3d );
      this.dataTensor3d = null;
    }

    if ( this.depthwiseConvFilters_OneStep ) {
      tf.dispose( this.depthwiseConvFilters_OneStep );
      this.depthwiseConvFilters_OneStep = null;
    }

    if ( this.depthwiseConv3x3Filters ) {
      tf.dispose( this.depthwiseConv3x3Filters );
      this.depthwiseConv3x3Filters = null;
    }
  }

  // Test max-pool
  test_MaxPool( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.maxPool( this.filterHeightWidth_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test avg-pool
  test_AvgPool( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.avgPool( this.filterHeightWidth_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D)
  test_DepthwiseConv2d_OneStep( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.depthwiseConv2d( this.depthwiseConvFilters_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D) by 3x3 filter with multi-step
  test_DepthwiseConv2d_3x3_MultiStep( bReturn ) {
    tf.tidy( () => {
      let steps = this.stepsBy3x3MultiSteps;
      let t = this.dataTensor3d;
      for ( let i = 0; i < steps; ++i ) {
        let tNew = t.depthwiseConv2d( this.depthwiseConv3x3Filters, 1, "valid" );
        tf.dispose( t );
        t = tNew;
      }
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D) by 3x3 filter with stride 2
  test_DepthwiseConv2d_3x3_Stride2( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.depthwiseConv2d( this.depthwiseConv3x3Filters, 2, "same" );
      if ( bReturn )
        return t;
    });
  }

  // Test rsize-bilinear
  test_ResizeBilinear( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.resizeBilinear( this.targetSize, true );
      if ( bReturn )
        return t;
    });
  }

  // Test rsize-nearest-neighbor
  test_ResizeNearestNeighbor( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.resizeNearestNeighbor( this.targetSize, true );
      if ( bReturn )
        return t;
    });
  }

  // Testing whether the results of different implementation are the same.
  testResultSame() {
    tf.tidy( () => {
      let quarterTensor1 = this.test_MaxPool( true );
      let quarterTensor2 = this.test_AvgPool( true );
      let quarterTensor3 = this.test_DepthwiseConv2d_OneStep( true );
      let quarterTensor4 = this.test_DepthwiseConv2d_3x3_MultiStep( true );
      let quarterTensor5 = this.test_DepthwiseConv2d_3x3_Stride2( true );
      let quarterTensor6 = this.test_ResizeBilinear( true );
      let quarterTensor7 = this.test_ResizeNearestNeighbor( true );

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor1.shape, quarterTensor2.shape ),
        `MaxPool() != AvgPool()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor2.shape, quarterTensor3.shape ),
        `AvgPool() != DepthwiseConv2d_OneStep()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor3.shape, quarterTensor4.shape ),
        `DepthwiseConv2d_OneStep() != DepthwiseConv2d_3x3_MultiStep()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor4.shape, quarterTensor5.shape ),
        `DepthwiseConv2d_3x3_MultiStep() != DepthwiseConv2d_3x3_Stride2()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor5.shape, quarterTensor6.shape ),
        `DepthwiseConv2d_3x3_Stride2() != ResizeBilinear()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor6.shape, quarterTensor7.shape ),
        `ResizeBilinear() != ResizeNearestNeighbor()`);
    });

    tf.tidy( () => {
      this.logProfile( "MaxPool", this.test_MaxPool.bind( this ) );
      this.logProfile( "AvgPool", this.test_AvgPool.bind( this ) );
      this.logProfile( "DepthwiseConv2d_OneStep", this.test_DepthwiseConv2d_OneStep.bind( this ) );
      this.logProfile( "DepthwiseConv2d_3x3_MultiStep", this.test_DepthwiseConv2d_3x3_MultiStep.bind( this ) );
      this.logProfile( "DepthwiseConv2d_3x3_Stride2", this.test_DepthwiseConv2d_3x3_Stride2.bind( this ) );
      this.logProfile( "ResizeBilinear", this.test_ResizeBilinear.bind( this ) );
      this.logProfile( "ResizeNearestNeighbor", this.test_ResizeNearestNeighbor.bind( this ) );
    });
  }

  logProfile( title, func ) {
    tf.profile( func ).then( profile => {
      tf.time( func ).then( time => {

        console.log(
           `${title}: `

         + `newBytes: ${profile.newBytes}, `
         + `newTensors: ${profile.newTensors}, `
         + `peakBytes: ${profile.peakBytes}, `
         + `byte usage over all kernels: ${profile.kernels.map(k => k.totalBytesSnapshot)}, `

         + `kernelMs: ${time.kernelMs}, wallTimeMs: ${time.wallMs}`);
      });
    });
  }
}
<commit_msg>Update HeightWidthDepth.js<commit_after>//import * as TensorTools from "../util/TensorTools.js";

export { Base };

/**
 * A test set.
 */
class Base {

  /**
   * @param {number} height      image height
   * @param {number} width       image width
   * @param {number} depth       image channel count
   */
  constructor( height, width, depth ) {

    this.height = height;
    this.width = width;
    this.depth = depth;

    this.valueCount = height * width * depth;

    //this.concatenatedShape = [ height, width, depth ];

    // About half, but the difference should be divisable by 2.
    this.targetSize = [ Math.floor( height / 20 ) * 10, Math.floor( width / 20 ) * 10 ];

    // The filter size for achieving target size in one step.
    let filterHeight_OneStep =      this.targetSize[ 0 ] + 1;
    let filterWidth_OneStep =       this.targetSize[ 1 ] + 1;
    this.filterHeightWidth_OneStep = [ filterHeight_OneStep, filterWidth_OneStep ];

    // How many steps if achieving target size in many 3x3 depthwise convolution.
    let sizeReducedPerStepBy3x3 = 2;
    this.stepsBy3x3MultiSteps = ( this.height - this.targetSize[ 0 ] ) / sizeReducedPerStepBy3x3;

    this.dataTensor3d = tf.tidy( () => {
      let dataTensor1d = tf.linspace( 0, this.valueCount - 1, this.valueCount );
      let dataTensor3d = dataTensor1d.reshape( [ height, width, depth ] );
      return dataTensor3d;
    });

    let inChannels =        depth;
    let channelMultiplier = 1;

    this.depthwiseConvFilters_OneStep = tf.tidy( () => {
      let filtersShape = [ filterHeight_OneStep, filterWidth_OneStep, inChannels, channelMultiplier ];
      let filtersValueCount = tf.util.sizeFromShape( filtersShape );

      let filtersTensor1d = tf.range( 0, filtersValueCount, 1 );
      let filtersTensor4d = filtersTensor1d.reshape( filtersShape );
      return filtersTensor4d;
    });

    this.depthwiseConv3x3Filters = tf.tidy( () => {
      let filtersShape = [ 3, 3, inChannels, channelMultiplier ];
      let filtersValueCount = tf.util.sizeFromShape( filtersShape );

      let filtersTensor1d = tf.range( 0, filtersValueCount, 1 );
      let filtersTensor4d = filtersTensor1d.reshape( filtersShape );
      return filtersTensor4d;
    });
  }

  disposeTensors() {
    if ( this.dataTensor3d ) {
      tf.dispose( this.dataTensor3d );
      this.dataTensor3d = null;
    }

    if ( this.depthwiseConvFilters_OneStep ) {
      tf.dispose( this.depthwiseConvFilters_OneStep );
      this.depthwiseConvFilters_OneStep = null;
    }

    if ( this.depthwiseConv3x3Filters ) {
      tf.dispose( this.depthwiseConv3x3Filters );
      this.depthwiseConv3x3Filters = null;
    }
  }

  // Test max-pool
  test_MaxPool( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.maxPool( this.filterHeightWidth_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test avg-pool
  test_AvgPool( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.avgPool( this.filterHeightWidth_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D)
  test_DepthwiseConv2d_OneStep( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.depthwiseConv2d( this.depthwiseConvFilters_OneStep, 1, "valid" );
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D) by 3x3 filter with multi-step
  test_DepthwiseConv2d_3x3_MultiStep( bReturn ) {
    tf.tidy( () => {
      let steps = this.stepsBy3x3MultiSteps;
      let t = this.dataTensor3d;
      for ( let i = 0; i < steps; ++i ) {
        let tNew = t.depthwiseConv2d( this.depthwiseConv3x3Filters, 1, "valid" );
        tf.dispose( t );
        t = tNew;
      }
      if ( bReturn )
        return t;
    });
  }

  // Test depthwise convolution (2D) by 3x3 filter with stride 2
  test_DepthwiseConv2d_3x3_Stride2( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.depthwiseConv2d( this.depthwiseConv3x3Filters, 2, "same" );
      if ( bReturn )
        return t;
    });
  }

  // Test rsize-bilinear
  test_ResizeBilinear( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.resizeBilinear( this.targetSize, true );
      if ( bReturn )
        return t;
    });
  }

  // Test rsize-nearest-neighbor
  test_ResizeNearestNeighbor( bReturn ) {
    tf.tidy( () => {
      let t = this.dataTensor3d.resizeNearestNeighbor( this.targetSize, true );
      if ( bReturn )
        return t;
    });
  }

  // Testing whether the results of different implementation are the same.
  testResultSame() {
    tf.tidy( () => {
      let quarterTensor1 = this.test_MaxPool( true );
      let quarterTensor2 = this.test_AvgPool( true );
      let quarterTensor3 = this.test_DepthwiseConv2d_OneStep( true );
      let quarterTensor4 = this.test_DepthwiseConv2d_3x3_MultiStep( true );
      let quarterTensor5 = this.test_DepthwiseConv2d_3x3_Stride2( true );
      let quarterTensor6 = this.test_ResizeBilinear( true );
      let quarterTensor7 = this.test_ResizeNearestNeighbor( true );

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor1.shape, quarterTensor2.shape ),
        `MaxPool() != AvgPool()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor2.shape, quarterTensor3.shape ),
        `AvgPool() != DepthwiseConv2d_OneStep()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor3.shape, quarterTensor4.shape ),
        `DepthwiseConv2d_OneStep() != DepthwiseConv2d_3x3_MultiStep()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor4.shape, quarterTensor5.shape ),
        `DepthwiseConv2d_3x3_MultiStep() != DepthwiseConv2d_3x3_Stride2()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor5.shape, quarterTensor6.shape ),
        `DepthwiseConv2d_3x3_Stride2() != ResizeBilinear()`);

      tf.util.assert(
        tf.util.arraysEqual( quarterTensor6.shape, quarterTensor7.shape ),
        `ResizeBilinear() != ResizeNearestNeighbor()`);
    });

    tf.tidy( () => {
      this.logProfile( "MaxPool", this.test_MaxPool.bind( this ) );
      this.logProfile( "AvgPool", this.test_AvgPool.bind( this ) );
      this.logProfile( "DepthwiseConv2d_OneStep", this.test_DepthwiseConv2d_OneStep.bind( this ) );
      this.logProfile( "DepthwiseConv2d_3x3_MultiStep", this.test_DepthwiseConv2d_3x3_MultiStep.bind( this ) );
      this.logProfile( "DepthwiseConv2d_3x3_Stride2", this.test_DepthwiseConv2d_3x3_Stride2.bind( this ) );
      this.logProfile( "ResizeBilinear", this.test_ResizeBilinear.bind( this ) );
      this.logProfile( "ResizeNearestNeighbor", this.test_ResizeNearestNeighbor.bind( this ) );
    });
  }

  logProfile( title, func ) {
    tf.profile( func ).then( profile => {
      tf.time( func ).then( time => {

        console.log(
           `${title}: `

         + `newBytes: ${profile.newBytes}, `
         + `newTensors: ${profile.newTensors}, `
         + `peakBytes: ${profile.peakBytes}, `
         + `byte usage over all kernels: ${profile.kernels.map(k => k.totalBytesSnapshot)}, `

         + `kernelMs: ${time.kernelMs}, wallTimeMs: ${time.wallMs}`);
      });
    });
  }
}
<|endoftext|>


<commit_before>// Don't put secret values nto the source tree.
module.exports = {

};<commit_msg>Config: Read Stripe key from the file system<commit_after>var fs = require('fs');
module.exports = function () {

    var stripeKeyPath = '/home/ubuntu/stripe/private.key';

     try {
        var values = {
            stripePrivateKey: fs.readFileSync(stripeKeyPath)
        };
    }
    catch (err) {
        console.log(err);
        return {
            stripePrivateKey: ""
        }
    }

    return values;
};<|endoftext|>


<commit_before>"use strict";

/**
 * Page revision API module
 *
 * Main tasks:
 * - keep track of titles, and provide listings for them
 * - keep track of MediaWiki revisions and their metadata
 * - translate MediaWiki revisions into timeuuid ranges for property queries
 * - detect edit conflicts
 */

var rbUtil = require('../lib/rbUtil.js');
var URI = require('swagger-router').URI;

// TODO: move to module
var fs = require('fs');
var yaml = require('js-yaml');
var spec = yaml.safeLoad(fs.readFileSync(__dirname + '/page_revisions.yaml'));

// Title Revision Service
function PRS (options) {
    this.options = options;
    this.log = options.log || function(){};
}

PRS.prototype.tableName = 'title_revisions';
PRS.prototype.tableURI = function(domain) {
    return new URI([domain,'sys','table',this.tableName,'']);
};

// Get the schema for the revision table
PRS.prototype.getTableSchema = function () {
    return {
        table: this.tableName,
        attributes: {
            // listing: /titles.rev/Barack_Obama/master/
            // @specific time: /titles.rev/Barack_Obama?ts=20140312T20:22:33.3Z
            title: 'string',
            rev: 'int',             // MediaWiki oldid
            latest_rev: 'int',      // Latest MediaWiki revision
            tid: 'timeuuid',
            // revision deletion or suppression, can be:
            // - sha1hidden, commenthidden, texthidden
            hidden: 'set<string>',
            // Revision tags. Examples:
            // - minor revision
            tags: 'set<string>',
            // Page renames. null, to:destination or from:source
            // Followed for linear history, possibly useful for branches / drafts
            renames: 'set<string>',
            nextrev_tid: 'timeuuid',// tid of next revision, or null
            latest_tid: 'timeuuid', // static, CAS synchronization point
            // revision metadata in individual attributes for ease of indexing
            user_id: 'int',         // stable for contributions etc
            user_text: 'string',
            comment: 'string'
        },
        index: [
            { attribute: 'title', type: 'hash' },
            { attribute: 'rev', type: 'range', order: 'desc' },
            { attribute: 'latest_rev', type: 'static' },
            { attribute: 'tid', type: 'range', order: 'desc' }
        ],
        secondaryIndexes: {
            by_rev: [
                { attribute: 'rev', type: 'hash' },
                { attribute: 'tid', type: 'range', order: 'desc' },
                { attribute: 'title', type: 'range', order: 'asc' },
                { attribute: 'hidden', type: 'proj' },
                { attribute: 'tags', type: 'proj' }
            ]
        }
    };
};

// /page/
PRS.prototype.listTitles = function(restbase, req, options) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            proj: ['title'],
            distinct: true
        }
    };

    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.title;
            });
        }
        return res;
    });
};

PRS.prototype.fetchAndStoreMWRevision = function (restbase, req) {
    var self = this;
    var rp = req.params;
    // Try to resolve MW oldids to tids
    var apiReq = {
        uri: new URI([rp.domain,'sys','action','query']),
        body: {
            format: 'json',
            action: 'query',
            prop: 'revisions',
            continue: '',
            rvprop: 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|tags'
        }
    };
    if (/^[0-9]+$/.test(rp.revision)) {
        apiReq.body.revids = rp.revision;
    } else {
        apiReq.body.titles = rp.title;
    }
    return restbase.post(apiReq)
    .then(function(apiRes) {
        var items = apiRes.body.items;
        if (!items.length || !items[0].revisions) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiResponse: apiRes
                }
            });
        }
        // the response item
        var dataResp = apiRes.body.items[0];
        // the revision info
        var apiRev = dataResp.revisions[0];
        // are there any hidden fields set?
        var hidden = Object.keys(apiRev).filter(function(key) { return /hidden$/.test(key) });
        // the tid to store this info under
        var tid = rbUtil.tidFromDate(apiRev.timestamp);
        return restbase.put({ // Save / update the revision entry
            uri: self.tableURI(rp.domain),
            body: {
                table: self.tableName,
                attributes: {
                    // FIXME: if a title has been given, check it
                    // matches the one returned by the MW API
                    // cf. https://phabricator.wikimedia.org/T87393
                    title: dataResp.title,
                    rev: parseInt(apiRev.revid),
                    tid: tid,
                    user_id: apiRev.userid,
                    user_text: apiRev.user,
                    comment: apiRev.comment,
                    tags: apiRev.tags,
                    hidden: hidden
                }
            }
        })
        .then(function() {
            rp.revision = apiRev.revid + '';
            rp.title = dataResp.title;
            return self.getTitleRevision(restbase, req);
        });
    });
};

PRS.prototype.getTitleRevision = function(restbase, req) {
    var self = this;
    var rp = req.params;
    var revisionRequest;
    if (/^[0-9]+$/.test(rp.revision)) {
        // Check the local db
        revisionRequest = restbase.get({
            uri: this.tableURI(rp.domain),
            body: {
                table: this.tableName,
                attributes: {
                    title: rp.title,
                    rev: parseInt(rp.revision)
                },
                limit: 1
            }
        })
        .catch(function(e) {
            if (e.status !== 404) {
                throw e;
            }
            return self.fetchAndStoreMWRevision(restbase, req);
        });
    } else if (rp.revision === 'latest') {
        revisionRequest = self.fetchAndStoreMWRevision(restbase, req);
    } else {
        throw new Error("Invalid revision: " + rp.revision);
    }
    return revisionRequest
    .then(function(res) {
        if (!res.headers) {
            res.headers = {};
        }
        res.headers.etag = res.body.items[0].tid;
        return res;
    });
    // TODO: handle other revision formats (tid)
};

PRS.prototype.listTitleRevisions = function(restbase, req) {
    var rp = req.params;
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                title: rp.title
            },
            proj: ['rev']
        }
    })
    .then(function(res) {
        // Flatten to an array of revisions rather than an array of objects
        res.body.items = res.body.items.map(function(row) {
            return row.rev;
        });
        return res;
    });
};

PRS.prototype.getRevision = function(restbase, req) {
    // TODO TODO TODO
    // Currently, we cannot query Cassandra without specifying
    // relevant fields in the primary key, which is what we nned
    // to do in this case (as we do not have a title, but only a
    // revision id). Therefore, until this issue is resolved, we
    // have to ask the MW API directly about the revision info and
    // store it before giving it to the user, even if we have that
    // info stored already.
    return this.fetchAndStoreMWRevision(restbase, req);
    // END TODO END
    var rp = req.params;
    if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
        // ask the MW API directly and
        // store and return its result
        return this.fetchAndStoreMWRevision(restbase, req);
    }
    // sanity check
    if (!/^[0-9]+$/.test(rp.revision)) {
        throw new rbUtil.HTTPError({
            status: 400,
            body: {
                type: 'invalidRevision',
                description: 'Invalid revision specified.'
            }
        });
    }
    // check the storage, and, if no match is found
    // ask the MW API about the revision
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                rev: parseInt(rp.revision)
            },
            limit: 1
        }
    })
    .catch(function(e) {
        if (e.status !== 404) {
            throw e;
        }
        return self.fetchAndStoreMWRevision(restbase, req);
    });
};

module.exports = function(options) {
    var prs = new PRS(options);
    // XXX: add docs
    return {
        spec: spec,
        operations: {
            listTitles: prs.listTitles.bind(prs),
            listTitleRevisions: prs.listTitleRevisions.bind(prs),
            getTitleRevision: prs.getTitleRevision.bind(prs),
            //getTitleRevisionId: prs.getTitleRevisionId.bind(prs)
            getRevision: prs.getRevision.bind(prs),
        },
        resources: [
            {
                // Revision table
                uri: '/{domain}/sys/table/' + prs.tableName,
                body: prs.getTableSchema()
            }
        ]
    };
};
<commit_msg>Minor code rearrangement<commit_after>"use strict";

/**
 * Page revision API module
 *
 * Main tasks:
 * - keep track of titles, and provide listings for them
 * - keep track of MediaWiki revisions and their metadata
 * - translate MediaWiki revisions into timeuuid ranges for property queries
 * - detect edit conflicts
 */

var rbUtil = require('../lib/rbUtil.js');
var URI = require('swagger-router').URI;

// TODO: move to module
var fs = require('fs');
var yaml = require('js-yaml');
var spec = yaml.safeLoad(fs.readFileSync(__dirname + '/page_revisions.yaml'));

// Title Revision Service
function PRS (options) {
    this.options = options;
    this.log = options.log || function(){};
}

PRS.prototype.tableName = 'title_revisions';
PRS.prototype.tableURI = function(domain) {
    return new URI([domain,'sys','table',this.tableName,'']);
};

// Get the schema for the revision table
PRS.prototype.getTableSchema = function () {
    return {
        table: this.tableName,
        attributes: {
            // listing: /titles.rev/Barack_Obama/master/
            // @specific time: /titles.rev/Barack_Obama?ts=20140312T20:22:33.3Z
            title: 'string',
            rev: 'int',             // MediaWiki oldid
            latest_rev: 'int',      // Latest MediaWiki revision
            tid: 'timeuuid',
            // revision deletion or suppression, can be:
            // - sha1hidden, commenthidden, texthidden
            hidden: 'set<string>',
            // Revision tags. Examples:
            // - minor revision
            tags: 'set<string>',
            // Page renames. null, to:destination or from:source
            // Followed for linear history, possibly useful for branches / drafts
            renames: 'set<string>',
            nextrev_tid: 'timeuuid',// tid of next revision, or null
            latest_tid: 'timeuuid', // static, CAS synchronization point
            // revision metadata in individual attributes for ease of indexing
            user_id: 'int',         // stable for contributions etc
            user_text: 'string',
            comment: 'string'
        },
        index: [
            { attribute: 'title', type: 'hash' },
            { attribute: 'rev', type: 'range', order: 'desc' },
            { attribute: 'latest_rev', type: 'static' },
            { attribute: 'tid', type: 'range', order: 'desc' }
        ],
        secondaryIndexes: {
            by_rev: [
                { attribute: 'rev', type: 'hash' },
                { attribute: 'tid', type: 'range', order: 'desc' },
                { attribute: 'title', type: 'range', order: 'asc' },
                { attribute: 'hidden', type: 'proj' },
                { attribute: 'tags', type: 'proj' }
            ]
        }
    };
};

// /page/
PRS.prototype.listTitles = function(restbase, req, options) {
    var rp = req.params;
    var listReq = {
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            proj: ['title'],
            distinct: true
        }
    };

    return restbase.get(listReq)
    .then(function(res) {
        if (res.status === 200) {
            res.body.items = res.body.items.map(function(row) {
                return row.title;
            });
        }
        return res;
    });
};

PRS.prototype.fetchAndStoreMWRevision = function (restbase, req) {
    var self = this;
    var rp = req.params;
    // Try to resolve MW oldids to tids
    var apiReq = {
        uri: new URI([rp.domain,'sys','action','query']),
        body: {
            format: 'json',
            action: 'query',
            prop: 'revisions',
            continue: '',
            rvprop: 'ids|timestamp|user|userid|size|sha1|contentmodel|comment|tags'
        }
    };
    if (/^[0-9]+$/.test(rp.revision)) {
        apiReq.body.revids = rp.revision;
    } else {
        apiReq.body.titles = rp.title;
    }
    return restbase.post(apiReq)
    .then(function(apiRes) {
        var items = apiRes.body.items;
        if (!items.length || !items[0].revisions) {
            throw new rbUtil.HTTPError({
                status: 404,
                body: {
                    type: 'not_found#page_revisions',
                    description: 'Page or revision not found.',
                    apiResponse: apiRes
                }
            });
        }
        // the response item
        var dataResp = apiRes.body.items[0];
        // the revision info
        var apiRev = dataResp.revisions[0];
        // are there any hidden fields set?
        var hidden = Object.keys(apiRev).filter(function(key) { return /hidden$/.test(key) });
        // the tid to store this info under
        var tid = rbUtil.tidFromDate(apiRev.timestamp);
        return restbase.put({ // Save / update the revision entry
            uri: self.tableURI(rp.domain),
            body: {
                table: self.tableName,
                attributes: {
                    // FIXME: if a title has been given, check it
                    // matches the one returned by the MW API
                    // cf. https://phabricator.wikimedia.org/T87393
                    title: dataResp.title,
                    rev: parseInt(apiRev.revid),
                    tid: tid,
                    user_id: apiRev.userid,
                    user_text: apiRev.user,
                    comment: apiRev.comment,
                    tags: apiRev.tags,
                    hidden: hidden
                }
            }
        })
        .then(function() {
            rp.revision = apiRev.revid + '';
            rp.title = dataResp.title;
            return self.getTitleRevision(restbase, req);
        });
    });
};

PRS.prototype.getTitleRevision = function(restbase, req) {
    var self = this;
    var rp = req.params;
    var revisionRequest;
    if (/^[0-9]+$/.test(rp.revision)) {
        // Check the local db
        revisionRequest = restbase.get({
            uri: this.tableURI(rp.domain),
            body: {
                table: this.tableName,
                attributes: {
                    title: rp.title,
                    rev: parseInt(rp.revision)
                },
                limit: 1
            }
        })
        .catch(function(e) {
            if (e.status !== 404) {
                throw e;
            }
            return self.fetchAndStoreMWRevision(restbase, req);
        });
    } else if (rp.revision === 'latest') {
        revisionRequest = self.fetchAndStoreMWRevision(restbase, req);
    } else {
        throw new Error("Invalid revision: " + rp.revision);
    }
    return revisionRequest
    .then(function(res) {
        if (!res.headers) {
            res.headers = {};
        }
        res.headers.etag = res.body.items[0].tid;
        return res;
    });
    // TODO: handle other revision formats (tid)
};

PRS.prototype.listTitleRevisions = function(restbase, req) {
    var rp = req.params;
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                title: rp.title
            },
            proj: ['rev']
        }
    })
    .then(function(res) {
        // Flatten to an array of revisions rather than an array of objects
        res.body.items = res.body.items.map(function(row) {
            return row.rev;
        });
        return res;
    });
};

PRS.prototype.getRevision = function(restbase, req) {
    var rp = req.params;
    // sanity check
    if (!/^[0-9]+$/.test(rp.revision)) {
        throw new rbUtil.HTTPError({
            status: 400,
            body: {
                type: 'invalidRevision',
                description: 'Invalid revision specified.'
            }
        });
    }
    // TODO TODO TODO
    // Currently, we cannot query Cassandra without specifying
    // relevant fields in the primary key, which is what we nned
    // to do in this case (as we do not have a title, but only a
    // revision id). Therefore, until this issue is resolved, we
    // have to ask the MW API directly about the revision info and
    // store it before giving it to the user, even if we have that
    // info stored already.
    return this.fetchAndStoreMWRevision(restbase, req);
    // END TODO END
    if (req.headers && /no-cache/.test(req.headers['cache-control'])) {
        // ask the MW API directly and
        // store and return its result
        return this.fetchAndStoreMWRevision(restbase, req);
    }
    // check the storage, and, if no match is found
    // ask the MW API about the revision
    return restbase.get({
        uri: this.tableURI(rp.domain),
        body: {
            table: this.tableName,
            attributes: {
                rev: parseInt(rp.revision)
            },
            limit: 1
        }
    })
    .catch(function(e) {
        if (e.status !== 404) {
            throw e;
        }
        return self.fetchAndStoreMWRevision(restbase, req);
    });
};

module.exports = function(options) {
    var prs = new PRS(options);
    // XXX: add docs
    return {
        spec: spec,
        operations: {
            listTitles: prs.listTitles.bind(prs),
            listTitleRevisions: prs.listTitleRevisions.bind(prs),
            getTitleRevision: prs.getTitleRevision.bind(prs),
            //getTitleRevisionId: prs.getTitleRevisionId.bind(prs)
            getRevision: prs.getRevision.bind(prs),
        },
        resources: [
            {
                // Revision table
                uri: '/{domain}/sys/table/' + prs.tableName,
                body: prs.getTableSchema()
            }
        ]
    };
};
<|endoftext|>


<commit_before>
var argv = require('minimist')(process.argv.slice(2));

var l = require('../utils/log').l;
var m = require('moment');

var flattenObj = require('../utils/objs').flattenObj;

var colls = require('./collections');
var deq = require('deep-equal');

var upsertOpts = { upsert: true, returnOriginal: false };
var upsertCb = function(event) {
  return function(err, val) {
    if (err) {
      l.error("ERROR (%s): %O", event, err);
    } else {
      l.debug("Added %s: %O", event, val);
    }
  }
};

var UpsertStats = require('./upsert-stats').UpsertStats;
var upsertStats = new UpsertStats();

var useRealInc = !argv.f && !argv['fake-inc'];

function getProp(root, key, create) {
  var segments = key.split('.');
  var parentObj = segments.reduce(function(obj, segment, idx) {
    if (!obj || idx + 1 == segments.length) {
      return obj;
    }
    if (!(segment in obj)) {
      if (create) {
        obj[segment] = {};
      } else {
        return null;
      }
    }
    return obj[segment];
  }, root);
  var name = segments[segments.length - 1];
  var exists = parentObj && (name in parentObj);
  return {
    obj: parentObj,
    name: name,
    val: parentObj && parentObj[name],
    exists: exists,
    set: function(val) {
      if (parentObj) {
        parentObj[name] = val;
      }
    },
    delete: function() {
      if (exists) {
        delete parentObj[name];
      }
    }
  };
}

function addSetProp(clazz, className) {
  clazz.prototype.getProp = function(key, create) {
    return getProp(this.propsObj, key, create);
  };
  clazz.prototype.set = function(key, val, allowExtant) {
    if (typeof key == 'string') {
      if (val === undefined) return this;
      var prop = this.getProp(key, true);
      if (prop.exists) {
        if (!deq(prop.val, val)) {
          if (!allowExtant) {
            l.error(
                  "%s(%s) overwriting %s: %s -> %s",
                  className,
                  JSON.stringify(this.findObj),
                  key,
                  JSON.stringify(prop.val),
                  JSON.stringify(val)
            );
          }
          prop.set(val);
          this.toSyncObj[key] = val;
          this.dirty = true;
        }
      } else {
        prop.set(val);
        this.toSyncObj[key] = val;
        this.dirty = true;
      }
    } else if (typeof key == 'object') {
      for (k in key) {
        this.set(k, key[k], !!val);
      }
    } else {
      l.error(
            "%s(%s): Invalid .set() argument: %s",
            className,
            JSON.stringify(this.findObj),
            JSON.stringify(key)
      );
    }
    return this;
  }
}

function addUnset(clazz) {
  clazz.prototype.unset = function(key) {
    var prop = this.getProp(key);
    if (prop.exists) {
      this.unsetKeys = this.unsetKeys || [];
      this.unsetKeys.push(key);
      prop.delete();
      getProp(this.toSyncObj, key).delete();
      delete this.toSyncObj[key];
      this.dirty = true;
    }
    return this;
  }
}

function addIncProp(clazz) {
  clazz.prototype.inc = function(key, i) {
    if (typeof key === 'object') {
      var flattened = flattenObj(key);
      for (var k in flattened) {
        this.inc(k, flattened[k]);
      }
      return this;
    }
    if (i === undefined) {
      i = 1;
    }
    if (i == 0) return this;
    if (useRealInc) {
      if (!this.incObj) this.incObj = {};
      this.incObj[key] = (this.incObj[key] || 0) + i;
      var prop = this.getProp(key, true);
      prop.set((prop.val || 0) + i);
      this.dirty = true;
      return this;
    } else {
      return this.set(key, (this.get(key) || 0) + i, true);
    }
  };
}

function addDecProp(clazz) {
  clazz.prototype.dec = function(key, i) {
    if (i === undefined) {
      i = 1;
    }
    if (i == 0) return this;
    return this.inc(key, -i);
  };
}

function addAddToSetProp(clazz) {
  clazz.prototype.addToSet = function(key, val) {
    var prop = this.getProp(key, true);
    if (!prop.exists || !prop.val) {
      prop.obj[prop.name] = {};
    }
    if (!(val in prop.obj[prop.name])) {
      prop.obj[prop.name][val] = true;

      this.addToSetObj = this.addToSetObj || {};
      if (!(key in this.addToSetObj)) {
        this.addToSetObj[key] = {};
      }
      this.addToSetObj[key][val] = true;

      this.dirty = true;
    }
    return this;
  };
}

function addSetDuration(clazz) {
  clazz.prototype.maybeIncrementAggregatedDurations = function(delta) {
    if (this.durationAggregationKey) {
      (this.durationAggregationObjs || []).forEach(function (obj) {
        var before = obj.get(this.durationAggregationKey);
        obj.inc(this.durationAggregationKey, delta);
      }.bind(this));
    }
  };
  clazz.prototype.setDuration = function() {
    var start = this.get('time.start');
    if (start) {
      var end = this.get('time.end');
      if (end) {
        if (!this.get('duration')) {
          this.set('duration', end - start, true);
          this.maybeIncrementAggregatedDurations(end - start);
        }
      } else {
        var newDuration = Math.max(0, (m().unix() * 1000) - start);
        var prevDuration = this.get('duration') || 0;
        var delta = newDuration - prevDuration;
        this.set('duration', newDuration, true);
        this.maybeIncrementAggregatedDurations(delta);
      }
    }
    return this;
  }
}

function addGetProp(clazz) {
  clazz.prototype.get = function(key) {
    return this.getProp(key).val;
  }
}

function addHasProp(clazz) {
  clazz.prototype.has = function(key) {
    return this.getProp(key).exists;
  }
}

function isEmptyObject(o) {
  for (k in o) return false;
  return true;
}

function addUpsert(clazz, className, collectionName) {
  clazz.prototype.upsert = function() {
    if (!this.dirty) return this;
    if (this.applyRateLimit) {
      if (this.blocking) {
        this.blocked = true;
        return this;
      } else {
        this.blocking = true;
      }
    }

    this.setDuration();
    var upsertObj = {};
    if (!isEmptyObject(this.toSyncObj)) {
      upsertObj['$set'] = this.toSyncObj;
    }
    if (this.unsetKeys) {
      upsertObj['$unset'] = {};
      this.unsetKeys.forEach(function(unsetKey) {
        upsertObj['$unset'][unsetKey] = 1;
      });
      this.unsetKeys = null;
    }
    if (this.incObj && !isEmptyObject(this.incObj)) {
      upsertObj['$inc'] = this.incObj;
    }
    if (!upsertObj['$inc']) {
      upsertObj['$inc'] = {};
    }
    upsertObj['$inc']['n'] = 1;

    if (this.addToSetObj && !isEmptyObject(this.addToSetObj)) {
      var addToSetObj = {};
      for (var k in this.addToSetObj) {
        var keyObj = this.addToSetObj[k];
        var elems = [];
        for (var ok in keyObj) {
          elems.push(ok);
        }
        if (elems.length == 1) {
          addToSetObj[k] = elems[0];
        } else if (elems.length > 1) {
          addToSetObj[k] = { $each: elems }
        }
      }
      upsertObj['$addToSet'] = addToSetObj;
    }

    upsertStats.inc();
    var b = {
      t: m(),
      started: upsertStats.started,
      ended: upsertStats.ended
    };
    colls[collectionName].findOneAndUpdate(
          this.findObj,
          upsertObj,
          upsertOpts,
          function(err, val) {
            var after = m();
            upsertStats.dec();
            this.blocking = false;
            if (err) {
              l.error("ERROR (%s): %O", className, err);
            } else {
              l.debug("Added %s: %O", className, val);
              if (className == 'Stage') {
                var v = val.value;
                l.info(
                      'After upsert (started %d->%d, ended %d->%d, in flight %d->%d): Stage %d: %d running, %d succeeded, took %d ms',
                      b.started, upsertStats.started,
                      b.ended, upsertStats.ended,
                      b.started - b.ended, upsertStats.started - upsertStats.ended,
                      v.id,
                      v.taskCounts && v.taskCounts.running || 0,
                      v.taskCounts && v.taskCounts.succeeded || 0,
                      after - b.t
                );
              }
              if (this.blocked) {
                this.blocked = false;
                this.upsert();
              }
            }
          }.bind(this)
    );
    this.toSyncObj = {};
    this.addToSetObj = {};
    this.incObj = {};
    this.dirty = false;
    return this;
  };
}

function addInit(clazz, className) {
  clazz.prototype.init = function(findKeys, durationAggregationKey, durationAggregationObjs) {
    this.findObj = {};
    this.findKeys = findKeys;
    this.durationAggregationKey = durationAggregationKey;
    this.durationAggregationObjs = durationAggregationObjs;
    this.clazz = className;

    if (!findKeys || !findKeys.length) {
      l.error('%s: no findKeys set', className);
    }

    findKeys.forEach(function(key) {
      if (!(key in this)) {
        l.error("%s: missing findKey %s.", className, key, JSON.stringify(this));
      } else {
        this.findObj[key] = this[key];
      }
    }.bind(this));

    this.propsObj = {};
    this.toSyncObj = {};
    this.dirty = true;
    this.applyRateLimit = true;
  };
}

function addFromMongo(clazz) {
  clazz.prototype.fromMongo = function(mongoRecord) {
    if (!mongoRecord) return this;
    for (var k in mongoRecord) {
      if (!(k in this.findObj) && k != '_id') {
        this.propsObj[k] = mongoRecord[k];
      }
    }
    return this;
  };
}

function mixinMongoMethods(clazz, className, collectionName) {
  addInit(clazz, className);
  addFromMongo(clazz);
  addSetProp(clazz, className);
  addUnset(clazz);
  addIncProp(clazz);
  addDecProp(clazz);
  addGetProp(clazz);
  addHasProp(clazz);
  addUpsert(clazz, className, collectionName);
  addSetDuration(clazz);
  addAddToSetProp(clazz);
}

module.exports = {
  upsertOpts: upsertOpts,
  upsertCb: upsertCb,
  mixinMongoMethods: mixinMongoMethods,
  addSetDuration: addSetDuration,
  addAddToSetProp: addAddToSetProp,
  addHasProp: addHasProp
};
<commit_msg>add "last updated" time to all records<commit_after>
var argv = require('minimist')(process.argv.slice(2));

var l = require('../utils/log').l;
var m = require('moment');

var flattenObj = require('../utils/objs').flattenObj;

var colls = require('./collections');
var deq = require('deep-equal');

var upsertOpts = { upsert: true, returnOriginal: false };
var upsertCb = function(event) {
  return function(err, val) {
    if (err) {
      l.error("ERROR (%s): %O", event, err);
    } else {
      l.debug("Added %s: %O", event, val);
    }
  }
};

var UpsertStats = require('./upsert-stats').UpsertStats;
var upsertStats = new UpsertStats();

var useRealInc = !argv.f && !argv['fake-inc'];

function getProp(root, key, create) {
  var segments = key.split('.');
  var parentObj = segments.reduce(function(obj, segment, idx) {
    if (!obj || idx + 1 == segments.length) {
      return obj;
    }
    if (!(segment in obj)) {
      if (create) {
        obj[segment] = {};
      } else {
        return null;
      }
    }
    return obj[segment];
  }, root);
  var name = segments[segments.length - 1];
  var exists = parentObj && (name in parentObj);
  return {
    obj: parentObj,
    name: name,
    val: parentObj && parentObj[name],
    exists: exists,
    set: function(val) {
      if (parentObj) {
        parentObj[name] = val;
      }
    },
    delete: function() {
      if (exists) {
        delete parentObj[name];
      }
    }
  };
}

function addSetProp(clazz, className) {
  clazz.prototype.getProp = function(key, create) {
    return getProp(this.propsObj, key, create);
  };
  clazz.prototype.set = function(key, val, allowExtant) {
    if (typeof key == 'string') {
      if (val === undefined) return this;
      var prop = this.getProp(key, true);
      if (prop.exists) {
        if (!deq(prop.val, val)) {
          if (!allowExtant) {
            l.error(
                  "%s(%s) overwriting %s: %s -> %s",
                  className,
                  JSON.stringify(this.findObj),
                  key,
                  JSON.stringify(prop.val),
                  JSON.stringify(val)
            );
          }
          prop.set(val);
          this.toSyncObj[key] = val;
          this.dirty = true;
        }
      } else {
        prop.set(val);
        this.toSyncObj[key] = val;
        this.dirty = true;
      }
    } else if (typeof key == 'object') {
      for (k in key) {
        this.set(k, key[k], !!val);
      }
    } else {
      l.error(
            "%s(%s): Invalid .set() argument: %s",
            className,
            JSON.stringify(this.findObj),
            JSON.stringify(key)
      );
    }
    return this;
  }
}

function addUnset(clazz) {
  clazz.prototype.unset = function(key) {
    var prop = this.getProp(key);
    if (prop.exists) {
      this.unsetKeys = this.unsetKeys || [];
      this.unsetKeys.push(key);
      prop.delete();
      getProp(this.toSyncObj, key).delete();
      delete this.toSyncObj[key];
      this.dirty = true;
    }
    return this;
  }
}

function addIncProp(clazz) {
  clazz.prototype.inc = function(key, i) {
    if (typeof key === 'object') {
      var flattened = flattenObj(key);
      for (var k in flattened) {
        this.inc(k, flattened[k]);
      }
      return this;
    }
    if (i === undefined) {
      i = 1;
    }
    if (i == 0) return this;
    if (useRealInc) {
      if (!this.incObj) this.incObj = {};
      this.incObj[key] = (this.incObj[key] || 0) + i;
      var prop = this.getProp(key, true);
      prop.set((prop.val || 0) + i);
      this.dirty = true;
      return this;
    } else {
      return this.set(key, (this.get(key) || 0) + i, true);
    }
  };
}

function addDecProp(clazz) {
  clazz.prototype.dec = function(key, i) {
    if (i === undefined) {
      i = 1;
    }
    if (i == 0) return this;
    return this.inc(key, -i);
  };
}

function addAddToSetProp(clazz) {
  clazz.prototype.addToSet = function(key, val) {
    var prop = this.getProp(key, true);
    if (!prop.exists || !prop.val) {
      prop.obj[prop.name] = {};
    }
    if (!(val in prop.obj[prop.name])) {
      prop.obj[prop.name][val] = true;

      this.addToSetObj = this.addToSetObj || {};
      if (!(key in this.addToSetObj)) {
        this.addToSetObj[key] = {};
      }
      this.addToSetObj[key][val] = true;

      this.dirty = true;
    }
    return this;
  };
}

function addSetDuration(clazz) {
  clazz.prototype.maybeIncrementAggregatedDurations = function(delta) {
    if (this.durationAggregationKey) {
      (this.durationAggregationObjs || []).forEach(function (obj) {
        var before = obj.get(this.durationAggregationKey);
        obj.inc(this.durationAggregationKey, delta);
      }.bind(this));
    }
  };
  clazz.prototype.setDuration = function() {
    var start = this.get('time.start');
    if (start) {
      var end = this.get('time.end');
      if (end) {
        if (!this.get('duration')) {
          this.set('duration', end - start, true);
          this.maybeIncrementAggregatedDurations(end - start);
        }
      } else {
        var newDuration = Math.max(0, (m().unix() * 1000) - start);
        var prevDuration = this.get('duration') || 0;
        var delta = newDuration - prevDuration;
        this.set('duration', newDuration, true);
        this.maybeIncrementAggregatedDurations(delta);
      }
    }
    return this;
  }
}

function addGetProp(clazz) {
  clazz.prototype.get = function(key) {
    return this.getProp(key).val;
  }
}

function addHasProp(clazz) {
  clazz.prototype.has = function(key) {
    return this.getProp(key).exists;
  }
}

function isEmptyObject(o) {
  for (k in o) return false;
  return true;
}

function addUpsert(clazz, className, collectionName) {
  clazz.prototype.upsert = function() {
    if (!this.dirty) return this;
    if (this.applyRateLimit) {
      if (this.blocking) {
        this.blocked = true;
        return this;
      } else {
        this.blocking = true;
      }
    }

    this.setDuration();
    var upsertObj = {};
    if (!isEmptyObject(this.toSyncObj)) {
      upsertObj['$set'] = this.toSyncObj;
    }
    if (this.unsetKeys) {
      upsertObj['$unset'] = {};
      this.unsetKeys.forEach(function(unsetKey) {
        upsertObj['$unset'][unsetKey] = 1;
      });
      this.unsetKeys = null;
    }
    if (this.incObj && !isEmptyObject(this.incObj)) {
      upsertObj['$inc'] = this.incObj;
    }
    if (!upsertObj['$inc']) {
      upsertObj['$inc'] = {};
    }
    upsertObj['$inc']['n'] = 1;

    if (this.addToSetObj && !isEmptyObject(this.addToSetObj)) {
      var addToSetObj = {};
      for (var k in this.addToSetObj) {
        var keyObj = this.addToSetObj[k];
        var elems = [];
        for (var ok in keyObj) {
          elems.push(ok);
        }
        if (elems.length == 1) {
          addToSetObj[k] = elems[0];
        } else if (elems.length > 1) {
          addToSetObj[k] = { $each: elems }
        }
      }
      upsertObj['$addToSet'] = addToSetObj;
    }

    var now = m();
    if (!upsertObj['$set']) {
      upsertObj['$set'] = {};
    }
    upsertObj['$set'].l = now.unix() * 1000;

    upsertStats.inc();
    var b = {
      t: now,
      started: upsertStats.started,
      ended: upsertStats.ended
    };
    colls[collectionName].findOneAndUpdate(
          this.findObj,
          upsertObj,
          upsertOpts,
          function(err, val) {
            var after = m();
            upsertStats.dec();
            this.blocking = false;
            if (err) {
              l.error("ERROR (%s): %O", className, err);
            } else {
              l.debug("Added %s: %O", className, val);
              if (className == 'Stage') {
                var v = val.value;
                l.info(
                      'After upsert (started %d->%d, ended %d->%d, in flight %d->%d): Stage %d: %d running, %d succeeded, took %d ms',
                      b.started, upsertStats.started,
                      b.ended, upsertStats.ended,
                      b.started - b.ended, upsertStats.started - upsertStats.ended,
                      v.id,
                      v.taskCounts && v.taskCounts.running || 0,
                      v.taskCounts && v.taskCounts.succeeded || 0,
                      after - b.t
                );
              }
              if (this.blocked) {
                this.blocked = false;
                this.upsert();
              }
            }
          }.bind(this)
    );
    this.toSyncObj = {};
    this.addToSetObj = {};
    this.incObj = {};
    this.dirty = false;
    return this;
  };
}

function addInit(clazz, className) {
  clazz.prototype.init = function(findKeys, durationAggregationKey, durationAggregationObjs) {
    this.findObj = {};
    this.findKeys = findKeys;
    this.durationAggregationKey = durationAggregationKey;
    this.durationAggregationObjs = durationAggregationObjs;
    this.clazz = className;

    if (!findKeys || !findKeys.length) {
      l.error('%s: no findKeys set', className);
    }

    findKeys.forEach(function(key) {
      if (!(key in this)) {
        l.error("%s: missing findKey %s.", className, key, JSON.stringify(this));
      } else {
        this.findObj[key] = this[key];
      }
    }.bind(this));

    this.propsObj = {};
    this.toSyncObj = {};
    this.dirty = true;
    this.applyRateLimit = true;
  };
}

function addFromMongo(clazz) {
  clazz.prototype.fromMongo = function(mongoRecord) {
    if (!mongoRecord) return this;
    for (var k in mongoRecord) {
      if (!(k in this.findObj) && k != '_id') {
        this.propsObj[k] = mongoRecord[k];
      }
    }
    return this;
  };
}

function mixinMongoMethods(clazz, className, collectionName) {
  addInit(clazz, className);
  addFromMongo(clazz);
  addSetProp(clazz, className);
  addUnset(clazz);
  addIncProp(clazz);
  addDecProp(clazz);
  addGetProp(clazz);
  addHasProp(clazz);
  addUpsert(clazz, className, collectionName);
  addSetDuration(clazz);
  addAddToSetProp(clazz);
}

module.exports = {
  upsertOpts: upsertOpts,
  upsertCb: upsertCb,
  mixinMongoMethods: mixinMongoMethods,
  addSetDuration: addSetDuration,
  addAddToSetProp: addAddToSetProp,
  addHasProp: addHasProp
};
<|endoftext|>