<commit_before>/*
 * Ext JS Connect
 * Copyright(c) 2010 Sencha Inc.
 * MIT Licensed
 *
 * Modified by Yahoo!
 * Copyright (c) 2011-2012, Yahoo! Inc.  All rights reserved.
 * Yahoo! Copyrights licensed under the New BSD License.
 * See the accompanying LICENSE file for terms.
 */

/*
 * Connect staticProvider middleware adapted for Mojito *
 ********************************************************
 * This was modified to allow load all combo urls from the
 * Mojito development environment instead of one static
 * directory.
 ********************************************************
 */

/*jslint anon:true, sloppy:true, nomen:true, stupid:true, node: true*/

'use strict';

/*
DECLAIMER: this is VERY experimental, and the purpose of this
middleware is to provide an easy way to load yui modules by their
names rather than the real static path.
    from: @caridy
*/

/*
 * Module dependencies.
 */
var libfs = require('fs'),
    mime = require('mime'),
    libpath = require('path'),
    YUI = require(libpath.join(__dirname, '..', '..', 'yui-sandbox.js')).getYUI(),
    parseUrl = require('url').parse,
    logger,
    NAME = 'ComboHandler',

    MODULE_META_ENTRIES          = ['requires', 'use', 'optional', 'skinnable', 'after', 'condition'],
    // TODO: revisit this list with @davglass
    MODULE_META_PRIVATE_ENTRIES  = ['after', 'expanded', 'supersedes', 'ext', '_parsed', '_inspected',
                                    'skinCache', 'langCache'],

    REGEX_LANG_TOKEN = /\"\{langToken\}\"/g,
    REGEX_LANG_PATH  = /\{langPath\}/g,
    REGEX_LOCALE     = /\_([a-z]{2}(-[A-Z]{2})?)$/,

    DEFAULT_HEADERS = {
        '.js': {
            'Content-Type': 'application/javascript; charset=utf-8'
        },
        '.css': {
            'Content-Type': 'text/css; charset=utf-8'
        }
    },

    MODULE_TEMPLATES = {
        'loader-app-base':
            'YUI.add("loader-app-base",function(Y){' +
                'Y.applyConfig({groups:{app:{' +
                    'combine:true,' +
                    'maxURLLength:1024,' +
                    'base:"/static/",' +
                    'comboBase:"/static/combo?",' +
                    'root:"",' +
                    'modules:{app-base}' +
                '}}});' +
            '},"",{requires:["loader-base"]});',
        'loader-app-full':
            'YUI.add("loader-app-full",function(Y){' +
                'Y.applyConfig({groups:{app:{' +
                    'combine:true,' +
                    'maxURLLength:1024,' +
                    'base:"/static/",' +
                    'comboBase:"/static/combo?",' +
                    'root:"",' +
                    'modules:{app-full}' +
                '}}});' +
            '},"",{requires:["loader-base"]});',

        'loader':
            'YUI.add("loader",function(Y){' +
            '},"",{requires:["loader-base","loader-yui3","loader-app-base"]});',

        'loader-lock':
            'YUI.add("loader",function(Y){' +
                // TODO: we should use YUI.applyConfig() instead of the internal
                //       YUI.Env API, but that's pending due a bug in YUI:
                //       http://yuilibrary.com/projects/yui3/ticket/2532854
                'YUI.Env[Y.version].modules=YUI.Env[Y.version].modules||' +
                '{yui-base};' +
            '},"",{requires:["loader-base","loader-app-base"]});',

        'loader-full':
            'YUI.add("loader",function(Y){' +
                // TODO: we should use YUI.applyConfig() instead of the internal
                //       YUI.Env API, but that's pending due a bug in YUI:
                //       http://yuilibrary.com/projects/yui3/ticket/2532854
                'YUI.Env[Y.version].modules=YUI.Env[Y.version].modules||' +
                '{yui-full};' +
            '},"",{requires:["loader-base","loader-app-full"]});'
    };

/*
 * Check if `req` and response `headers`.
 *
 * @param {IncomingMessage} req
 * @param {Object} headers
 * @return {Boolean}
 * @api private
 */
function modified(req, headers) {
    var modifiedSince = req.headers['if-modified-since'],
        lastModified = headers['Last-Modified'],
        noneMatch = req.headers['if-none-match'],
        etag = headers.ETag;

    // Check If-None-Match
    if (noneMatch && etag && noneMatch === etag) {
        return false;
    }

    // Check If-Modified-Since
    if (modifiedSince && lastModified) {
        modifiedSince = new Date(modifiedSince);
        lastModified = new Date(lastModified);
        // Ignore invalid dates
        if (!isNaN(modifiedSince.getTime())) {
            if (lastModified <= modifiedSince) {
                return false;
            }
        }
    }

    return true;
}

/*
 * Check if `req` is a conditional GET request.
 *
 * @method conditionalGET
 * @param {IncomingMessage} req
 * @return {Boolean}
 * @api private
 */
function conditionalGET(req) {
    return req.headers['if-modified-since'] ||
        req.headers['if-none-match'];
}

/*
 * Return an ETag in the form of size-mtime.
 *
 * @method etag
 * @param {Object} stat
 * @return {String}
 * @api private
 */
function etag(stat) {
    return stat.size + '-' + Number(stat.mtime);
}

/*
 * Respond with 304 "Not Modified".
 *
 * @method notModified
 * @param {ServerResponse} res
 * @param {Object} headers
 * @api private
 */
function notModified(res, headers) {
    // Strip Content-* headers
    Object.keys(headers).forEach(function(field) {
        if (0 === field.indexOf('Content')) {
            delete headers[field];
        }
    });
    res.writeHead(304, headers);
    res.end();
}

/*
 * Respond with 403 "Forbidden".
 *
 * @method forbidden
 * @param {ServerResponse} res
 * @api private
 */
function forbidden(res) {
    var body = 'Forbidden';
    res.writeHead(403, {
        'Content-Type': 'text/plain',
        'Content-Length': body.length
    });
    res.end(body);
}

function getModuleResources(store) {
    var ress,
        m,
        mojit,
        mojits,
        moduleRess = {};

    function processRess(ress) {
        var r,
            res;
        for (r = 0; r < ress.length; r += 1) {
            res = ress[r];
            if ('common' !== res.affinity.affinity) {
                continue;
            }
            if (res.yui && res.yui.name) {
                moduleRess[res.yui.name] = res;
            }
        }
        for (r = 0; r < ress.length; r += 1) {
            res = ress[r];
            if ('client' !== res.affinity.affinity) {
                continue;
            }
            if (res.yui && res.yui.name) {
                if (moduleRess[res.yui.name]) {
                    logger.log('YUI Modules should have unique name per affinity. ' +
                               'Module [' + res.yui.name + '] has both common and ' +
                               'client affinity.', 'error', NAME);
                }
                moduleRess[res.yui.name] = res;
            }
        }
    }

    ress = store.getResourceVersions({});
    processRess(ress);

    mojits = store.listAllMojits();
    mojits.push('shared');
    for (m = 0; m < mojits.length; m += 1) {
        mojit = mojits[m];
        ress = store.getResourceVersions({mojit: mojit});
        processRess(ress);
    }

    return moduleRess;
}

/*
 * Static file server.
 *
 * Options:
 *
 *   - `root`     Root path from which to serve static files.
 *   - `maxAge`   Browser cache maxAge in milliseconds, defaults to 0
 *   - `cache`    When true cache files in memory indefinitely,
 *                until invalidated by a conditional GET request.
 *                When given, maxAge will be derived from this value.
 *
 * @param {Object} options
 * @return {Function}
 * @api public
 */
function staticProvider(store, globalLogger) {
    var appConfig = store.getAppConfig(),
        options = appConfig.staticHandling || {},
        cache = options.cache,
        maxAge = options.maxAge,
        moduleRess,
        Y,
        loader,
        i;

    logger = globalLogger;

    moduleRess = getModuleResources(store);

    Y = YUI({
        fetchCSS: true,
        combine: true,
        base: "/static/combo?",
        comboBase: "/static/combo?",
        root: ""
    }, ((appConfig.yui && appConfig.yui.config && appConfig.yui.config.config) || {}));

    // used to find the the modules in YUI itself
    Y.use('loader');
    loader = new Y.Loader();

    if (cache && !maxAge) {
        maxAge = cache;
    }
    maxAge = maxAge || 0;

    return function(req, res, next) {
        if (req.method !== 'GET' && req.method !== 'HEAD') {
            return next();
        }

        var url = parseUrl(req.url),
            files = [],
            module,
            ext,
            result = [],
            failures = 0,
            counter = 0,
            i = 0,
            head = (req.method === 'HEAD');

        // only combo requests are allow here
        if (libpath.basename(url.pathname) !== 'combo' || !url.query) {
            return next();
        }

        logger.log('serving combo url: ' + url.query, 'debug', NAME);

        // YIV might be messing around with the querystring params
        // trying to formalize them by adding = and transforming /
        // so we need to revert back to / and remove the =
        // TODO: this might not work in Windows
        files = url.query.replace(/[=]/g, '').replace(/%2F/g, '/').split('&');

        if (files.length === 0) {
            // probably an empty /combo? request
            res.writeHead(400);
            res.end(undefined);
            return;
        }

        // combo response's content-type should be
        // resolved from the first file in the list
        ext = libpath.extname(files[0]);

        if (!ext || !DEFAULT_HEADERS.hasOwnProperty(ext)) {
            // probably an invalid request
            res.writeHead(400);
            res.end(undefined);
            return;
        }

        function readHandler(index, module) {
            return function (err, data) {
                var headers,
                    content = '',
                    j;

                counter += 1;
                if (err) {
                    logger.log('NOT FOUND: ' + module, 'error', NAME);
                    failures += 1;
                } else {
                    result[index].content = data;
                }
                if (counter === files.length) {
                    if (failures) {
                        res.writeHead(400);
                        res.end(undefined);
                        return;
                    }
                    for (j = 0; j < counter; j += 1) {
                        content += result[j].content;
                    }
                    // FUTURE:  use libfs.stat for last-modified
                    headers = Y.merge({
                        'Content-Length': content.length,
                        'Last-Modified': new Date().toUTCString(),
                        'Cache-Control': 'public max-age=' + (maxAge / 1000)
                    }, (DEFAULT_HEADERS[ext] || {}));

                    res.writeHead(200, headers);
                    res.end(head ? undefined : content);
                }
            };
        }

        // validating all files before doing anything else
        // so errors can be found early on.
        for (i = 0; i < files.length; i += 1) {
            // something like:
            // - foo/bar-min.js becomes "bar"
            // - foo/lang/bar_en-US.js becomes "lang/bar_en-US"
            module = (files[i].indexOf('/lang/') >= 0 ? 'lang/' : '') +
                libpath.basename(files[i], ext).replace(/\-(min|debug)$/, '');

            if (moduleRess[module]) {
                // geting an app module
                result[i] = {
                    module: module,
                    res: moduleRess[module]
                };
            } else if (loader.moduleInfo[module] && loader.moduleInfo[module].path) {
                // getting a yui module
                result[i] = {
                    module: module,
                    fullpath: libpath.join(__dirname,
                        '../../../node_modules/yui', loader.moduleInfo[module].path)
                };
            } else {
                logger.log('Invalid module name: ' + module, 'error', NAME);
                res.writeHead(400);
                res.end(undefined);
                return;
            }
        }

        // async queue implementation
        if (files.length > 0 && (result.length === files.length)) {
            for (i = 0; i < result.length; i += 1) {
                if (result[i].content) {
                    // if we already have content in memory, let's
                    // just use it directly.
                    readHandler(i, module)(null, result[i].content);
                } else if (result[i].res) {
                    store.getResourceContent(result[i].res, readHandler(i, module));
                } else if (result[i].fullpath) {
                    try {
                        libfs.readFile(result[i].fullpath, readHandler(i, module));
                    } catch (err) {
                        logger.log('Error reading: ' + result[i].module, 'error', NAME);
                        // this should never happen, if the module is in loader
                        // meta, it should be in disk as well.
                        res.writeHead(400);
                        res.end(undefined);
                        break;
                    }
                }
                // no "else" clause since result[i] either has .res or .fullpath
            }
        }

    };  // middleware handler
}

/**
 * Export function to create the static handler.
 * @param {Object} config The configuration data for the handler.
 * @return {Object} A static handler.
 */
module.exports = function(config) {
    return staticProvider(config.store, config.logger, config.Y);
};
<commit_msg>trimmed unused stuff<commit_after>/*
 * Ext JS Connect
 * Copyright(c) 2010 Sencha Inc.
 * MIT Licensed
 *
 * Modified by Yahoo!
 * Copyright (c) 2011-2012, Yahoo! Inc.  All rights reserved.
 * Yahoo! Copyrights licensed under the New BSD License.
 * See the accompanying LICENSE file for terms.
 */

/*
 * Connect staticProvider middleware adapted for Mojito *
 ********************************************************
 * This was modified to allow load all combo urls from the
 * Mojito development environment instead of one static
 * directory.
 ********************************************************
 */

/*jslint anon:true, sloppy:true, nomen:true, stupid:true, node: true*/

'use strict';

/*
DECLAIMER: this is VERY experimental, and the purpose of this
middleware is to provide an easy way to load yui modules by their
names rather than the real static path.
    from: @caridy
*/

/*
 * Module dependencies.
 */
var libfs = require('fs'),
    libpath = require('path'),
    YUI = require(libpath.join(__dirname, '..', '..', 'yui-sandbox.js')).getYUI(),
    parseUrl = require('url').parse,
    logger,
    NAME = 'ComboHandler',

    DEFAULT_HEADERS = {
        '.js': {
            'Content-Type': 'application/javascript; charset=utf-8'
        },
        '.css': {
            'Content-Type': 'text/css; charset=utf-8'
        }
    };

function getModuleResources(store) {
    var ress,
        m,
        mojit,
        mojits,
        moduleRess = {};

    function processRess(ress) {
        var r,
            res;
        for (r = 0; r < ress.length; r += 1) {
            res = ress[r];
            if ('common' !== res.affinity.affinity) {
                continue;
            }
            if (res.yui && res.yui.name) {
                moduleRess[res.yui.name] = res;
            }
        }
        for (r = 0; r < ress.length; r += 1) {
            res = ress[r];
            if ('client' !== res.affinity.affinity) {
                continue;
            }
            if (res.yui && res.yui.name) {
                if (moduleRess[res.yui.name]) {
                    logger.log('YUI Modules should have unique name per affinity. ' +
                               'Module [' + res.yui.name + '] has both common and ' +
                               'client affinity.', 'error', NAME);
                }
                moduleRess[res.yui.name] = res;
            }
        }
    }

    ress = store.getResourceVersions({});
    processRess(ress);

    mojits = store.listAllMojits();
    mojits.push('shared');
    for (m = 0; m < mojits.length; m += 1) {
        mojit = mojits[m];
        ress = store.getResourceVersions({mojit: mojit});
        processRess(ress);
    }

    return moduleRess;
}

/*
 * Static file server.
 *
 * Options:
 *
 *   - `root`     Root path from which to serve static files.
 *   - `maxAge`   Browser cache maxAge in milliseconds, defaults to 0
 *   - `cache`    When true cache files in memory indefinitely,
 *                until invalidated by a conditional GET request.
 *                When given, maxAge will be derived from this value.
 *
 * @param {Object} options
 * @return {Function}
 * @api public
 */
function staticProvider(store, globalLogger) {
    var appConfig = store.getAppConfig(),
        options = appConfig.staticHandling || {},
        cache = options.cache,
        maxAge = options.maxAge,
        moduleRess,
        Y,
        loader,
        i;

    logger = globalLogger;

    moduleRess = getModuleResources(store);

    Y = YUI({
        fetchCSS: true,
        combine: true,
        base: "/static/combo?",
        comboBase: "/static/combo?",
        root: ""
    }, ((appConfig.yui && appConfig.yui.config && appConfig.yui.config.config) || {}));

    // used to find the the modules in YUI itself
    Y.use('loader');
    loader = new Y.Loader();

    if (cache && !maxAge) {
        maxAge = cache;
    }
    maxAge = maxAge || 0;

    return function(req, res, next) {
        if (req.method !== 'GET' && req.method !== 'HEAD') {
            return next();
        }

        var url = parseUrl(req.url),
            files = [],
            module,
            ext,
            result = [],
            failures = 0,
            counter = 0,
            i = 0,
            head = (req.method === 'HEAD');

        // only combo requests are allow here
        if (libpath.basename(url.pathname) !== 'combo' || !url.query) {
            return next();
        }

        logger.log('serving combo url: ' + url.query, 'debug', NAME);

        // YIV might be messing around with the querystring params
        // trying to formalize them by adding = and transforming /
        // so we need to revert back to / and remove the =
        // TODO: this might not work in Windows
        files = url.query.replace(/[=]/g, '').replace(/%2F/g, '/').split('&');

        if (files.length === 0) {
            // probably an empty /combo? request
            res.writeHead(400);
            res.end(undefined);
            return;
        }

        // combo response's content-type should be
        // resolved from the first file in the list
        ext = libpath.extname(files[0]);

        if (!ext || !DEFAULT_HEADERS.hasOwnProperty(ext)) {
            // probably an invalid request
            res.writeHead(400);
            res.end(undefined);
            return;
        }

        function readHandler(index, module) {
            return function (err, data) {
                var headers,
                    content = '',
                    j;

                counter += 1;
                if (err) {
                    logger.log('NOT FOUND: ' + module, 'error', NAME);
                    failures += 1;
                } else {
                    result[index].content = data;
                }
                if (counter === files.length) {
                    if (failures) {
                        res.writeHead(400);
                        res.end(undefined);
                        return;
                    }
                    for (j = 0; j < counter; j += 1) {
                        content += result[j].content;
                    }
                    // FUTURE:  use libfs.stat for last-modified
                    headers = Y.merge({
                        'Content-Length': content.length,
                        'Last-Modified': new Date().toUTCString(),
                        'Cache-Control': 'public max-age=' + (maxAge / 1000)
                    }, (DEFAULT_HEADERS[ext] || {}));

                    res.writeHead(200, headers);
                    res.end(head ? undefined : content);
                }
            };
        }

        // validating all files before doing anything else
        // so errors can be found early on.
        for (i = 0; i < files.length; i += 1) {
            // something like:
            // - foo/bar-min.js becomes "bar"
            // - foo/lang/bar_en-US.js becomes "lang/bar_en-US"
            module = (files[i].indexOf('/lang/') >= 0 ? 'lang/' : '') +
                libpath.basename(files[i], ext).replace(/\-(min|debug)$/, '');

            if (moduleRess[module]) {
                // geting an app module
                result[i] = {
                    module: module,
                    res: moduleRess[module]
                };
            } else if (loader.moduleInfo[module] && loader.moduleInfo[module].path) {
                // getting a yui module
                result[i] = {
                    module: module,
                    fullpath: libpath.join(__dirname,
                        '../../../node_modules/yui', loader.moduleInfo[module].path)
                };
            } else {
                logger.log('Invalid module name: ' + module, 'error', NAME);
                res.writeHead(400);
                res.end(undefined);
                return;
            }
        }

        // async queue implementation
        if (files.length > 0 && (result.length === files.length)) {
            for (i = 0; i < result.length; i += 1) {
                if (result[i].content) {
                    // if we already have content in memory, let's
                    // just use it directly.
                    readHandler(i, module)(null, result[i].content);
                } else if (result[i].res) {
                    store.getResourceContent(result[i].res, readHandler(i, module));
                } else if (result[i].fullpath) {
                    try {
                        libfs.readFile(result[i].fullpath, readHandler(i, module));
                    } catch (err) {
                        logger.log('Error reading: ' + result[i].module, 'error', NAME);
                        // this should never happen, if the module is in loader
                        // meta, it should be in disk as well.
                        res.writeHead(400);
                        res.end(undefined);
                        break;
                    }
                }
                // no "else" clause since result[i] either has .res or .fullpath
            }
        }

    };  // middleware handler
}

/**
 * Export function to create the static handler.
 * @param {Object} config The configuration data for the handler.
 * @return {Object} A static handler.
 */
module.exports = function(config) {
    return staticProvider(config.store, config.logger, config.Y);
};
<|endoftext|>


<commit_before>test(
  'ValueSchemaRawTest',

  [
    'ephox.agar.api.Logger',
    'ephox.agar.api.RawAssertions',
    'ephox.boulder.api.FieldPresence',
    'ephox.boulder.api.FieldSchema',
    'ephox.boulder.api.ValueSchema',
    'ephox.katamari.api.Result',
    'ephox.sand.api.JSON'
  ],

  function (Logger, RawAssertions, FieldPresence, FieldSchema, ValueSchema, Result, Json) {
    var checkErr = function (label, expectedPart, input, processor) {
      ValueSchema.asRaw(label, processor, input).fold(function (err) {
        var message = ValueSchema.formatError(err);
        RawAssertions.assertEq(label + '. Was looking to see if contained: ' + expectedPart + '.\nWas: ' + message, true, message.indexOf(expectedPart) > -1);
      }, function (val) {
        assert.fail(label + '\nExpected error: ' + expectedPart + '\nWas success(' + Json.stringify(val, null, 2) + ')');
      });
    };

    var check = function (label, input, processor) {
      var actual = ValueSchema.asRawOrDie(label, processor, input);
      RawAssertions.assertEq(label, input, actual);
    };

    var checkIs = function (label, expected, input, processor) {
      var actual = ValueSchema.asRawOrDie(label, processor, input);
      RawAssertions.assertEq(label, expected, actual);
    };

    check('test.1', 10, ValueSchema.anyValue());

    check('test.2', [ 10, 20, 50 ], ValueSchema.arrOfVal());

    check('test.3', {
      a: 'a',
      b: 'b'
    }, ValueSchema.objOf([
      FieldSchema.strict('a'),
      FieldSchema.strict('b')
    ]));

    checkIs('test.3 (with extra)', {
      a: 'a',
      b: 'b'
    }, {
      a: 'a',
      b: 'b',
      c: 'c'
    }, ValueSchema.objOf([
      FieldSchema.strict('a'),
      FieldSchema.strict('b')
    ]));

    checkErr(
      'test.3 (with extra and only) and two fields',
      'unsupported fields: [c]',
      {
        a: 'a',
        b: 'b',
        c: 'c'
      },
      ValueSchema.objOfOnly([
        FieldSchema.strict('a'),
        FieldSchema.strict('b')
      ])
    );

    checkErr(
      'test.3 (with extra and only) and no fields',
      'unsupported fields: [aa]',
      {
        aa: 'aa'
      },
      ValueSchema.objOfOnly([

      ])
    );

    check(
      'test.3 with no forbidden field',
      {
        a: 'a',
        b: 'b'
      },
      ValueSchema.objOfOnly([
        FieldSchema.strict('a'),
        FieldSchema.strict('b')
      ])
    );

    checkErr(
      'test.3 with 1 forbidden field',
      'Do not use c. Use b',
      {
        a: 'a',
        b: 'b',
        c: 'c'
      },
      ValueSchema.objOf([
        FieldSchema.strict('a'),
        FieldSchema.strict('b'),
        FieldSchema.forbid('c', 'Do not use c. Use b')
      ])
    );

    check('test.4', {
      urls: [
        { url: 'hi', fresh: 'true' },
        { url: 'hi', fresh: 'true' }
      ]
    }, ValueSchema.objOf([
      FieldSchema.strictArrayOfObj('urls', [
        FieldSchema.strict('url'),
        FieldSchema.defaulted('fresh', '10')
      ])
    ]));

    check('test.5', {
      urls: [ 'dog', 'cat' ]
    }, ValueSchema.objOf([
      FieldSchema.strictArrayOf('urls', ValueSchema.anyValue())
    ]));

    checkErr('test.6 should fail because fields do not both start with f',
      'start-with-f',
      {
        'fieldA': { val: 'a' },
        'cieldB': { val: 'b' }
      }, ValueSchema.setOf(function (key) {
        return key.indexOf('f') > -1 ? Result.value(key) : Result.error('start-with-f error');
      }, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    checkErr('test.6b should fail because values do not contain "val"',
      'val',
      {
        'fieldA': { val2: 'a' },
        'fieldB': { val2: 'b' }
      }, ValueSchema.setOf(Result.value, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    check('test.7',
      {
        'fieldA': { val: 'a' },
        'fieldB': { val: 'b' }
      }, ValueSchema.setOf(Result.value, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    check('test.8',
      {
        prop: {
          'merged': true,
          other: 'yes'
        }
      }, ValueSchema.objOf([
        FieldSchema.field('prop', 'prop', FieldPresence.mergeWith({ merged: true }), ValueSchema.anyValue())
      ])
    );

    checkIs('test.9 (defaulted thunk) with no value supplied',
      {
        name: 'Dr Jekyll'
      },
      {
        surname: 'Jekyll'
      }, ValueSchema.objOf([
        FieldSchema.field('name', 'name', FieldPresence.defaultedThunk(function (s) {
          return 'Dr ' + s.surname;
        }), ValueSchema.anyValue())
      ])
    );

    checkIs('test.10 (defaulted thunk) with value supplied',
      {
        name: 'Hyde'
      },
      {
        name: 'Hyde'
      }, ValueSchema.objOf([
        FieldSchema.field('name', 'name', FieldPresence.defaultedThunk(function (s) {
          return 'Dr ' + s.surname;
        }), ValueSchema.anyValue())
      ])
    );

    var optionValue = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
      FieldSchema.option('alpha')
    ]), {});
    RawAssertions.assertEq('alpha should be none', true, optionValue.alpha.isNone());

    var optionValue2 = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
      FieldSchema.option('alpha')
    ]), { alpha: 'beta' });
    RawAssertions.assertEq('alpha should be some', true, optionValue2.alpha.isSome());

    var optionValue3 = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
      FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
    ]), { alpha: 'beta' });
    RawAssertions.assertEq('fallback.opt: alpha:beta should be some(beta)', 'beta', optionValue3.alpha.getOrDie());

    var optionValue4 = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
      FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
    ]), { alpha: true });
    RawAssertions.assertEq('fallback.opt: alpha:true should be some(fallback)', 'fallback', optionValue4.alpha.getOrDie());

    var optionValue5 = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
      FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
    ]), {  });
    RawAssertions.assertEq('fallback.opt: no alpha should be none', true, optionValue5.alpha.isNone());

    Logger.sync('asDefaultedOptionThunk not supplied', function () {
      var optionValue6 = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk' }
      );
      RawAssertions.assertEq('fallback.opt: no alpha should be none', true, optionValue6.alpha.isNone());
    });

    Logger.sync('asDefaultedOptionThunk supplied as true', function () {
      var v = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk', alpha: true }
      );
      RawAssertions.assertEq('Checking output', 'defaulted thunk.fallback', v.alpha.getOrDie(
        'Alpha should be some'
      ));
    });

    Logger.sync('asDefaultedOptionThunk supplied', function () {
      var v = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk', alpha: 'alpha.value' }
      );
      RawAssertions.assertEq('Checking output', 'alpha.value', v.alpha.getOrDie(
        'Alpha should be some'
      ));
    });

    Logger.sync(
      'Checking choose',
      function () {

        var processor = ValueSchema.choose(
          'type',
          {
            'general': [
              FieldSchema.strict('cards')
            ],
            'other': [
              FieldSchema.strict('houses')
            ]
          }
        );

        checkIs(
          'Checking choose(other)',
          { houses: '10' },
          {
            type: 'other',
            houses: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(other) without everything',
          'Could not find valid *strict* value for "houses"',
          {
            type: 'other',
            house: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(other) without wrong schema',
          'Could not find valid *strict* value for "houses"',
          {
            type: 'other',
            cards: '10'
          },
          processor
        );

        checkIs(
          'Checking choose(general)',
          { cards: '10' },
          {
            type: 'general',
            cards: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(general) without everything',
          'Could not find valid *strict* value for "cards"',
          {
            type: 'general',
            mech: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(general) with wrong schema',
          'Could not find valid *strict* value for "cards"',
          {
            type: 'general',
            houses: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(dog)',
          'chosen schema: "dog" did not exist',
          {
            type: 'dog',
            houses: '10'
          },
          processor
        );

      }
    );

  }
);<commit_msg>TM-60: wrapped up option schema tests in logger.sync<commit_after>test(
  'ValueSchemaRawTest',

  [
    'ephox.agar.api.Logger',
    'ephox.agar.api.RawAssertions',
    'ephox.boulder.api.FieldPresence',
    'ephox.boulder.api.FieldSchema',
    'ephox.boulder.api.ValueSchema',
    'ephox.katamari.api.Result',
    'ephox.sand.api.JSON'
  ],

  function (Logger, RawAssertions, FieldPresence, FieldSchema, ValueSchema, Result, Json) {
    var checkErr = function (label, expectedPart, input, processor) {
      ValueSchema.asRaw(label, processor, input).fold(function (err) {
        var message = ValueSchema.formatError(err);
        RawAssertions.assertEq(label + '. Was looking to see if contained: ' + expectedPart + '.\nWas: ' + message, true, message.indexOf(expectedPart) > -1);
      }, function (val) {
        assert.fail(label + '\nExpected error: ' + expectedPart + '\nWas success(' + Json.stringify(val, null, 2) + ')');
      });
    };

    var check = function (label, input, processor) {
      var actual = ValueSchema.asRawOrDie(label, processor, input);
      RawAssertions.assertEq(label, input, actual);
    };

    var checkIs = function (label, expected, input, processor) {
      var actual = ValueSchema.asRawOrDie(label, processor, input);
      RawAssertions.assertEq(label, expected, actual);
    };

    check('test.1', 10, ValueSchema.anyValue());

    check('test.2', [ 10, 20, 50 ], ValueSchema.arrOfVal());

    check('test.3', {
      a: 'a',
      b: 'b'
    }, ValueSchema.objOf([
      FieldSchema.strict('a'),
      FieldSchema.strict('b')
    ]));

    checkIs('test.3 (with extra)', {
      a: 'a',
      b: 'b'
    }, {
      a: 'a',
      b: 'b',
      c: 'c'
    }, ValueSchema.objOf([
      FieldSchema.strict('a'),
      FieldSchema.strict('b')
    ]));

    checkErr(
      'test.3 (with extra and only) and two fields',
      'unsupported fields: [c]',
      {
        a: 'a',
        b: 'b',
        c: 'c'
      },
      ValueSchema.objOfOnly([
        FieldSchema.strict('a'),
        FieldSchema.strict('b')
      ])
    );

    checkErr(
      'test.3 (with extra and only) and no fields',
      'unsupported fields: [aa]',
      {
        aa: 'aa'
      },
      ValueSchema.objOfOnly([

      ])
    );

    check(
      'test.3 with no forbidden field',
      {
        a: 'a',
        b: 'b'
      },
      ValueSchema.objOfOnly([
        FieldSchema.strict('a'),
        FieldSchema.strict('b')
      ])
    );

    checkErr(
      'test.3 with 1 forbidden field',
      'Do not use c. Use b',
      {
        a: 'a',
        b: 'b',
        c: 'c'
      },
      ValueSchema.objOf([
        FieldSchema.strict('a'),
        FieldSchema.strict('b'),
        FieldSchema.forbid('c', 'Do not use c. Use b')
      ])
    );

    check('test.4', {
      urls: [
        { url: 'hi', fresh: 'true' },
        { url: 'hi', fresh: 'true' }
      ]
    }, ValueSchema.objOf([
      FieldSchema.strictArrayOfObj('urls', [
        FieldSchema.strict('url'),
        FieldSchema.defaulted('fresh', '10')
      ])
    ]));

    check('test.5', {
      urls: [ 'dog', 'cat' ]
    }, ValueSchema.objOf([
      FieldSchema.strictArrayOf('urls', ValueSchema.anyValue())
    ]));

    checkErr('test.6 should fail because fields do not both start with f',
      'start-with-f',
      {
        'fieldA': { val: 'a' },
        'cieldB': { val: 'b' }
      }, ValueSchema.setOf(function (key) {
        return key.indexOf('f') > -1 ? Result.value(key) : Result.error('start-with-f error');
      }, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    checkErr('test.6b should fail because values do not contain "val"',
      'val',
      {
        'fieldA': { val2: 'a' },
        'fieldB': { val2: 'b' }
      }, ValueSchema.setOf(Result.value, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    check('test.7',
      {
        'fieldA': { val: 'a' },
        'fieldB': { val: 'b' }
      }, ValueSchema.setOf(Result.value, ValueSchema.objOf([
        FieldSchema.strict('val')
      ]))
    );

    check('test.8',
      {
        prop: {
          'merged': true,
          other: 'yes'
        }
      }, ValueSchema.objOf([
        FieldSchema.field('prop', 'prop', FieldPresence.mergeWith({ merged: true }), ValueSchema.anyValue())
      ])
    );

    checkIs('test.9 (defaulted thunk) with no value supplied',
      {
        name: 'Dr Jekyll'
      },
      {
        surname: 'Jekyll'
      }, ValueSchema.objOf([
        FieldSchema.field('name', 'name', FieldPresence.defaultedThunk(function (s) {
          return 'Dr ' + s.surname;
        }), ValueSchema.anyValue())
      ])
    );

    checkIs('test.10 (defaulted thunk) with value supplied',
      {
        name: 'Hyde'
      },
      {
        name: 'Hyde'
      }, ValueSchema.objOf([
        FieldSchema.field('name', 'name', FieldPresence.defaultedThunk(function (s) {
          return 'Dr ' + s.surname;
        }), ValueSchema.anyValue())
      ])
    );

    Logger.sync('option, value not supplied', function () {
      var v = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
        FieldSchema.option('alpha')
      ]), {});
      RawAssertions.assertEq('alpha should be none', true, v.alpha.isNone());
    });

    Logger.sync('option, value supplied', function () {
      var v = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
        FieldSchema.option('alpha')
      ]), { alpha: 'beta' });
      RawAssertions.assertEq('alpha should be some(beta)', 'beta', v.alpha.getOrDie('expected some'));
    });

    Logger.sync('defaulted option(fallback), value supplied', function () {
      var v = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
        FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
      ]), { alpha: 'beta' });
      RawAssertions.assertEq('fallback.opt: alpha:beta should be some(beta)', 'beta', v.alpha.getOrDie());
    });

    Logger.sync('defaulted option(fallback), value supplied as true', function () {
      var v = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
        FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
      ]), { alpha: true });
      RawAssertions.assertEq('fallback.opt: alpha:true should be some(fallback)', 'fallback', v.alpha.getOrDie());
    });

    Logger.sync('defaulted option(fallback), value not supplied', function () {
      var v = ValueSchema.asRawOrDie('test.option', ValueSchema.objOf([
        FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOption('fallback'), ValueSchema.anyValue())
      ]), {  });
      RawAssertions.assertEq('fallback.opt: no alpha should be none', true, v.alpha.isNone());
    });

    Logger.sync('asDefaultedOptionThunk not supplied', function () {
      var v = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk' }
      );
      RawAssertions.assertEq('fallback.opt: no alpha should be none', true, v.alpha.isNone());
    });

    Logger.sync('asDefaultedOptionThunk supplied as true', function () {
      var v = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk', alpha: true }
      );
      RawAssertions.assertEq('Checking output', 'defaulted thunk.fallback', v.alpha.getOrDie(
        'Alpha should be some'
      ));
    });

    Logger.sync('asDefaultedOptionThunk supplied', function () {
      var v = ValueSchema.asRawOrDie(
        'test.option',
        ValueSchema.objOf([
          FieldSchema.field('alpha', 'alpha', FieldPresence.asDefaultedOptionThunk(function (s) {
            return s.label + '.' + 'fallback';
          }), ValueSchema.anyValue())
        ]),
        { label: 'defaulted thunk', alpha: 'alpha.value' }
      );
      RawAssertions.assertEq('Checking output', 'alpha.value', v.alpha.getOrDie(
        'Alpha should be some'
      ));
    });

    Logger.sync(
      'Checking choose',
      function () {

        var processor = ValueSchema.choose(
          'type',
          {
            'general': [
              FieldSchema.strict('cards')
            ],
            'other': [
              FieldSchema.strict('houses')
            ]
          }
        );

        checkIs(
          'Checking choose(other)',
          { houses: '10' },
          {
            type: 'other',
            houses: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(other) without everything',
          'Could not find valid *strict* value for "houses"',
          {
            type: 'other',
            house: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(other) without wrong schema',
          'Could not find valid *strict* value for "houses"',
          {
            type: 'other',
            cards: '10'
          },
          processor
        );

        checkIs(
          'Checking choose(general)',
          { cards: '10' },
          {
            type: 'general',
            cards: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(general) without everything',
          'Could not find valid *strict* value for "cards"',
          {
            type: 'general',
            mech: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(general) with wrong schema',
          'Could not find valid *strict* value for "cards"',
          {
            type: 'general',
            houses: '10'
          },
          processor
        );

        checkErr(
          'Checking choose(dog)',
          'chosen schema: "dog" did not exist',
          {
            type: 'dog',
            houses: '10'
          },
          processor
        );

      }
    );

  }
);<|endoftext|>


<commit_before>import filter from 'lodash.filter'
import intersection from 'lodash.intersection'
import uniq from 'lodash.uniq'
import includes from 'lodash.includes'
import { CronJob } from 'cron'
import { default as mapToArray } from 'lodash.map'

// ===================================================================

const PERFORMANCE_MODE = 0
const DENSITY_MODE = 1

const LOW_BEHAVIOR = 0
const NORMAL_BEHAVIOR = 1
const AGGRESSIVE_BEHAVIOR = 2

// Delay between each ressources evaluation in minutes.
// Must be less than MINUTES_OF_HISTORICAL_DATA.
const EXECUTION_DELAY = 1
const MINUTES_OF_HISTORICAL_DATA = 30

// Threshold cpu in percent.
// const CRITICAL_THRESHOLD_CPU = 90
const HIGH_THRESHOLD_CPU = 0 // 76.5
// const LOW_THRESHOLD_CPU = 22.5

// const CRITICAL_THRESHOLD_FREE_MEMORY = 51
const HIGH_THRESHOLD_FREE_MEMORY = 63.75
// const LOW_THRESHOLD_FREE_MEMORY = 1020

// ===================================================================

export const configurationSchema = {
  type: 'object',

  properties: {
    plans: {
      type: 'array',
      description: 'an array of plans',

      items: {
        type: 'object',
        title: 'plan',

        properties: {
          name: {
            type: 'string'
          },

          mode: {
            type: 'object',

            properties: {
              performance: { type: 'boolean' },
              density: { type: 'boolean' }
            },

            oneOf: [
              { required: ['performance'] },
              { required: ['density'] }
            ]
          },

          behavior: {
            type: 'object',

            properties: {
              low: { type: 'boolean' },
              normal: { type: 'boolean' },
              aggressive: { type: 'boolean' }
            },

            oneOf: [
              { required: ['low'] },
              { required: ['normal'] },
              { required: ['aggressive'] }
            ]
          },

          pools: {
            type: 'array',
            description: 'list of pools id where to apply the policy',

            items: {
              type: 'string',
              $objectType: 'pool'
            },

            minItems: 1,
            uniqueItems: true
          }
        }
      },
      minItems: 1
    }
  },

  additionalProperties: false
}

// ===================================================================

const makeCronJob = (cronPattern, fn) => {
  let running

  const job = new CronJob(cronPattern, async () => {
    if (running) {
      return
    }

    running = true

    try {
      await fn()
    } catch (error) {
      console.error('[WARN] scheduled function:', error && error.stack || error)
    } finally {
      running = false
    }
  })

  return job
}

function computeAverage (values, nPoints = values.length) {
  let sum = 0
  let tot = 0

  const { length } = values

  for (let i = length - nPoints; i < length; i++) {
    const value = values[i]

    sum += value || 0

    if (value) {
      tot += 1
    }
  }

  return sum / tot
}

function computeRessourcesAverage (objects, objectsStats, nPoints) {
  const averages = {}

  for (const object of objects) {
    const { id } = object
    const { stats } = objectsStats[id]
    const objectAverages = averages[id] = {}

    objectAverages.cpus = computeAverage(
      mapToArray(stats.cpus, cpu => computeAverage(cpu, nPoints))
    )
    objectAverages.memoryFree = computeAverage(stats.memoryFree, nPoints)
    objectAverages.memoryUsed = computeAverage(stats.memoryUsed, nPoints)
  }

  return averages
}

function checkRessourcesThresholds (objects, averages) {
  return filter(objects, object => {
    const objectAverages = averages[object.id]

    return (
      objectAverages.cpus >= HIGH_THRESHOLD_CPU ||
      objectAverages.memoryFree >= HIGH_THRESHOLD_FREE_MEMORY
    )
  })
}

function computeRessourcesAverageWithWeight (averages1, averages2, ratio) {
  const averages = {}

  for (const id in averages1) {
    const objectAverages = averages[id] = {}

    for (const averageName in averages1[id]) {
      objectAverages[averageName] = averages1[id][averageName] * ratio + averages2[id][averageName] * (1 - ratio)
    }
  }

  return averages
}

function setRealCpuAverageOfVms (vms, vmsAverages) {
  for (const vm of vms) {
    vmsAverages[vm.id].cpus /= vm.CPUs.number
  }
}

// ===================================================================

class Plan {
  constructor (xo, { name, mode, behavior, poolIds }) {
    this.xo = xo
    this._name = name // Useful ?
    this._mode = mode
    this._behavior = behavior
    this._poolIds = poolIds
  }

  async execute () {
    if (this._mode === PERFORMANCE_MODE) {
      await this._executeInPerformanceMode()
    } else {
      await this._executeInDensityMode()
    }
  }

  async _executeInPerformanceMode () {
    const hosts = this._getHosts()
    const hostsStats = await this._getHostsStats(hosts, 'minutes')

    // 1. Check if a ressource's utilization exceeds threshold.
    const avgNow = computeRessourcesAverage(hosts, hostsStats, EXECUTION_DELAY)
    let exceededHosts = checkRessourcesThresholds(hosts, avgNow)

    // No ressource's utilization problem.
    if (exceededHosts.length === 0) {
      return
    }

    // 2. Check in the last 30 min interval with ratio.
    const avgBefore = computeRessourcesAverage(hosts, hostsStats, MINUTES_OF_HISTORICAL_DATA)
    const avgWithRatio = computeRessourcesAverageWithWeight(avgNow, avgBefore, 0.75)
    exceededHosts = checkRessourcesThresholds(exceededHosts, avgWithRatio)

    // No ressource's utilization problem.
    if (exceededHosts.length === 0) {
      return
    }

    // 3. Reorder the exceeded hosts by priority.
    exceededHosts.sort((a, b) => {
      a = avgWithRatio[a.id]
      b = avgWithRatio[b.id]

      return (b.cpus - a.cpus) || (a.memoryFree - b.memoryFree)
    })

    // 4. Search bests combinations for the worst host.
    const toOptimize = exceededHosts[0]
    const optimizations = await this._computeOptimizations(
      toOptimize,
      filter(hosts, host => host.id !== toOptimize.id),
      avgWithRatio
    )

    // 5. Apply optimizations if necessary.
    await this._applyOptimizations(optimizations)
  }

  async _executeInDensityMode () {
    throw new Error('not yet implemented')
  }

  async _computeOptimizations (exceededHost, hostsAverages) {
    const vms = await this._getVms(exceededHost.id)
    const vmsStats = await this._getVmsStats(vms, 'minutes')
    const vmsAverages = computeRessourcesAverageWithWeight(
      computeRessourcesAverage(vms, vmsStats, EXECUTION_DELAY),
      computeRessourcesAverage(vms, vmsStats, MINUTES_OF_HISTORICAL_DATA),
      0.75
    )

    // Compute real CPU usage. Virtuals cpus to real cpu.
    setRealCpuAverageOfVms(vms, vmsAverages)

    // TODO
  }

  async _applyOptimizations (optimizations) {
    // throw new Error('not yet implemented')
  }

  // Compute hosts for each pool. They can change over time.
  _getHosts () {
    return filter(this.xo.getObjects(), object =>
      object.type === 'host' && includes(this._poolIds, object.$poolId)
    )
  }

  async _getHostsStats (hosts, granularity) {
    const hostsStats = {}

    await Promise.all(mapToArray(hosts, host =>
      this.xo.getXapiHostStats(host, granularity).then(hostStats => {
        hostsStats[host.id] = {
          nPoints: hostStats.stats.cpus[0].length,
          stats: hostStats.stats,
          averages: {}
        }
      })
    ))

    return hostsStats
  }

  async _getVms (hostId) {
    return filter(this.xo.getObjects(), object =>
      object.type === 'VM' &&
      object.power_state === 'Running' &&
      object.$container === hostId
    )
  }

  async _getVmsStats (vms, granularity) {
    const vmsStats = {}

    await Promise.all(mapToArray(vms, vm =>
      this.xo.getXapiVmStats(vm, granularity).then(vmStats => {
        vmsStats[vm.id] = {
          nPoints: vmStats.stats.cpus[0].length,
          stats: vmStats.stats,
          averages: {}
        }
      })
    ))

    return vmsStats
  }
}

// ===================================================================

class LoadBalancerPlugin {
  constructor (xo) {
    this.xo = xo
    this._cronJob = makeCronJob(`*/${EXECUTION_DELAY} * * * *`, ::this._executePlans)
  }

  async configure ({ plans }) {
    const cronJob = this._cronJob
    const enabled = cronJob.running

    if (enabled) {
      cronJob.stop()
    }

    // Wait until all old plans stopped running.
    await this._plansPromise

    this._plans = []
    this._poolIds = [] // Used pools.

    if (plans) {
      for (const plan of plans) {
        const mode = plan.mode.performance
          ? PERFORMANCE_MODE
          : DENSITY_MODE

        const { behavior: planBehavior } = plan
        let behavior

        if (planBehavior.low) {
          behavior = LOW_BEHAVIOR
        } else if (planBehavior.normal) {
          behavior = NORMAL_BEHAVIOR
        } else {
          behavior = AGGRESSIVE_BEHAVIOR
        }

        this._addPlan({ name: plan.name, mode, behavior, poolIds: plan.pools })
      }
    }

    // TMP
    this._addPlan({
      name: 'Test plan',
      mode: PERFORMANCE_MODE,
      behavior: AGGRESSIVE_BEHAVIOR,
      poolIds: [ '313624ab-0958-bb1e-45b5-7556a463a10b' ]
    })

    if (enabled) {
      cronJob.start()
    }
  }

  load () {
    this._cronJob.start()
  }

  unload () {
    this._cronJob.stop()
  }

  _addPlan (plan) {
    const poolIds = plan.poolIds = uniq(plan.poolIds)

    // Check already used pools.
    if (intersection(poolIds, this._poolIds) > 0) {
      throw new Error(`Pool(s) already included in an other plan: ${poolIds}`)
    }

    this._plans.push(new Plan(this.xo, plan))
  }

  _executePlans () {
    return (this._plansPromise = Promise.all(
      mapToArray(this._plans, plan => plan.execute())
    ))
  }
}

// ===================================================================

export default ({ xo }) => new LoadBalancerPlugin(xo)
<commit_msg>Unsable: _computeOptimizations<commit_after>import filter from 'lodash.filter'
import includes from 'lodash.includes'
import intersection from 'lodash.intersection'
import uniq from 'lodash.uniq'

import { CronJob } from 'cron'
import { default as mapToArray } from 'lodash.map'

// ===================================================================

const PERFORMANCE_MODE = 0
const DENSITY_MODE = 1

const LOW_BEHAVIOR = 0
const NORMAL_BEHAVIOR = 1
const AGGRESSIVE_BEHAVIOR = 2

// Delay between each ressources evaluation in minutes.
// Must be less than MINUTES_OF_HISTORICAL_DATA.
const EXECUTION_DELAY = 1
const MINUTES_OF_HISTORICAL_DATA = 30

// Threshold cpu in percent.
// const CRITICAL_THRESHOLD_CPU = 90
const HIGH_THRESHOLD_CPU = 0 // 76.5
// const LOW_THRESHOLD_CPU = 22.5

// const CRITICAL_THRESHOLD_FREE_MEMORY = 51
const HIGH_THRESHOLD_FREE_MEMORY = 63.75
// const LOW_THRESHOLD_FREE_MEMORY = 1020

// ===================================================================

export const configurationSchema = {
  type: 'object',

  properties: {
    plans: {
      type: 'array',
      description: 'an array of plans',

      items: {
        type: 'object',
        title: 'plan',

        properties: {
          name: {
            type: 'string'
          },

          mode: {
            type: 'object',

            properties: {
              performance: { type: 'boolean' },
              density: { type: 'boolean' }
            },

            oneOf: [
              { required: ['performance'] },
              { required: ['density'] }
            ]
          },

          behavior: {
            type: 'object',

            properties: {
              low: { type: 'boolean' },
              normal: { type: 'boolean' },
              aggressive: { type: 'boolean' }
            },

            oneOf: [
              { required: ['low'] },
              { required: ['normal'] },
              { required: ['aggressive'] }
            ]
          },

          pools: {
            type: 'array',
            description: 'list of pools id where to apply the policy',

            items: {
              type: 'string',
              $objectType: 'pool'
            },

            minItems: 1,
            uniqueItems: true
          }
        }
      },
      minItems: 1
    }
  },

  additionalProperties: false
}

// ===================================================================

const makeCronJob = (cronPattern, fn) => {
  let running

  const job = new CronJob(cronPattern, async () => {
    if (running) {
      return
    }

    running = true

    try {
      await fn()
    } catch (error) {
      console.error('[WARN] scheduled function:', error && error.stack || error)
    } finally {
      running = false
    }
  })

  return job
}

function computeAverage (values, nPoints = values.length) {
  let sum = 0
  let tot = 0

  const { length } = values

  for (let i = length - nPoints; i < length; i++) {
    const value = values[i]

    sum += value || 0

    if (value) {
      tot += 1
    }
  }

  return sum / tot
}

function computeRessourcesAverage (objects, objectsStats, nPoints) {
  const averages = {}

  for (const object of objects) {
    const { id } = object
    const { stats } = objectsStats[id]
    const objectAverages = averages[id] = {}

    objectAverages.cpus = computeAverage(
      mapToArray(stats.cpus, cpu => computeAverage(cpu, nPoints))
    )
    objectAverages.memoryFree = computeAverage(stats.memoryFree, nPoints)
    objectAverages.memoryUsed = computeAverage(stats.memoryUsed, nPoints)
  }

  return averages
}

function checkRessourcesThresholds (objects, averages) {
  return filter(objects, object => {
    const objectAverages = averages[object.id]

    return (
      objectAverages.cpus >= HIGH_THRESHOLD_CPU ||
      objectAverages.memoryFree >= HIGH_THRESHOLD_FREE_MEMORY
    )
  })
}

function computeRessourcesAverageWithWeight (averages1, averages2, ratio) {
  const averages = {}

  for (const id in averages1) {
    const objectAverages = averages[id] = {}

    for (const averageName in averages1[id]) {
      objectAverages[averageName] = averages1[id][averageName] * ratio + averages2[id][averageName] * (1 - ratio)
    }
  }

  return averages
}

function setRealCpuAverageOfVms (vms, vmsAverages) {
  for (const vm of vms) {
    vmsAverages[vm.id].cpus /= vm.CPUs.number
  }
}

function searchObject (objects, fun) {
  let object = 0

  for (let i = 1; i < objects.length; i++) {
    if (fun(object, objects[i]) > 0) {
      object = objects[i]
    }
  }

  return object
}

// ===================================================================

class Plan {
  constructor (xo, { name, mode, behavior, poolIds }) {
    this.xo = xo
    this._name = name // Useful ?
    this._mode = mode
    this._behavior = behavior
    this._poolIds = poolIds
  }

  async execute () {
    if (this._mode === PERFORMANCE_MODE) {
      await this._executeInPerformanceMode()
    } else {
      await this._executeInDensityMode()
    }
  }

  async _executeInPerformanceMode () {
    const hosts = this._getHosts()
    const hostsStats = await this._getHostsStats(hosts, 'minutes')

    // 1. Check if a ressource's utilization exceeds threshold.
    const avgNow = computeRessourcesAverage(hosts, hostsStats, EXECUTION_DELAY)
    let exceededHosts = checkRessourcesThresholds(hosts, avgNow)

    // No ressource's utilization problem.
    if (exceededHosts.length === 0) {
      return
    }

    // 2. Check in the last 30 min interval with ratio.
    const avgBefore = computeRessourcesAverage(hosts, hostsStats, MINUTES_OF_HISTORICAL_DATA)
    const avgWithRatio = computeRessourcesAverageWithWeight(avgNow, avgBefore, 0.75)
    exceededHosts = checkRessourcesThresholds(exceededHosts, avgWithRatio)

    // No ressource's utilization problem.
    if (exceededHosts.length === 0) {
      return
    }

    // 3. Search the worst exceeded host by priority.
    const toOptimize = searchObject(exceededHosts, (a, b) => {
      a = avgWithRatio[a.id]
      b = avgWithRatio[b.id]

      return (b.cpus - a.cpus) || (a.memoryFree - b.memoryFree)
    })

    // 4. Search bests combinations for the worst host.
    const optimizations = await this._computeOptimizations(
      toOptimize,
      filter(hosts, host => host.id !== toOptimize.id),
      avgWithRatio
    )

    // 5. Apply optimizations if necessary.
    await this._applyOptimizations(optimizations)
  }

  async _executeInDensityMode () {
    throw new Error('not yet implemented')
  }

  async _computeOptimizations (exceededHost, hosts, hostsAverages) {
    const vms = await this._getVms(exceededHost.id)
    const vmsStats = await this._getVmsStats(vms, 'minutes')
    const vmsAverages = computeRessourcesAverageWithWeight(
      computeRessourcesAverage(vms, vmsStats, EXECUTION_DELAY),
      computeRessourcesAverage(vms, vmsStats, MINUTES_OF_HISTORICAL_DATA),
      0.75
    )

    // Compute real CPU usage. Virtuals cpus to reals cpus.
    setRealCpuAverageOfVms(vms, vmsAverages)

    const optimizations = {}

    // Sort vms by cpu usage. (higher to lower)
    vms.sort((a, b) =>
      vmsAverages[b.id].cpus - vmsAverages[a.id].cpus
    )

    const exceededAverages = hostsAverages[exceededHosts.id]

    for (const vm of vms) {
      // Search host with lower cpu usage.
      const destination = searchObject(hosts, (a, b) =>
        hostsAverages[b.id].cpus - hostsAverages[a.id]
                                      )
      const destinationAverages = hostsAverages[destination.id]
      const vmAverages = vmsAverages[vm.id]

      // Unable to move the vm.
      if (
        exceededAverages.cpus - vmAverages.cpu < destinationAverages.cpu + vmAverages.cpu ||

      ) {
        continue
      }
    }

    return optimizations
  }

  async _applyOptimizations (optimizations) {
    // throw new Error('not yet implemented')
  }

  // Compute hosts for each pool. They can change over time.
  _getHosts () {
    return filter(this.xo.getObjects(), object =>
      object.type === 'host' && includes(this._poolIds, object.$poolId)
    )
  }

  async _getHostsStats (hosts, granularity) {
    const hostsStats = {}

    await Promise.all(mapToArray(hosts, host =>
      this.xo.getXapiHostStats(host, granularity).then(hostStats => {
        hostsStats[host.id] = {
          nPoints: hostStats.stats.cpus[0].length,
          stats: hostStats.stats,
          averages: {}
        }
      })
    ))

    return hostsStats
  }

  async _getVms (hostId) {
    return filter(this.xo.getObjects(), object =>
      object.type === 'VM' &&
      object.power_state === 'Running' &&
      object.$container === hostId
    )
  }

  async _getVmsStats (vms, granularity) {
    const vmsStats = {}

    await Promise.all(mapToArray(vms, vm =>
      this.xo.getXapiVmStats(vm, granularity).then(vmStats => {
        vmsStats[vm.id] = {
          nPoints: vmStats.stats.cpus[0].length,
          stats: vmStats.stats,
          averages: {}
        }
      })
    ))

    return vmsStats
  }
}

// ===================================================================

class LoadBalancerPlugin {
  constructor (xo) {
    this.xo = xo
    this._cronJob = makeCronJob(`*/${EXECUTION_DELAY} * * * *`, ::this._executePlans)
  }

  async configure ({ plans }) {
    const cronJob = this._cronJob
    const enabled = cronJob.running

    if (enabled) {
      cronJob.stop()
    }

    // Wait until all old plans stopped running.
    await this._plansPromise

    this._plans = []
    this._poolIds = [] // Used pools.

    if (plans) {
      for (const plan of plans) {
        const mode = plan.mode.performance
          ? PERFORMANCE_MODE
          : DENSITY_MODE

        const { behavior: planBehavior } = plan
        let behavior

        if (planBehavior.low) {
          behavior = LOW_BEHAVIOR
        } else if (planBehavior.normal) {
          behavior = NORMAL_BEHAVIOR
        } else {
          behavior = AGGRESSIVE_BEHAVIOR
        }

        this._addPlan({ name: plan.name, mode, behavior, poolIds: plan.pools })
      }
    }

    // TMP
    this._addPlan({
      name: 'Test plan',
      mode: PERFORMANCE_MODE,
      behavior: AGGRESSIVE_BEHAVIOR,
      poolIds: [ '313624ab-0958-bb1e-45b5-7556a463a10b' ]
    })

    if (enabled) {
      cronJob.start()
    }
  }

  load () {
    this._cronJob.start()
  }

  unload () {
    this._cronJob.stop()
  }

  _addPlan (plan) {
    const poolIds = plan.poolIds = uniq(plan.poolIds)

    // Check already used pools.
    if (intersection(poolIds, this._poolIds) > 0) {
      throw new Error(`Pool(s) already included in an other plan: ${poolIds}`)
    }

    this._plans.push(new Plan(this.xo, plan))
  }

  _executePlans () {
    return (this._plansPromise = Promise.all(
      mapToArray(this._plans, plan => plan.execute())
    ))
  }
}

// ===================================================================

export default ({ xo }) => new LoadBalancerPlugin(xo)
<|endoftext|>


<commit_before>'use strict';

var path = require('path');
var fs = require('fs');
var argv = require('minimist')(process.argv.slice(2));
var tildify = require('tildify');
var gulp = require('gulp');
var gutil = require('gulp-util');
var replace = require('gulp-replace');
var rename = require('gulp-rename');
var stringUtils = require('../utils/string');

function validateComponentName (filename) {
  if (filename.indexOf('-') === -1) {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(filename),
      gutil.colors.red(' must be a dashize string. ex: my-component')
    );
    gutil.log(
      gutil.colors.red('[-Error:]  Generate task has been canceled')
    );
    process.exit(0);
  }
}

function generatorEngine (type, srcPath, moduleName, fileName, finalPath, destPath) {
  var ext = (type === 'template') ? '.hbs' : '.js';
  var fullFilePath = destPath + '/' + fileName + ext;

  // if the file has existed, it will abort the task
  if (fs.existsSync(fullFilePath)) {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(fileName + ext),
      gutil.colors.red('has existed at '),
      gutil.colors.magenta(tildify(destPath))
    );
    gutil.log(
      gutil.colors.red('[-Error:]  Generate task has been canceled')
    );
    process.exit(0);
  }

  var dasherizeName = '';
  var classifyName = '';
  var matcher;

  // if generating any testing files, need to clean up moduleName without "Test"
  if (type.indexOf('test') > -1) {
    if (type.indexOf('model-test') > -1) {
      moduleName = moduleName.replace(/ModelTest(\s+)?$/, '');
      dasherizeName = stringUtils.dasherize(moduleName);
    } else {
      matcher = stringUtils.classify(type);
      var dasherizeModuleName = moduleName.replace(new RegExp(matcher), '');

      moduleName = moduleName.replace(/Test(\s+)?$/, '');
      dasherizeName = stringUtils.dasherize(dasherizeModuleName);
    }
  } else {
    matcher = stringUtils.capitalize(type);
    var localModuleName = moduleName.replace(new RegExp(matcher), '');

    dasherizeName = stringUtils.dasherize(localModuleName);
    classifyName = stringUtils.classify(localModuleName);
  }

  // @TODO when generate multiple files on certain type
  // moduleName is not being defined correctly.
  // fine for now, since multiple file generation are only template file
  return gulp.src(srcPath)
    .pipe(replace(/__NAMESPACE__/g, moduleName))
    // __DASHERIZE_NAMESPACE__  mainly used in `-test` generator
    .pipe(replace(/__DASHERIZE_NAMESPACE__/g, dasherizeName))
    // __CLASSIFY_NAMESPACE__ mainly used in regular generator
    .pipe(replace(/__CLASSIFY_NAMESPACE__/g, classifyName))
    .pipe(rename({
      basename: fileName,
      extname:  ext
    }))
    .on('end', function () {
      gutil.log(
        gutil.colors.green('[-done:] Generate'),
        gutil.colors.cyan(fileName + ext),
        gutil.colors.green('at'),
        gutil.colors.magenta(tildify(destPath))
      );
    })
    .pipe(gulp.dest(destPath));
}

function setupTask (generator) {
  // task: gen
  // @describe	generate an model,view,store,controller from base template
  return gulp.task('gen', function () {
    var type = generator.type;
    var name = generator.name;
    var pathName = '';
    var moduleName = '';
    var i = 0;
    var pathNested; // Boolean
    var fileName; // setup the fileName which used for rename module

    if (name.indexOf('/') > -1) {
      name = name.split('/');
      pathNested = true;
      fileName = name.pop();
    } else {
      pathNested = false;
      fileName = name;
    }

    // handle the error case when arg is `component:foo`
    // component name has to be dash separated string
    // case 1: `em g component:name`         <= simple case
    // case 2: `em g component:nested/name`  <= nested case
    if (type === 'component') {
      validateComponentName(fileName);
    }
    // when type is template, name[0] is component, name of nest path has to be dashized string
    // case 3: `em g template:component/name`  <= nested case in template
    if (type === 'template' && pathNested && name[0] === 'component') {
      validateComponentName(fileName);
    }

    if (pathNested) {
      // build up the nested path
      for (; i < name.length; i++) {
        // 'component' and 'components' resolve as a 'app/templates/components/'
        if (type === 'template' && name[0] === 'component') {
          name[i] = 'components';
        }
        pathName += '/' + name[i];
        moduleName += stringUtils.capitalize(name[i]);
      }
      // append fileName to the moduleName string
      moduleName += stringUtils.capitalize(fileName);
    } else {
      pathName += name;
      moduleName += stringUtils.capitalize(name);
    }

    moduleName += stringUtils.capitalize(type);

    // if it has dashized moduleName, it will camelize
    moduleName = stringUtils.classify(moduleName);

    // ignore the 'store' case, since it is already created
    var typeFolder = path.resolve('client/app', type + 's');

    // if client/app/[type](s) is not existed and it is not a test generator, simply create one
    if (!fs.existsSync(typeFolder) && type.indexOf('test') === -1) {
      fs.mkdirSync(typeFolder);
      gutil.log(
        gutil.colors.gray('[-log:] Created a new folder at '),
        gutil.colors.cyan('~/client/app/' + type + 's')
      );
    }

    // when generator type is route or component
    // it will also generate the template as well
    var srcPath =
      (type === 'route' || type === 'component') ?
        [{
          type:          type,
          generatorPath: path.join(__dirname, '..', 'skeletons/generators', type) + '.js'
        }, {
          type:          'template',
          injection:     ( type === 'component' ) ? 'components' : void 0,
          generatorPath: path.join(__dirname, '..', 'skeletons/generators', 'template.js')
        }]
        : path.join(__dirname, '..', 'skeletons/generators', type) + '.js';

    var dirName, finalDirName, finalPath, destPath;

    // if type is test, or route-test or any sorts, it should append `-test` to the filename
    fileName = (type.indexOf('test') > -1) ? fileName + '-test' : fileName;

    // if it is a string, simple call generatorEngine once
    // else it is an object(array), repeat the generatorEngine call
    if (typeof srcPath === 'string') {
      dirName = (type === 'store') ? type : ( type.slice(-1) === 's' ) ? type : type + 's';

      // Figure out the type is testing generator
      if (type.indexOf('test') > -1) {
        // Is it an Unit Test generator or Integration Test generator
        if (type.indexOf('-test') > -1) {
          var typeArray = type.split('-');
          finalDirName = 'tests/unit/' + typeArray[0] + 's';
        } else {
          finalDirName = dirName + '/integration';
        }
      } else {
        finalDirName = dirName;
      }

      finalPath = pathNested ? finalDirName + pathName : finalDirName;
      destPath = (type.indexOf('test') > -1) ?
      path.resolve('client') + '/' + finalPath
        : path.resolve('client/app') + '/' + finalPath;

      generatorEngine(type, srcPath, moduleName, fileName, finalPath, destPath);
    } else {
      for (var j = 0, l = srcPath.length; j < l; j++) {
        var _type = srcPath[j].type;
        // when original type is 'component'
        // it will create a template file at 'templates/components' folder
        var injection = srcPath[j].injection;

        dirName = (_type === 'store') ? _type : ( _type.slice(-1) === 's' ) ? _type : _type + 's';
        dirName = ( injection ) ? dirName + '/' + injection : dirName;

        finalPath = pathNested ? dirName + pathName : dirName;
        destPath = path.resolve('client/app') + '/' + finalPath;

        generatorEngine(
          _type, srcPath[j].generatorPath, moduleName, fileName, finalPath, destPath
        );
      }
    }
  });
}

// Check the fullname attribute is correct or not
var VALID_FULL_NAME_REGEXP = /^[^:]+.+:[^:]+$/;

function errorHandler (fullName) {
  gutil.log(
    gutil.colors.red('[-Error:] Invalid argument, expected: `type:name` got: '),
    gutil.colors.bold(fullName)
  );

  gutil.log(
    '[-Syntax:]',
    gutil.colors.cyan('type:name'), ' ex: em generate route:post'
  );

  gutil.log(
    gutil.colors.red('[-Error:]'),
    'See \'em generate --help\''
  );
  process.exit(0);
}

var generate = function (options) {
  // if the folder 'client/app' is not existed
  // can assume that the project may not be created by Ember Rocks
  if (!fs.existsSync('client') && !fs.existsSync('client/app')) {
    gutil.log(
      gutil.colors.red(
        '[-Error:] This project may not be created by \'Ember-Rocks\'\n'
      ),
      gutil.colors.red(
        '[-Error:] `em new [dirName]` does not install the NPM packages dependencies correctly'
      )
    );
    process.exit(1);
  }

  // Error out when user did not provide any argument
  if (argv._.length < 2) {
    gutil.log(gutil.colors.red('[-Error:] Missing type:name argument.'), 'ex: em new route:post');
    gutil.log(gutil.colors.red('[-Error:]'), 'See \'em generate --help\'');
    process.exit(0);
  }

  var typeAndName = argv._.slice()[1];

  if (!VALID_FULL_NAME_REGEXP.test(typeAndName)) {
    errorHandler(typeAndName);
  }

  var validTypes = [
    'adapter', 'component', 'controller', 'helper', 'initializer', 'mixin', 'model',
    'route', 'serializer', 'template', 'transform', 'util', 'view',
    'test', 'adapter-test', 'component-test', 'controller-test', 'helper-test',
    'initializer-test', 'mixin-test', 'model-test', 'route-test', 'serializer-test',
    'transform-test', 'util-test', 'view-test'
  ];
  var gen;
  var generatorAndTasks = typeAndName.split(':', 2);
  var type = generatorAndTasks[0];
  var name = generatorAndTasks[1];

  // type could be either route or routes
  type = (type.slice(-1) === 's') ? type.substring(0, type.length - 1) : type;

  // Type must be in the `validTypes` array
  if (validTypes.indexOf(type) > -1) {
    // Name must be a valid string
    if (name.length > 0) {
      gen = {
        type: type,
        name: name
      };
    } else {
      gutil.log(
        gutil.colors.red('[-Error:] '),
        gutil.colors.cyan(name),
        gutil.colors.red(' must be a valid string.')
      );
      gutil.log(gutil.colors.red('[-Error:]'), 'See \'em generate --help\'');
      process.exit(0);
    }
  } else {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(type),
      gutil.colors.red(' is not a valid type.')
    );
    gutil.log(
      gutil.colors.bold('[-note:] valid types are'),
      gutil.colors.cyan(validTypes.join(', '))
    );
    process.exit(0);
  }

  setupTask(gen);
  // Trigger the generator task
  gulp.start('gen');
};

module.exports = generate;
<commit_msg>update the comment for setupTask function<commit_after>'use strict';

var path = require('path');
var fs = require('fs');
var argv = require('minimist')(process.argv.slice(2));
var tildify = require('tildify');
var gulp = require('gulp');
var gutil = require('gulp-util');
var replace = require('gulp-replace');
var rename = require('gulp-rename');
var stringUtils = require('../utils/string');

function validateComponentName (filename) {
  if (filename.indexOf('-') === -1) {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(filename),
      gutil.colors.red(' must be a dashize string. ex: my-component')
    );
    gutil.log(
      gutil.colors.red('[-Error:]  Generate task has been canceled')
    );
    process.exit(0);
  }
}

function generatorEngine (type, srcPath, moduleName, fileName, finalPath, destPath) {
  var ext = (type === 'template') ? '.hbs' : '.js';
  var fullFilePath = destPath + '/' + fileName + ext;

  // if the file has existed, it will abort the task
  if (fs.existsSync(fullFilePath)) {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(fileName + ext),
      gutil.colors.red('has existed at '),
      gutil.colors.magenta(tildify(destPath))
    );
    gutil.log(
      gutil.colors.red('[-Error:]  Generate task has been canceled')
    );
    process.exit(0);
  }

  var dasherizeName = '';
  var classifyName = '';
  var matcher;

  // if generating any testing files, need to clean up moduleName without "Test"
  if (type.indexOf('test') > -1) {
    if (type.indexOf('model-test') > -1) {
      moduleName = moduleName.replace(/ModelTest(\s+)?$/, '');
      dasherizeName = stringUtils.dasherize(moduleName);
    } else {
      matcher = stringUtils.classify(type);
      var dasherizeModuleName = moduleName.replace(new RegExp(matcher), '');

      moduleName = moduleName.replace(/Test(\s+)?$/, '');
      dasherizeName = stringUtils.dasherize(dasherizeModuleName);
    }
  } else {
    matcher = stringUtils.capitalize(type);
    var localModuleName = moduleName.replace(new RegExp(matcher), '');

    dasherizeName = stringUtils.dasherize(localModuleName);
    classifyName = stringUtils.classify(localModuleName);
  }

  // @TODO when generate multiple files on certain type
  // moduleName is not being defined correctly.
  // fine for now, since multiple file generation are only template file
  return gulp.src(srcPath)
    .pipe(replace(/__NAMESPACE__/g, moduleName))
    // __DASHERIZE_NAMESPACE__  mainly used in `-test` generator
    .pipe(replace(/__DASHERIZE_NAMESPACE__/g, dasherizeName))
    // __CLASSIFY_NAMESPACE__ mainly used in regular generator
    .pipe(replace(/__CLASSIFY_NAMESPACE__/g, classifyName))
    .pipe(rename({
      basename: fileName,
      extname:  ext
    }))
    .on('end', function () {
      gutil.log(
        gutil.colors.green('[-done:] Generate'),
        gutil.colors.cyan(fileName + ext),
        gutil.colors.green('at'),
        gutil.colors.magenta(tildify(destPath))
      );
    })
    .pipe(gulp.dest(destPath));
}

function setupTask (generator) {
  // task: gen
  // @describe	generate an model,view,store,controller from base template
  return gulp.task('gen', function () {
    var type = generator.type;
    var name = generator.name;
    var pathName = '';
    var moduleName = '';
    var i = 0;
    var pathNested; // Boolean
    var fileName; // setup the fileName which used for rename module

    // based on the passing name arguments, to determine it is an nested folder structure
    // or it is a simple file generation. assign a var `fileName` for current file name
    if (name.indexOf('/') > -1) {
      name = name.split('/');
      pathNested = true;
      fileName = name.pop();
    } else {
      pathNested = false;
      fileName = name;
    }

    // handle the error case when arg is `component:foo`
    // component name has to be dash separated string
    // case 1: `em g component:name`         <= simple case
    // case 2: `em g component:nested/name`  <= nested case
    if (type === 'component') {
      validateComponentName(fileName);
    }
    // when type is template, name[0] is component, name of nest path has to be dashized string
    // case 3: `em g template:component/name`  <= nested case in template
    if (type === 'template' && pathNested && name[0] === 'component') {
      validateComponentName(fileName);
    }

    if (pathNested) {
      // build up the nested path
      for (; i < name.length; i++) {
        // 'component' and 'components' resolve as a 'app/templates/components/'
        if (type === 'template' && name[0] === 'component') {
          name[i] = 'components';
        }
        pathName += '/' + name[i];
        moduleName += stringUtils.capitalize(name[i]);
      }
      // append fileName to the moduleName string
      moduleName += stringUtils.capitalize(fileName);
    } else {
      pathName += name;
      moduleName += stringUtils.capitalize(name);
    }

    moduleName += stringUtils.capitalize(type);

    // if it has dashized moduleName, it will camelize
    moduleName = stringUtils.classify(moduleName);

    // ignore the 'store' case, since it is already created
    var typeFolder = path.resolve('client/app', type + 's');

    // if client/app/[type](s) is not existed and it is not a test generator, simply create one
    if (!fs.existsSync(typeFolder) && type.indexOf('test') === -1) {
      fs.mkdirSync(typeFolder);
      gutil.log(
        gutil.colors.gray('[-log:] Created a new folder at '),
        gutil.colors.cyan('~/client/app/' + type + 's')
      );
    }

    // when generator type is route or component
    // it will also generate the template as well
    var srcPath =
      (type === 'route' || type === 'component') ?
        [{
          type:          type,
          generatorPath: path.join(__dirname, '..', 'skeletons/generators', type) + '.js'
        }, {
          type:          'template',
          injection:     ( type === 'component' ) ? 'components' : void 0,
          generatorPath: path.join(__dirname, '..', 'skeletons/generators', 'template.js')
        }]
        : path.join(__dirname, '..', 'skeletons/generators', type) + '.js';

    var dirName, finalDirName, finalPath, destPath;

    // if type is test, or route-test or any sorts, it should append `-test` to the filename
    fileName = (type.indexOf('test') > -1) ? fileName + '-test' : fileName;

    // if it is a string, simple call generatorEngine once
    // else it is an object(array), repeat the generatorEngine call
    if (typeof srcPath === 'string') {
      dirName = (type === 'store') ? type : ( type.slice(-1) === 's' ) ? type : type + 's';

      // Figure out the type is testing generator
      if (type.indexOf('test') > -1) {
        // Is it an Unit Test generator or Integration Test generator
        if (type.indexOf('-test') > -1) {
          var typeArray = type.split('-');
          finalDirName = 'tests/unit/' + typeArray[0] + 's';
        } else {
          finalDirName = dirName + '/integration';
        }
      } else {
        finalDirName = dirName;
      }

      finalPath = pathNested ? finalDirName + pathName : finalDirName;
      destPath = (type.indexOf('test') > -1) ?
      path.resolve('client') + '/' + finalPath
        : path.resolve('client/app') + '/' + finalPath;

      generatorEngine(type, srcPath, moduleName, fileName, finalPath, destPath);
    } else {
      for (var j = 0, l = srcPath.length; j < l; j++) {
        var _type = srcPath[j].type;
        // when original type is 'component'
        // it will create a template file at 'templates/components' folder
        var injection = srcPath[j].injection;

        dirName = (_type === 'store') ? _type : ( _type.slice(-1) === 's' ) ? _type : _type + 's';
        dirName = ( injection ) ? dirName + '/' + injection : dirName;

        finalPath = pathNested ? dirName + pathName : dirName;
        destPath = path.resolve('client/app') + '/' + finalPath;

        generatorEngine(
          _type, srcPath[j].generatorPath, moduleName, fileName, finalPath, destPath
        );
      }
    }
  });
}

// Check the fullname attribute is correct or not
var VALID_FULL_NAME_REGEXP = /^[^:]+.+:[^:]+$/;

function errorHandler (fullName) {
  gutil.log(
    gutil.colors.red('[-Error:] Invalid argument, expected: `type:name` got: '),
    gutil.colors.bold(fullName)
  );

  gutil.log(
    '[-Syntax:]',
    gutil.colors.cyan('type:name'), ' ex: em generate route:post'
  );

  gutil.log(
    gutil.colors.red('[-Error:]'),
    'See \'em generate --help\''
  );
  process.exit(0);
}

var generate = function (options) {
  // if the folder 'client/app' is not existed
  // can assume that the project may not be created by Ember Rocks
  if (!fs.existsSync('client') && !fs.existsSync('client/app')) {
    gutil.log(
      gutil.colors.red(
        '[-Error:] This project may not be created by \'Ember-Rocks\'\n'
      ),
      gutil.colors.red(
        '[-Error:] `em new [dirName]` does not install the NPM packages dependencies correctly'
      )
    );
    process.exit(1);
  }

  // Error out when user did not provide any argument
  if (argv._.length < 2) {
    gutil.log(gutil.colors.red('[-Error:] Missing type:name argument.'), 'ex: em new route:post');
    gutil.log(gutil.colors.red('[-Error:]'), 'See \'em generate --help\'');
    process.exit(0);
  }

  var typeAndName = argv._.slice()[1];

  if (!VALID_FULL_NAME_REGEXP.test(typeAndName)) {
    errorHandler(typeAndName);
  }

  var validTypes = [
    'adapter', 'component', 'controller', 'helper', 'initializer', 'mixin', 'model',
    'route', 'serializer', 'template', 'transform', 'util', 'view',
    'test', 'adapter-test', 'component-test', 'controller-test', 'helper-test',
    'initializer-test', 'mixin-test', 'model-test', 'route-test', 'serializer-test',
    'transform-test', 'util-test', 'view-test'
  ];
  var gen;
  var generatorAndTasks = typeAndName.split(':', 2);
  var type = generatorAndTasks[0];
  var name = generatorAndTasks[1];

  // type could be either route or routes
  type = (type.slice(-1) === 's') ? type.substring(0, type.length - 1) : type;

  // Type must be in the `validTypes` array
  if (validTypes.indexOf(type) > -1) {
    // Name must be a valid string
    if (name.length > 0) {
      gen = {
        type: type,
        name: name
      };
    } else {
      gutil.log(
        gutil.colors.red('[-Error:] '),
        gutil.colors.cyan(name),
        gutil.colors.red(' must be a valid string.')
      );
      gutil.log(gutil.colors.red('[-Error:]'), 'See \'em generate --help\'');
      process.exit(0);
    }
  } else {
    gutil.log(
      gutil.colors.red('[-Error:] '),
      gutil.colors.cyan(type),
      gutil.colors.red(' is not a valid type.')
    );
    gutil.log(
      gutil.colors.bold('[-note:] valid types are'),
      gutil.colors.cyan(validTypes.join(', '))
    );
    process.exit(0);
  }

  setupTask(gen);
  // Trigger the generator task
  gulp.start('gen');
};

module.exports = generate;
<|endoftext|>


<commit_before>
module.exports.up = function () {
  require('util.promisify/shim')();
  const log = require('migrate/lib/log');
  const path = require('path');
  const fs = require('fs');
  const util = require('util');

  const writeFile = util.promisify(fs.writeFile);
  const copyFile = util.promisify(fs.copyFile);
  const access = util.promisify(fs.access);

  let modelPath;
  if (process.env.EG_CONFIG_DIR) {
    log('modelPath', 'EG_CONFIG_DIR set.');
    modelPath = path.join(process.env.EG_CONFIG_DIR, 'models');
  } else {
    log('modelPath', 'EG_CONFIG_DIR not set. Guessing the config dir…');
    modelPath = path.join(__dirname, '../../../', 'config/models');
  }

  log('modelPath', `set to: ${modelPath}`);

  return access(modelPath)
    .then(() => Promise.all(
      [copyFile(path.join(__dirname, '../lib/config/models/credentials.json'), path.join(modelPath, 'credentials.json'))]
        .concat(['users', 'applications']
          .map(model => ({ modelName: model, modelDefinition: require(path.join(modelPath, path.format({ name: model, ext: '.js' }))) }))
          .map(({ modelName, modelDefinition }) => {
            const newModel = {
              $id: `http://express-gateway.io/models/${modelName}.json`,
              type: 'object',
              properties: {},
              required: []
            };

            if (modelDefinition.properties) {
              Object.entries(modelDefinition.properties).forEach(([propertyName, propertyValue]) => {
                newModel.properties[propertyName] = {
                  type: 'string'
                };

                if (propertyName === 'email') {
                  newModel.properties[propertyName].format = 'email';
                }

                if (propertyName === 'redirectUri') {
                  newModel.properties[propertyName].format = 'uri';
                }

                if (propertyValue.isRequired) {
                  newModel.required.push(propertyName);
                }
              });
            }

            if (modelName === 'users') {
              newModel.required.push('username');
              newModel.properties.username = {
                type: 'string'
              };
            }

            return { modelName, modelDefinition: newModel };
          }).map(({ modelName, modelDefinition }) =>
            writeFile(
              path.join(modelPath, path.format({ name: modelName, ext: '.json' })),
              JSON.stringify(modelDefinition, null, 2)
            )
          )
        )
    ).then(() => Promise.resolve()));
};

module.exports.down = function (next) {
  throw new Error('I\'m sorry, I can\'t make that happen');
};
<commit_msg>Tell the user what to do after the migration<commit_after>
module.exports.up = function () {
  require('util.promisify/shim')();
  const log = require('migrate/lib/log');
  const path = require('path');
  const fs = require('fs');
  const util = require('util');

  const writeFile = util.promisify(fs.writeFile);
  const copyFile = util.promisify(fs.copyFile);
  const access = util.promisify(fs.access);

  let modelPath;
  if (process.env.EG_CONFIG_DIR) {
    log('modelPath', 'EG_CONFIG_DIR set.');
    modelPath = path.join(process.env.EG_CONFIG_DIR, 'models');
  } else {
    log('modelPath', 'EG_CONFIG_DIR not set. Guessing the config dir…');
    modelPath = path.join(__dirname, '../../../', 'config/models');
  }

  log('modelPath', `set to: ${modelPath}`);

  return access(modelPath)
    .then(() => Promise.all(
      [copyFile(path.join(__dirname, '../lib/config/models/credentials.json'), path.join(modelPath, 'credentials.json'))]
        .concat(['users', 'applications']
          .map(model => ({ modelName: model, modelDefinition: require(path.join(modelPath, path.format({ name: model, ext: '.js' }))) }))
          .map(({ modelName, modelDefinition }) => {
            const newModel = {
              $id: `http://express-gateway.io/models/${modelName}.json`,
              type: 'object',
              properties: {},
              required: []
            };

            if (modelDefinition.properties) {
              Object.entries(modelDefinition.properties).forEach(([propertyName, propertyValue]) => {
                newModel.properties[propertyName] = {
                  type: 'string'
                };

                if (propertyName === 'email') {
                  newModel.properties[propertyName].format = 'email';
                }

                if (propertyName === 'redirectUri') {
                  newModel.properties[propertyName].format = 'uri';
                }

                if (propertyValue.isRequired) {
                  newModel.required.push(propertyName);
                }
              });
            }

            if (modelName === 'users') {
              newModel.required.push('username');
              newModel.properties.username = {
                type: 'string'
              };
            }

            return { modelName, modelDefinition: newModel };
          }).map(({ modelName, modelDefinition }) =>
            writeFile(
              path.join(modelPath, path.format({ name: modelName, ext: '.json' })),
              JSON.stringify(modelDefinition, null, 2)
            )
          )
        )
    ).then(() => {
      log('Complete', 'You can now remove the old .js files');
      Promise.resolve();
    }));
};

module.exports.down = function (next) {
  throw new Error('I\'m sorry, I can\'t make that happen');
};
<|endoftext|>