<commit_before>var express = require('express');
var router = express.Router();
var async = require('async');
var dateFormat = require('dateformat');

/*
 * GET /mf/deployments/ms2/t2.1/cluster
 */
router.get('/:workflow/:task/:platform', function(req, res, next) {
    var client = req.app.get('elastic'),
      workflow = req.params.workflow.toLowerCase(),
      task = req.params.task.toLowerCase(),
      platform = req.params.platform.toLowerCase(),
      deployment = req.params.deployment, // we keep the original case
      mf_server = req.app.get('mf_server'),
      dreamcloud_pwm_idx = req.app.get('pwm_idx'),
      expand = req.query.expand,
      experiments = [],
      predicted_execution_time = 0,
      json = [];

    client.search({
        index: 'deployment_on_' + platform,
        type: workflow + '_' + task,
        searchType: 'count'
    }, function(error, response) {
        if (error) {
            res.status(500);
            return next(error);
        }

        if (response.hits !== undefined) {
            size = response.hits.total;
        }

        client.search({
            index: 'deployment_on_' + platform,
            type: workflow + '_' + task,
            size: size,
        }, function(error, response) {
            if (error) {
                res.status(500);
                return next(error);
            }
            if (response.hits !== undefined) {
                var results = response.hits.hits;
                async.each(results, function(deployment, outer_callback) {
                    deployment = deployment._id;

                    /* (1) get deployment plan for the given hashvalue */
                    client.get({
                        index: 'deployment_on_' + platform,
                        type: workflow + '_' + task,
                        id: deployment
                    }, function(error, response) {
                        if (error) {
                            return next(error);
                        }

                        /* parse all experiments that use this deployment plan */
                        if (response.found) {
                            var source = response._source;
                            experiments = source.experiments;
                            predicted_execution_time = source.estimatedTime;
                        } else {
                            var message = {};
                            message.error = "Could not find deployment plan in the database";
                            res.json(message);
                            return;
                        }

                        var index = workflow + '_' + task;
                        async.forEachOf(experiments, function(value, experiment, callback) {

                            /* get start of experiment */
                            client.search({
                                index: index,
                                type: experiment,
                                size: 1,
                                sort: ['@timestamp:asc']
                            }, function(error, response) {
                                var data = {};
                                if (error) {
                                    callback(null);
                                    return;
                                }
                                var start, end;
                                if (response.hits !== undefined) {
                                    var only_results = response.hits.hits;
                                    var keys = Object.keys(only_results);
                                    keys.forEach(function(key) {
                                        var metric_data = only_results[key]._source;
                                        start = metric_data['@timestamp'];
                                        start = start.replace(/\s/g, '0');
                                    });
                                }

                                /* get end of experiment */
                                client.search({
                                    index: index,
                                    type: experiment,
                                    size: 1,
                                    sort: ['@timestamp:desc']
                                }, function(error, response) {
                                    if (error) {
                                        callback(null);
                                        return;
                                    }
                                    if (response.hits !== undefined) {
                                        var only_results = response.hits.hits;
                                        var keys = Object.keys(only_results);
                                        keys.forEach(function(key) {
                                            var metric_data = only_results[key]._source;
                                            end = metric_data['@timestamp'];
                                            end = end.replace(/\s/g, '0');
                                        });
                                    }

                                    if (!is_defined(start) || !is_defined(end)) {
                                        callback(null);
                                        return;
                                    }

                                    /* add runtime statistics to response object */
                                    data.experiment_id = experiment;
                                    data.workflow_id = workflow;
                                    data.task_id = task;
                                    data.deployment_id = deployment;
                                    data.runtime = {};
                                    data.runtime.start_time = start;
                                    data.runtime.end_time = end;
                                    data.runtime.actual_time = (new Date(end) - new Date(start)) / 1000;
                                    data.runtime.predicted_time = predicted_execution_time;
                                    if (!data.runtime.actual_time) {
                                        data.runtime.actual_time = 0;
                                    }

                                    /* get energy measurments */
                                    var body = compute_average_on("NODE01", "NODE02", "NODE03", start, end);
                                    client.search({
                                        index: dreamcloud_pwm_idx,
                                        searchType: 'count',
                                        body: body
                                    }, function(error, response) {
                                        if (error) {
                                            // do nothing
                                        } else {
                                            var answer = {},
                                                aggs = response.aggregations;

                                            data.energy = {};
                                            var power_data = {};
                                            var average = aggs.power_metrics.NODE01.value;
                                            data.energy.NODE01 = {};
                                            data.energy.NODE01.avg_watt_consumption = average;
                                            var duration = (new Date(end) - new Date(start)) / 1000;
                                            var joule = average * duration;
                                            data.energy.NODE01.total_energy_consumption = joule;

                                            average = aggs.power_metrics.NODE02.value;
                                            data.energy.NODE02 = {};
                                            data.energy.NODE02.avg_watt_consumption = average;
                                            joule = average * duration;
                                            data.energy.NODE02.total_energy_consumption = joule;

                                            average = aggs.power_metrics.NODE03.value;
                                            data.energy.NODE03 = {};
                                            data.energy.NODE03.avg_watt_consumption = average;
                                            joule = average * duration;
                                            data.energy.NODE03.total_energy_consumption = joule;

                                            /* no energy measurements available */
                                            if (!is_defined(average) || average === null) {
                                                data.energy = [];
                                            }
                                        }

                                        /* retrieve all metric counters as a set */
                                        var index = workflow + '_' + task;
                                        var metric_keys = {};
                                        data.metrics = {};
                                        client.search({
                                            index: index,
                                            type: experiment,
                                            size: 20 /* should be enough results to capture all metrics */
                                        }, function(error, response) {
                                            if (error) {
                                                callback(null);
                                                return;
                                            }
                                            if (response.hits !== undefined) {
                                                var results = response.hits.hits,
                                                  keys = Object.keys(results),
                                                  items = {};
                                                /* filter keys like @timestamp, host, task, and type */
                                                keys.forEach(function(key) {
                                                    items = results[key]._source;
                                                    if (items.type == 'progress') {
                                                        return;
                                                    }
                                                    delete items['@timestamp'];
                                                    delete items.host;
                                                    delete items.task;
                                                    delete items.type;
                                                    delete items.platform;
                                                    for (var item in items) {
                                                        metric_keys[item] = item;
                                                    }
                                                });

                                                /* compute statistics for each identified metric */
                                                async.each(metric_keys, function(metric, inner_callback) {
                                                    client.search({
                                                        index: index,
                                                        type: experiment,
                                                        searchType: 'count',
                                                        body: aggregation_by(metric)
                                                    }, function(error, response) {
                                                        if (error) {
                                                            inner_callback(null);
                                                        }
                                                        var aggs = response.aggregations;
                                                        data.metrics[metric] = aggs[metric + '_Stats'];
                                                        inner_callback(null);
                                                    });
                                                }, function(error) {
                                                    json.push(data);
                                                    /* finished */
                                                    callback(null);
                                                });
                                            }
                                        });
                                    });
                                });
                            });
                        }, function(error) {
                            if (error) {
                                return next(error);
                            }

                            //json.push(plan);
                            outer_callback(null);
                        });
                    });
                }, function(error) {
                    res.json(json);
                });
            } else {
                var message = {};
                message = 'No deployment plans available';
                res.json(message);
            }
        });
    });
});

/*
 * GET /mf/deployments/ms2/t2.1/cluster/asij122109klad
 */
router.get('/:workflow/:task/:platform/:deployment', function(req, res, next) {
    var client = req.app.get('elastic'),
      workflow = req.params.workflow.toLowerCase(),
      task = req.params.task.toLowerCase(),
      platform = req.params.platform.toLowerCase(),
      deployment = req.params.deployment, // we keep the original case
      mf_server = req.app.get('mf_server'),
      dreamcloud_pwm_idx = req.app.get('pwm_idx'),
      expand = req.query.expand,
      experiments = [],
      predicted_execution_time = 0,
      json = [];

    /* (1) get deployment plan for the given hashvalue */
    client.get({
        index: 'deployment_on_' + platform,
        type: workflow + '_' + task,
        id: deployment
    }, function(error, response) {
        if (error) {
            return next(error);
        }
        /* parse all experiments that use this deployment plan */
        if (response.found) {
            var source = response._source;
            experiments = source.experiments;
            predicted_execution_time = source.estimatedTime;
        } else {
            var message = {};
            message.error = "Could not find deployment plan in the database";
            res.json(message);
            return;
        }

        var index = workflow + '_' + task;
        async.forEachOf(experiments, function(value, experiment, callback) {
            /* get start of experiment */
            client.search({
                index: index,
                type: experiment,
                size: 1,
                sort: ['@timestamp:asc']
            }, function(error, response) {
                if (error) {
                    callback(null);
                    return;
                }
                var start, end;
                if (response.hits !== undefined) {
                    var only_results = response.hits.hits;
                    var keys = Object.keys(only_results);
                    keys.forEach(function(key) {
                        var metric_data = only_results[key]._source;
                        start = metric_data['@timestamp'];
                        start = start.replace(/\s/g, '0');
                    });
                }

                /* get end of experiment */
                client.search({
                    index: index,
                    type: experiment,
                    size: 1,
                    sort: ['@timestamp:desc']
                }, function(error, response) {
                    if (error) {
                        callback(null);
                        return;
                    }
                    if (response.hits !== undefined) {
                        var only_results = response.hits.hits;
                        var keys = Object.keys(only_results);
                        keys.forEach(function(key) {
                            var metric_data = only_results[key]._source;
                            end = metric_data['@timestamp'];
                            end = end.replace(/\s/g, '0');
                        });
                    }

                    var data = {};
                    data.experiment_id = experiment;
                    data.workflow_id = workflow;
                    data.task_id = task;
                    data.deployment_id = deployment;

                    if (!is_defined(start) || !is_defined(end)) {
                        json.push(data);
                        callback(null);
                        return;
                    }

                    /* add runtime statistics to response object */
                    data.runtime = {};
                    data.runtime.start_time = start;
                    data.runtime.end_time = end;
                    data.runtime.actual_time = (new Date(end) - new Date(start)) / 1000;
                    data.runtime.predicted_time = predicted_execution_time;
                    if (!data.runtime.actual_time) {
                        data.runtime.actual_time = 0;
                    }

                    /* get energy measurments */
                    var body = compute_average_on("NODE01", "NODE02", "NODE03", start, end);
                    client.search({
                        index: dreamcloud_pwm_idx,
                        searchType: 'count',
                        body: body
                    }, function(error, response) {
                        if (error) {
                            callback(null);
                            return;
                        }
                        var answer = {},
                            aggs = response.aggregations;

                        data.energy = {};
                        var power_data = {};
                        var average = aggs.power_metrics.NODE01.value;
                        data.energy.NODE01 = {};
                        data.energy.NODE01.avg_watt_consumption = average;
                        var duration = (new Date(end) - new Date(start)) / 1000;
                        var joule = average * duration;
                        data.energy.NODE01.total_energy_consumption = joule;

                        average = aggs.power_metrics.NODE02.value;
                        data.energy.NODE02 = {};
                        data.energy.NODE02.avg_watt_consumption = average;
                        joule = average * duration;
                        data.energy.NODE02.total_energy_consumption = joule;

                        average = aggs.power_metrics.NODE03.value;
                        data.energy.NODE03 = {};
                        data.energy.NODE03.avg_watt_consumption = average;
                        joule = average * duration;
                        data.energy.NODE03.total_energy_consumption = joule;

                        /* no energy measurements available */
                        if (!is_defined(average) || average === null) {
                            data.energy = [];
                        }

                        /* retrieve all metric counters as a set */
                        var index = workflow + '_' + task;
                        var metric_keys = {};
                        data.metrics = {};
                        client.search({
                            index: index,
                            type: experiment,
                            size: 20 /* should be enough results to capture all metrics */
                        }, function(error, response) {
                            if (error) {
                                callback(null);
                                return;
                            }
                            if (response.hits !== undefined) {
                                var results = response.hits.hits,
                                  keys = Object.keys(results),
                                  items = {};
                                /* filter keys like @timestamp, host, task, and type */
                                keys.forEach(function(key) {
                                    items = results[key]._source;
                                    if (items.type == 'progress') {
                                        return;
                                    }
                                    delete items['@timestamp'];
                                    delete items.host;
                                    delete items.task;
                                    delete items.type;
                                    for (var item in items) {
                                        metric_keys[item] = item;
                                    }
                                });

                                /* compute statistics for each identified metric */
                                async.each(metric_keys, function(metric, inner_callback) {
                                    client.search({
                                        index: index,
                                        type: experiment,
                                        searchType: 'count',
                                        body: aggregation_by(metric)
                                    }, function(error, response) {
                                        if (error) {
                                            inner_callback(null);
                                        }
                                        var aggs = response.aggregations;
                                        data.metrics[metric] = aggs[metric + '_Stats'];
                                        inner_callback(null);
                                    });
                                }, function(error) {
                                    json.push(data);
                                    /* finished */
                                    callback(null);
                                });
                            }
                        });
                    });
                });
            });
        }, function(error) {
            if (error) {
                return next(error);
            }

            res.json(json);
        });
    });
});

function is_defined(variable) {
    return (typeof variable !== 'undefined');
}

function compute_average_on(metric_name_a, metric_name_b, metric_name_c, from, to) {
    var query = {
        "aggs": {
            "power_metrics": {
                "filter": {
                    "and": [{
                        "or": [{
                            "exists": {
                                "field": metric_name_a
                            }
                        }, {
                            "exists": {
                                "field": metric_name_b
                            }
                        }, {
                            "exists": {
                                "field": metric_name_c
                            }
                        }]
                    }, {
                        "range": {
                            "@timestamp": {
                                "gte": from.toString(),
                                "lte": to.toString()
                            }
                        }
                    }]
                },
                "aggs": {
                    "NODE01": {
                        "avg": {
                            "field": metric_name_a
                        }
                    },
                    "NODE02": {
                        "avg": {
                            "field": metric_name_b
                        }
                    },
                    "NODE03": {
                        "avg": {
                            "field": metric_name_c
                        }
                    }
                }
            }
        }
    };
    return query;
}

function aggregation_by(field_name) {
    return '{' +
        '"aggs": {' +
            '"' + field_name + '_Stats" : {' +
                '"stats" : {' +
                    '"field" : "' + field_name + '"' +
                '}' +
            '},' +
            '"Minimum_' + field_name + '": {' +
                '"top_hits": {' +
                    '"size": 1,' +
                    '"sort": [' +
                        '{' +
                            '"' + field_name + '": {' +
                                '"order": "asc"' +
                            '}' +
                        '}' +
                    ']' +
                '}' +
            '},' +
            '"Maximum_' + field_name + '": {' +
                '"top_hits": {' +
                    '"size": 1,' +
                    '"sort": [' +
                        '{' +
                            '"' + field_name + '": {' +
                                '"order": "desc"' +
                            '}' +
                        '}' +
                    ']' +
                '}' +
            '}' +
        '}' +
    '}';
}

module.exports = router;
<commit_msg>added function to delete optional keys in the metric profile prefixed with 'info_'<commit_after>var express = require('express');
var router = express.Router();
var async = require('async');
var dateFormat = require('dateformat');

/*
 * GET /mf/deployments/ms2/t2.1/cluster
 */
router.get('/:workflow/:task/:platform', function(req, res, next) {
    var client = req.app.get('elastic'),
      workflow = req.params.workflow.toLowerCase(),
      task = req.params.task.toLowerCase(),
      platform = req.params.platform.toLowerCase(),
      deployment = req.params.deployment, // we keep the original case
      mf_server = req.app.get('mf_server'),
      dreamcloud_pwm_idx = req.app.get('pwm_idx'),
      expand = req.query.expand,
      experiments = [],
      predicted_execution_time = 0,
      json = [];

    client.search({
        index: 'deployment_on_' + platform,
        type: workflow + '_' + task,
        searchType: 'count'
    }, function(error, response) {
        if (error) {
            res.status(500);
            return next(error);
        }

        if (response.hits !== undefined) {
            size = response.hits.total;
        }

        client.search({
            index: 'deployment_on_' + platform,
            type: workflow + '_' + task,
            size: size,
        }, function(error, response) {
            if (error) {
                res.status(500);
                return next(error);
            }
            if (response.hits !== undefined) {
                var results = response.hits.hits;
                async.each(results, function(deployment, outer_callback) {
                    deployment = deployment._id;

                    /* (1) get deployment plan for the given hashvalue */
                    client.get({
                        index: 'deployment_on_' + platform,
                        type: workflow + '_' + task,
                        id: deployment
                    }, function(error, response) {
                        if (error) {
                            return next(error);
                        }

                        /* parse all experiments that use this deployment plan */
                        if (response.found) {
                            var source = response._source;
                            experiments = source.experiments;
                            predicted_execution_time = source.estimatedTime;
                        } else {
                            var message = {};
                            message.error = "Could not find deployment plan in the database";
                            res.json(message);
                            return;
                        }

                        var index = workflow + '_' + task;
                        async.forEachOf(experiments, function(value, experiment, callback) {

                            /* get start of experiment */
                            client.search({
                                index: index,
                                type: experiment,
                                size: 1,
                                sort: ['@timestamp:asc']
                            }, function(error, response) {
                                var data = {};
                                if (error) {
                                    callback(null);
                                    return;
                                }
                                var start, end;
                                if (response.hits !== undefined) {
                                    var only_results = response.hits.hits;
                                    var keys = Object.keys(only_results);
                                    keys.forEach(function(key) {
                                        var metric_data = only_results[key]._source;
                                        start = metric_data['@timestamp'];
                                        start = start.replace(/\s/g, '0');
                                    });
                                }

                                /* get end of experiment */
                                client.search({
                                    index: index,
                                    type: experiment,
                                    size: 1,
                                    sort: ['@timestamp:desc']
                                }, function(error, response) {
                                    if (error) {
                                        callback(null);
                                        return;
                                    }
                                    if (response.hits !== undefined) {
                                        var only_results = response.hits.hits;
                                        var keys = Object.keys(only_results);
                                        keys.forEach(function(key) {
                                            var metric_data = only_results[key]._source;
                                            end = metric_data['@timestamp'];
                                            end = end.replace(/\s/g, '0');
                                        });
                                    }

                                    if (!is_defined(start) || !is_defined(end)) {
                                        callback(null);
                                        return;
                                    }

                                    /* add runtime statistics to response object */
                                    data.experiment_id = experiment;
                                    data.workflow_id = workflow;
                                    data.task_id = task;
                                    data.deployment_id = deployment;
                                    data.runtime = {};
                                    data.runtime.start_time = start;
                                    data.runtime.end_time = end;
                                    data.runtime.actual_time = (new Date(end) - new Date(start)) / 1000;
                                    data.runtime.predicted_time = predicted_execution_time;
                                    if (!data.runtime.actual_time) {
                                        data.runtime.actual_time = 0;
                                    }

                                    /* get energy measurments */
                                    var body = compute_average_on("NODE01", "NODE02", "NODE03", start, end);
                                    client.search({
                                        index: dreamcloud_pwm_idx,
                                        searchType: 'count',
                                        body: body
                                    }, function(error, response) {
                                        if (error) {
                                            // do nothing
                                        } else {
                                            var answer = {},
                                                aggs = response.aggregations;

                                            data.energy = {};
                                            var power_data = {};
                                            var average = aggs.power_metrics.NODE01.value;
                                            data.energy.NODE01 = {};
                                            data.energy.NODE01.avg_watt_consumption = average;
                                            var duration = (new Date(end) - new Date(start)) / 1000;
                                            var joule = average * duration;
                                            data.energy.NODE01.total_energy_consumption = joule;

                                            average = aggs.power_metrics.NODE02.value;
                                            data.energy.NODE02 = {};
                                            data.energy.NODE02.avg_watt_consumption = average;
                                            joule = average * duration;
                                            data.energy.NODE02.total_energy_consumption = joule;

                                            average = aggs.power_metrics.NODE03.value;
                                            data.energy.NODE03 = {};
                                            data.energy.NODE03.avg_watt_consumption = average;
                                            joule = average * duration;
                                            data.energy.NODE03.total_energy_consumption = joule;

                                            /* no energy measurements available */
                                            if (!is_defined(average) || average === null) {
                                                data.energy = [];
                                            }
                                        }

                                        /* retrieve all metric counters as a set */
                                        var index = workflow + '_' + task;
                                        var metric_keys = {};
                                        data.metrics = {};
                                        client.search({
                                            index: index,
                                            type: experiment,
                                            size: 20 /* should be enough results to capture all metrics */
                                        }, function(error, response) {
                                            if (error) {
                                                callback(null);
                                                return;
                                            }
                                            if (response.hits !== undefined) {
                                                var results = response.hits.hits,
                                                  keys = Object.keys(results),
                                                  items = {};
                                                /* filter keys like @timestamp, host, task, and type */
                                                keys.forEach(function(key) {
                                                    items = results[key]._source;
                                                    if (items.type == 'progress') {
                                                        return;
                                                    }
                                                    delete items['@timestamp'];
                                                    delete items.host;
                                                    delete items.task;
                                                    delete items.type;
                                                    delete items.platform;
                                                    deleteKeysByPrefix(items, "info_");
                                                    for (var item in items) {
                                                        metric_keys[item] = item;
                                                    }
                                                });

                                                /* compute statistics for each identified metric */
                                                async.each(metric_keys, function(metric, inner_callback) {
                                                    client.search({
                                                        index: index,
                                                        type: experiment,
                                                        searchType: 'count',
                                                        body: aggregation_by(metric)
                                                    }, function(error, response) {
                                                        if (error) {
                                                            inner_callback(null);
                                                        }
                                                        var aggs = response.aggregations;
                                                        data.metrics[metric] = aggs[metric + '_Stats'];
                                                        inner_callback(null);
                                                    });
                                                }, function(error) {
                                                    json.push(data);
                                                    /* finished */
                                                    callback(null);
                                                });
                                            }
                                        });
                                    });
                                });
                            });
                        }, function(error) {
                            if (error) {
                                return next(error);
                            }

                            //json.push(plan);
                            outer_callback(null);
                        });
                    });
                }, function(error) {
                    res.json(json);
                });
            } else {
                var message = {};
                message = 'No deployment plans available';
                res.json(message);
            }
        });
    });
});

/*
 * GET /mf/deployments/ms2/t2.1/cluster/asij122109klad
 */
router.get('/:workflow/:task/:platform/:deployment', function(req, res, next) {
    var client = req.app.get('elastic'),
      workflow = req.params.workflow.toLowerCase(),
      task = req.params.task.toLowerCase(),
      platform = req.params.platform.toLowerCase(),
      deployment = req.params.deployment, // we keep the original case
      mf_server = req.app.get('mf_server'),
      dreamcloud_pwm_idx = req.app.get('pwm_idx'),
      expand = req.query.expand,
      experiments = [],
      predicted_execution_time = 0,
      json = [];

    /* (1) get deployment plan for the given hashvalue */
    client.get({
        index: 'deployment_on_' + platform,
        type: workflow + '_' + task,
        id: deployment
    }, function(error, response) {
        if (error) {
            return next(error);
        }
        /* parse all experiments that use this deployment plan */
        if (response.found) {
            var source = response._source;
            experiments = source.experiments;
            predicted_execution_time = source.estimatedTime;
        } else {
            var message = {};
            message.error = "Could not find deployment plan in the database";
            res.json(message);
            return;
        }

        var index = workflow + '_' + task;
        async.forEachOf(experiments, function(value, experiment, callback) {
            /* get start of experiment */
            client.search({
                index: index,
                type: experiment,
                size: 1,
                sort: ['@timestamp:asc']
            }, function(error, response) {
                if (error) {
                    callback(null);
                    return;
                }
                var start, end;
                if (response.hits !== undefined) {
                    var only_results = response.hits.hits;
                    var keys = Object.keys(only_results);
                    keys.forEach(function(key) {
                        var metric_data = only_results[key]._source;
                        start = metric_data['@timestamp'];
                        start = start.replace(/\s/g, '0');
                    });
                }

                /* get end of experiment */
                client.search({
                    index: index,
                    type: experiment,
                    size: 1,
                    sort: ['@timestamp:desc']
                }, function(error, response) {
                    if (error) {
                        callback(null);
                        return;
                    }
                    if (response.hits !== undefined) {
                        var only_results = response.hits.hits;
                        var keys = Object.keys(only_results);
                        keys.forEach(function(key) {
                            var metric_data = only_results[key]._source;
                            end = metric_data['@timestamp'];
                            end = end.replace(/\s/g, '0');
                        });
                    }

                    var data = {};
                    data.experiment_id = experiment;
                    data.workflow_id = workflow;
                    data.task_id = task;
                    data.deployment_id = deployment;

                    if (!is_defined(start) || !is_defined(end)) {
                        json.push(data);
                        callback(null);
                        return;
                    }

                    /* add runtime statistics to response object */
                    data.runtime = {};
                    data.runtime.start_time = start;
                    data.runtime.end_time = end;
                    data.runtime.actual_time = (new Date(end) - new Date(start)) / 1000;
                    data.runtime.predicted_time = predicted_execution_time;
                    if (!data.runtime.actual_time) {
                        data.runtime.actual_time = 0;
                    }

                    /* get energy measurments */
                    var body = compute_average_on("NODE01", "NODE02", "NODE03", start, end);
                    client.search({
                        index: dreamcloud_pwm_idx,
                        searchType: 'count',
                        body: body
                    }, function(error, response) {
                        if (error) {
                            callback(null);
                            return;
                        }
                        var answer = {},
                            aggs = response.aggregations;

                        data.energy = {};
                        var power_data = {};
                        var average = aggs.power_metrics.NODE01.value;
                        data.energy.NODE01 = {};
                        data.energy.NODE01.avg_watt_consumption = average;
                        var duration = (new Date(end) - new Date(start)) / 1000;
                        var joule = average * duration;
                        data.energy.NODE01.total_energy_consumption = joule;

                        average = aggs.power_metrics.NODE02.value;
                        data.energy.NODE02 = {};
                        data.energy.NODE02.avg_watt_consumption = average;
                        joule = average * duration;
                        data.energy.NODE02.total_energy_consumption = joule;

                        average = aggs.power_metrics.NODE03.value;
                        data.energy.NODE03 = {};
                        data.energy.NODE03.avg_watt_consumption = average;
                        joule = average * duration;
                        data.energy.NODE03.total_energy_consumption = joule;

                        /* no energy measurements available */
                        if (!is_defined(average) || average === null) {
                            data.energy = [];
                        }

                        /* retrieve all metric counters as a set */
                        var index = workflow + '_' + task;
                        var metric_keys = {};
                        data.metrics = {};
                        client.search({
                            index: index,
                            type: experiment,
                            size: 20 /* should be enough results to capture all metrics */
                        }, function(error, response) {
                            if (error) {
                                callback(null);
                                return;
                            }
                            if (response.hits !== undefined) {
                                var results = response.hits.hits,
                                  keys = Object.keys(results),
                                  items = {};
                                /* filter keys like @timestamp, host, task, and type */
                                keys.forEach(function(key) {
                                    items = results[key]._source;
                                    if (items.type == 'progress') {
                                        return;
                                    }
                                    delete items['@timestamp'];
                                    delete items.host;
                                    delete items.task;
                                    delete items.type;
                                    delete items.platform;
                                    deleteKeysByPrefix(items, "info_");
                                    for (var item in items) {
                                        metric_keys[item] = item;
                                    }
                                });

                                /* compute statistics for each identified metric */
                                async.each(metric_keys, function(metric, inner_callback) {
                                    client.search({
                                        index: index,
                                        type: experiment,
                                        searchType: 'count',
                                        body: aggregation_by(metric)
                                    }, function(error, response) {
                                        if (error) {
                                            inner_callback(null);
                                        }
                                        var aggs = response.aggregations;
                                        data.metrics[metric] = aggs[metric + '_Stats'];
                                        inner_callback(null);
                                    });
                                }, function(error) {
                                    json.push(data);
                                    /* finished */
                                    callback(null);
                                });
                            }
                        });
                    });
                });
            });
        }, function(error) {
            if (error) {
                return next(error);
            }

            res.json(json);
        });
    });
});

function is_defined(variable) {
    return (typeof variable !== 'undefined');
}

function deleteKeysByPrefix(object, prefix) {
    for (var property in object) {
        if (object.hasOwnProperty(property) && property.toString().startsWith(prefix)) {
            delete object.property;
        }
    }
}

function compute_average_on(metric_name_a, metric_name_b, metric_name_c, from, to) {
    var query = {
        "aggs": {
            "power_metrics": {
                "filter": {
                    "and": [{
                        "or": [{
                            "exists": {
                                "field": metric_name_a
                            }
                        }, {
                            "exists": {
                                "field": metric_name_b
                            }
                        }, {
                            "exists": {
                                "field": metric_name_c
                            }
                        }]
                    }, {
                        "range": {
                            "@timestamp": {
                                "gte": from.toString(),
                                "lte": to.toString()
                            }
                        }
                    }]
                },
                "aggs": {
                    "NODE01": {
                        "avg": {
                            "field": metric_name_a
                        }
                    },
                    "NODE02": {
                        "avg": {
                            "field": metric_name_b
                        }
                    },
                    "NODE03": {
                        "avg": {
                            "field": metric_name_c
                        }
                    }
                }
            }
        }
    };
    return query;
}

function aggregation_by(field_name) {
    return '{' +
        '"aggs": {' +
            '"' + field_name + '_Stats" : {' +
                '"stats" : {' +
                    '"field" : "' + field_name + '"' +
                '}' +
            '},' +
            '"Minimum_' + field_name + '": {' +
                '"top_hits": {' +
                    '"size": 1,' +
                    '"sort": [' +
                        '{' +
                            '"' + field_name + '": {' +
                                '"order": "asc"' +
                            '}' +
                        '}' +
                    ']' +
                '}' +
            '},' +
            '"Maximum_' + field_name + '": {' +
                '"top_hits": {' +
                    '"size": 1,' +
                    '"sort": [' +
                        '{' +
                            '"' + field_name + '": {' +
                                '"order": "desc"' +
                            '}' +
                        '}' +
                    ']' +
                '}' +
            '}' +
        '}' +
    '}';
}

module.exports = router;
<|endoftext|>


<commit_before>$(document).ready(function() {
	$('#item-ver-becas').addClass('active');

    $('#item-ver-becas').click(function(e) {
    	e.preventDefault();
    });

  	$('#submenu-becas').addClass('in');

    $('#hiddenDiv').hide();

    var table = $('#becasTable').DataTable({
    	"searching": false,
        "scrollX": true,
        "scrollY": "200px",
        "scrollCollapse": true,
        "order": [[ 7, "asc" ]], //por nombre
	    "language": {
	        "lengthMenu": "Mostrando _MENU_ registros por página",
	        "zeroRecords": "No se encontraron registros",
	        "info": "Mostrando página _PAGE_ de _PAGES_",
	        "infoEmpty": "No hay registros disponibles",
	        "infoFiltered": "(de _MAX_ registros totales)",
	        "search": "Buscar",
	        "paginate": {
	          "previous": "Anterior",
	          "next": "Siguiente"
	        }
        }
    });

    new $.fn.dataTable.FixedColumns(table);

    var search = function(query, periodo) {
        $.ajax({
            url: '/searchSolicitud?periodo=' + periodo + '&q=' + query,
            success: function(result,status,xhr) {
                var arr = result.map(function(obj) {
                    var hiddenButton = $('#hiddenButton').clone();
                    hiddenButton.removeClass();
                    hiddenButton.addClass('btn btn-xs');
                    var btnClass = '';

                    switch(obj.estatus_solicitud) {
                        case 'ACEPTADO': btnClass = 'btn-success'; break;
                        case 'RECHAZADO': btnClass = 'btn-danger'; break;
                        case 'LISTA DE ESPERA': btnClass = 'btn-warning'; break;
                        default: btnClass = 'btn-info';
                    }

                    hiddenButton.addClass(btnClass);
                    hiddenButton.html(obj.estatus_solicitud);
                    hiddenButton.attr('data-toggle', 'modal');
                    hiddenButton.attr('data-target', '#modalCambioEstatus');
                    hiddenButton.attr('data-id', obj.id);
                    hiddenButton.attr('data-nombre', obj.nombre);
                    hiddenButton.attr('data-estatus', obj.estatus_solicitud);

                    var editarButton = $('#hiddenButton').clone();
                    editarButton.removeClass();
                    editarButton.empty();
                    editarButton.html('Editar')
                    editarButton.addClass('btn btn-xs btn-link editar-solicitud');
                    editarButton.attr('data-id', obj.id);

                    var promedio = parseFloat(obj.promedio).toFixed(2);
                    var ingresos = parseFloat(obj.ingresos).toFixed(2);
                    var ingresoMinimo = parseFloat(obj.ingreso_minimo).toFixed(2);
                    var dependientes = parseInt(obj.dependientes);
                    var relacionIngresosDependientes = (ingresos / dependientes / ingresoMinimo).toFixed(2);

                    if(isNaN(promedio)) promedio = null;
                    if(isNaN(ingresos)) ingresos = null;
                    if(isNaN(ingresoMinimo)) ingresoMinimo = null;
                    if(isNaN(dependientes)) dependientes = null;

                    var hiddenMessage = $('#hiddenMessage').clone();
                    hiddenMessage.removeClass();

                    if(isNaN(relacionIngresosDependientes)) {
                        relacionIngresosDependientes = null;
                    } else if(relacionIngresosDependientes > 4.0) {
                        hiddenMessage.addClass('alert alert-danger text-center');
                    } else if(relacionIngresosDependientes > 0 && relacionIngresosDependientes <= 4.0) {
                        hiddenMessage.addClass('alert alert-success text-center');
                    }

                    hiddenMessage.html(relacionIngresosDependientes);

                    return[
                        hiddenButton.get(0).outerHTML,
                        editarButton.get(0).outerHTML,
                        obj.folio,
                        obj.etiqueta,
                        obj.boleta,
                        obj.curp,
                        obj.genero,
                        obj.nombre,
                        obj.carrera,
                        obj.semestre,
                        promedio,
                        obj.estatus_estudiante,
                        obj.carga,
                        obj.estatus_becario,
                        obj.beca_anterior,
                        obj.beca_solicitada,
                        obj.folio_manutencion,
                        obj.folio_transporte,
                        obj.mapa,
                        obj.fecha_recibido,
                        obj.comprobante_ingresos,
                        ingresos,
                        dependientes,
                        ingresoMinimo,
                        hiddenMessage.get(0).outerHTML,
                        obj.oriundo,
                        obj.email,
                        obj.telefono,
                        obj.observaciones
                    ];
                });
                table.clear();
                table.rows.add(arr).draw();

            }
        });
    };

    $('#modalCambioEstatus').on('show.bs.modal', function(e) {
        var nombre = e.relatedTarget.dataset.nombre;
        var id = e.relatedTarget.dataset.id;
        var estatus = e.relatedTarget.dataset.estatus;

        $('#modalCambioEstatus #nombre').text(nombre);
        $('#modalCambioEstatus #estatus').val(estatus);
        $('#btnCambioEstatus').removeAttr('disabled');

        $('#btnCambioEstatus').unbind('click').click(function(e) {
            $(this).attr('disabled', 'disabled');
            $.ajax({
                url: '/updateEstatusSolicitud/' + id,
                type: 'POST',
                data: {
                    '_token': $('#modalCambioEstatus #_token').val(),
                    'estatus_solicitud': $('#modalCambioEstatus #estatus').val()
                },
                success: function(result,status,xhr) {
                    $('#modalCambioEstatus').modal('hide');
                    $('#modalMessage #message').removeClass('alert-danger');
                    $('#modalMessage #message').addClass('alert-success');
                    $('#modalMessage #message').html(
                        'Se ha actualizado el estatus de la solicitud con folio ' + result.folio +
                        ' a ' + result.estatus_solicitud
                    );
                    $('#modalMessage').modal('show');
                    search($('#search').val(), $('#periodo').val());

                },
                error: function() {
                    $('#modalCambioEstatus').modal('hide');
                    $('#modalMessage #message').removeClass('alert-success');
                    $('#modalMessage #message').addClass('alert-danger');
                    $('#modalMessage #message').html('No fue posible actualizar el estatus de la solicitud');
                    $('#modalMessage').modal('show');
                }
            });
        });
    });

    $('#search').keyup(function() {
        search($(this).val(), $('#periodo').val());
    });

    $('#periodo').change(function() {
        search($('#search').val(), $(this).val());
    });

    var toggleColumns = function() {
        $('.toggle-column').each(function() {
            var column = table.column($(this).attr('data-column'));
            column.visible($(this).is(':checked'));
        });
    };

    $('.toggle-column').change(function(e) {
        e.preventDefault();
        var column = table.column($(this).attr('data-column'));
        column.visible($(this).is(':checked'));
    });

    $('#selectAll').click(function(e) {
        e.preventDefault();
        $('.toggle-column').prop('checked', true);
        toggleColumns();

    });

    $('#deselectAll').click(function(e) {
        e.preventDefault();
        $('.toggle-column').prop('checked', false);
        toggleColumns();
    });

});<commit_msg>La columna de Editar se vuelve sticky.<commit_after>$(document).ready(function() {
	$('#item-ver-becas').addClass('active');

    $('#item-ver-becas').click(function(e) {
    	e.preventDefault();
    });

  	$('#submenu-becas').addClass('in');

    $('#hiddenDiv').hide();

    var table = $('#becasTable').DataTable({
    	"searching": false,
        "scrollX": true,
        "scrollY": "200px",
        "scrollCollapse": true,
        "order": [[ 7, "asc" ]], //por nombre
        "fixedColumns": {
            "leftColumns": 2
        },
	    "language": {
	        "lengthMenu": "Mostrando _MENU_ registros por página",
	        "zeroRecords": "No se encontraron registros",
	        "info": "Mostrando página _PAGE_ de _PAGES_",
	        "infoEmpty": "No hay registros disponibles",
	        "infoFiltered": "(de _MAX_ registros totales)",
	        "search": "Buscar",
	        "paginate": {
	          "previous": "Anterior",
	          "next": "Siguiente"
	        }
        }
    });

    var search = function(query, periodo) {
        $.ajax({
            url: '/searchSolicitud?periodo=' + periodo + '&q=' + query,
            success: function(result,status,xhr) {
                var arr = result.map(function(obj) {
                    var hiddenButton = $('#hiddenButton').clone();
                    hiddenButton.removeClass();
                    hiddenButton.addClass('btn btn-xs');
                    var btnClass = '';

                    switch(obj.estatus_solicitud) {
                        case 'ACEPTADO': btnClass = 'btn-success'; break;
                        case 'RECHAZADO': btnClass = 'btn-danger'; break;
                        case 'LISTA DE ESPERA': btnClass = 'btn-warning'; break;
                        default: btnClass = 'btn-info';
                    }

                    hiddenButton.addClass(btnClass);
                    hiddenButton.html(obj.estatus_solicitud);
                    hiddenButton.attr('data-toggle', 'modal');
                    hiddenButton.attr('data-target', '#modalCambioEstatus');
                    hiddenButton.attr('data-id', obj.id);
                    hiddenButton.attr('data-nombre', obj.nombre);
                    hiddenButton.attr('data-estatus', obj.estatus_solicitud);

                    var editarButton = $('#hiddenButton').clone();
                    editarButton.removeClass();
                    editarButton.empty();
                    editarButton.html('Editar')
                    editarButton.addClass('btn btn-xs btn-link editar-solicitud');
                    editarButton.attr('data-id', obj.id);

                    var promedio = parseFloat(obj.promedio).toFixed(2);
                    var ingresos = parseFloat(obj.ingresos).toFixed(2);
                    var ingresoMinimo = parseFloat(obj.ingreso_minimo).toFixed(2);
                    var dependientes = parseInt(obj.dependientes);
                    var relacionIngresosDependientes = (ingresos / dependientes / ingresoMinimo).toFixed(2);

                    if(isNaN(promedio)) promedio = null;
                    if(isNaN(ingresos)) ingresos = null;
                    if(isNaN(ingresoMinimo)) ingresoMinimo = null;
                    if(isNaN(dependientes)) dependientes = null;

                    var hiddenMessage = $('#hiddenMessage').clone();
                    hiddenMessage.removeClass();

                    if(isNaN(relacionIngresosDependientes)) {
                        relacionIngresosDependientes = null;
                    } else if(relacionIngresosDependientes > 4.0) {
                        hiddenMessage.addClass('alert alert-danger text-center');
                    } else if(relacionIngresosDependientes > 0 && relacionIngresosDependientes <= 4.0) {
                        hiddenMessage.addClass('alert alert-success text-center');
                    }

                    hiddenMessage.html(relacionIngresosDependientes);

                    return[
                        hiddenButton.get(0).outerHTML,
                        editarButton.get(0).outerHTML,
                        obj.folio,
                        obj.etiqueta,
                        obj.boleta,
                        obj.curp,
                        obj.genero,
                        obj.nombre,
                        obj.carrera,
                        obj.semestre,
                        promedio,
                        obj.estatus_estudiante,
                        obj.carga,
                        obj.estatus_becario,
                        obj.beca_anterior,
                        obj.beca_solicitada,
                        obj.folio_manutencion,
                        obj.folio_transporte,
                        obj.mapa,
                        obj.fecha_recibido,
                        obj.comprobante_ingresos,
                        ingresos,
                        dependientes,
                        ingresoMinimo,
                        hiddenMessage.get(0).outerHTML,
                        obj.oriundo,
                        obj.email,
                        obj.telefono,
                        obj.observaciones
                    ];
                });
                table.clear();
                table.rows.add(arr).draw();

            }
        });
    };

    $('#modalCambioEstatus').on('show.bs.modal', function(e) {
        var nombre = e.relatedTarget.dataset.nombre;
        var id = e.relatedTarget.dataset.id;
        var estatus = e.relatedTarget.dataset.estatus;

        $('#modalCambioEstatus #nombre').text(nombre);
        $('#modalCambioEstatus #estatus').val(estatus);
        $('#btnCambioEstatus').removeAttr('disabled');

        $('#btnCambioEstatus').unbind('click').click(function(e) {
            $(this).attr('disabled', 'disabled');
            $.ajax({
                url: '/updateEstatusSolicitud/' + id,
                type: 'POST',
                data: {
                    '_token': $('#modalCambioEstatus #_token').val(),
                    'estatus_solicitud': $('#modalCambioEstatus #estatus').val()
                },
                success: function(result,status,xhr) {
                    $('#modalCambioEstatus').modal('hide');
                    $('#modalMessage #message').removeClass('alert-danger');
                    $('#modalMessage #message').addClass('alert-success');
                    $('#modalMessage #message').html(
                        'Se ha actualizado el estatus de la solicitud con folio ' + result.folio +
                        ' a ' + result.estatus_solicitud
                    );
                    $('#modalMessage').modal('show');
                    search($('#search').val(), $('#periodo').val());

                },
                error: function() {
                    $('#modalCambioEstatus').modal('hide');
                    $('#modalMessage #message').removeClass('alert-success');
                    $('#modalMessage #message').addClass('alert-danger');
                    $('#modalMessage #message').html('No fue posible actualizar el estatus de la solicitud');
                    $('#modalMessage').modal('show');
                }
            });
        });
    });

    $('#search').keyup(function() {
        search($(this).val(), $('#periodo').val());
    });

    $('#periodo').change(function() {
        search($('#search').val(), $(this).val());
    });

    var toggleColumns = function() {
        $('.toggle-column').each(function() {
            var column = table.column($(this).attr('data-column'));
            column.visible($(this).is(':checked'));
        });
    };

    $('.toggle-column').change(function(e) {
        e.preventDefault();
        var column = table.column($(this).attr('data-column'));
        column.visible($(this).is(':checked'));
    });

    $('#selectAll').click(function(e) {
        e.preventDefault();
        $('.toggle-column').prop('checked', true);
        toggleColumns();

    });

    $('#deselectAll').click(function(e) {
        e.preventDefault();
        $('.toggle-column').prop('checked', false);
        toggleColumns();
    });

});<|endoftext|>


<commit_before>window.ScheduleView = Backbone.View.extend({
    el: $("#content"),
    initialize: function () {
        console.log ("rendering schedule");
        this.render();
    },
    render: function () {
        var self = this;
        $(this.el).html(template.schedule());

        new UniverseListView({
            collection: Universe,
            el: $("#universe"),
            draggable: true,
        });

	var date = new Date();
	var d = date.getDate();
	var m = date.getMonth();
	var y = date.getFullYear();

        $('#calendar', this.el).fullCalendar({
	    editable: true,
            droppable: true,
            disableResizing: true,
            aspectRatio: 2,
            drop: function(date, allDay) {
                var list = Universe.get(this.id);
                var start = moment(date);
                var end   = moment(start);

                end.add('ms', list.get('duration'));
                var event ={
                    title: list.get('name'),
                    start: start.toDate(),
                    end:   end.toDate(),
                    allDay: allDay,
                };

                var item = self.collection.create (list);
                event.model = item;

                console.log (this, list, event);
                // render the event on the calendar
		// the last `true` argument determines if the event "sticks" (http://arshaw.com/fullcalendar/docs/event_rendering/renderEvent/)
		$('#calendar', self.el).fullCalendar('renderEvent', event, true);
            },
            eventDrop: function(event,dayDelta,minuteDelta,allDay,revertFunc) {
                console.log ('changed', event.list, event.id);

                if (!confirm("Are you sure about this change?")) {
                    revertFunc();
                }
            },
	    header: {
		left: 'prev,next today',
		center: 'title',
		right: 'month,agendaWeek,agendaDay'
	    },
	    defaultView: 'agendaWeek',
	    events: [
		{
		    title: 'WALLKINTUN SALE AL AIRE !',
		    start: new Date(2012, 11, 7, 22, 0),
		    end:   new Date(2012, 11, 8, 6, 0),
                    allDay: false,
		},
	    ]
        });

        setTimeout (function () {
            $('#calendar table', this.el).addClass("table table-bordered");
        }, 500);
        return this;
    },
});
<commit_msg>medialist view in calendar ? yes, that's possible<commit_after>window.ScheduleView = Backbone.View.extend({
    el: $("#content"),
    initialize: function () {
        console.log ("rendering schedule");
        this.render();
    },
    render: function () {
        var self = this;
        $(this.el).html(template.schedule());

        new UniverseListView({
            collection: Universe,
            el: $("#universe"),
            draggable: true,
        });

	var date = new Date();
	var d = date.getDate();
	var m = date.getMonth();
	var y = date.getFullYear();

        $('#calendar', this.el).fullCalendar({
	    editable: true,
            droppable: true,
            disableResizing: true,
            aspectRatio: 2,
            drop: function(date, allDay) {
                var list = Universe.get(this.id);
                var start = moment(date);
                var end   = moment(start);

                end.add('ms', list.get('duration'));
                var event ={
                    title: list.get('name'),
                    start: start.toDate(),
                    end:   end.toDate(),
                    allDay: allDay,
                };

                var item = self.collection.create (list);
                event.model = item;

                console.log (this, list, event);
                // render the event on the calendar
		// the last `true` argument determines if the event "sticks" (http://arshaw.com/fullcalendar/docs/event_rendering/renderEvent/)
		$('#calendar', self.el).fullCalendar('renderEvent', event, true);
            },
            eventDrop: function(event,dayDelta,minuteDelta,allDay,revertFunc) {
                console.log ('changed', event.list, event.id);

                if (!confirm("Are you sure about this change?")) {
                    revertFunc();
                }
            },
	    header: {
		left: 'prev,next today',
		center: 'title',
		right: 'month,agendaWeek,agendaDay'
	    },
	    defaultView: 'agendaWeek',
	    events: [
		{
		    title: 'WALLKINTUN SALE AL AIRE !',
		    start: new Date(2012, 11, 7, 22, 0),
		    end:   new Date(2012, 11, 8, 6, 0),
                    allDay: false,
		},
	    ],
            eventRender: function(event, element) {
                console.log ('----o---', element);

                element.tooltip({
                    trigger: 'click',
                    placement: 'left',
                    title: function () {
                        var target = document.createElement('div');
                        var view = new MediaListView({model: event.model,
                                                      noSearch:true,
                                                      el: target});
                        return view.render().el;
                    },
                });
            }
        });

        setTimeout (function () {
            $('#calendar table', this.el).addClass("table table-bordered");
        }, 500);
        return this;
    },
});
<|endoftext|>


<commit_before>'use strict';
var argv = require('minimist')(process.argv.slice(2));
var MongoClient = require('mongodb').MongoClient;
var https = require('https');
var assert = require('assert');
var fs = require('fs');
var cli = {};
var arg_handler = {
    'user': ['u', 'user'],
    'repo': ['r', 'repo'],
    'user_agent': ['a', 'user_agent']
};
cli.verbose = argv.v || argv.verbose;

// read the command line argv and assign keys found in arg_handler to cli object
for (var key in arg_handler) {
    try {
        arg_handler[key].forEach(function (value, index, array) {
            if (!cli[key]) cli[key] = argv[value];
        });
        if (!cli[key]) {
            // throw new Error('Please specify a ' + key);
            console.error('Please specify a ' + key);
            process.exit(1);
        }
    } catch (exception) {
        console.error(exception);
        process.exit(1);
    }
}

if (cli.verbose) {
    console.log('Running with settings:\n'+
                'user: ' + cli.user + '\n'+
                'repo: ' + cli.repo + '\n'+
                'user_agent: ' + cli.user_agent);
}

var httpOptions = {
    hostname: 'api.github.com',
    port: 443,
    path: '/repos/' + cli.user + '/' + cli.repo + '/releases',
    headers: {
        'User-Agent': cli.user_agent
    }
};

function httpCallback (response) {
    var content = '';

    if (cli.verbose) {
        console.log('HTTP response statusCode: ', response.statusCode);
        console.log('HTTP response headers: ', response.headers);
    }

    if (response.statusCode !== 200) {
        throw new Error(response.statusCode + ': ' + response.statusMessage);
    }

    response
        .on('error', function (err) {
            throw new Error(err);
        })
        .on('end', function () {
            formatHttpResponse(content);
        })
        .on('data', function (chunk) {
            content += chunk;
        });
}

// formatHttpResponse accepts the http reponse content as string
function formatHttpResponse (rawResponse) {
    try {
        var httpResponse = JSON.parse(rawResponse);
    } catch (exception) {
        console.error('JSON.parse error');
        console.error(exception);
        process.exit(1);
    }

    // delete unneeded information like author objs, uploader objs, and supplimentary urls
    httpResponse.forEach(function (release, index, httpResponse) {
        delete httpResponse[index].body;
        delete httpResponse[index].author;
        delete httpResponse[index].published_at;
        delete httpResponse[index].target_commitish;
        (function deleteExtraInfo (httpResponse) {
            for (var key in httpResponse) {
                if (key.indexOf('url')      > -1 ||
                    key.indexOf('uploader') > -1 ||
                    key.indexOf('updated_at')>-1 ||
                    key.indexOf('state')    > -1 ||
                    key.indexOf('label')    > -1 ||
                    key.indexOf('size')     > -1 ||
                    key.indexOf('content_type') > -1) {
                        delete httpResponse[key];
                } else if (typeof httpResponse[key] === 'object') {
                    deleteExtraInfo(httpResponse[key]);
                }
            }
        })(httpResponse[index])
    });
    writeToMongo(httpResponse);
}

// writeToMongo accepts an object and attempts to write it to a local mongo instance
function writeToMongo (httpResponse) {
    if (cli.verbose) console.log('Attempting to write to Mongo');

    MongoClient.connect('mongodb://127.0.0.1:27017/github-downloads', function(err, db) {
        assert.equal(null, err);

        if (cli.verbose) console.log('Connected to Mongo database');

        db.collection('downloads').insert(httpResponse, function(err, docs) {
            assert.equal(null, err);

            var today = new Date();
            console.log('Successful inserted at ' + today.toUTCString());

            db.collection('downloads').find({}).toArray(function(err, result) {
                assert.equal(null, err);
                assert.ok(result.length);

                fs.writeFile('openice/github-downloads-data.txt', JSON.stringify(result), function (err) {
                    assert.equal(null, err);

                    if (verbose) console.log('Successful file write');
                });

                db.close();
            });
        });
    });
}

https.get(httpOptions, httpCallback);
<commit_msg>changed to match gfish user<commit_after>'use strict';
var argv = require('minimist')(process.argv.slice(2));
var MongoClient = require('mongodb').MongoClient;
var https = require('https');
var assert = require('assert');
var fs = require('fs');
var cli = {};
var arg_handler = {
    'user': ['u', 'user'],
    'repo': ['r', 'repo'],
    'user_agent': ['a', 'user_agent']
};
cli.verbose = argv.v || argv.verbose;

// read the command line argv and assign keys found in arg_handler to cli object
for (var key in arg_handler) {
    try {
        arg_handler[key].forEach(function (value, index, array) {
            if (!cli[key]) cli[key] = argv[value];
        });
        if (!cli[key]) {
            // throw new Error('Please specify a ' + key);
            console.error('Please specify a ' + key);
            process.exit(1);
        }
    } catch (exception) {
        console.error(exception);
        process.exit(1);
    }
}

if (cli.verbose) {
    console.log('Running with settings:\n'+
                'user: ' + cli.user + '\n'+
                'repo: ' + cli.repo + '\n'+
                'user_agent: ' + cli.user_agent);
}

var httpOptions = {
    hostname: 'api.github.com',
    port: 443,
    path: '/repos/' + cli.user + '/' + cli.repo + '/releases',
    headers: {
        'User-Agent': cli.user_agent
    }
};

function httpCallback (response) {
    var content = '';

    if (cli.verbose) {
        console.log('HTTP response statusCode: ', response.statusCode);
        console.log('HTTP response headers: ', response.headers);
    }

    if (response.statusCode !== 200) {
        throw new Error(response.statusCode + ': ' + response.statusMessage);
    }

    response
        .on('error', function (err) {
            throw new Error(err);
        })
        .on('end', function () {
            formatHttpResponse(content);
        })
        .on('data', function (chunk) {
            content += chunk;
        });
}

// formatHttpResponse accepts the http reponse content as string
function formatHttpResponse (rawResponse) {
    try {
        var httpResponse = JSON.parse(rawResponse);
    } catch (exception) {
        console.error('JSON.parse error');
        console.error(exception);
        process.exit(1);
    }

    // delete unneeded information like author objs, uploader objs, and supplimentary urls
    httpResponse.forEach(function (release, index, httpResponse) {
        delete httpResponse[index].body;
        delete httpResponse[index].author;
        delete httpResponse[index].published_at;
        delete httpResponse[index].target_commitish;
        (function deleteExtraInfo (httpResponse) {
            for (var key in httpResponse) {
                if (key.indexOf('url')      > -1 ||
                    key.indexOf('uploader') > -1 ||
                    key.indexOf('updated_at')>-1 ||
                    key.indexOf('state')    > -1 ||
                    key.indexOf('label')    > -1 ||
                    key.indexOf('size')     > -1 ||
                    key.indexOf('content_type') > -1) {
                        delete httpResponse[key];
                } else if (typeof httpResponse[key] === 'object') {
                    deleteExtraInfo(httpResponse[key]);
                }
            }
        })(httpResponse[index])
    });
    writeToMongo(httpResponse);
}

// writeToMongo accepts an object and attempts to write it to a local mongo instance
function writeToMongo (httpResponse) {
    if (cli.verbose) console.log('Attempting to write to Mongo');

    MongoClient.connect('mongodb://127.0.0.1:27017/github-downloads', function(err, db) {
        assert.equal(null, err);

        if (cli.verbose) console.log('Connected to Mongo database');

        db.collection('downloads').insert(httpResponse, function(err, docs) {
            assert.equal(null, err);

            var today = new Date();
            console.log('Successful inserted at ' + today.toUTCString());

            db.collection('downloads').find({}).toArray(function(err, result) {
                assert.equal(null, err);
                assert.ok(result.length);

                fs.writeFile('/home/gfish/openice/github-downloads-data.txt', JSON.stringify(result), function (err) {
                    assert.equal(null, err);

                    if (cli.verbose) console.log('Successful file write');
                });

                db.close();
            });
        });
    });
}

https.get(httpOptions, httpCallback);
<|endoftext|>


<commit_before>const Pagetest = require('./Pagetest');
const credentials = require('./credentials');
const download = require('./download');

exports.call = function (db, data, req) {
    const API = new Pagetest.API(credentials.wpt_dns, credentials.wpt_api_key);
    const testId = data.id;
    db.log.info('Pingback received for ' + testId);

    return db.TestResult.find().equal('testId', testId).count().then(exists => {
        if (exists > 0) {
            db.log.info('Result already exists for ' + testId);
            return;
        }

        let testResult;
        return API.getTestResults(testId, {
            requests: false,
            breakdown: false,
            domains: false,
            pageSpeed: false,
        }).then(result => {
            db.log.info('Saving test result for ' + testId, result);

            testResult = createTestResult(testId, result.data, db);
            return testResult.save();
        }).then(ignored => {
            if(testResult.testDataMissing)
                throw new Error('Test Data Missing');

            db.log.info('creating video for ' + testId);
            return Promise.all([API.createVideo(testId + '-r:1-c:0'), API.createVideo(testId + '-r:1-c:1')]);
        }).then(result => {
            db.log.info('videos created for ' + testId, {data: result});

            testResult.videoIdFirstView = result[0].data.videoId;
            const videoFirstViewPromise = download.toFile(db,
                constructVideoLink(testId, testResult.videoIdFirstView), '/www/videoFirstView/' + testId +'.mp4');

            let videoRepeatViewPromise = Promise.resolve(true);
            if (result[1].data && result[1].data.videoId) {
                testResult.videoIdRepeatedView = result[1].data.videoId;
                videoRepeatViewPromise = download.toFile(db,
                    constructVideoLink(testId, testResult.videoIdRepeatedView), '/www/videoRepeatView/' + testId + '.mp4');
            }

            return Promise.all([videoFirstViewPromise, videoRepeatViewPromise]).then((values) => {
                const [videoFirstView, videoRepeatView] = values;
                testResult.videoFileFirstView = videoFirstView;
                testResult.videoFileRepeatView = videoRepeatView;
                return testResult.save();
            })
        }).catch(error => {
            db.log.info('Error handling test result for ' + testId + ' with error: ' + error);
        });
    });
};

function constructVideoLink(testId, videoId) {
    const date = testId.substr(0, 2) + '/' + testId.substr(2, 2) + '/' + testId.substr(4, 2);
    return 'http://' + credentials.wpt_dns + '/results/video/' + date + '/' +
        videoId.substr(videoId.indexOf('_') + 1, videoId.length) + '/video.mp4';
}

function createTestResult(testId, testResult, db) {
    const result = new db.TestResult();
    result.testId = testId;
    result.location = testResult.location;
    result.url = testResult.testUrl;
    result.summaryUrl = testResult.summary;
    result.firstView = createRun(testResult.runs['1'].firstView, db);
    result.testDataMissing = result.firstView.lastVisualChange <= 0;
    if (testResult.runs['1'].repeatView) {
        result.repeatView = createRun(testResult.runs['1'].repeatView, db);
        result.testDataMissing = result.repeatView.lastVisualChange <= 0;
    }
    return result;
}

function createRun(data, db) {
    const run = new db.Run();
    run.loadTime = data.loadTime;
    run.ttfb = data.TTFB;
    run.domLoaded = data.domContentLoadedEventStart;
    run.load = data.loadEventStart;
    run.fullyLoaded = data.fullyLoaded;
    run.firstPaint = data.firstPaint;
    run.startRender = data.render;
    run.lastVisualChange = data.lastVisualChange;
    run.speedIndex = data.SpeedIndex;
    run.requests = data.requests;
    run.bytes = data.bytesOut;
    run.domElements = data.domElements;
    run.basePageCDN = data.base_page_cdn;

    const completeness = new db.Completeness();
    completeness.p85 = data.visualComplete85;
    completeness.p90 = data.visualComplete90;
    completeness.p95 = data.visualComplete95;
    completeness.p99 = data.visualComplete99;
    completeness.p100 = data.visualComplete;
    run.visualCompleteness = completeness;
    return run;
}<commit_msg>added counting of hits, misses and others<commit_after>const Pagetest = require('./Pagetest');
const credentials = require('./credentials');
const download = require('./download');
const _ = require('underscore');

exports.call = function (db, data, req) {
    const API = new Pagetest.API(credentials.wpt_dns, credentials.wpt_api_key);
    const testId = data.id;
    db.log.info('Pingback received for ' + testId);

    return db.TestResult.find().equal('testId', testId).count().then(exists => {
        if (exists > 0) {
            db.log.info('Result already exists for ' + testId);
            return;
        }

        let testResult;
        return API.getTestResults(testId, {
            requests: true,
            breakdown: false,
            domains: false,
            pageSpeed: false,
        }).then(result => {
            db.log.info('Saving test result for ' + testId, result);

            testResult = createTestResult(testId, result.data, db);
            return testResult.save();
        }).then(ignored => {
            if(testResult.testDataMissing)
                throw new Error('Test Data Missing');

            db.log.info('creating video for ' + testId);
            return Promise.all([API.createVideo(testId + '-r:1-c:0'), API.createVideo(testId + '-r:1-c:1')]);
        }).then(result => {
            db.log.info('videos created for ' + testId, {data: result});

            testResult.videoIdFirstView = result[0].data.videoId;
            const videoFirstViewPromise = download.toFile(db,
                constructVideoLink(testId, testResult.videoIdFirstView), '/www/videoFirstView/' + testId +'.mp4');

            let videoRepeatViewPromise = Promise.resolve(true);
            if (result[1].data && result[1].data.videoId) {
                testResult.videoIdRepeatedView = result[1].data.videoId;
                videoRepeatViewPromise = download.toFile(db,
                    constructVideoLink(testId, testResult.videoIdRepeatedView), '/www/videoRepeatView/' + testId + '.mp4');
            }

            return Promise.all([videoFirstViewPromise, videoRepeatViewPromise]).then((values) => {
                const [videoFirstView, videoRepeatView] = values;
                testResult.videoFileFirstView = videoFirstView;
                testResult.videoFileRepeatView = videoRepeatView;
                return testResult.save();
            })
        }).catch(error => {
            db.log.info('Error handling test result for ' + testId + ' with error: ' + error);
        });
    });
};

function constructVideoLink(testId, videoId) {
    const date = testId.substr(0, 2) + '/' + testId.substr(2, 2) + '/' + testId.substr(4, 2);
    return 'http://' + credentials.wpt_dns + '/results/video/' + date + '/' +
        videoId.substr(videoId.indexOf('_') + 1, videoId.length) + '/video.mp4';
}

function createTestResult(testId, testResult, db) {
    const result = new db.TestResult();
    result.testId = testId;
    result.location = testResult.location;
    result.url = testResult.testUrl;
    result.summaryUrl = testResult.summary;
    result.firstView = createRun(testResult.runs['1'].firstView, db);
    result.testDataMissing = result.firstView.lastVisualChange <= 0;
    if (testResult.runs['1'].repeatView) {
        result.repeatView = createRun(testResult.runs['1'].repeatView, db);
        result.testDataMissing = result.repeatView.lastVisualChange <= 0;
    }
    return result;
}

function createRun(data, db) {
    const run = new db.Run();
    run.loadTime = data.loadTime;
    run.ttfb = data.TTFB;
    run.domLoaded = data.domContentLoadedEventStart;
    run.load = data.loadEventStart;
    run.fullyLoaded = data.fullyLoaded;
    run.firstPaint = data.firstPaint;
    run.startRender = data.render;
    run.lastVisualChange = data.lastVisualChange;
    run.speedIndex = data.SpeedIndex;
    run.requests = data.requests.length;
    run.hits = new db.Hits(countHits(data));
    run.bytes = data.bytesOut;
    run.domElements = data.domElements;
    run.basePageCDN = data.base_page_cdn;

    const completeness = new db.Completeness();
    completeness.p85 = data.visualComplete85;
    completeness.p90 = data.visualComplete90;
    completeness.p95 = data.visualComplete95;
    completeness.p99 = data.visualComplete99;
    completeness.p100 = data.visualComplete;
    run.visualCompleteness = completeness;
    return run;
}

function countHits(data) {
    return _.countBy(data.requests, req => {
        const headers = req.headers.response.join(" ").toLowerCase();
        if(headers.indexOf("x-cache") != -1) {
            return headers.indexOf('hit') != -1  ? 'hit': 'miss';
        } else {
            return 'other';
        }
    });
}<|endoftext|>