<commit_before>/*!
 * Django Autocomplete Light
 */

var yl = yl || {};
yl.functions = {};
/**
 * Register your own JS function for DAL.
 *
 * @param name The name of your function. This should be the same as the widget
 *             `autocomplete_function` property value.
 * @param func The callback that will initialize your custom autocomplete.
 */
yl.registerFunction = function (name, func) {
    if (this.functions.hasOwnProperty(name)) {
        // This function already exists to show an error and skip.
        console.error('The DAL function "' + name + '" has already been registered.');
        return
    }
    if (typeof func != 'function') {
        // It's not a function kill it.
        throw new Error('The custom DAL function must be a function.');
    }
    this.functions[name] = func;
    var event = new CustomEvent('dal-function-registered.' + name, {detail: {name: name, func: func}})
    window.dispatchEvent(event);
};

window.addEventListener("load", function () {

    (function ($) {
        $.fn.getFormPrefix = function () {
            /* Get the form prefix for a field.
             *
             * For example:
             *
             *     $(':input[name$=owner]').getFormsetPrefix()
             *
             * Would return an empty string for an input with name 'owner' but would return
             * 'inline_model-0-' for an input named 'inline_model-0-owner'.
             */
            var parts = $(this).attr('name').split('-');
            var prefix = '';

            for (var i in parts) {
                var testPrefix = parts.slice(0, -i).join('-');
                if (!testPrefix.length) continue;
                testPrefix += '-';

                var result = $(':input[name^=' + testPrefix + ']')

                if (result.length) {
                    return testPrefix;
                }
            }

            return '';
        }

        $.fn.getFormPrefixes = function () {
            /*
             * Get the form prefixes for a field, from the most specific to the least.
             *
             * For example:
             *
             *      $(':input[name$=owner]').getFormPrefixes()
             *
             * Would return:
             * - [''] for an input named 'owner'.
             * - ['inline_model-0-', ''] for an input named 'inline_model-0-owner' (i.e. nested with a nested inline).
             * - ['sections-0-items-0-', 'sections-0-', ''] for an input named 'sections-0-items-0-product'
             *   (i.e. nested multiple time with django-nested-admin).
             */
            var parts = $(this).attr('name').split('-').slice(0, -1);
            var prefixes = [];

            for (i = 0; i < parts.length; i += 2) {
                var testPrefix = parts.slice(0, -i || parts.length).join('-');
                if (!testPrefix.length)
                    continue;

                testPrefix += '-';

                var result = $(':input[name^=' + testPrefix + ']')

                if (result.length)
                    prefixes.push(testPrefix);
            }

            prefixes.push('');

            return prefixes;
        }

        /*
         * This ensures the Language file is loaded and passes it our jQuery.
         */
        if (typeof dalLoadLanguage !== 'undefined') {
            dalLoadLanguage($);
        } else {
            document.addEventListener('dal-language-loaded', function (e) {
                // `e.lang` is the language that was loaded.
                dalLoadLanguage($);
            })
        }

        // Fire init event for yl.registerFunction() execution.
        var event = new CustomEvent('dal-init-function');
        document.dispatchEvent(event);

        var initialized = [];

        /**
         * Initialize a field element. This function calls the registered init function
         * and ensures that the element is only initialized once.
         *
         * @param element The field to be initialized
         */
        function initialize(element) {
            if (typeof element === 'undefined' || typeof element === 'number') {
                element = this;
            }

            // Ensure element is not already initialized.
            if (initialized.indexOf(element) >= 0) {
                return;
            }

            // The DAL function to execute.
            var dalFunction = $(element).attr('data-autocomplete-light-function');

            if (yl.functions.hasOwnProperty(dalFunction) && typeof yl.functions[dalFunction] == 'function') {
                // If the function has been registered call it.
                yl.functions[dalFunction]($, element);
            } else if (yl.functions.hasOwnProperty(dalFunction)) {
                // If the function exists but has not been registered wait for it to be registered.
                window.addEventListener('dal-function-registered.' + dalFunction, function (e) {
                    yl.functions[dalFunction]($, element);
                })
            } else {
                // Otherwise notify that the function should be registered.
                console.warn('Your custom DAL function "' + dalFunction + '" uses a deprecated event listener that will be removed in future versions. https://django-autocomplete-light.readthedocs.io/en/master/tutorial.html#overriding-javascript-code')
            }

            // Fire init event for custom function execution.
            // DEPRECATED
            $(element).trigger('autocompleteLightInitialize');

            // Add element to the array of already initialized fields
            initialized.push(element);
        }

        if (!window.__dal__initialize) {
            window.__dal__initialize = initialize;

            $(document).ready(function () {
                $('[data-autocomplete-light-function]:not([id*="__prefix__"])').each(initialize);
            });

            if ('MutationObserver' in window) {
                new MutationObserver(function (mutations) {
                    var mutationRecord;
                    var addedNode;

                    for (var i = 0; i < mutations.length; i++) {
                        mutationRecord = mutations[i];

                        if (mutationRecord.addedNodes.length > 0) {
                            for (var j = 0; j < mutationRecord.addedNodes.length; j++) {
                                addedNode = mutationRecord.addedNodes[j];

                                $(addedNode).find('[data-autocomplete-light-function]').each(initialize);
                            }
                        }
                    }

                }).observe(document.documentElement, {childList: true, subtree: true});
            } else {
                $(document).on('DOMNodeInserted', function (e) {
                    $(e.target).find('[data-autocomplete-light-function]').each(initialize);
                });
            }
        }

        // using jQuery
        function getCookie(name) {
            var cookieValue = null;
            if (document.cookie && document.cookie != '') {
                var cookies = document.cookie.split(';');
                for (var i = 0; i < cookies.length; i++) {
                    var cookie = $.trim(cookies[i]);
                    // Does this cookie string begin with the name we want?
                    if (cookie.substring(0, name.length + 1) == (name + '=')) {
                        cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
                        break;
                    }
                }
            }
            return cookieValue;
        }

        document.csrftoken = getCookie('csrftoken');
        if (document.csrftoken === null) {
            // Try to get CSRF token from DOM when cookie is missing
            var $csrf = $('form :input[name="csrfmiddlewaretoken"]');
            if ($csrf.length > 0) {
                document.csrftoken = $csrf[0].value;
            }
        }
    })(django.jQuery);

    // Does the same thing as django's admin/js/autocomplete.js, but uses yl.jQuery.
    (function ($) {
        'use strict';
        var init = function ($element, options) {
            var settings = $.extend({
                ajax: {
                    data: function (params) {
                        return {
                            term: params.term,
                            page: params.page
                        };
                    }
                }
            }, options);
            $element.select2(settings);
        };

        $.fn.djangoAdminSelect2 = function (options) {
            var settings = $.extend({}, options);
            $.each(this, function (i, element) {
                var $element = $(element);
                init($element, settings);
            });
            return this;
        };

        $(function () {
            // Initialize all autocomplete widgets except the one in the template
            // form used when a new formset is added.
            $('.admin-autocomplete').not('[name*=__prefix__]').djangoAdminSelect2();
        });

        $(document).on('formset:added', (function () {
            return function (event, $newFormset) {
                return $newFormset.find('.admin-autocomplete').djangoAdminSelect2();
            };
        })(this));
    }(django.jQuery));

    (function ($, yl) {
        yl.forwardHandlerRegistry = yl.forwardHandlerRegistry || {};

        yl.registerForwardHandler = function (name, handler) {
            yl.forwardHandlerRegistry[name] = handler;
        };

        yl.getForwardHandler = function (name) {
            return yl.forwardHandlerRegistry[name];
        };

        function getForwardStrategy(element) {
            var checkForCheckboxes = function () {
                var all = true;
                $.each(element, function (ix, e) {
                    if ($(e).attr("type") !== "checkbox") {
                        all = false;
                    }
                });
                return all;
            };

            if (element.length === 1 &&
                element.attr("type") === "checkbox" &&
                element.attr("value") === undefined) {
                // Single checkbox without 'value' attribute
                // Boolean field
                return "exists";
            } else if (element.length === 1 &&
                element.attr("multiple") !== undefined) {
                // Multiple by HTML semantics. E. g. multiple select
                // Multiple choice field
                return "multiple";
            } else if (checkForCheckboxes()) {
                // Multiple checkboxes or one checkbox with 'value' attribute.
                // Multiple choice field represented by checkboxes
                return "multiple";
            } else {
                // Other cases
                return "single";
            }
        }

        /**
         * Get fields with name `name` relative to `element` with considering form
         * prefixes.
         * @param element the element
         * @param name name of the field
         * @returns jQuery object with found fields or empty jQuery object if no
         * field was found
         */
        yl.getFieldRelativeTo = function (element, name) {
            var prefixes = $(element).getFormPrefixes();

            for (var i = 0; i < prefixes.length; i++) {
                var fieldSelector = "[name=" + prefixes[i] + name + "]";
                var field = $(fieldSelector);

                if (field.length) {
                    return field;
                }
            }

            return $();
        };

        /**
         * Get field value which is put to forwarded dictionary
         * @param field the field
         * @returns forwarded value
         */
        yl.getValueFromField = function (field) {
            var strategy = getForwardStrategy(field);
            var serializedField = $(field).serializeArray();

            if ((serializedField == false) && ($(field).prop('disabled'))) {
                $(field).prop('disabled', false);
                serializedField = $(field).serializeArray();
                $(field).prop('disabled', true);
            }

            var getSerializedFieldElementAt = function (index) {
                // Return serializedField[index]
                // or null if something went wrong
                if (serializedField.length > index) {
                    return serializedField[index];
                } else {
                    return null;
                }
            };

            var getValueOf = function (elem) {
                // Return elem.value
                // or null if something went wrong
                if (elem.hasOwnProperty("value") &&
                    elem.value !== undefined
                ) {
                    return elem.value;
                } else {
                    return null;
                }
            };

            var getSerializedFieldValueAt = function (index) {
                // Return serializedField[index].value
                // or null if something went wrong
                var elem = getSerializedFieldElementAt(index);
                if (elem !== null) {
                    return getValueOf(elem);
                } else {
                    return null;
                }
            };

            if (strategy === "multiple") {
                return serializedField.map(
                    function (item) {
                        return getValueOf(item);
                    }
                );
            } else if (strategy === "exists") {
                return serializedField.length > 0;
            } else {
                return getSerializedFieldValueAt(0);
            }
        };

        yl.getForwards = function (element) {
            var forwardElem,
                forwardList,
                forwardedData,
                divSelector,
                form;
            divSelector = "div.dal-forward-conf#dal-forward-conf-for-" +
                element.attr("id") + ", " +
                "div.dal-forward-conf#dal-forward-conf-for_" +
                element.attr("id");
            form = element.length > 0 ? $(element[0].form) : $();

            forwardElem =
                form.find(divSelector).find('script');
            if (forwardElem.length === 0) {
                return;
            }
            try {
                forwardList = JSON.parse(forwardElem.text());
            } catch (e) {
                return;
            }

            if (!Array.isArray(forwardList)) {
                return;
            }

            forwardedData = {};

            $.each(forwardList, function (ix, field) {
                var srcName, dstName;
                if (field.type === "const") {
                    forwardedData[field.dst] = field.val;
                } else if (field.type === "self") {
                    if (field.hasOwnProperty("dst")) {
                        dstName = field.dst;
                    } else {
                        dstName = "self";
                    }
                    forwardedData[dstName] = yl.getValueFromField(element);
                } else if (field.type === "field") {
                    srcName = field.src;
                    if (field.hasOwnProperty("dst")) {
                        dstName = field.dst;
                    } else {
                        dstName = srcName;
                    }
                    var forwardedField = yl.getFieldRelativeTo(element, srcName);

                    if (!forwardedField.length) {
                        return;
                    }

                    forwardedData[dstName] = yl.getValueFromField(forwardedField);
                } else if (field.type === "javascript") {
                    var handler = yl.getForwardHandler(field.handler);
                    forwardedData[field.dst || field.handler] = handler(element);
                }

            });
            return JSON.stringify(forwardedData);
        };

    })(django.jQuery, yl);
});<commit_msg>Fixed django.jQuery missing outside admin<commit_after>/*!
 * Django Autocomplete Light
 */

var yl = yl || {};
yl.functions = {};
/**
 * Register your own JS function for DAL.
 *
 * @param name The name of your function. This should be the same as the widget
 *             `autocomplete_function` property value.
 * @param func The callback that will initialize your custom autocomplete.
 */
yl.registerFunction = function (name, func) {
    if (this.functions.hasOwnProperty(name)) {
        // This function already exists to show an error and skip.
        console.error('The DAL function "' + name + '" has already been registered.');
        return
    }
    if (typeof func != 'function') {
        // It's not a function kill it.
        throw new Error('The custom DAL function must be a function.');
    }
    this.functions[name] = func;
    var event = new CustomEvent('dal-function-registered.' + name, {detail: {name: name, func: func}})
    window.dispatchEvent(event);
};

window.addEventListener("load", function () {

    // Check if `django.jQuery` exists otherwise set `django.jQuery` to non namespaced jQuery.
    window.django = window.django || {};
    if (!django.hasOwnProperty('jQuery') && jQuery !== 'undefined') {
        django.jQuery = jQuery;
    }

    (function ($) {
        $.fn.getFormPrefix = function () {
            /* Get the form prefix for a field.
             *
             * For example:
             *
             *     $(':input[name$=owner]').getFormsetPrefix()
             *
             * Would return an empty string for an input with name 'owner' but would return
             * 'inline_model-0-' for an input named 'inline_model-0-owner'.
             */
            var parts = $(this).attr('name').split('-');
            var prefix = '';

            for (var i in parts) {
                var testPrefix = parts.slice(0, -i).join('-');
                if (!testPrefix.length) continue;
                testPrefix += '-';

                var result = $(':input[name^=' + testPrefix + ']')

                if (result.length) {
                    return testPrefix;
                }
            }

            return '';
        }

        $.fn.getFormPrefixes = function () {
            /*
             * Get the form prefixes for a field, from the most specific to the least.
             *
             * For example:
             *
             *      $(':input[name$=owner]').getFormPrefixes()
             *
             * Would return:
             * - [''] for an input named 'owner'.
             * - ['inline_model-0-', ''] for an input named 'inline_model-0-owner' (i.e. nested with a nested inline).
             * - ['sections-0-items-0-', 'sections-0-', ''] for an input named 'sections-0-items-0-product'
             *   (i.e. nested multiple time with django-nested-admin).
             */
            var parts = $(this).attr('name').split('-').slice(0, -1);
            var prefixes = [];

            for (i = 0; i < parts.length; i += 2) {
                var testPrefix = parts.slice(0, -i || parts.length).join('-');
                if (!testPrefix.length)
                    continue;

                testPrefix += '-';

                var result = $(':input[name^=' + testPrefix + ']')

                if (result.length)
                    prefixes.push(testPrefix);
            }

            prefixes.push('');

            return prefixes;
        }

        /*
         * This ensures the Language file is loaded and passes it our jQuery.
         */
        if (typeof dalLoadLanguage !== 'undefined') {
            dalLoadLanguage($);
        } else {
            document.addEventListener('dal-language-loaded', function (e) {
                // `e.lang` is the language that was loaded.
                dalLoadLanguage($);
            })
        }

        // Fire init event for yl.registerFunction() execution.
        var event = new CustomEvent('dal-init-function');
        document.dispatchEvent(event);

        var initialized = [];

        /**
         * Initialize a field element. This function calls the registered init function
         * and ensures that the element is only initialized once.
         *
         * @param element The field to be initialized
         */
        function initialize(element) {
            if (typeof element === 'undefined' || typeof element === 'number') {
                element = this;
            }

            // Ensure element is not already initialized.
            if (initialized.indexOf(element) >= 0) {
                return;
            }

            // The DAL function to execute.
            var dalFunction = $(element).attr('data-autocomplete-light-function');

            if (yl.functions.hasOwnProperty(dalFunction) && typeof yl.functions[dalFunction] == 'function') {
                // If the function has been registered call it.
                yl.functions[dalFunction]($, element);
            } else if (yl.functions.hasOwnProperty(dalFunction)) {
                // If the function exists but has not been registered wait for it to be registered.
                window.addEventListener('dal-function-registered.' + dalFunction, function (e) {
                    yl.functions[dalFunction]($, element);
                })
            } else {
                // Otherwise notify that the function should be registered.
                console.warn('Your custom DAL function "' + dalFunction + '" uses a deprecated event listener that will be removed in future versions. https://django-autocomplete-light.readthedocs.io/en/master/tutorial.html#overriding-javascript-code')
            }

            // Fire init event for custom function execution.
            // DEPRECATED
            $(element).trigger('autocompleteLightInitialize');

            // Add element to the array of already initialized fields
            initialized.push(element);
        }

        if (!window.__dal__initialize) {
            window.__dal__initialize = initialize;

            $(document).ready(function () {
                $('[data-autocomplete-light-function]:not([id*="__prefix__"])').each(initialize);
            });

            if ('MutationObserver' in window) {
                new MutationObserver(function (mutations) {
                    var mutationRecord;
                    var addedNode;

                    for (var i = 0; i < mutations.length; i++) {
                        mutationRecord = mutations[i];

                        if (mutationRecord.addedNodes.length > 0) {
                            for (var j = 0; j < mutationRecord.addedNodes.length; j++) {
                                addedNode = mutationRecord.addedNodes[j];

                                $(addedNode).find('[data-autocomplete-light-function]').each(initialize);
                            }
                        }
                    }

                }).observe(document.documentElement, {childList: true, subtree: true});
            } else {
                $(document).on('DOMNodeInserted', function (e) {
                    $(e.target).find('[data-autocomplete-light-function]').each(initialize);
                });
            }
        }

        // using jQuery
        function getCookie(name) {
            var cookieValue = null;
            if (document.cookie && document.cookie != '') {
                var cookies = document.cookie.split(';');
                for (var i = 0; i < cookies.length; i++) {
                    var cookie = $.trim(cookies[i]);
                    // Does this cookie string begin with the name we want?
                    if (cookie.substring(0, name.length + 1) == (name + '=')) {
                        cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
                        break;
                    }
                }
            }
            return cookieValue;
        }

        document.csrftoken = getCookie('csrftoken');
        if (document.csrftoken === null) {
            // Try to get CSRF token from DOM when cookie is missing
            var $csrf = $('form :input[name="csrfmiddlewaretoken"]');
            if ($csrf.length > 0) {
                document.csrftoken = $csrf[0].value;
            }
        }
    })(django.jQuery);

    // Does the same thing as django's admin/js/autocomplete.js, but uses yl.jQuery.
    (function ($) {
        'use strict';
        var init = function ($element, options) {
            var settings = $.extend({
                ajax: {
                    data: function (params) {
                        return {
                            term: params.term,
                            page: params.page
                        };
                    }
                }
            }, options);
            $element.select2(settings);
        };

        $.fn.djangoAdminSelect2 = function (options) {
            var settings = $.extend({}, options);
            $.each(this, function (i, element) {
                var $element = $(element);
                init($element, settings);
            });
            return this;
        };

        $(function () {
            // Initialize all autocomplete widgets except the one in the template
            // form used when a new formset is added.
            $('.admin-autocomplete').not('[name*=__prefix__]').djangoAdminSelect2();
        });

        $(document).on('formset:added', (function () {
            return function (event, $newFormset) {
                return $newFormset.find('.admin-autocomplete').djangoAdminSelect2();
            };
        })(this));
    }(django.jQuery));

    (function ($, yl) {
        yl.forwardHandlerRegistry = yl.forwardHandlerRegistry || {};

        yl.registerForwardHandler = function (name, handler) {
            yl.forwardHandlerRegistry[name] = handler;
        };

        yl.getForwardHandler = function (name) {
            return yl.forwardHandlerRegistry[name];
        };

        function getForwardStrategy(element) {
            var checkForCheckboxes = function () {
                var all = true;
                $.each(element, function (ix, e) {
                    if ($(e).attr("type") !== "checkbox") {
                        all = false;
                    }
                });
                return all;
            };

            if (element.length === 1 &&
                element.attr("type") === "checkbox" &&
                element.attr("value") === undefined) {
                // Single checkbox without 'value' attribute
                // Boolean field
                return "exists";
            } else if (element.length === 1 &&
                element.attr("multiple") !== undefined) {
                // Multiple by HTML semantics. E. g. multiple select
                // Multiple choice field
                return "multiple";
            } else if (checkForCheckboxes()) {
                // Multiple checkboxes or one checkbox with 'value' attribute.
                // Multiple choice field represented by checkboxes
                return "multiple";
            } else {
                // Other cases
                return "single";
            }
        }

        /**
         * Get fields with name `name` relative to `element` with considering form
         * prefixes.
         * @param element the element
         * @param name name of the field
         * @returns jQuery object with found fields or empty jQuery object if no
         * field was found
         */
        yl.getFieldRelativeTo = function (element, name) {
            var prefixes = $(element).getFormPrefixes();

            for (var i = 0; i < prefixes.length; i++) {
                var fieldSelector = "[name=" + prefixes[i] + name + "]";
                var field = $(fieldSelector);

                if (field.length) {
                    return field;
                }
            }

            return $();
        };

        /**
         * Get field value which is put to forwarded dictionary
         * @param field the field
         * @returns forwarded value
         */
        yl.getValueFromField = function (field) {
            var strategy = getForwardStrategy(field);
            var serializedField = $(field).serializeArray();

            if ((serializedField == false) && ($(field).prop('disabled'))) {
                $(field).prop('disabled', false);
                serializedField = $(field).serializeArray();
                $(field).prop('disabled', true);
            }

            var getSerializedFieldElementAt = function (index) {
                // Return serializedField[index]
                // or null if something went wrong
                if (serializedField.length > index) {
                    return serializedField[index];
                } else {
                    return null;
                }
            };

            var getValueOf = function (elem) {
                // Return elem.value
                // or null if something went wrong
                if (elem.hasOwnProperty("value") &&
                    elem.value !== undefined
                ) {
                    return elem.value;
                } else {
                    return null;
                }
            };

            var getSerializedFieldValueAt = function (index) {
                // Return serializedField[index].value
                // or null if something went wrong
                var elem = getSerializedFieldElementAt(index);
                if (elem !== null) {
                    return getValueOf(elem);
                } else {
                    return null;
                }
            };

            if (strategy === "multiple") {
                return serializedField.map(
                    function (item) {
                        return getValueOf(item);
                    }
                );
            } else if (strategy === "exists") {
                return serializedField.length > 0;
            } else {
                return getSerializedFieldValueAt(0);
            }
        };

        yl.getForwards = function (element) {
            var forwardElem,
                forwardList,
                forwardedData,
                divSelector,
                form;
            divSelector = "div.dal-forward-conf#dal-forward-conf-for-" +
                element.attr("id") + ", " +
                "div.dal-forward-conf#dal-forward-conf-for_" +
                element.attr("id");
            form = element.length > 0 ? $(element[0].form) : $();

            forwardElem =
                form.find(divSelector).find('script');
            if (forwardElem.length === 0) {
                return;
            }
            try {
                forwardList = JSON.parse(forwardElem.text());
            } catch (e) {
                return;
            }

            if (!Array.isArray(forwardList)) {
                return;
            }

            forwardedData = {};

            $.each(forwardList, function (ix, field) {
                var srcName, dstName;
                if (field.type === "const") {
                    forwardedData[field.dst] = field.val;
                } else if (field.type === "self") {
                    if (field.hasOwnProperty("dst")) {
                        dstName = field.dst;
                    } else {
                        dstName = "self";
                    }
                    forwardedData[dstName] = yl.getValueFromField(element);
                } else if (field.type === "field") {
                    srcName = field.src;
                    if (field.hasOwnProperty("dst")) {
                        dstName = field.dst;
                    } else {
                        dstName = srcName;
                    }
                    var forwardedField = yl.getFieldRelativeTo(element, srcName);

                    if (!forwardedField.length) {
                        return;
                    }

                    forwardedData[dstName] = yl.getValueFromField(forwardedField);
                } else if (field.type === "javascript") {
                    var handler = yl.getForwardHandler(field.handler);
                    forwardedData[field.dst || field.handler] = handler(element);
                }

            });
            return JSON.stringify(forwardedData);
        };

    })(django.jQuery, yl);
});<|endoftext|>


<commit_before>/* @flow */

import * as path from "path";
import { readFile } from "fs";
import _ from "lodash";

import { promisify } from "util";
const { dialog } = require("electron").remote;
import { parseNotebook, fromJS } from "@nteract/commutable";
import type { Notebook, Cell } from "@nteract/commutable";

import store from "./store";
import {
  getCommentStartString,
  getBreakpoints,
  escapeBlankRows
} from "./code-manager";
import { importResult } from "./result";

const readFileP = promisify(readFile);
const linesep = process.platform === "win32" ? "\r\n" : "\n";

export function ipynbOpener(uri: string) {
  if (
    path.extname(uri).toLowerCase() === ".ipynb" &&
    atom.config.get("Hydrogen.importNotebookURI") === true
  ) {
    return _loadNotebook(uri);
  }
}

export function importNotebook(event?: atom$CustomEvent) {
  // Use selected filepath if called from tree-view context menu
  const filenameFromTreeView = _.get(event, "target.dataset.path");
  if (filenameFromTreeView && path.extname(filenameFromTreeView) === ".ipynb") {
    return _loadNotebook(filenameFromTreeView);
  }

  dialog.showOpenDialog(
    {
      properties: ["openFile"],
      filters: [{ name: "Notebooks", extensions: ["ipynb"] }]
    },
    (filenames: ?Array<string>) => {
      if (!filenames) {
        atom.notifications.addError("No filenames selected");
        return;
      }
      const filename = filenames[0];
      if (path.extname(filename) !== ".ipynb") {
        atom.notifications.addError("Selected file must have extension .ipynb");
        return;
      }

      _loadNotebook(filename);
    }
  );
}

export async function _loadNotebook(filename: string) {
  let data;
  let nb;
  try {
    data = await readFileP(filename);
    nb = fromJS(JSON.parse(data));
    console.log(nb);
  } catch (err) {
    if (err.name === "SyntaxError") {
      atom.notifications.addError("Error not a valid notebook", {
        detail: err.stack
      });
    } else {
      atom.notifications.addError("Error reading file", {
        detail: err
      });
    }
    return;
  }
  if (nb.nbformat < 4) {
    atom.notifications.addError("Only notebook version 4 currently supported");
    return;
  }
  const editor = await atom.workspace.open();
  const grammar = getGrammarForNotebook(nb);
  if (!grammar) return;
  editor.setGrammar(grammar);
  const commentStartString = getCommentStartString(editor);
  if (!commentStartString) {
    atom.notifications.addError("No comment symbol defined in root scope");
    return;
  }
  const nbCells = nb.cellMap.valueSeq().toArray();
  const sources = _.map(nbCells, cell => {
    return getCellSource(cell, commentStartString + " ");
  });
  editor.setText(sources.join(linesep));
  let markers = store.markersMapping.get(editor.id);
  markers = markers ? markers : store.newMarkerStore(editor.id);
  let cellBreakpoints = getBreakpoints(editor);
  let cellNumber = 1;
  for (let cell of nbCells) {
    if (cell.cell_type == "code" && cell.outputs.toArray().length > 0) {
      importResult(
        { editor, markers },
        {
          outputs: cell.outputs.toArray(),
          row: escapeBlankRows(
            editor,
            cellBreakpoints[cellNumber - 1].row,
            cellBreakpoints[cellNumber].row - 1
          )
        }
      );
    }
    cellNumber++;
  }
}

function getGrammarForNotebook(nb: Notebook) {
  if (!nb.metadata.kernelspec || !nb.metadata.language_info) {
    atom.notifications.addWarning(
      "No language metadata in notebook; assuming Python"
    );
    return atom.grammars.grammarForScopeName("source.python");
  }

  let matchedGrammar = null;
  // metadata.language_info.file_extension is not a required metadata field, but
  // if it exists is the best way to match with Atom Grammar
  if (nb.metadata.language_info && nb.metadata.language_info.file_extension) {
    matchedGrammar = getGrammarForFileExtension(
      nb.metadata.language_info.file_extension
    );
    if (matchedGrammar) return matchedGrammar;
  }

  // If metadata exists, then metadata.kernelspec.name is required (in v4)
  if (nb.metadata.kernelspec.name) {
    matchedGrammar = getGrammarForKernelspecName(nb.metadata.kernelspec.name);
    if (matchedGrammar) return matchedGrammar;
  }

  atom.notifications.addWarning("Unable to determine correct language grammar");
  return atom.grammars.grammarForScopeName("source.python");
}

function getGrammarForFileExtension(ext: string): ?atom$Grammar {
  ext = ext.startsWith(".") ? ext.slice(1) : ext;
  const grammars = atom.grammars.getGrammars();
  return _.find(grammars, grammar => {
    return _.includes(grammar.fileTypes, ext);
  });
}

function getGrammarForKernelspecName(name: string): ?atom$Grammar {
  // Check if there exists an Atom grammar named source.${name}
  const grammars = atom.grammars.getGrammars();
  const matchedGrammar = _.find(grammars, { scopeName: `source.${name}` });
  if (matchedGrammar) return matchedGrammar;

  // Otherwise attempt manual matching from kernelspec name to Atom scope
  const crosswalk = {
    python2: "source.python",
    python3: "source.python",
    bash: "source.shell",
    javascript: "source.js",
    ir: "source.r"
  };
  if (crosswalk[name]) {
    return atom.grammars.grammarForScopeName(crosswalk[name]);
  }
}

function getCellSource(cell: Cell, commentStartString: string): string {
  console.log(cell);
  const cellType = cell.cell_type;
  const cellMarkerKeyword = cellType === "markdown" ? "markdown" : null;
  const cellMarker = getCellMarker(commentStartString, cellMarkerKeyword);
  let source = cell.source;
  /*if (cellType === "markdown") {
    source = _.map(source, line => commentStartString + line);
  }*/
  return cellMarker + linesep + source;
}

function getCellMarker(commentStartString: string, keyword: ?string) {
  const marker = commentStartString + "%%";
  return keyword ? marker + ` ${keyword}` : marker;
}
<commit_msg>prototype v3 - support markdown<commit_after>/* @flow */

import * as path from "path";
import { readFile } from "fs";
import _ from "lodash";

import { promisify } from "util";
const { dialog } = require("electron").remote;
import { parseNotebook, fromJS } from "@nteract/commutable";
import type { Notebook, Cell } from "@nteract/commutable";

import store from "./store";
import {
  getCommentStartString,
  getBreakpoints,
  escapeBlankRows
} from "./code-manager";
import { importResult } from "./result";

const readFileP = promisify(readFile);
const linesep = process.platform === "win32" ? "\r\n" : "\n";

export function ipynbOpener(uri: string) {
  if (
    path.extname(uri).toLowerCase() === ".ipynb" &&
    atom.config.get("Hydrogen.importNotebookURI") === true
  ) {
    return _loadNotebook(uri);
  }
}

export function importNotebook(event?: atom$CustomEvent) {
  // Use selected filepath if called from tree-view context menu
  const filenameFromTreeView = _.get(event, "target.dataset.path");
  if (filenameFromTreeView && path.extname(filenameFromTreeView) === ".ipynb") {
    return _loadNotebook(filenameFromTreeView);
  }

  dialog.showOpenDialog(
    {
      properties: ["openFile"],
      filters: [{ name: "Notebooks", extensions: ["ipynb"] }]
    },
    (filenames: ?Array<string>) => {
      if (!filenames) {
        atom.notifications.addError("No filenames selected");
        return;
      }
      const filename = filenames[0];
      if (path.extname(filename) !== ".ipynb") {
        atom.notifications.addError("Selected file must have extension .ipynb");
        return;
      }

      _loadNotebook(filename);
    }
  );
}

export async function _loadNotebook(filename: string) {
  let data;
  let nb;
  try {
    data = await readFileP(filename);
    nb = fromJS(JSON.parse(data));
    console.log(nb);
  } catch (err) {
    if (err.name === "SyntaxError") {
      atom.notifications.addError("Error not a valid notebook", {
        detail: err.stack
      });
    } else {
      atom.notifications.addError("Error reading file", {
        detail: err
      });
    }
    return;
  }
  if (nb.nbformat < 4) {
    atom.notifications.addError("Only notebook version 4 currently supported");
    return;
  }
  const editor = await atom.workspace.open();
  const grammar = getGrammarForNotebook(nb);
  if (!grammar) return;
  editor.setGrammar(grammar);
  const commentStartString = getCommentStartString(editor);
  if (!commentStartString) {
    atom.notifications.addError("No comment symbol defined in root scope");
    return;
  }
  const nbCells = nb.cellMap.valueSeq().toArray();
  const sources = _.map(nbCells, cell => {
    return getCellSource(cell, commentStartString + " ");
  });
  editor.setText(sources.join(linesep));
  let markers = store.markersMapping.get(editor.id);
  markers = markers ? markers : store.newMarkerStore(editor.id);
  let cellBreakpoints = getBreakpoints(editor);
  let cellNumber = 1;
  for (let cell of nbCells) {
    const cellEnd = escapeBlankRows(
      editor,
      cellBreakpoints[cellNumber - 1].row,
      cellBreakpoints[cellNumber].row - 1
    );

    switch (cell.cell_type) {
      case "code":
        if (cell.outputs.toArray().length > 0) {
          importResult(
            { editor, markers },
            {
              outputs: cell.outputs.toArray(),
              row: cellEnd
            }
          );
        }
        break;
      case "markdown":
        importResult(
          { editor, markers },
          {
            outputs: [
              {
                output_type: "display_data",
                data: {
                  "text/markdown": cell.source
                },
                metadata: {}
              }
            ],
            row: cellEnd
          }
        );
        break;
    }
    cellNumber++;
  }
}

function getGrammarForNotebook(nb: Notebook) {
  if (!nb.metadata.kernelspec || !nb.metadata.language_info) {
    atom.notifications.addWarning(
      "No language metadata in notebook; assuming Python"
    );
    return atom.grammars.grammarForScopeName("source.python");
  }

  let matchedGrammar = null;
  // metadata.language_info.file_extension is not a required metadata field, but
  // if it exists is the best way to match with Atom Grammar
  if (nb.metadata.language_info && nb.metadata.language_info.file_extension) {
    matchedGrammar = getGrammarForFileExtension(
      nb.metadata.language_info.file_extension
    );
    if (matchedGrammar) return matchedGrammar;
  }

  // If metadata exists, then metadata.kernelspec.name is required (in v4)
  if (nb.metadata.kernelspec.name) {
    matchedGrammar = getGrammarForKernelspecName(nb.metadata.kernelspec.name);
    if (matchedGrammar) return matchedGrammar;
  }

  atom.notifications.addWarning("Unable to determine correct language grammar");
  return atom.grammars.grammarForScopeName("source.python");
}

function getGrammarForFileExtension(ext: string): ?atom$Grammar {
  ext = ext.startsWith(".") ? ext.slice(1) : ext;
  const grammars = atom.grammars.getGrammars();
  return _.find(grammars, grammar => {
    return _.includes(grammar.fileTypes, ext);
  });
}

function getGrammarForKernelspecName(name: string): ?atom$Grammar {
  // Check if there exists an Atom grammar named source.${name}
  const grammars = atom.grammars.getGrammars();
  const matchedGrammar = _.find(grammars, { scopeName: `source.${name}` });
  if (matchedGrammar) return matchedGrammar;

  // Otherwise attempt manual matching from kernelspec name to Atom scope
  const crosswalk = {
    python2: "source.python",
    python3: "source.python",
    bash: "source.shell",
    javascript: "source.js",
    ir: "source.r"
  };
  if (crosswalk[name]) {
    return atom.grammars.grammarForScopeName(crosswalk[name]);
  }
}

function getCellSource(cell: Cell, commentStartString: string): string {
  console.log(cell);
  const cellType = cell.cell_type;
  const cellMarkerKeyword = cellType === "markdown" ? "markdown" : null;
  const cellMarker = getCellMarker(commentStartString, cellMarkerKeyword);
  let source = cell.source;
  /*if (cellType === "markdown") {
    source = _.map(source, line => commentStartString + line);
  }*/
  return cellMarker + linesep + source;
}

function getCellMarker(commentStartString: string, keyword: ?string) {
  const marker = commentStartString + "%%";
  return keyword ? marker + ` ${keyword}` : marker;
}
<|endoftext|>


<commit_before>var fs = require('fs')
  , path = require('path')
  , lconfig = require("lconfig")
  , spawn = require('child_process').spawn
  , ldatastore = require('ldatastore')
  , datastore = {}
  , async = require('async')
  , url = require('url')
  , lutil = require('lutil')
  , EventEmitter = require('events').EventEmitter
  , levents = require(__dirname + '/levents')
  , logger = require("./logger.js");

// this works, but feels like it should be a cleaner abstraction layer on top of the datastore instead of this garbage
datastore.init = function (callback) {
    ldatastore.init('synclets', callback);
};

datastore.addCollection = function (collectionKey, id, mongoId) {
    ldatastore.addCollection('synclets', collectionKey, id, mongoId);
};

datastore.removeObject = function (collectionKey, id, ts, callback) {
    if (typeof(ts) === 'function') {
        ldatastore.removeObject('synclets', collectionKey, id, {timeStamp: Date.now()}, ts);
    } else {
        ldatastore.removeObject('synclets', collectionKey, id, ts, callback);
    }
};

datastore.addObject = function (collectionKey, obj, ts, callback) {
    ldatastore.addObject('synclets', collectionKey, obj, ts, callback);
};

// core syncmanager init function, need to talk to serviceManager
var serviceManager;
exports.init = function (sman, callback) {
    serviceManager = sman;
    datastore.init(callback);
};

var executeable = true;
exports.setExecuteable = function (e) {
    executeable = e;
};

exports.syncNow = function (serviceId, syncletId, post, callback) {
    if(typeof syncletId == "function")
    {
        callback = syncletId;
        syncletId = false;
    }
    var js = serviceManager.map(serviceId);
    if (!js || !js.synclets) return callback("no synclets like that installed");
    async.forEachSeries(js.synclets, function (synclet, cb) {
        if(syncletId && synclet.name != syncletId) return cb();
        if(post)
        {
            if(!Array.isArray(synclet.posts)) synclet.posts = [];
            synclet.posts.push(post);
        }
        executeSynclet(js, synclet, cb, true);
    }, callback);
};

// run all synclets that have a tolerance and reset them
exports.flushTolerance = function(callback, force) {
    var map = serviceManager.map();
    async.forEach(Object.keys(map), function(service, cb){ // do all services in parallel
        if(!map[service].synclets) return cb();
        async.forEachSeries(map[service].synclets, function(synclet, cb2) { // do each synclet in series
            if (!force && (!synclet.tolAt || synclet.tolAt === 0)) return cb2();
            synclet.tolAt = 0;
            executeSynclet(map[service], synclet, cb2);
        }, cb);
    }, callback);
};

/**
* Add a timeout to run a synclet
*/
var scheduled = {};
exports.scheduleRun = function(info, synclet) {
    synclet.status = "waiting";
    if (!synclet.frequency) return;

    var key = info.id + "-" + synclet.name;
    logger.verbose("scheduling "+key);
    if(scheduled[key]) clearTimeout(scheduled[key]); // remove any existing timer

    // run from a clean state
    var force = false;
    function run() {
        delete scheduled[key];
        executeSynclet(info, synclet, function(){}, force);
    }

    // the synclet is paging and needs to run again immediately, forcefully
    if(info.config && info.config.nextRun === -1)
    {
        force = true;
        delete info.config.nextRun;
        return process.nextTick(run);
    }

    // validation check
    if(synclet.nextRun && typeof synclet.nextRun != "number") delete synclet.nextRun;

    // had a schedule and missed it, run it now
    if(synclet.nextRun && synclet.nextRun <= Date.now()) return process.nextTick(run);

    // if no schedule, in the future with 10% fuzz
    if(!synclet.nextRun)
    {
        var milliFreq = parseInt(synclet.frequency) * 1000;
        synclet.nextRun = parseInt(Date.now() + milliFreq + (((Math.random() - 0.5) * 0.1) * milliFreq));
    }
    scheduled[key] = setTimeout(run, synclet.nextRun - Date.now());
};

function localError(base, err) {
    logger.error(base+"\t"+err);
}

/**
* Executes a synclet
*/
function executeSynclet(info, synclet, callback, force) {
    if(!callback) callback = function(err){
        if(err) return logger.error(err);
        logger.debug("finished processing "+synclet.name);
    };
    if (synclet.status === 'running') return callback('already running');
    delete synclet.nextRun; // cancel any schedule
    // we're put on hold from running any for some reason, re-schedule them
    // this is a workaround for making synclets available in the map separate from scheduling them which could be done better
    if (!force && !executeable) {
        setTimeout(function() {
            executeSynclet(info, synclet, callback);
        }, 1000);
        return;
    }
    if(!synclet.tolMax) {
        synclet.tolAt = 0;
        synclet.tolMax = 0;
    }
    // if we can have tolerance, try again later
    if(!force && synclet.tolAt > 0) {
        synclet.tolAt--;
        logger.verbose("tolerance now at "+synclet.tolAt+" synclet "+synclet.name+" for "+info.id);
        exports.scheduleRun(info, synclet);
        return callback();
    }
    // if another synclet is running, come back a little later, don't overlap!
    if (info.status == 'running') {
        logger.verbose("delaying "+synclet.name);
        setTimeout(function() {
            executeSynclet(info, synclet, callback, force);
        }, 10000);
        return;
    }
    logger.info("Synclet "+synclet.name+" starting for "+info.id);
    info.status = synclet.status = "running";
    var run;
    var env = process.env;
    if (!synclet.run) {
        env["NODE_PATH"] = path.join(lconfig.lockerDir, 'Common', 'node') + ":" + path.join(lconfig.lockerDir, "node_modules");
        run = ["node", lconfig.lockerDir + "/Common/node/synclet/client.js"];
    } else if (synclet.run.substr(-3) == ".py") {
        env["PYTHONPATH"] = path.join(lconfig.lockerDir, 'Common', 'python');
        run = ["python", lconfig.lockerDir + "/Common/python/synclet/client.py"];
    } else {
        env["NODE_PATH"] = path.join(lconfig.lockerDir, 'Common', 'node') + ":" + path.join(lconfig.lockerDir, "node_modules");
        run = ["node", path.join(lconfig.lockerDir, info.srcdir, synclet.run)];
    }

    var dataResponse = '';
    var cwd = (info.srcdir.charAt(0) == '/') ? info.srcdir : path.join(lconfig.lockerDir, info.srcdir);
    var app = spawn(run.shift(), run, {cwd: cwd, env:env});

    // edge case backup, max 30 min runtime by default
    var timer = setTimeout(function(){
        logger.error("Having to kill long-running "+synclet.name+" synclet of "+info.id);
        info.status = synclet.status = 'failed : timeout';
        process.kill(app.pid); // will fire exit event below and cleanup
    }, (synclet.timeout) ? synclet.timeout : 30*60*1000);

    app.stderr.on('data', function (data) {
        localError(info.title+" "+synclet.name + " error:",data.toString());
    });

    var tstart;
    app.stdout.on('data',function (data) {
        if(!tstart) tstart = Date.now();
        dataResponse += data;
    });

    app.on('exit', function (code,signal) {
        clearTimeout(timer);
        var response;
        try {
            response = JSON.parse(dataResponse);
        } catch (E) {
            if (info.status != 'failed : timeout') {
                localError("Service : " + info.title + ", synclet: "+synclet.name, ". Failed to parse the response from the synclet.  Error was: '" + E + "' and the response from the synclet was '" + dataResponse + "'");
                info.status = synclet.status = 'failed : ' + E;
            }
            if (callback) callback(E);
            return;
        }
        logger.info("Synclet "+synclet.name+" finished for "+info.id+" timing "+(Date.now() - tstart));
        info.status = synclet.status = 'processing data';
        var deleteIDs = compareIDs(info.config, response.config);
        info.auth = lutil.extend(true, info.auth, response.auth); // for refresh tokens and profiles
        info.config = lutil.extend(true, info.config, response.config);
        exports.scheduleRun(info, synclet);
        serviceManager.mapDirty(info.id); // save out to disk
        processResponse(deleteIDs, info, synclet, response, function(err){
            info.status = 'waiting';
            callback(err);
        });
    });
    if (!info.config) info.config = {};

    info.syncletToRun = synclet;
    info.syncletToRun.workingDirectory = path.join(lconfig.lockerDir, lconfig.me, info.id);
    info.lockerUrl = lconfig.lockerBase;
    app.stdin.on('error',function(err){
        localError(info.title+" "+synclet.name, "stdin closed: "+err);
    });
    app.stdin.write(JSON.stringify(info)+"\n"); // Send them the process information
    if(synclet.posts) synclet.posts = []; // they're serialized, empty the queue
    delete info.syncletToRun;
}

function compareIDs (originalConfig, newConfig) {
    var resp = {};
    if (originalConfig && originalConfig.ids && newConfig && newConfig.ids) {
        for (var i in newConfig.ids) {
            if (!originalConfig.ids[i]) break;
            var newSet = newConfig.ids[i];
            var oldSet = originalConfig.ids[i];
            var seenIDs = {};
            resp[i] = [];
            for (var j = 0; j < newSet.length; j++) seenIDs[newSet[j]] = true;
            for (var j = 0; j < oldSet.length; j++) {
                if (!seenIDs[oldSet[j]]) resp[i].push(oldSet[j]);
            }
        }
    }
    return resp;
}

function processResponse(deleteIDs, info, synclet, response, callback) {
    datastore.init(function() {
        synclet.status = 'waiting';

        var dataKeys = [];
        if (typeof(response.data) === 'string') {
            return callback('bad data from synclet');
        }
        for (var i in response.data) {
            if(!Array.isArray(response.data[i])) continue;
            dataKeys.push(i);
        }
        for (var i in deleteIDs) {
            if (!dataKeys[i]) dataKeys.push(i);
        }
        synclet.deleted = synclet.added = synclet.updated = 0;
        async.forEach(dataKeys, function(key, cb) { processData(deleteIDs[key], info, synclet, key, response.data[key], cb); }, function(err){
            if(err) logger.error("err processing data: "+err);
            // here we roughly compromise a multiplier up or down based on the threshold being met
            var threshold = synclet.threshold || lconfig.tolerance.threshold;
            var total = synclet.deleted + synclet.added + synclet.updated;
            if(total < threshold)
            {
                if(synclet.tolMax < lconfig.tolerance.maxstep) synclet.tolMax++; // max 10x scheduled
                synclet.tolAt = synclet.tolMax;
            }else{
                if(synclet.tolMax > 0) synclet.tolMax--;
                synclet.tolAt = synclet.tolMax;
            }
            logger.info("total of "+synclet.added+"+"+synclet.updated+"+"+synclet.deleted+" and threshold "+threshold+" so setting tolerance to "+synclet.tolMax);
            callback(err);
        });
    });
}

function processData (deleteIDs, info, synclet, key, data, callback) {
    // this extra (handy) log breaks the synclet tests somehow??
    var len = (data)?data.length:0;
    var type = (info.types && info.types[key]) ? info.types[key] : key; // try to map the key to a generic data type for the idr
    var idr = lutil.idrNew(type, info.provider, undefined, key, info.id);
    if(len > 0) logger.info("processing synclet data from "+idr+" of length "+len);
    var collection = info.id + "_" + key;

    if (key.indexOf('/') !== -1) {
        console.error("DEPRECATED, dropping! "+key);
        return callback();
    }

    var mongoId;
    if (typeof info.mongoId === 'string') mongoId = info.mongoId;
    else if (info.mongoId) mongoId = info.mongoId[key + 's'] || info.mongoId[key] || 'id';
    else mongoId = 'id';

    datastore.addCollection(key, info.id, mongoId);

    if (deleteIDs && deleteIDs.length > 0 && data) {
        addData(collection, mongoId, data, info, synclet, idr, function(err) {
            if(err) {
                callback(err);
            } else {
                deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback);
            }
        });
    } else if (data && data.length > 0) {
        addData(collection, mongoId, data, info, synclet, idr, callback);
    } else if (deleteIDs && deleteIDs.length > 0) {
        deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback);
    } else {
        callback();
    }
}

function deleteData (collection, mongoId, deleteIds, info, synclet, idr, callback) {
    var q = async.queue(function(id, cb) {
        var r = url.parse(idr);
        r.hash = id.toString();
        levents.fireEvent(url.format(r), 'delete');
        synclet.deleted++;
        datastore.removeObject(collection, id, {timeStamp: Date.now()}, cb);
    }, 5);
    deleteIds.forEach(q.push);
    q.drain = callback;
}

function addData (collection, mongoId, data, info, synclet, idr, callback) {
    var errs = [];
    var q = async.queue(function(item, cb) {
        var object = (item.obj) ? item : {obj: item};
        if (object.obj) {
            if(object.obj[mongoId] === null || object.obj[mongoId] === undefined) {
                localError(info.title + ' ' + url.format(idr), "missing primary key (" + mongoId + ") value: "+JSON.stringify(object.obj));
                errs.push({"message":"no value for primary key", "obj": object.obj});
                return cb();
            }
            var r = url.parse(idr);
            r.hash = object.obj[mongoId].toString();
            if (object.type === 'delete') {
                levents.fireEvent(url.format(r), 'delete');
                synclet.deleted++;
                datastore.removeObject(collection, object.obj[mongoId], {timeStamp: object.timestamp}, cb);
            } else {
                var source = r.pathname.substring(1);
                var options = {timeStamp: object.timestamp};
                if(info.strip && info.strip[source]) options.strip = info.strip[source];
                datastore.addObject(collection, object.obj, options, function(err, type, doc) {
                    if (type === 'same') return cb();
                    if (type === 'new') synclet.added++;
                    if (type === 'update') synclet.updated++;
                    levents.fireEvent(url.format(r), type, doc);
                    return cb();
                });
            }
        } else {
            cb();
        }
    }, 5);
    data.forEach(function(d){ q.push(d, errs.push); }); // hehe fun
    q.drain = function() {
        if (errs.length > 0) {
            callback(errs);
        } else {
            callback();
        }
    };
}

<commit_msg>make sure there's lots of padding around rescheduling<commit_after>var fs = require('fs')
  , path = require('path')
  , lconfig = require("lconfig")
  , spawn = require('child_process').spawn
  , ldatastore = require('ldatastore')
  , datastore = {}
  , async = require('async')
  , url = require('url')
  , lutil = require('lutil')
  , EventEmitter = require('events').EventEmitter
  , levents = require(__dirname + '/levents')
  , logger = require("./logger.js");

// this works, but feels like it should be a cleaner abstraction layer on top of the datastore instead of this garbage
datastore.init = function (callback) {
    ldatastore.init('synclets', callback);
};

datastore.addCollection = function (collectionKey, id, mongoId) {
    ldatastore.addCollection('synclets', collectionKey, id, mongoId);
};

datastore.removeObject = function (collectionKey, id, ts, callback) {
    if (typeof(ts) === 'function') {
        ldatastore.removeObject('synclets', collectionKey, id, {timeStamp: Date.now()}, ts);
    } else {
        ldatastore.removeObject('synclets', collectionKey, id, ts, callback);
    }
};

datastore.addObject = function (collectionKey, obj, ts, callback) {
    ldatastore.addObject('synclets', collectionKey, obj, ts, callback);
};

// core syncmanager init function, need to talk to serviceManager
var serviceManager;
exports.init = function (sman, callback) {
    serviceManager = sman;
    datastore.init(callback);
};

var executeable = true;
exports.setExecuteable = function (e) {
    executeable = e;
};

exports.syncNow = function (serviceId, syncletId, post, callback) {
    if(typeof syncletId == "function")
    {
        callback = syncletId;
        syncletId = false;
    }
    var js = serviceManager.map(serviceId);
    if (!js || !js.synclets) return callback("no synclets like that installed");
    async.forEachSeries(js.synclets, function (synclet, cb) {
        if(syncletId && synclet.name != syncletId) return cb();
        if(post)
        {
            if(!Array.isArray(synclet.posts)) synclet.posts = [];
            synclet.posts.push(post);
        }
        executeSynclet(js, synclet, cb, true);
    }, callback);
};

// run all synclets that have a tolerance and reset them
exports.flushTolerance = function(callback, force) {
    var map = serviceManager.map();
    async.forEach(Object.keys(map), function(service, cb){ // do all services in parallel
        if(!map[service].synclets) return cb();
        async.forEachSeries(map[service].synclets, function(synclet, cb2) { // do each synclet in series
            if (!force && (!synclet.tolAt || synclet.tolAt === 0)) return cb2();
            synclet.tolAt = 0;
            executeSynclet(map[service], synclet, cb2);
        }, cb);
    }, callback);
};

/**
* Add a timeout to run a synclet
*/
var scheduled = {};
exports.scheduleRun = function(info, synclet) {
    synclet.status = "waiting";
    if (!synclet.frequency) return;

    var key = info.id + "-" + synclet.name;
    logger.verbose("scheduling "+key);
    if(scheduled[key]) clearTimeout(scheduled[key]); // remove any existing timer

    // run from a clean state
    var force = false;
    function run() {
        delete scheduled[key];
        executeSynclet(info, synclet, function(){}, force);
    }

    // the synclet is paging and needs to run again immediately, forcefully
    if(info.config && info.config.nextRun === -1)
    {
        force = true;
        delete info.config.nextRun;
        return process.nextTick(run);
    }

    // validation check
    if(synclet.nextRun && typeof synclet.nextRun != "number") delete synclet.nextRun;

    // had a schedule and missed it, run it now
    if(synclet.nextRun && synclet.nextRun <= Date.now()) return process.nextTick(run);

    // if no schedule, in the future with 10% fuzz
    if(!synclet.nextRun)
    {
        var milliFreq = parseInt(synclet.frequency) * 1000;
        synclet.nextRun = parseInt(Date.now() + milliFreq + (((Math.random() - 0.5) * 0.5) * milliFreq)); // 50% fuzz added or subtracted
    }
    scheduled[key] = setTimeout(run, synclet.nextRun - Date.now());
};

function localError(base, err) {
    logger.error(base+"\t"+err);
}

/**
* Executes a synclet
*/
function executeSynclet(info, synclet, callback, force) {
    if(!callback) callback = function(err){
        if(err) return logger.error(err);
        logger.debug("finished processing "+synclet.name);
    };
    if (synclet.status === 'running') return callback('already running');
    delete synclet.nextRun; // cancel any schedule
    // we're put on hold from running any for some reason, re-schedule them
    // this is a workaround for making synclets available in the map separate from scheduling them which could be done better
    if (!force && !executeable) {
        setTimeout(function() {
            executeSynclet(info, synclet, callback);
        }, 1000);
        return;
    }
    if(!synclet.tolMax) {
        synclet.tolAt = 0;
        synclet.tolMax = 0;
    }
    // if we can have tolerance, try again later
    if(!force && synclet.tolAt > 0) {
        synclet.tolAt--;
        logger.verbose("tolerance now at "+synclet.tolAt+" synclet "+synclet.name+" for "+info.id);
        exports.scheduleRun(info, synclet);
        return callback();
    }
    // if another synclet is running, come back a little later, don't overlap!
    if (info.status == 'running') {
        logger.verbose("delaying "+synclet.name);
        setTimeout(function() {
            executeSynclet(info, synclet, callback, force);
        }, 10000);
        return;
    }
    logger.info("Synclet "+synclet.name+" starting for "+info.id);
    info.status = synclet.status = "running";
    var run;
    var env = process.env;
    if (!synclet.run) {
        env["NODE_PATH"] = path.join(lconfig.lockerDir, 'Common', 'node') + ":" + path.join(lconfig.lockerDir, "node_modules");
        run = ["node", lconfig.lockerDir + "/Common/node/synclet/client.js"];
    } else if (synclet.run.substr(-3) == ".py") {
        env["PYTHONPATH"] = path.join(lconfig.lockerDir, 'Common', 'python');
        run = ["python", lconfig.lockerDir + "/Common/python/synclet/client.py"];
    } else {
        env["NODE_PATH"] = path.join(lconfig.lockerDir, 'Common', 'node') + ":" + path.join(lconfig.lockerDir, "node_modules");
        run = ["node", path.join(lconfig.lockerDir, info.srcdir, synclet.run)];
    }

    var dataResponse = '';
    var cwd = (info.srcdir.charAt(0) == '/') ? info.srcdir : path.join(lconfig.lockerDir, info.srcdir);
    var app = spawn(run.shift(), run, {cwd: cwd, env:env});

    // edge case backup, max 30 min runtime by default
    var timer = setTimeout(function(){
        logger.error("Having to kill long-running "+synclet.name+" synclet of "+info.id);
        info.status = synclet.status = 'failed : timeout';
        process.kill(app.pid); // will fire exit event below and cleanup
    }, (synclet.timeout) ? synclet.timeout : 30*60*1000);

    app.stderr.on('data', function (data) {
        localError(info.title+" "+synclet.name + " error:",data.toString());
    });

    var tstart;
    app.stdout.on('data',function (data) {
        if(!tstart) tstart = Date.now();
        dataResponse += data;
    });

    app.on('exit', function (code,signal) {
        clearTimeout(timer);
        var response;
        try {
            response = JSON.parse(dataResponse);
        } catch (E) {
            if (info.status != 'failed : timeout') {
                localError("Service : " + info.title + ", synclet: "+synclet.name, ". Failed to parse the response from the synclet.  Error was: '" + E + "' and the response from the synclet was '" + dataResponse + "'");
                info.status = synclet.status = 'failed : ' + E;
            }
            if (callback) callback(E);
            return;
        }
        logger.info("Synclet "+synclet.name+" finished for "+info.id+" timing "+(Date.now() - tstart));
        info.status = synclet.status = 'processing data';
        var deleteIDs = compareIDs(info.config, response.config);
        info.auth = lutil.extend(true, info.auth, response.auth); // for refresh tokens and profiles
        info.config = lutil.extend(true, info.config, response.config);
        exports.scheduleRun(info, synclet);
        serviceManager.mapDirty(info.id); // save out to disk
        processResponse(deleteIDs, info, synclet, response, function(err){
            info.status = 'waiting';
            callback(err);
        });
    });
    if (!info.config) info.config = {};

    info.syncletToRun = synclet;
    info.syncletToRun.workingDirectory = path.join(lconfig.lockerDir, lconfig.me, info.id);
    info.lockerUrl = lconfig.lockerBase;
    app.stdin.on('error',function(err){
        localError(info.title+" "+synclet.name, "stdin closed: "+err);
    });
    app.stdin.write(JSON.stringify(info)+"\n"); // Send them the process information
    if(synclet.posts) synclet.posts = []; // they're serialized, empty the queue
    delete info.syncletToRun;
}

function compareIDs (originalConfig, newConfig) {
    var resp = {};
    if (originalConfig && originalConfig.ids && newConfig && newConfig.ids) {
        for (var i in newConfig.ids) {
            if (!originalConfig.ids[i]) break;
            var newSet = newConfig.ids[i];
            var oldSet = originalConfig.ids[i];
            var seenIDs = {};
            resp[i] = [];
            for (var j = 0; j < newSet.length; j++) seenIDs[newSet[j]] = true;
            for (var j = 0; j < oldSet.length; j++) {
                if (!seenIDs[oldSet[j]]) resp[i].push(oldSet[j]);
            }
        }
    }
    return resp;
}

function processResponse(deleteIDs, info, synclet, response, callback) {
    datastore.init(function() {
        synclet.status = 'waiting';

        var dataKeys = [];
        if (typeof(response.data) === 'string') {
            return callback('bad data from synclet');
        }
        for (var i in response.data) {
            if(!Array.isArray(response.data[i])) continue;
            dataKeys.push(i);
        }
        for (var i in deleteIDs) {
            if (!dataKeys[i]) dataKeys.push(i);
        }
        synclet.deleted = synclet.added = synclet.updated = 0;
        async.forEach(dataKeys, function(key, cb) { processData(deleteIDs[key], info, synclet, key, response.data[key], cb); }, function(err){
            if(err) logger.error("err processing data: "+err);
            // here we roughly compromise a multiplier up or down based on the threshold being met
            var threshold = synclet.threshold || lconfig.tolerance.threshold;
            var total = synclet.deleted + synclet.added + synclet.updated;
            if(total < threshold)
            {
                if(synclet.tolMax < lconfig.tolerance.maxstep) synclet.tolMax++; // max 10x scheduled
                synclet.tolAt = synclet.tolMax;
            }else{
                if(synclet.tolMax > 0) synclet.tolMax--;
                synclet.tolAt = synclet.tolMax;
            }
            logger.info("total of "+synclet.added+"+"+synclet.updated+"+"+synclet.deleted+" and threshold "+threshold+" so setting tolerance to "+synclet.tolMax);
            callback(err);
        });
    });
}

function processData (deleteIDs, info, synclet, key, data, callback) {
    // this extra (handy) log breaks the synclet tests somehow??
    var len = (data)?data.length:0;
    var type = (info.types && info.types[key]) ? info.types[key] : key; // try to map the key to a generic data type for the idr
    var idr = lutil.idrNew(type, info.provider, undefined, key, info.id);
    if(len > 0) logger.info("processing synclet data from "+idr+" of length "+len);
    var collection = info.id + "_" + key;

    if (key.indexOf('/') !== -1) {
        console.error("DEPRECATED, dropping! "+key);
        return callback();
    }

    var mongoId;
    if (typeof info.mongoId === 'string') mongoId = info.mongoId;
    else if (info.mongoId) mongoId = info.mongoId[key + 's'] || info.mongoId[key] || 'id';
    else mongoId = 'id';

    datastore.addCollection(key, info.id, mongoId);

    if (deleteIDs && deleteIDs.length > 0 && data) {
        addData(collection, mongoId, data, info, synclet, idr, function(err) {
            if(err) {
                callback(err);
            } else {
                deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback);
            }
        });
    } else if (data && data.length > 0) {
        addData(collection, mongoId, data, info, synclet, idr, callback);
    } else if (deleteIDs && deleteIDs.length > 0) {
        deleteData(collection, mongoId, deleteIDs, info, synclet, idr, callback);
    } else {
        callback();
    }
}

function deleteData (collection, mongoId, deleteIds, info, synclet, idr, callback) {
    var q = async.queue(function(id, cb) {
        var r = url.parse(idr);
        r.hash = id.toString();
        levents.fireEvent(url.format(r), 'delete');
        synclet.deleted++;
        datastore.removeObject(collection, id, {timeStamp: Date.now()}, cb);
    }, 5);
    deleteIds.forEach(q.push);
    q.drain = callback;
}

function addData (collection, mongoId, data, info, synclet, idr, callback) {
    var errs = [];
    var q = async.queue(function(item, cb) {
        var object = (item.obj) ? item : {obj: item};
        if (object.obj) {
            if(object.obj[mongoId] === null || object.obj[mongoId] === undefined) {
                localError(info.title + ' ' + url.format(idr), "missing primary key (" + mongoId + ") value: "+JSON.stringify(object.obj));
                errs.push({"message":"no value for primary key", "obj": object.obj});
                return cb();
            }
            var r = url.parse(idr);
            r.hash = object.obj[mongoId].toString();
            if (object.type === 'delete') {
                levents.fireEvent(url.format(r), 'delete');
                synclet.deleted++;
                datastore.removeObject(collection, object.obj[mongoId], {timeStamp: object.timestamp}, cb);
            } else {
                var source = r.pathname.substring(1);
                var options = {timeStamp: object.timestamp};
                if(info.strip && info.strip[source]) options.strip = info.strip[source];
                datastore.addObject(collection, object.obj, options, function(err, type, doc) {
                    if (type === 'same') return cb();
                    if (type === 'new') synclet.added++;
                    if (type === 'update') synclet.updated++;
                    levents.fireEvent(url.format(r), type, doc);
                    return cb();
                });
            }
        } else {
            cb();
        }
    }, 5);
    data.forEach(function(d){ q.push(d, errs.push); }); // hehe fun
    q.drain = function() {
        if (errs.length > 0) {
            callback(errs);
        } else {
            callback();
        }
    };
}

<|endoftext|>