<commit_before>/*globals qq*/
/**
 * Base upload handler module.  Delegates to more specific handlers.
 *
 * @param o Options.  Passed along to the specific handler submodule as well.
 * @param namespace [optional] Namespace for the specific handler.
 */
qq.UploadHandlerController = function(o, namespace) {
    "use strict";

    var controller = this,
        queue = [],
        chunkingPossible = false,
        chunking, preventRetryResponse, log, handler,

    // Default options, can be overridden by the user
    options = {
        paramsStore: {},
        maxConnections: 3, // maximum number of concurrent uploads
        chunking: {
            enabled: false,
            partSize: 2000000 //bytes
        },
        log: function(str, level) {},
        onProgress: function(id, fileName, loaded, total){},
        onComplete: function(id, fileName, response, xhr){},
        onCancel: function(id, fileName){},
        onUploadPrep: function(id){}, // Called if non-trivial operations will be performed before onUpload
        onUpload: function(id, fileName){},
        onUploadChunk: function(id, fileName, chunkData){},
        onUploadChunkSuccess: function(id, chunkData, response, xhr){},
        onAutoRetry: function(id, fileName, response, xhr){},
        onResume: function(id, fileName, chunkData){},
        onUuidChanged: function(id, newUuid){},
        getName: function(id) {},
        setSize: function(id, newSize) {},
        isQueued: function(id) {},
        getIdsInProxyGroup: function(id) {},
        getIdsInBatch: function(id) {}
    },

    chunked = {
        done: function(id, chunkIdx, response, xhr) {
            var chunkData = handler._getChunkData(id, chunkIdx),
                estRequestOverhead = handler._getFileState(id).lastRequestOverhead || 0;

            handler._getFileState(id).attemptingResume = false;
            handler._getFileState(id).loaded += chunkData.size + estRequestOverhead;

            options.onUploadChunkSuccess(id, handler._getChunkDataForCallback(chunkData), response, xhr);
        },

        nextPart: function(id) {
            var nextIdx = handler._getFileState(id).chunking.lastSent;

            if (nextIdx >= 0) {
                nextIdx = nextIdx + 1;
            }
            else {
                nextIdx = 0;
            }

            if (nextIdx >= handler._getTotalChunks(id)) {
                nextIdx = null;
            }

            return nextIdx;
        },

        reset: function(id) {
            log("Server or callback has ordered chunking effort to be restarted on next attempt for item ID " + id, "error");

            handler._maybeDeletePersistedChunkData(id);
            handler.reevaluateChunking(id);
            handler._getFileState(id).loaded = 0;
        },

        send: function(id) {
            var size = options.getSize(id),
                name = options.getName(id),
                chunkIdx = chunked.nextPart(id),
                chunkData = handler._getChunkData(id, chunkIdx),
                resuming = handler._getFileState(id).attemptingResume;

            if (handler._getFileState(id).loaded === undefined) {
                handler._getFileState(id).loaded = 0;
            }

            // Don't follow-through with the resume attempt if the integrator returns false from onResume
            if (resuming && options.onResume(id, name, chunkData) === false) {
                chunked.reset(id);
                resuming = false;
                chunkIdx = chunked.nextPart(id);
                chunkData = handler._getChunkData(id, chunkIdx);
            }

            options.onUploadChunk(id, name, handler._getChunkDataForCallback(chunkData));

            if (chunkData.part > 0) {
                handler._maybePersistChunkedState(id);
            }

            handler.uploadChunk(id, chunkIdx, resuming).then(
                function(response, xhr) {
                    // Make sure the success property is true, since other higher-level code still
                    // depends on this value to determine status.
                    response.success = true;

                    chunked.done(id, chunkIdx, response, xhr);
                    handler._getFileState(id).chunking.lastSent = chunkIdx;

                    var nextChunkIdx = chunked.nextPart(id);

                    if (nextChunkIdx === null) {
                        options.onProgress(id, name, size, size);
                        handler._maybeDeletePersistedChunkData(id);
                        upload.maybeNewUuid(id, response);
                        upload.cleanup(id, response, xhr);
                    }
                    else {
                        chunked.send(id);
                    }
                },

                function(response, xhr) {
                    // Make sure the success property is not true, since other higher-level code still
                    // depends on this value to determine status.
                    response.succcess = false;

                    if (response.reset) {
                        chunked.reset(id);
                    }

                    if (!options.onAutoRetry(id, name, response, xhr)) {
                        upload.cleanup(id, response, xhr);
                    }
                }
            );
        }
    },

    simple = {
        send: function(id, name) {
            handler._getFileState(id).loaded = 0;

            handler.uploadFile(id).then(
                function(response, opt_xhr) {
                    // Make sure the success property is true, since other higher-level code still
                    // depends on this value to determine status.
                    response.success = true;

                    var size = options.getSize(id);

                    options.onProgress(id, name, size, size);
                    upload.maybeNewUuid(id, response);
                    upload.cleanup(id, response, opt_xhr);
                },

                function(response, opt_xhr) {
                    // Make sure the success property is not true, since other higher-level code still
                    // depends on this value to determine status.
                    response.success = false;

                    if (!options.onAutoRetry(id, name, response, opt_xhr)) {
                        upload.cleanup(id, response, opt_xhr);
                    }
                }
            );
        }
    },

    upload = {
        cancel: function(id) {
            log("Cancelling " + id);
            options.paramsStore.remove(id);
            upload.dequeue(id);
        },

        cleanup: function(id, response, opt_xhr) {
            var name = options.getName(id);

            options.onComplete(id, name, response, opt_xhr);

            if (handler._getFileState(id)) {
                delete handler._getFileState(id).xhr;
            }

            upload.dequeue(id);
        },

        /**
         * Removes element from queue, starts upload of next
         */
        dequeue: function(id) {
            var i = qq.indexOf(queue, id),
                max = options.maxConnections,
                nextId;

            if (upload.getProxyOrBlob(id) instanceof qq.BlobProxy) {
                log("Generated blob upload has ended for " + id + ", disposing generated blob.");
                delete handler._getFileState(id).file;
            }

            if (i >= 0) {
                queue.splice(i, 1);

                if (queue.length >= max && i < max) {
                    nextId = queue[max-1];
                    upload.start(nextId);
                }
            }
        },

        // Returns a qq.BlobProxy, or an actual File/Blob if no proxy is involved, or undefined
        // if none of these are available for the ID
        getProxyOrBlob: function(id) {
            return (handler.getProxy && handler.getProxy(id)) ||
                (handler.getFile && handler.getFile(id));
        },

        initHandler: function() {
            var handlerType = namespace ? qq[namespace] : qq.traditional,
                handlerModuleSubtype = qq.supportedFeatures.ajaxUploading ? "Xhr" : "Form";

            handler = new handlerType[handlerModuleSubtype + "UploadHandler"](
                options,
                {
                    getDataByUuid: options.getDataByUuid,
                    getName: options.getName,
                    getSize: options.getSize,
                    getUuid: options.getUuid,
                    log: log,
                    onCancel: options.onCancel,
                    onProgress: options.onProgress,
                    onUuidChanged: options.onUuidChanged
                }
            );

            if (handler._removeExpiredChunkingRecords) {
                handler._removeExpiredChunkingRecords();
            }
        },

        isDeferredEligibleForUpload: function(id) {
            return options.isQueued(id);
        },

        // For Blobs that are part of a group of generated images, along with a reference image,
        // this will ensure the blobs in the group are uploaded in the order they were triggered,
        // even if some async processing must be completed on one or more Blobs first.
        maybeDefer: function(id, blob) {
            // If we don't have a file/blob yet & no file/blob exists for this item, request it,
            // and then submit the upload to the specific handler once the blob is available.
            // ASSUMPTION: This condition will only ever be true if XHR uploading is supported.
            if (blob && !handler.getFile(id) && blob instanceof qq.BlobProxy) {

                // Blob creation may take some time, so the caller may want to update the
                // UI to indicate that an operation is in progress, even before the actual
                // upload begins and an onUpload callback is invoked.
                options.onUploadPrep(id);

                log("Attempting to generate a blob on-demand for " + id);
                blob.create().then(function(generatedBlob) {
                    log("Generated an on-demand blob for " + id);

                    // Update record associated with this file by providing the generated Blob
                    handler.updateBlob(id, generatedBlob);

                    // Propagate the size for this generated Blob
                    options.setSize(id, generatedBlob.size);

                    // Order handler to recalculate chunking possibility, if applicable
                    handler.reevaluateChunking(id);

                    upload.maybeSendDeferredFiles(id);
                },

                // Blob could not be generated.  Fail the upload & attempt to prevent retries.  Also bubble error message.
                function(errorMessage) {
                    var errorResponse = {};

                    if (errorMessage) {
                        errorResponse.error = errorMessage;
                    }

                    log(qq.format("Failed to generate blob for ID {}.  Error message: {}.", id, errorMessage), "error");

                    options.onComplete(id, options.getName(id), qq.extend(errorResponse, preventRetryResponse), null);
                    upload.maybeSendDeferredFiles(id);
                    upload.dequeue(id);
                });
            }
            else {
                return upload.maybeSendDeferredFiles(id);
            }

            return false;
        },

        // Upload any grouped blobs, in the proper order, that are ready to be uploaded
        maybeSendDeferredFiles: function(id) {
            var idsInGroup = options.getIdsInProxyGroup(id),
                uploadedThisId = false;

            if (idsInGroup && idsInGroup.length) {
                log("Maybe ready to upload proxy group file " + id);

                qq.each(idsInGroup, function(idx, idInGroup) {
                    if (upload.isDeferredEligibleForUpload(idInGroup) && !!handler.getFile(idInGroup)) {
                        uploadedThisId = idInGroup === id;
                        upload.now(idInGroup);
                    }
                    else if (upload.isDeferredEligibleForUpload(idInGroup)) {
                        return false;
                    }
                });
            }
            else {
                uploadedThisId = true;
                upload.now(id);
            }

            return uploadedThisId;
        },

        maybeNewUuid: function (id, response) {
            if (response.newUuid !== undefined) {
                options.onUuidChanged(id, response.newUuid);
            }
        },

        now: function(id) {
            var name = options.getName(id);

            if (!controller.isValid(id)) {
                throw new qq.Error(id + " is not a valid file ID to upload!");
            }

            options.onUpload(id, name);

            if (chunkingPossible && handler._shouldChunkThisFile(id)) {
                chunked.send(id);
            }
            else {
                simple.send(id, name);
            }
        },

        start: function(id) {
            var blobToUpload = upload.getProxyOrBlob(id);

            if (blobToUpload) {
                return upload.maybeDefer(id, blobToUpload);
            }
            else {
                upload.now(id);
                return true;
            }
        }
    };

    qq.extend(this, {
        /**
         * Adds file or file input to the queue
         * @returns id
         **/
        add: function(id, file) {
            return handler.add.apply(this, arguments);
        },

        /**
         * Sends the file identified by id
         */
        upload: function(id) {
            var len = queue.push(id);

            // if too many active uploads, wait...
            if (len <= options.maxConnections) {
                return upload.start(id);
            }

            return false;
        },

        retry: function(id) {
            var i = qq.indexOf(queue, id);

            if (i >= 0) {
                return upload.start(id);
            }
            else {
                return controller.upload(id);
            }
        },

        /**
         * Cancels file upload by id
         */
        cancel: function(id) {
            var cancelRetVal = handler.cancel(id);

            if (cancelRetVal instanceof qq.Promise) {
                cancelRetVal.then(function() {
                    upload.cancel(id);
                });
            }
            else if (cancelRetVal !== false) {
                upload.cancel(id);
            }
        },

        /**
         * Cancels all queued or in-progress uploads
         */
        cancelAll: function() {
            var self = this,
                queueCopy = [];

            qq.extend(queueCopy, queue);
            qq.each(queueCopy, function(idx, fileId) {
                self.cancel(fileId);
            });

            queue = [];
        },

        // Returns a File, Blob, or the Blob/File for the reference/parent file if the targeted blob is a proxy.
        // Undefined if no file record is available.
        getFile: function(id) {
            if (handler.getProxy && handler.getProxy(id)) {
                return handler.getProxy(id).referenceBlob;
            }

            return handler.getFile && handler.getFile(id);
        },

        // Returns true if the Blob associated with the ID is related to a proxy s
        isProxied: function(id) {
            return !!(handler.getProxy && handler.getProxy(id));
        },

        getInput: function(id) {
            if (handler.getInput) {
                return handler.getInput(id);
            }
        },

        reset: function() {
            log("Resetting upload handler");
            controller.cancelAll();
            queue = [];
            handler.reset();
        },

        expunge: function(id) {
            if (controller.isValid(id)) {
                return handler.expunge(id);
            }
        },

        /**
         * Determine if the file exists.
         */
        isValid: function(id) {
            return handler.isValid(id);
        },

        getResumableFilesData: function() {
            if (handler.getResumableFilesData) {
                return handler.getResumableFilesData();
            }
            return [];
        },

        /**
         * This may or may not be implemented, depending on the handler.  For handlers where a third-party ID is
         * available (such as the "key" for Amazon S3), this will return that value.  Otherwise, the return value
         * will be undefined.
         *
         * @param id Internal file ID
         * @returns {*} Some identifier used by a 3rd-party service involved in the upload process
         */
        getThirdPartyFileId: function(id) {
            if (controller.isValid(id)) {
                return handler.getThirdPartyFileId(id);
            }
        },

        /**
         * Attempts to pause the associated upload if the specific handler supports this and the file is "valid".
         * @param id ID of the upload/file to pause
         * @returns {boolean} true if the upload was paused
         */
        pause: function(id) {
            if (controller.isResumable(id) && handler.pause && controller.isValid(id) && handler.pause(id)) {
                upload.dequeue(id);
                return true;
            }
        },

        // True if the file is eligible for pause/resume.
        isResumable: function(id) {
            return !!handler.isResumable && handler.isResumable(id);
        }
    });

    qq.extend(options, o);

    log = options.log;

    preventRetryResponse = (function() {
        var response = {};

        response[options.preventRetryParam] = true;

        return response;
    }());

    chunkingPossible = options.chunking.enabled && qq.supportedFeatures.chunking;

    upload.initHandler();
};
<commit_msg>fix($handler-cleanup): address regressions in Safari, Azure #1157<commit_after>/*globals qq*/
/**
 * Base upload handler module.  Delegates to more specific handlers.
 *
 * @param o Options.  Passed along to the specific handler submodule as well.
 * @param namespace [optional] Namespace for the specific handler.
 */
qq.UploadHandlerController = function(o, namespace) {
    "use strict";

    var controller = this,
        queue = [],
        chunkingPossible = false,
        chunking, preventRetryResponse, log, handler,

    // Default options, can be overridden by the user
    options = {
        paramsStore: {},
        maxConnections: 3, // maximum number of concurrent uploads
        chunking: {
            enabled: false,
            partSize: 2000000 //bytes
        },
        log: function(str, level) {},
        onProgress: function(id, fileName, loaded, total){},
        onComplete: function(id, fileName, response, xhr){},
        onCancel: function(id, fileName){},
        onUploadPrep: function(id){}, // Called if non-trivial operations will be performed before onUpload
        onUpload: function(id, fileName){},
        onUploadChunk: function(id, fileName, chunkData){},
        onUploadChunkSuccess: function(id, chunkData, response, xhr){},
        onAutoRetry: function(id, fileName, response, xhr){},
        onResume: function(id, fileName, chunkData){},
        onUuidChanged: function(id, newUuid){},
        getName: function(id) {},
        setSize: function(id, newSize) {},
        isQueued: function(id) {},
        getIdsInProxyGroup: function(id) {},
        getIdsInBatch: function(id) {}
    },

    chunked = {
        done: function(id, chunkIdx, response, xhr) {
            var chunkData = handler._getChunkData(id, chunkIdx),
                estRequestOverhead = handler._getFileState(id).lastRequestOverhead || 0;

            handler._getFileState(id).attemptingResume = false;
            handler._getFileState(id).loaded += chunkData.size + estRequestOverhead;

            options.onUploadChunkSuccess(id, handler._getChunkDataForCallback(chunkData), response, xhr);
        },

        nextPart: function(id) {
            var nextIdx = handler._getFileState(id).chunking.lastSent;

            if (nextIdx >= 0) {
                nextIdx = nextIdx + 1;
            }
            else {
                nextIdx = 0;
            }

            if (nextIdx >= handler._getTotalChunks(id)) {
                nextIdx = null;
            }

            return nextIdx;
        },

        reset: function(id) {
            log("Server or callback has ordered chunking effort to be restarted on next attempt for item ID " + id, "error");

            handler._maybeDeletePersistedChunkData(id);
            handler.reevaluateChunking(id);
            handler._getFileState(id).loaded = 0;
        },

        send: function(id) {
            var size = options.getSize(id),
                name = options.getName(id),
                chunkIdx = chunked.nextPart(id),
                chunkData = handler._getChunkData(id, chunkIdx),
                resuming = handler._getFileState(id).attemptingResume;

            if (handler._getFileState(id).loaded === undefined) {
                handler._getFileState(id).loaded = 0;
            }

            // Don't follow-through with the resume attempt if the integrator returns false from onResume
            if (resuming && options.onResume(id, name, chunkData) === false) {
                chunked.reset(id);
                resuming = false;
                chunkIdx = chunked.nextPart(id);
                chunkData = handler._getChunkData(id, chunkIdx);
            }

            options.onUploadChunk(id, name, handler._getChunkDataForCallback(chunkData));

            if (chunkData.part > 0) {
                handler._maybePersistChunkedState(id);
            }

            handler.uploadChunk(id, chunkIdx, resuming).then(
                function(response, xhr) {
                    var responseToReport = upload.normalizeResponse(response, true);

                    chunked.done(id, chunkIdx, responseToReport, xhr);
                    handler._getFileState(id).chunking.lastSent = chunkIdx;

                    var nextChunkIdx = chunked.nextPart(id);

                    if (nextChunkIdx === null) {
                        options.onProgress(id, name, size, size);
                        handler._maybeDeletePersistedChunkData(id);
                        upload.maybeNewUuid(id, responseToReport);
                        upload.cleanup(id, responseToReport, xhr);
                    }
                    else {
                        chunked.send(id);
                    }
                },

                function(response, xhr) {
                    var responseToReport = upload.normalizeResponse(response, false);

                    if (responseToReport.reset) {
                        chunked.reset(id);
                    }

                    if (!options.onAutoRetry(id, name, responseToReport, xhr)) {
                        upload.cleanup(id, responseToReport, xhr);
                    }
                }
            );
        }
    },

    simple = {
        send: function(id, name) {
            handler._getFileState(id).loaded = 0;

            handler.uploadFile(id).then(
                function(response, opt_xhr) {
                    var responseToReport = upload.normalizeResponse(response, true);

                    var size = options.getSize(id);

                    options.onProgress(id, name, size, size);
                    upload.maybeNewUuid(id, responseToReport);
                    upload.cleanup(id, responseToReport, opt_xhr);
                },

                function(response, opt_xhr) {
                    var responseToReport = upload.normalizeResponse(response, false);

                    if (!options.onAutoRetry(id, name, responseToReport, opt_xhr)) {
                        upload.cleanup(id, responseToReport, opt_xhr);
                    }
                }
            );
        }
    },

    upload = {
        cancel: function(id) {
            log("Cancelling " + id);
            options.paramsStore.remove(id);
            upload.dequeue(id);
        },

        cleanup: function(id, response, opt_xhr) {
            var name = options.getName(id);

            options.onComplete(id, name, response, opt_xhr);

            if (handler._getFileState(id)) {
                delete handler._getFileState(id).xhr;
            }

            upload.dequeue(id);
        },

        /**
         * Removes element from queue, starts upload of next
         */
        dequeue: function(id) {
            var i = qq.indexOf(queue, id),
                max = options.maxConnections,
                nextId;

            if (upload.getProxyOrBlob(id) instanceof qq.BlobProxy) {
                log("Generated blob upload has ended for " + id + ", disposing generated blob.");
                delete handler._getFileState(id).file;
            }

            if (i >= 0) {
                queue.splice(i, 1);

                if (queue.length >= max && i < max) {
                    nextId = queue[max-1];
                    upload.start(nextId);
                }
            }
        },

        // Returns a qq.BlobProxy, or an actual File/Blob if no proxy is involved, or undefined
        // if none of these are available for the ID
        getProxyOrBlob: function(id) {
            return (handler.getProxy && handler.getProxy(id)) ||
                (handler.getFile && handler.getFile(id));
        },

        initHandler: function() {
            var handlerType = namespace ? qq[namespace] : qq.traditional,
                handlerModuleSubtype = qq.supportedFeatures.ajaxUploading ? "Xhr" : "Form";

            handler = new handlerType[handlerModuleSubtype + "UploadHandler"](
                options,
                {
                    getDataByUuid: options.getDataByUuid,
                    getName: options.getName,
                    getSize: options.getSize,
                    getUuid: options.getUuid,
                    log: log,
                    onCancel: options.onCancel,
                    onProgress: options.onProgress,
                    onUuidChanged: options.onUuidChanged
                }
            );

            if (handler._removeExpiredChunkingRecords) {
                handler._removeExpiredChunkingRecords();
            }
        },

        isDeferredEligibleForUpload: function(id) {
            return options.isQueued(id);
        },

        // For Blobs that are part of a group of generated images, along with a reference image,
        // this will ensure the blobs in the group are uploaded in the order they were triggered,
        // even if some async processing must be completed on one or more Blobs first.
        maybeDefer: function(id, blob) {
            // If we don't have a file/blob yet & no file/blob exists for this item, request it,
            // and then submit the upload to the specific handler once the blob is available.
            // ASSUMPTION: This condition will only ever be true if XHR uploading is supported.
            if (blob && !handler.getFile(id) && blob instanceof qq.BlobProxy) {

                // Blob creation may take some time, so the caller may want to update the
                // UI to indicate that an operation is in progress, even before the actual
                // upload begins and an onUpload callback is invoked.
                options.onUploadPrep(id);

                log("Attempting to generate a blob on-demand for " + id);
                blob.create().then(function(generatedBlob) {
                    log("Generated an on-demand blob for " + id);

                    // Update record associated with this file by providing the generated Blob
                    handler.updateBlob(id, generatedBlob);

                    // Propagate the size for this generated Blob
                    options.setSize(id, generatedBlob.size);

                    // Order handler to recalculate chunking possibility, if applicable
                    handler.reevaluateChunking(id);

                    upload.maybeSendDeferredFiles(id);
                },

                // Blob could not be generated.  Fail the upload & attempt to prevent retries.  Also bubble error message.
                function(errorMessage) {
                    var errorResponse = {};

                    if (errorMessage) {
                        errorResponse.error = errorMessage;
                    }

                    log(qq.format("Failed to generate blob for ID {}.  Error message: {}.", id, errorMessage), "error");

                    options.onComplete(id, options.getName(id), qq.extend(errorResponse, preventRetryResponse), null);
                    upload.maybeSendDeferredFiles(id);
                    upload.dequeue(id);
                });
            }
            else {
                return upload.maybeSendDeferredFiles(id);
            }

            return false;
        },

        // Upload any grouped blobs, in the proper order, that are ready to be uploaded
        maybeSendDeferredFiles: function(id) {
            var idsInGroup = options.getIdsInProxyGroup(id),
                uploadedThisId = false;

            if (idsInGroup && idsInGroup.length) {
                log("Maybe ready to upload proxy group file " + id);

                qq.each(idsInGroup, function(idx, idInGroup) {
                    if (upload.isDeferredEligibleForUpload(idInGroup) && !!handler.getFile(idInGroup)) {
                        uploadedThisId = idInGroup === id;
                        upload.now(idInGroup);
                    }
                    else if (upload.isDeferredEligibleForUpload(idInGroup)) {
                        return false;
                    }
                });
            }
            else {
                uploadedThisId = true;
                upload.now(id);
            }

            return uploadedThisId;
        },

        maybeNewUuid: function (id, response) {
            if (response.newUuid !== undefined) {
                options.onUuidChanged(id, response.newUuid);
            }
        },

        // The response coming from handler implementations may be in various formats.
        // Instead of hoping a promise nested 5 levels deep will always return an object
        // as its first param, let's just normalize the response here.
        normalizeResponse: function(originalResponse, successful) {
            var response = originalResponse;

            // The passed "response" param may not be a response at all.
            // It could be a string, detailing the error, for example.
            if (!qq.isObject(originalResponse)) {
                response = {};

                if (qq.isString(originalResponse) && !successful) {
                    response.error = originalResponse;
                }
            }

            response.success = successful;

            return response;
        },

        now: function(id) {
            var name = options.getName(id);

            if (!controller.isValid(id)) {
                throw new qq.Error(id + " is not a valid file ID to upload!");
            }

            options.onUpload(id, name);

            if (chunkingPossible && handler._shouldChunkThisFile(id)) {
                chunked.send(id);
            }
            else {
                simple.send(id, name);
            }
        },

        start: function(id) {
            var blobToUpload = upload.getProxyOrBlob(id);

            if (blobToUpload) {
                return upload.maybeDefer(id, blobToUpload);
            }
            else {
                upload.now(id);
                return true;
            }
        }
    };

    qq.extend(this, {
        /**
         * Adds file or file input to the queue
         * @returns id
         **/
        add: function(id, file) {
            return handler.add.apply(this, arguments);
        },

        /**
         * Sends the file identified by id
         */
        upload: function(id) {
            var len = queue.push(id);

            // if too many active uploads, wait...
            if (len <= options.maxConnections) {
                return upload.start(id);
            }

            return false;
        },

        retry: function(id) {
            var i = qq.indexOf(queue, id);

            if (i >= 0) {
                return upload.start(id);
            }
            else {
                return controller.upload(id);
            }
        },

        /**
         * Cancels file upload by id
         */
        cancel: function(id) {
            var cancelRetVal = handler.cancel(id);

            if (cancelRetVal instanceof qq.Promise) {
                cancelRetVal.then(function() {
                    upload.cancel(id);
                });
            }
            else if (cancelRetVal !== false) {
                upload.cancel(id);
            }
        },

        /**
         * Cancels all queued or in-progress uploads
         */
        cancelAll: function() {
            var self = this,
                queueCopy = [];

            qq.extend(queueCopy, queue);
            qq.each(queueCopy, function(idx, fileId) {
                self.cancel(fileId);
            });

            queue = [];
        },

        // Returns a File, Blob, or the Blob/File for the reference/parent file if the targeted blob is a proxy.
        // Undefined if no file record is available.
        getFile: function(id) {
            if (handler.getProxy && handler.getProxy(id)) {
                return handler.getProxy(id).referenceBlob;
            }

            return handler.getFile && handler.getFile(id);
        },

        // Returns true if the Blob associated with the ID is related to a proxy s
        isProxied: function(id) {
            return !!(handler.getProxy && handler.getProxy(id));
        },

        getInput: function(id) {
            if (handler.getInput) {
                return handler.getInput(id);
            }
        },

        reset: function() {
            log("Resetting upload handler");
            controller.cancelAll();
            queue = [];
            handler.reset();
        },

        expunge: function(id) {
            if (controller.isValid(id)) {
                return handler.expunge(id);
            }
        },

        /**
         * Determine if the file exists.
         */
        isValid: function(id) {
            return handler.isValid(id);
        },

        getResumableFilesData: function() {
            if (handler.getResumableFilesData) {
                return handler.getResumableFilesData();
            }
            return [];
        },

        /**
         * This may or may not be implemented, depending on the handler.  For handlers where a third-party ID is
         * available (such as the "key" for Amazon S3), this will return that value.  Otherwise, the return value
         * will be undefined.
         *
         * @param id Internal file ID
         * @returns {*} Some identifier used by a 3rd-party service involved in the upload process
         */
        getThirdPartyFileId: function(id) {
            if (controller.isValid(id)) {
                return handler.getThirdPartyFileId(id);
            }
        },

        /**
         * Attempts to pause the associated upload if the specific handler supports this and the file is "valid".
         * @param id ID of the upload/file to pause
         * @returns {boolean} true if the upload was paused
         */
        pause: function(id) {
            if (controller.isResumable(id) && handler.pause && controller.isValid(id) && handler.pause(id)) {
                upload.dequeue(id);
                return true;
            }
        },

        // True if the file is eligible for pause/resume.
        isResumable: function(id) {
            return !!handler.isResumable && handler.isResumable(id);
        }
    });

    qq.extend(options, o);

    log = options.log;

    preventRetryResponse = (function() {
        var response = {};

        response[options.preventRetryParam] = true;

        return response;
    }());

    chunkingPossible = options.chunking.enabled && qq.supportedFeatures.chunking;

    upload.initHandler();
};
<|endoftext|>


<commit_before>/**
  * The migrator interface provides the ability to handle all abilities which
  * are needed to successfully modify the DD of your tables.
  * This includes all DDL methods provided by SQL naturally.
  *
  * Considering the zero downtime deployment we are handling migrations like
  * the following:
  *
  * Zero Downtime notes
  *
  *  *) The very first step is always a full dump, for the sake of security
  *
  * - all operations that are deleting a currently existing column, e.g. renaming a column:
  *      - 1. duplicate the column
  *   - if renamed:
  *      -  1. create a trigger to update both columns
  *      -  2. disable/delete trigger after successful deployment
  *   - if deleted:
  *      - 1. apply steps from the renaming process
  *      - 2. create a backup of this column (+PK)/table and delete it afterwards (after the successful deployment)
  * - all operations that modify content
  *   - 1. Retrieve data to be applied
  *   - 2. Extract PKs of to be updated rows
  *   - 3. Create backup of to be updated rows
  *   - 4. Apply changes
  */

function dummy () {
  arguments[arguments.length - 1]('not implemented');
}

function MigratorInterface () {

}

MigratorInterface.prototype = {

  renameCollection: dummy,

  dropCollection: dummy,

  createCollection: dummy,

  checkDBMS: dummy,

  createDatabase: dummy,

  switchDatabase: dummy,

  dropDatabase: dummy,

  bindForeignKey: dummy,

  createTable: dummy,

  dropTable: dummy,

  renameTable: dummy,

  addColumn: dummy,

  removeColumn: dummy,

  renameColumn: dummy,

  changeColumn: dummy,

  addIndex: dummy,

  removeIndex: dummy,

  addForeignKey: dummy,

  removeForeignKey: dummy,

  createMigrationsTable: dummy,

  createSeedsTable: dummy,

  normalizeColumnSpec: dummy,

  runSql: dummy,

  createColumnDef: dummy,

  mapDataType: dummy,

  createColumnConstraint: dummy,

  recurseCallbackArray: dummy,

  allLoadedMigrations: dummy,

  allLoadedSeeds: dummy,

  addSeedRecord: dummy,

  startMigration: dummy,

  endMigration: dummy,

  addMigrationRecord: dummy,

  deleteMigration: dummy,

  deleteSeed: dummy,

  quoteDDLArr: dummy,

  quoteArr: dummy,

  escapeString: dummy,

  escape: dummy,

  escapeDDL: dummy,

  all: dummy,

  close: dummy

};

module.exports = MigratorInterface;
module.exports.deprecated = {

  insert: function () {
    arguments[arguments.length - 1]('not implemented');
  },

  insert_deprecation: 'is going to be completely removed, use version controlled seeds instead.'
};

module.exports.extending = {
};

module.exports.exclude = {

};
<commit_msg>refactor(deprecationNotice): remove insert deprecation<commit_after>/**
 * The migrator interface provides the ability to handle all abilities which
 * are needed to successfully modify the DD of your tables.
 * This includes all DDL methods provided by SQL naturally.
 *
 * Considering the zero downtime deployment we are handling migrations like
 * the following:
 *
 * Zero Downtime notes
 *
 *  *) The very first step is always a full dump, for the sake of security
 *
 * - all operations that are deleting a currently existing column, e.g. renaming a column:
 *      - 1. duplicate the column
 *   - if renamed:
 *      -  1. create a trigger to update both columns
 *      -  2. disable/delete trigger after successful deployment
 *   - if deleted:
 *      - 1. apply steps from the renaming process
 *      - 2. create a backup of this column (+PK)/table and delete it afterwards (after the successful deployment)
 * - all operations that modify content
 *   - 1. Retrieve data to be applied
 *   - 2. Extract PKs of to be updated rows
 *   - 3. Create backup of to be updated rows
 *   - 4. Apply changes
 */

function dummy () {
  arguments[arguments.length - 1]('not implemented');
}

function MigratorInterface () {}

MigratorInterface.prototype = {
  renameCollection: dummy,

  dropCollection: dummy,

  createCollection: dummy,

  checkDBMS: dummy,

  createDatabase: dummy,

  switchDatabase: dummy,

  dropDatabase: dummy,

  bindForeignKey: dummy,

  createTable: dummy,

  dropTable: dummy,

  renameTable: dummy,

  addColumn: dummy,

  removeColumn: dummy,

  renameColumn: dummy,

  changeColumn: dummy,

  addIndex: dummy,

  removeIndex: dummy,

  addForeignKey: dummy,

  removeForeignKey: dummy,

  createMigrationsTable: dummy,

  createSeedsTable: dummy,

  normalizeColumnSpec: dummy,

  runSql: dummy,

  createColumnDef: dummy,

  mapDataType: dummy,

  createColumnConstraint: dummy,

  recurseCallbackArray: dummy,

  allLoadedMigrations: dummy,

  allLoadedSeeds: dummy,

  addSeedRecord: dummy,

  startMigration: dummy,

  endMigration: dummy,

  addMigrationRecord: dummy,

  deleteMigration: dummy,

  deleteSeed: dummy,

  quoteDDLArr: dummy,

  quoteArr: dummy,

  escapeString: dummy,

  escape: dummy,

  escapeDDL: dummy,

  all: dummy,

  close: dummy
};

module.exports = MigratorInterface;
module.exports.deprecated = {};

module.exports.extending = {};

module.exports.exclude = {};
<|endoftext|>


<commit_before>let utils = require('../../utils/utils.js');
let fs = require('../../core/fs.js');

class DeployTracker {

  constructor(embark, options) {
    this.logger = embark.logger;
    this.events = embark.events;

    // TODO: unclear where it comes from
    this.env = options.env;
    //this.chainConfig = options.chainConfig;
    this.chainConfig = embark.config.chainTracker;
    this.registerEvents();
  }

  registerEvents() {
    const self = this;

    this.events.on("deploy:beforeAll", this.setCurrentChain.bind(this));

    this.events.on("deploy:contract:deployed", (contract) => {
      self.trackContract(contract.className, contract.realRuntimeBytecode, contract.realArgs, contract.address);
      self.save();
    });

    this.events.setCommandHandler("deploy:contract:shouldDeploy", (contract, cb) => {
      let trackedContract = self.getContract(contract.className, contract.realRuntimeBytecode, contract.realArgs);
      cb(trackedContract);
    });
  }

  // TODO: just an event might not be enought to the async nature
  // it needs to be a plugin api before deploy, that makes the deployment wait
  setCurrentChain() {
    const self = this;
    if (this.chainConfig === false) {
      this.currentChain = {contracts: []};
      //return cb();
    }

    this.events.request("blockchain:block:byNumber", 0, function(_err, block) {
      let chainId = block.hash;

      if (self.chainConfig[chainId] === undefined) {
        self.chainConfig[chainId] = {contracts: {}};
      }

      self.currentChain = self.chainConfig[chainId];

      self.currentChain.name = self.env;
      //cb();
    });
  }

  loadConfig(config) {
    this.chainConfig = config;
    return this;
  }

  trackContract(contractName, code, args, address) {
    if (!this.currentChain) return false;
    this.currentChain.contracts[utils.sha3(code + contractName + args.join(','))] = {
      name: contractName,
      address: address
    };
  }

  getContract(contractName, code, args) {
    if (!this.currentChain) return false;
    let contract = this.currentChain.contracts[utils.sha3(code + contractName + args.join(','))];
    if (contract && contract.address === undefined) {
      return false;
    }
    return contract;
  }

  // TODO: abstract this
  // chainConfig can be an abstract PersistentObject
  save() {
    if (this.chainConfig === false) {
      return;
    }
    fs.writeJSONSync("./chains.json", this.chainConfig, {spaces: 2});
  }

}

module.exports = DeployTracker;
<commit_msg>use contractAddress instead of address<commit_after>let utils = require('../../utils/utils.js');
let fs = require('../../core/fs.js');

class DeployTracker {

  constructor(embark, options) {
    this.logger = embark.logger;
    this.events = embark.events;

    // TODO: unclear where it comes from
    this.env = options.env;
    //this.chainConfig = options.chainConfig;
    this.chainConfig = embark.config.chainTracker;
    this.registerEvents();
  }

  registerEvents() {
    const self = this;

    this.events.on("deploy:beforeAll", this.setCurrentChain.bind(this));

    this.events.on("deploy:contract:deployed", (contract) => {
      self.trackContract(contract.className, contract.realRuntimeBytecode, contract.realArgs, contract.deployedAddress);
      self.save();
    });

    this.events.setCommandHandler("deploy:contract:shouldDeploy", (contract, cb) => {
      let trackedContract = self.getContract(contract.className, contract.realRuntimeBytecode, contract.realArgs);
      cb(trackedContract);
    });
  }

  // TODO: just an event might not be enought to the async nature
  // it needs to be a plugin api before deploy, that makes the deployment wait
  setCurrentChain() {
    const self = this;
    if (this.chainConfig === false) {
      this.currentChain = {contracts: []};
      //return cb();
    }

    this.events.request("blockchain:block:byNumber", 0, function(_err, block) {
      let chainId = block.hash;

      if (self.chainConfig[chainId] === undefined) {
        self.chainConfig[chainId] = {contracts: {}};
      }

      self.currentChain = self.chainConfig[chainId];

      self.currentChain.name = self.env;
      //cb();
    });
  }

  loadConfig(config) {
    this.chainConfig = config;
    return this;
  }

  trackContract(contractName, code, args, address) {
    if (!this.currentChain) return false;
    this.currentChain.contracts[utils.sha3(code + contractName + args.join(','))] = {
      name: contractName,
      address: address
    };
  }

  getContract(contractName, code, args) {
    if (!this.currentChain) return false;
    let contract = this.currentChain.contracts[utils.sha3(code + contractName + args.join(','))];
    if (contract && contract.address === undefined) {
      return false;
    }
    return contract;
  }

  // TODO: abstract this
  // chainConfig can be an abstract PersistentObject
  save() {
    if (this.chainConfig === false) {
      return;
    }
    fs.writeJSONSync("./chains.json", this.chainConfig, {spaces: 2});
  }

}

module.exports = DeployTracker;
<|endoftext|>


<commit_before>var fs = require('fs');
var mkdirp = require('mkdirp');
var path = require('path');
var sanatizeType = require('../util/sanatize-patterntype');

/**
 * Generates a search file from the current tree of processed pages.
 * @param {string} outFile - Path to write to.
 * @param {function} cb - Callback to run when the search file is written to disk.
 * @todo Make hashes for search result types configurable
 */
module.exports = function(outFile, cb) {
  var tree = this.data.patterns;
  var results = [];

  results = results.concat(this.searchOptions.extra);

  // Each item in the tree is a page
  for (var i in tree) {
    var item = tree[i];

    // for now... only add to search if we have a linkable pattern file at hand
    if ( (typeof item.fileName != 'undefined') && (item.fileName != '') ) {

        var link = sanatizeType(path.join(
        		this.options.basepath,
        		this.options.patternspath,
        		item.pattern.name,
        		this.options.pattern.target
        ), true);
        var type = 'page';

        // By default pages are classified as a "page"
        // If it has code associated with it, then it's a "component" instead.
        if (keysInObject(item, Object.keys(this.adapters))) {
          type = 'component';
        }

        // Check for special page types
        for (var t in this.searchOptions.pageTypes) {
          var func = this.searchOptions.pageTypes[t];
          if (func(item)) {
            type = t;
          }
        }

        // Add the page itself as a search result
        results.push({
          type: type,
          name: item.title,
          description: item.description,
          link: link,
          tags: item.tags || []
        });

        // Run search builders for each adapter
        for (var a in this.adapters) {
          if (this.adapters[a].search && item[a]) {
            results = results.concat(this.adapters[a].search(item[a], link));
          }
        }

    }
  }

  // Re-order search results based on search config
  results = results.sort(function(a, b) {
    return this.searchOptions.sort.indexOf(a.type) - this.searchOptions.sort.indexOf(b.type);
  }.bind(this));

  // Write the finished results to disk
  mkdirp(path.dirname(outFile), function(err) {
    if (err) throw err;
    fs.writeFile(outFile, JSON.stringify(results, null, '  '), cb);
  });
}

/**
 * Determines if any key in an array exists on an object.
 * @param {object} obj - Object to check for keys.
 * @param {array} keys - Keys to check.
 * @returns {boolean} `true` if any key is found on the object, or `false` if not.
 */
function keysInObject(obj, keys) {
  for (var i in keys) {
    if (keys[i] in obj) return true;
  }
  return false;
}
<commit_msg>PATLAB-2 removed useless self-assignment.<commit_after>var fs = require('fs');
var mkdirp = require('mkdirp');
var path = require('path');
var sanatizeType = require('../util/sanatize-patterntype');

/**
 * Generates a search file from the current tree of processed pages.
 * @param {string} outFile - Path to write to.
 * @param {function} cb - Callback to run when the search file is written to disk.
 * @todo Make hashes for search result types configurable
 */
module.exports = function(outFile, cb) {
  var tree = this.data.patterns;
  var results = [];

  results = results.concat(this.searchOptions.extra);

  // Each item in the tree is a page
  for (var i in tree) {
    var item = tree[i];

    // for now... only add to search if we have a linkable pattern file at hand
    if ( (typeof item.fileName != 'undefined') && (item.fileName != '') ) {

        var link = sanatizeType(path.join(
        		this.options.basepath,
        		this.options.patternspath,
        		item.pattern.name,
        		this.options.pattern.target
        ), true);
        var type = 'page';

        // By default pages are classified as a "page"
        // If it has code associated with it, then it's a "component" instead.
        if (keysInObject(item, Object.keys(this.adapters))) {
          type = 'component';
        }

        // Check for special page types
        for (var t in this.searchOptions.pageTypes) {
          var func = this.searchOptions.pageTypes[t];
          if (func(item)) {
            type = t;
          }
        }

        // Add the page itself as a search result
        results.push({
          type: type,
          name: item.title,
          description: item.description,
          link: link,
          tags: item.tags || []
        });

        // Run search builders for each adapter
        for (var a in this.adapters) {
          if (this.adapters[a].search && item[a]) {
            results = results.concat(this.adapters[a].search(item[a], link));
          }
        }

    }
  }

  // Re-order search results based on search config
  results.sort(function(a, b) {
    return this.searchOptions.sort.indexOf(a.type) - this.searchOptions.sort.indexOf(b.type);
  }.bind(this));

  // Write the finished results to disk
  mkdirp(path.dirname(outFile), function(err) {
    if (err) throw err;
    fs.writeFile(outFile, JSON.stringify(results, null, '  '), cb);
  });
}

/**
 * Determines if any key in an array exists on an object.
 * @param {object} obj - Object to check for keys.
 * @param {array} keys - Keys to check.
 * @returns {boolean} `true` if any key is found on the object, or `false` if not.
 */
function keysInObject(obj, keys) {
  for (var i in keys) {
    if (keys[i] in obj) return true;
  }
  return false;
}
<|endoftext|>


<commit_before>var debug = require('debug')('pigsty-moloch-plugin');
var PigstyPlugin = require('pigsty-plugin');
var os = require('os');
var util = require('util');

function ntoa(ipl) {
    return ((ipl >>> 24) + '.' +
        (ipl >> 16 & 255) + '.' +
        (ipl >> 8 & 255) + '.' +
        (ipl & 255));
};

function twoDigitString(value) {
  return (value < 10) ? ("0" + value) : value.toString();
}

molochPlugin.prototype = new PigstyPlugin();
molochPlugin.prototype.constructor = PigstyPlugin;
function molochPlugin(options) {
  PigstyPlugin.call(this, options);
  this.options = options;
  this.nodeName = os.hostname().split(".")[0];
  if (!this.options.eventQueueMaxSize) {
    this.options.eventQueueMaxSize = 1024;
  }
  if (!this.options.maxHeapSize){
    this.options.maxHeapSize = 1073741824/10;
  }
  if (!this.options.tag_prefix) {
    this.options.tag_prefix = 'suricata';
  }

};

molochPlugin.prototype.getMolochOption = function (key, defaultValue) {
  var node = this.nodeName;
  var options = this.options.moloch
  if (options[node] && options[node][key] !== undefined) {
    return options[node][key];
  }
  if (options[node] && options[node].nodeClass && options[options[node].nodeClass] && options[options[node].nodeClass][key]) {
    return options[options[node].nodeClass][key];
  }
  if (options["default"][key]) {
    return options["default"][key];
  }
  if (options[key]) {
    return options[key];
  }
  return defaultValue;
};

molochPlugin.prototype.start = function(callback) {

  if (this.options.min_free_memory) {
    var freemem = os.freemem();
    if (this.options.min_free_memory > freemem) {
      this.emit('error','Over the limit of minimal free memory. Limit:'+this.options.min_free_memory+' Free:'+freemem);
      return;
    }
  }
  //todo check memory max limit...

  var iniParserLocation = 'iniparser'; //default to npm install iniparser -g
  if (this.options.iniparser_module_location){
    iniParserLocation = this.options.iniparser_module_location;
  }
  try {
    var iniParser = require(iniParserLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  this.options.moloch = iniParser.parseSync(this.options.moloch_config_file_location);
  if (this.options.moloch["default"] === undefined) {
    this.emit('error', 'default setcion is missing in ' + this.options.moloch_config_file_location);
    return;
  }

  var tmp_escInfo = this.getMolochOption("elasticsearch");
  if (tmp_escInfo) {
     var escInfo = tmp_escInfo.split(':');
  } else {
    this.emit('error','elasticsearch data missing');
    return;
  }

  var asyncLocation = 'async'; //default to npm install async -g
  if (this.options.async_module_location){
    asyncLocation = this.options.async_module_location;
  }
  try {
    this.async = require(asyncLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  var KAALocation = 'keep-alive-agent'; //default to npm install keep-alive-agent -g
  if (this.options.keepaliveagent_module_location){
    KAALocation = this.options.keepaliveagent_module_location;
  }
  try {
    this.KAA = require(KAALocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  if (this.options.db_module_location){
    var dbLocation = this.options.db_module_location;
  } else {
    this.emit('error', 'missing db module location in your config');
    return;
  }
  try {
    this.Db = require(dbLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  this.esHttpAgent = new this.KAA({maxSockets: 20}); // todo options here..
  this.Db.initialize({
        host: escInfo[0],
        port: escInfo[1],
        nodeName: this.nodeName,
        agent: this.esHttpAgent,
        dontMapTags: this.getMolochOption("multiES", false)
    });
  var _this = this;
  this.Db.status("sessions-*", function (err, status) {
    if (err || status.error) {
      if (status.error) {
        var e = status.error;
      } else {
        var e = err;
      }
      this.emit('error', e);
      return;
    } else {
      _this.dbStatustatus = status;
  debug(status._shards);
    }
  });

  this.options.tcpSaveTimeout = this.getMolochOption('tcpSaveTimeout',3);
  this.eventQueue = [];
  this.eventsNotFound = [];
  this.tagsQueue = {};

  this.eventCount = 0;
  this.eventCountIN = 0;
  this.eventCountFounded = 0;
  this.inPost = 0;
  if (this.options.print_stats){
    this.eventCountINLast = 0;
    this.eventCountLast = 0;
    var _this = this;
    this.timer = setInterval(function () {
      var now = new Date;
      var paused = '';
      if ( _this.paused ) { paused = ' state: paused'; }
      var lastMinuteCountIN = _this.eventCountIN - _this.eventCountINLast;
      var lastMinuteRateIN = lastMinuteCountIN / 10;
      var lastMinuteCount = _this.eventCount - _this.eventCountLast;
      var dropedEventCount = lastMinuteCountIN - lastMinuteCount;
      console.log('' + now + 'in post:'+_this.inPost + ' in queue:'+_this.eventQueue.length + ' founded:'+_this.eventCountFounded+ ' not found:'+_this.eventsNotFound.length+' in total: '+_this.eventCountIN+' last minute:'+lastMinuteCountIN+' rate:'+lastMinuteRateIN+'eps'+' droped:'+dropedEventCount+paused);
      _this.eventCountINLast = _this.eventCountIN;
      _this.eventCountLast = _this.eventCount;
    }, 10 * 1000);
  }
  this.emit('ready');
};

molochPlugin.prototype.close = function(callback) {
  //flusth queue ...
  for (var i = 0; i < (this.eventQueue.length - 1); i++) {
    this.post(this.eventQueue.pop());
    debug('flushing: ' + this.eventQueue.length);
  }
  this.eventQueue = [];
  this.tagsQueue = {};
  this.emit('end');
};

molochPlugin.prototype.getUnixTime = function () {
    return Math.floor(new Date().getTime() / 1000);
};

molochPlugin.prototype.send = function(event) {
  var _event = {};
  this.eventCountIN += 1;
  if (event && event.event && event.event.event_second) {
    _event.time = event.event.event_second;
    _event.srcIP = event.event.source_ip;
    _event.srcPort = event.event.source_port;
    _event.dstIP = event.event.destination_ip;
    _event.dstPort = event.event.dest_port;
    _event.tags = [];
    //todo options for tags...
    _event.tags.push(this.options.tag_prefix + '_severity_' + event.event.classification.severity);
    _event.tags.push(this.options.tag_prefix + '_classification_' + event.event.classification.name);
    _event.tags.push(this.options.tag_prefix + '_signatureID_' + event.event.signature.id);
    _event.tags.push(this.options.tag_prefix + '_signature_' + event.event.signature.name);

    this.eventCount += 1;

    if (this.paused){
      this.eventQueue.push(_event);
      debug('emited full, but still getting events...');
    } else {
      var needPause = false;

      if (this.eventQueue.length > this.options.eventQueueMaxSize) {
        needPause = true;
        debug('queue full');
      }

      var freemem = os.freemem();
      if (this.options.min_free_memory > freemem) {
          needPause = true;
          debug('memory almost full' + freemem);
      }

      var memusage = process.memoryUsage();
      if (memusage.heapUsed > (this.options.maxHeapSize)) {
        needPause = true;
        debug('heap full ' + memusage.heapUsed);
      }

      var diff = 1;
      var now = this.getUnixTime();
      if ( (now - this.options.tcpSaveTimeout) < _event.endTime ) {
        diff = now - this.options.tcpSaveTimeout - _event.endTime ;
        needPause = true;
        debug('need to wait moloch saves session ' + now + ' event time: ' + _event.endTime + ' diff:' + diff);
      }
      if (this.inPost > 256) {
        needPause = true;
        //debug('post full');
      }

      if (needPause){
        this.emit('full');
        this.eventQueue.push(_event);
        var _this = this;
        this.paused = setInterval(function () {
                      //debug('flushing after pause: ' + _this.eventQueue.length);
                      for (var i = 0; i < (_this.eventQueue.length - 1); i++) {
                        _this.post(_this.eventQueue.pop());
                      }
                      clearInterval(_this.paused);
                      _this.paused = null;
                      _this.emit('ok');
                  }, (diff * 1000)+1000);
      } else {

        this.post(_event);

      }
    }
  }
};

//helper funcs for post
molochPlugin.prototype.getTagIdFromMolochDB = function(tag, cb){
  var _this = this;
  this.Db.tagNameToId(tag, function (tagid) {
            if (tagid === -1) {
                debug('not in db',tag);
                tagid = _this.Db.createTag(tag, function(tagid) {
                  debug('new tag',tag,tagid);
                  cb(tagid);
                });
            }
            cb(tagid);
          });
}

molochPlugin.prototype.getTagId = function(tag,cb){
    if (this.tagsQueue[tag]){
      cb(this.tagsQueue[tag]);
    } else {
      var _this = this;
      var id = this.getTagIdFromMolochDB(tag, function(id){
        debug('got tag id',id,' for ', tag);
        _this.tagsQueue[tag] = id;
        debug('tag id in queue',_this.tagsQueue.length);
        cb(id);
      });

    }
  }

molochPlugin.prototype.getSessionIDFromMolochDB = function(time,srcIP,srcPort,dstIP,dstPort,cb){
  // TODO proper indices support...
  var d = new Date(time*1000)
  var iname = 'sessions-' + twoDigitString(d.getUTCFullYear() % 100) + twoDigitString(d.getUTCMonth() + 1)+twoDigitString(d.getUTCDate());
  var ok = false;
  if (this.dbStatustatus.indices[iname] ) {
    ok = true;
  } else {
    //refresh status
    var _this = this;
    this.Db.status("sessions-*", function (err, status) {
      if (err || status.error) {
        if (status.error) {
          var e = status.error;
        } else {
          var e = err;
        }
        this.emit('error', e);
        return;
      } else {
        _this.dbStatustatus = status;
      }
    });
    if (_this.dbStatustatus.indices[iname] ) {
      ok = true;
    }
  }
  if (ok) {
    var query = {
                 "from":0,
                 "size":10,
                 "fields" : ["_id","fp","lp","a1","p1","no"],
                 "query":{
                    "filtered":{
                       "query":[
                                {"range":{"fp":{"lte":time}}},
                                {"range":{"lp":{"gte":time}}}
                              ]
                       ,
                       "filter":{
                          "bool":{
                             "must":[
                                    /*{"term":{"no":this.nodeName+'*'}},*/
                                    {"term":{"a1":srcIP}},
                                    {"term":{"p1":''+srcPort}},
                                    {"term":{"a2":dstIP}},
                                    {"term":{"p2":''+dstPort}}
                                    ]
                              }
                          }
                        }
                     }
                  }

    this.Db.searchPrimary(iname, 'session', query, function(err, result) {
      //debug(JSON.stringify(query));
      //debug(JSON.stringify(result));
      if (err || result.error) {
        console.log("ERROR - Could not fetch list of sessions.  Err: ", err,  " Result: ", result, "query:", query);
        cb([]);
      } else {
        cb(result.hits.hits);
      }
    });
  } else {
    cb([]);
    this.eventsNotFound.pop();
    this.emit('error','no indices for '+d);
    this.emit('end');
  }

}

molochPlugin.prototype.saveTags2Session = function(sessionId,tagIDs){
    return true;
}

molochPlugin.prototype.post = function(data) {
  this.inPost += 1;
  var _this = this;
  this.getSessionIDFromMolochDB(data.time, data.srcIP, data.srcPort, data.dstIP, data.dstPort, function(sessions){
  //debug('sessions',JSON.stringify(sessions));
    if ( sessions.length > 0 ){
      _this.eventCountFounded +=1;
      var sessionId;
      if ( sessions.length === 1 ) {
        sessionId = sessions[0]._id;
      } else {
          for (var i = 0; i < (sessions.length - 2); i++) {
               debug('sess time:',sessions[i].fields.fp,'<', data.time,'>', sessions[i].fields.lp,sessions[i].fields.no)
          }
      }
      var tagIDs = [];
      for (var i = 0; i < (data.tags.length - 1); i++) {
        _this.getTagId(data.tags[i], function(id){
          tagIDs.push(id);
        });
      }
      if (data.tags.length != tagIDs.length) {
        debug('something wrong with tag id',data.tags, tagIDs);
      }
      if (sessionId && tagIDs.length > 0 ) {
        //debug('updating session:',sessionId, tagIDs);
        var ok = _this.saveTags2Session(sessionId,tagIDs);
        if (!ok) {
          console.error('failed session tagging ;(')
        }
      } else {
        debug('should not never happen, founded session has no id or tags');
      }
    } else {
      _this.eventsNotFound.push(data);
      //TODO...
    }
    _this.inPost -= 1;
  });
};

module.exports = function(options) {
  return new molochPlugin(options);
};
<commit_msg>typo fix<commit_after>var debug = require('debug')('pigsty-moloch-plugin');
var PigstyPlugin = require('pigsty-plugin');
var os = require('os');
var util = require('util');

function ntoa(ipl) {
    return ((ipl >>> 24) + '.' +
        (ipl >> 16 & 255) + '.' +
        (ipl >> 8 & 255) + '.' +
        (ipl & 255));
};

function twoDigitString(value) {
  return (value < 10) ? ("0" + value) : value.toString();
}

molochPlugin.prototype = new PigstyPlugin();
molochPlugin.prototype.constructor = PigstyPlugin;
function molochPlugin(options) {
  PigstyPlugin.call(this, options);
  this.options = options;
  this.nodeName = os.hostname().split(".")[0];
  if (!this.options.eventQueueMaxSize) {
    this.options.eventQueueMaxSize = 1024;
  }
  if (!this.options.maxHeapSize){
    this.options.maxHeapSize = 1073741824/10;
  }
  if (!this.options.tag_prefix) {
    this.options.tag_prefix = 'suricata';
  }

};

molochPlugin.prototype.getMolochOption = function (key, defaultValue) {
  var node = this.nodeName;
  var options = this.options.moloch
  if (options[node] && options[node][key] !== undefined) {
    return options[node][key];
  }
  if (options[node] && options[node].nodeClass && options[options[node].nodeClass] && options[options[node].nodeClass][key]) {
    return options[options[node].nodeClass][key];
  }
  if (options["default"][key]) {
    return options["default"][key];
  }
  if (options[key]) {
    return options[key];
  }
  return defaultValue;
};

molochPlugin.prototype.start = function(callback) {

  if (this.options.min_free_memory) {
    var freemem = os.freemem();
    if (this.options.min_free_memory > freemem) {
      this.emit('error','Over the limit of minimal free memory. Limit:'+this.options.min_free_memory+' Free:'+freemem);
      return;
    }
  }
  //todo check memory max limit...

  var iniParserLocation = 'iniparser'; //default to npm install iniparser -g
  if (this.options.iniparser_module_location){
    iniParserLocation = this.options.iniparser_module_location;
  }
  try {
    var iniParser = require(iniParserLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  this.options.moloch = iniParser.parseSync(this.options.moloch_config_file_location);
  if (this.options.moloch["default"] === undefined) {
    this.emit('error', 'default setcion is missing in ' + this.options.moloch_config_file_location);
    return;
  }

  var tmp_escInfo = this.getMolochOption("elasticsearch");
  if (tmp_escInfo) {
     var escInfo = tmp_escInfo.split(':');
  } else {
    this.emit('error','elasticsearch data missing');
    return;
  }

  var asyncLocation = 'async'; //default to npm install async -g
  if (this.options.async_module_location){
    asyncLocation = this.options.async_module_location;
  }
  try {
    this.async = require(asyncLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  var KAALocation = 'keep-alive-agent'; //default to npm install keep-alive-agent -g
  if (this.options.keepaliveagent_module_location){
    KAALocation = this.options.keepaliveagent_module_location;
  }
  try {
    this.KAA = require(KAALocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  if (this.options.db_module_location){
    var dbLocation = this.options.db_module_location;
  } else {
    this.emit('error', 'missing db module location in your config');
    return;
  }
  try {
    this.Db = require(dbLocation);
  } catch (e) {
    this.emit('error',e);
    return;
  }

  this.esHttpAgent = new this.KAA({maxSockets: 20}); // todo options here..
  this.Db.initialize({
        host: escInfo[0],
        port: escInfo[1],
        nodeName: this.nodeName,
        agent: this.esHttpAgent,
        dontMapTags: this.getMolochOption("multiES", false)
    });
  var _this = this;
  this.Db.status("sessions-*", function (err, status) {
    if (err || status.error) {
      if (status.error) {
        var e = status.error;
      } else {
        var e = err;
      }
      this.emit('error', e);
      return;
    } else {
      _this.dbStatustatus = status;
  debug(status._shards);
    }
  });

  this.options.tcpSaveTimeout = this.getMolochOption('tcpSaveTimeout',3);
  this.eventQueue = [];
  this.eventsNotFound = [];
  this.tagsQueue = [];

  this.eventCount = 0;
  this.eventCountIN = 0;
  this.eventCountFounded = 0;
  this.inPost = 0;
  if (this.options.print_stats){
    this.eventCountINLast = 0;
    this.eventCountLast = 0;
    var _this = this;
    this.timer = setInterval(function () {
      var now = new Date;
      var paused = '';
      if ( _this.paused ) { paused = ' state: paused'; }
      var lastMinuteCountIN = _this.eventCountIN - _this.eventCountINLast;
      var lastMinuteRateIN = lastMinuteCountIN / 10;
      var lastMinuteCount = _this.eventCount - _this.eventCountLast;
      var dropedEventCount = lastMinuteCountIN - lastMinuteCount;
      console.log('' + now + 'in post:'+_this.inPost + ' in queue:'+_this.eventQueue.length + ' founded:'+_this.eventCountFounded+ ' not found:'+_this.eventsNotFound.length+' in total: '+_this.eventCountIN+' last minute:'+lastMinuteCountIN+' rate:'+lastMinuteRateIN+'eps'+' droped:'+dropedEventCount+paused);
      _this.eventCountINLast = _this.eventCountIN;
      _this.eventCountLast = _this.eventCount;
    }, 10 * 1000);
  }
  this.emit('ready');
};

molochPlugin.prototype.close = function(callback) {
  //flusth queue ...
  for (var i = 0; i < (this.eventQueue.length - 1); i++) {
    this.post(this.eventQueue.pop());
    debug('flushing: ' + this.eventQueue.length);
  }
  this.eventQueue = [];
  this.tagsQueue = [];
  this.emit('end');
};

molochPlugin.prototype.getUnixTime = function () {
    return Math.floor(new Date().getTime() / 1000);
};

molochPlugin.prototype.send = function(event) {
  var _event = {};
  this.eventCountIN += 1;
  if (event && event.event && event.event.event_second) {
    _event.time = event.event.event_second;
    _event.srcIP = event.event.source_ip;
    _event.srcPort = event.event.source_port;
    _event.dstIP = event.event.destination_ip;
    _event.dstPort = event.event.dest_port;
    _event.tags = [];
    //todo options for tags...
    _event.tags.push(this.options.tag_prefix + '_severity_' + event.event.classification.severity);
    _event.tags.push(this.options.tag_prefix + '_classification_' + event.event.classification.name);
    _event.tags.push(this.options.tag_prefix + '_signatureID_' + event.event.signature.id);
    _event.tags.push(this.options.tag_prefix + '_signature_' + event.event.signature.name);

    this.eventCount += 1;

    if (this.paused){
      this.eventQueue.push(_event);
      debug('emited full, but still getting events...');
    } else {
      var needPause = false;

      if (this.eventQueue.length > this.options.eventQueueMaxSize) {
        needPause = true;
        debug('queue full');
      }

      var freemem = os.freemem();
      if (this.options.min_free_memory > freemem) {
          needPause = true;
          debug('memory almost full' + freemem);
      }

      var memusage = process.memoryUsage();
      if (memusage.heapUsed > (this.options.maxHeapSize)) {
        needPause = true;
        debug('heap full ' + memusage.heapUsed);
      }

      var diff = 1;
      var now = this.getUnixTime();
      if ( (now - this.options.tcpSaveTimeout) < _event.endTime ) {
        diff = now - this.options.tcpSaveTimeout - _event.endTime ;
        needPause = true;
        debug('need to wait moloch saves session ' + now + ' event time: ' + _event.endTime + ' diff:' + diff);
      }
      if (this.inPost > 256) {
        needPause = true;
        //debug('post full');
      }

      if (needPause){
        this.emit('full');
        this.eventQueue.push(_event);
        var _this = this;
        this.paused = setInterval(function () {
                      //debug('flushing after pause: ' + _this.eventQueue.length);
                      for (var i = 0; i < (_this.eventQueue.length - 1); i++) {
                        _this.post(_this.eventQueue.pop());
                      }
                      clearInterval(_this.paused);
                      _this.paused = null;
                      _this.emit('ok');
                  }, (diff * 1000)+1000);
      } else {

        this.post(_event);

      }
    }
  }
};

//helper funcs for post
molochPlugin.prototype.getTagIdFromMolochDB = function(tag, cb){
  var _this = this;
  this.Db.tagNameToId(tag, function (tagid) {
            if (tagid === -1) {
                debug('not in db',tag);
                tagid = _this.Db.createTag(tag, function(tagid) {
                  debug('new tag',tag,tagid);
                  cb(tagid);
                });
            }

            cb(tagid);
          });
}

molochPlugin.prototype.getTagId = function(tag,cb){
    if (this.tagsQueue[tag]){
      cb(this.tagsQueue[tag]);
    } else {
      var _this = this;
      var id = this.getTagIdFromMolochDB(tag, function(id){
        debug('got tag id',id,' for ', tag);
        _this.tagsQueue[tag] = id;
        debug('tag id in queue',_this.tagsQueue.length);
        cb(id);
      });

    }
  }

molochPlugin.prototype.getSessionIDFromMolochDB = function(time,srcIP,srcPort,dstIP,dstPort,cb){
  // TODO proper indices support...
  var d = new Date(time*1000)
  var iname = 'sessions-' + twoDigitString(d.getUTCFullYear() % 100) + twoDigitString(d.getUTCMonth() + 1)+twoDigitString(d.getUTCDate());
  var ok = false;
  if (this.dbStatustatus.indices[iname] ) {
    ok = true;
  } else {
    //refresh status
    var _this = this;
    this.Db.status("sessions-*", function (err, status) {
      if (err || status.error) {
        if (status.error) {
          var e = status.error;
        } else {
          var e = err;
        }
        _this.emit('error', e);
        return;
      } else {
        _this.dbStatustatus = status;
      }
    });
    if (_this.dbStatustatus.indices[iname] ) {
      ok = true;
    }
  }
  if (ok) {
    var query = {
                 "from":0,
                 "size":10,
                 "fields" : ["_id","fp","lp","a1","p1","no"],
                 "query":{
                    "filtered":{
                       "query":[
                                {"range":{"fp":{"lte":time}}},
                                {"range":{"lp":{"gte":time}}}
                              ]
                       ,
                       "filter":{
                          "bool":{
                             "must":[
                                    /*{"term":{"no":this.nodeName+'*'}},*/
                                    {"term":{"a1":srcIP}},
                                    {"term":{"p1":''+srcPort}},
                                    {"term":{"a2":dstIP}},
                                    {"term":{"p2":''+dstPort}}
                                    ]
                              }
                          }
                        }
                     }
                  }

    this.Db.searchPrimary(iname, 'session', query, function(err, result) {
      //debug(JSON.stringify(query));
      //debug(JSON.stringify(result));
      if (err || result.error) {
        console.log("ERROR - Could not fetch list of sessions.  Err: ", err,  " Result: ", result, "query:", query);
        cb([]);
      } else {
        cb(result.hits.hits);
      }
    });
  } else {
    cb([]);
    this.eventsNotFound.pop();
    this.emit('error','no indices for '+d);
    this.emit('end');
  }

}

molochPlugin.prototype.saveTags2Session = function(sessionId,tagIDs){
    return true;
}

molochPlugin.prototype.post = function(data) {
  this.inPost += 1;
  var _this = this;
  this.getSessionIDFromMolochDB(data.time, data.srcIP, data.srcPort, data.dstIP, data.dstPort, function(sessions){
  //debug('sessions',JSON.stringify(sessions));
    if ( sessions.length > 0 ){
      _this.eventCountFounded +=1;
      var sessionId;
      if ( sessions.length === 1 ) {
        sessionId = sessions[0]._id;
      } else {
          for (var i = 0; i < (sessions.length - 2); i++) {
               debug('sess time:',sessions[i].fields.fp,'<', data.time,'>', sessions[i].fields.lp,sessions[i].fields.no)
          }
      }
      var tagIDs = [];
      for (var i = 0; i < (data.tags.length - 1); i++) {
        _this.getTagId(data.tags[i], function(id){
          tagIDs.push(id);
        });
      }
      if (data.tags.length != tagIDs.length) {
        debug('something wrong with tag id',data.tags, tagIDs);
      }
      if (sessionId && tagIDs.length > 0 ) {
        //debug('updating session:',sessionId, tagIDs);
        var ok = _this.saveTags2Session(sessionId,tagIDs);
        if (!ok) {
          console.error('failed session tagging ;(')
        }
      } else {
        debug('should not never happen, founded session has no id or tags');
      }
    } else {
      _this.eventsNotFound.push(data);
      //TODO...
    }
    _this.inPost -= 1;
  });
};

module.exports = function(options) {
  return new molochPlugin(options);
};
<|endoftext|>


<commit_before>'use strict';

const { relative_url } = require('hexo-util');

module.exports = function(from, to) {
  return relative_url.call(this, from, to);
};
<commit_msg>refactor(relative_url): 'this' binding is not required<commit_after>'use strict';

const { relative_url } = require('hexo-util');

module.exports = function(from, to) {
  return relative_url(from, to);
};
<|endoftext|>


<commit_before>/* eslint-disable consistent-return */
const { Transform } = require('stream');
const fs = require('fs-extra');
const path = require('path');
const { chain, omit, pick } = require('lodash');
const uuid = require('uuid');
const { testRequiredChunkFields } = require('../util');
const { CustomError, MissingRequiredParameter } = require('../errors');
const async = require('async');
const EventEmitter = require('events');

class FsCopyError extends CustomError {}
class FsDeleteError extends CustomError {}

class ChunkHandler extends EventEmitter {
    constructor(metadata, errors, options) {
        super();
        this.options = options;
        this._state = {
            tempStorage: pick(metadata, ['localTmpFilepath']),
            metadata: omit(metadata, ['localTmpFilepath']),
            errors: chain(errors).castArray().compact().value(),
        };

        async.series([
            /* eslint-disable no-confusing-arrow */
            cb => this._state.errors.length ? cb() : this.copyObject(cb),
            cb => this.deleteObject(cb),
            /* eslint-enable no-confusing-arrow */
        ], this._response.bind(this));
    }

    _response() {
        process.nextTick(() => {
            const { metadata, errors } = this._state;
            if (errors.length) {
                this.emit('done', { metadata, errors });
            } else {
                this.emit('done', { metadata });
            }
        });
    }

    copyObject(callback) {
        const { localTmpFilepath } = this._state.tempStorage;
        const filename = this.getTargetFilename();
        const filepath = path.join(this.options.dir, `${filename}`);

        fs.move(localTmpFilepath, filepath, (err) => {
            if (err) {
                this._state.errors.push(new FsCopyError(err.message));
            } else {
                Object.assign(this._state.metadata, {
                    storageLocalFilename: filename,
                    storageLocalFilepath: filepath,
                });
            }
            callback();
        });
    }

    deleteObject(callback) {
        const { localTmpFilepath } = this._state.tempStorage;

        fs.remove(localTmpFilepath, (err) => {
            if (err) {
                this._state.errors.push(new FsDeleteError(err.message));
            }
            callback();
        });
    }

    getTargetFilename() {
        let filename = '';
        if (typeof this.options.filenameFn === 'function') {
            filename = this.options.filenameFn(this._state.metadata);
        } else if (this.options.filenameProperty && this._state.metadata[this.options.filenameProperty]) {
            filename = this._state.metadata[this.options.filenameProperty];
        } else {
            filename = uuid.v1();
        }
        filename += path.extname(this._state.metadata.filename).toLowerCase();
        return filename;
    }
}

class StorageLocal extends Transform {
    /**
     * Save temporary files.
     * @param {Object} options
     *      StorageLocal options.
     * @param {string} options.dir
     *      Directory for uploaded files.
     * @param {boolean} [options.ensureDir=true]
     *      Ensure that dir exists, create if not.
     * @param {string} [options.filenameProperty]
     *      A `metadata` field name to select filename from.
     * @param {function(object):string} [options.filenameFn]
     *      A function for selecting target filename, `metadata` object passed as argument.
     * @extends Transform
     */
    constructor(options) {
        super({ objectMode: true });

        if (!options || !options.dir) {
            throw new MissingRequiredParameter('Missing required key `dir` in options');
        }

        this.options = Object.assign({
            dir: '',
            ensureDir: true,
            filenameProperty: null,
            filenameFn: null,
        }, options);

        if (this.options.ensureDir !== false) {
            fs.ensureDirSync(this.options.dir);
        }

        // eslint-disable-next-line no-bitwise
        fs.accessSync(options.dir, fs.constants.R_OK | fs.constants.W_OK);
    }

    _transform({ metadata, errors = [] }, encoding, callback) {
        const missingFields = testRequiredChunkFields(metadata, [
            'filename',
            'localTmpFilepath',
        ], this.constructor.name, 'metadata');

        if (missingFields) {
            return callback(null, { metadata, errors: errors.concat(missingFields) });
        }

        new ChunkHandler(metadata, errors, this.options)
            .once('done', result => callback(null, result));
    }

    /**
     * @function write
     * @memberof StorageLocal#
     * @param {StorageLocal~InputChunk} chunk
     * @param {function} callback
     */

    /**
     * @typedef {Object} StorageLocal~InputChunk
     * @property {Object} metadata
     * @property {string} metadata.filename
     * @property {string} metadata.localTmpFilepath
     * @property {*} [metadata....]
     *      Extra metadata fields
     * @property {Error|Array<Error>} [errors]
     */

    /**
     * @function read
     * @memberof StorageLocal#
     * @returns {?StorageLocal~OutputChunk}
     */

    /**
     * @typedef {Object} StorageLocal~OutputChunk
     * @property {Object} metadata
     * @property {string} metadata.storageLocalFilename
     * @property {string} metadata.storageLocalFilepath
     * @property {*} [metadata....]
     *      Extra metadata fields
     * @property {Error|Array<Error>} [errors]
     */
}

module.exports = StorageLocal;
<commit_msg>remove unnecessary eslint comment<commit_after>/* eslint-disable consistent-return */
const { Transform } = require('stream');
const fs = require('fs-extra');
const path = require('path');
const { chain, omit, pick } = require('lodash');
const uuid = require('uuid');
const { testRequiredChunkFields } = require('../util');
const { CustomError, MissingRequiredParameter } = require('../errors');
const async = require('async');
const EventEmitter = require('events');

class FsCopyError extends CustomError {}
class FsDeleteError extends CustomError {}

class ChunkHandler extends EventEmitter {
    constructor(metadata, errors, options) {
        super();
        this.options = options;
        this._state = {
            tempStorage: pick(metadata, ['localTmpFilepath']),
            metadata: omit(metadata, ['localTmpFilepath']),
            errors: chain(errors).castArray().compact().value(),
        };

        async.series([
            cb => (this._state.errors.length ? cb() : this.copyObject(cb)),
            cb => this.deleteObject(cb),
        ], this._response.bind(this));
    }

    _response() {
        process.nextTick(() => {
            const { metadata, errors } = this._state;
            if (errors.length) {
                this.emit('done', { metadata, errors });
            } else {
                this.emit('done', { metadata });
            }
        });
    }

    copyObject(callback) {
        const { localTmpFilepath } = this._state.tempStorage;
        const filename = this.getTargetFilename();
        const filepath = path.join(this.options.dir, `${filename}`);

        fs.move(localTmpFilepath, filepath, (err) => {
            if (err) {
                this._state.errors.push(new FsCopyError(err.message));
            } else {
                Object.assign(this._state.metadata, {
                    storageLocalFilename: filename,
                    storageLocalFilepath: filepath,
                });
            }
            callback();
        });
    }

    deleteObject(callback) {
        const { localTmpFilepath } = this._state.tempStorage;

        fs.remove(localTmpFilepath, (err) => {
            if (err) {
                this._state.errors.push(new FsDeleteError(err.message));
            }
            callback();
        });
    }

    getTargetFilename() {
        let filename = '';
        if (typeof this.options.filenameFn === 'function') {
            filename = this.options.filenameFn(this._state.metadata);
        } else if (this.options.filenameProperty && this._state.metadata[this.options.filenameProperty]) {
            filename = this._state.metadata[this.options.filenameProperty];
        } else {
            filename = uuid.v1();
        }
        filename += path.extname(this._state.metadata.filename).toLowerCase();
        return filename;
    }
}

class StorageLocal extends Transform {
    /**
     * Save temporary files.
     * @param {Object} options
     *      StorageLocal options.
     * @param {string} options.dir
     *      Directory for uploaded files.
     * @param {boolean} [options.ensureDir=true]
     *      Ensure that dir exists, create if not.
     * @param {string} [options.filenameProperty]
     *      A `metadata` field name to select filename from.
     * @param {function(object):string} [options.filenameFn]
     *      A function for selecting target filename, `metadata` object passed as argument.
     * @extends Transform
     */
    constructor(options) {
        super({ objectMode: true });

        if (!options || !options.dir) {
            throw new MissingRequiredParameter('Missing required key `dir` in options');
        }

        this.options = Object.assign({
            dir: '',
            ensureDir: true,
            filenameProperty: null,
            filenameFn: null,
        }, options);

        if (this.options.ensureDir !== false) {
            fs.ensureDirSync(this.options.dir);
        }

        // eslint-disable-next-line no-bitwise
        fs.accessSync(options.dir, fs.constants.R_OK | fs.constants.W_OK);
    }

    _transform({ metadata, errors = [] }, encoding, callback) {
        const missingFields = testRequiredChunkFields(metadata, [
            'filename',
            'localTmpFilepath',
        ], this.constructor.name, 'metadata');

        if (missingFields) {
            return callback(null, { metadata, errors: errors.concat(missingFields) });
        }

        new ChunkHandler(metadata, errors, this.options)
            .once('done', result => callback(null, result));
    }

    /**
     * @function write
     * @memberof StorageLocal#
     * @param {StorageLocal~InputChunk} chunk
     * @param {function} callback
     */

    /**
     * @typedef {Object} StorageLocal~InputChunk
     * @property {Object} metadata
     * @property {string} metadata.filename
     * @property {string} metadata.localTmpFilepath
     * @property {*} [metadata....]
     *      Extra metadata fields
     * @property {Error|Array<Error>} [errors]
     */

    /**
     * @function read
     * @memberof StorageLocal#
     * @returns {?StorageLocal~OutputChunk}
     */

    /**
     * @typedef {Object} StorageLocal~OutputChunk
     * @property {Object} metadata
     * @property {string} metadata.storageLocalFilename
     * @property {string} metadata.storageLocalFilepath
     * @property {*} [metadata....]
     *      Extra metadata fields
     * @property {Error|Array<Error>} [errors]
     */
}

module.exports = StorageLocal;
<|endoftext|>


<commit_before>/*global require, exports*/
var util = require('util'),
    _ = require('underscore'),
    extendWithGettersAndSetters = require('../util/extendWithGettersAndSetters'),
    Relation = require('./Relation'),
    HtmlRelation = require('./HtmlRelation');

function HtmlRequireJsMain(config) {
    HtmlRelation.call(this, config);
}

util.inherits(HtmlRequireJsMain, HtmlRelation);

extendWithGettersAndSetters(HtmlRequireJsMain.prototype, {
    get href() {
        return this.node.getAttribute('data-main').replace(/(?:\.js)?$/, '.js');
    },

    set href(href) {
        this.node.setAttribute('data-main', href.replace(/\.js$/, ""));
    },

    inline: function () {
        throw new Error("relations.HtmlRequireJsMain.inline: Not supported.");
    },

    attach: function (asset, position, adjacentRelation) {
        throw new Error("relations.HtmlRequireJsMain.attach: Not supported.");
    },

    detach: function () {
        this.node.removeAttribute('data-main');
        delete this.node;
        return Relation.prototype.detach.call(this);
    }
});

module.exports = HtmlRequireJsMain;
<commit_msg>HtmlRequireJsMain<commit_after>var util = require('util'),
    extendWithGettersAndSetters = require('../util/extendWithGettersAndSetters'),
    Relation = require('./Relation'),
    HtmlRelation = require('./HtmlRelation');

function HtmlRequireJsMain(config) {
    HtmlRelation.call(this, config);
}

util.inherits(HtmlRequireJsMain, HtmlRelation);

extendWithGettersAndSetters(HtmlRequireJsMain.prototype, {
    get href() {
        return this.node.getAttribute('data-main').replace(/(?:\.js)?$/, '.js');
    },

    set href(href) {
        this.node.setAttribute('data-main', href.replace(/\.js$/, ''));
    },

    inline: function () {
        throw new Error('relations.HtmlRequireJsMain.inline: Not supported.');
    },

    attach: function () {
        throw new Error('relations.HtmlRequireJsMain.attach: Not supported.');
    },

    detach: function () {
        this.node.removeAttribute('data-main');
        delete this.node;
        return Relation.prototype.detach.call(this);
    }
});

module.exports = HtmlRequireJsMain;
<|endoftext|>


<commit_before>
const getStorage  = () => {
  const data = window.localStorage.getItem('__chrome.storage.sync__')
  if (data != null) {
    return JSON.parse(data)
  } else {
    return {}
  }
}

const setStorage = (storage) => {
  const json = JSON.stringify(storage)
  window.localStorage.setItem('__chrome.storage.sync__', json)
}

module.exports = {
  sync: {
    get (keys, callback) {
      const storage = getStorage()

      if (keys === {} || keys === []) return {}
      if (keys === null) return storage

      if (typeof keys === 'string') keys = [keys]

      let items = {}
      // ['foo'] or ['foo', 'bar'] or [{foo: 'bar'}]
      keys.forEach(function (key) {
        if (typeof key === 'string') {
          items[key] = storage[key]
        }
        else {
          const objKey = Object.keys(key)
          if (!storage[objKey] || storage[objKey] === '') {
            items[objKey] = key[objKey]
          } else {
            items[objKey] = storage[objKey]
          }
        }
      })

      setTimeout(function () {
        callback(items)
      })
    },

    set (items, callback) {
      const storage = getStorage()

      Object.keys(items).forEach(function (name) {
        storage[name] = items[name]
      })

      setStorage(storage)

      setTimeout(callback)
    }
  }
}
<commit_msg>Handle the three get cases<commit_after>
const getStorage  = () => {
  const data = window.localStorage.getItem('__chrome.storage.sync__')
  if (data != null) {
    return JSON.parse(data)
  } else {
    return {}
  }
}

const setStorage = (storage) => {
  const json = JSON.stringify(storage)
  window.localStorage.setItem('__chrome.storage.sync__', json)
}

module.exports = {
  sync: {
    get (keys, callback) {
      const storage = getStorage()
      if (keys == null) return storage

      let defaults = {}
      switch (typeof keys) {
        case 'string':
          keys = [keys]
          break
        case 'object':
          if (!Array.isArray(keys)) {
            defaults = keys
            keys = Object.keys(keys)
          }
          break;
      }
      if (keys.length === 0 ) return {}

      let items = {}
      keys.forEach(function (key) {
        var value = storage[key]
        if (value == null) value = defaults[key]
        items[key] = value
      })

      setTimeout(function () {
        callback(items)
      })
    },

    set (items, callback) {
      const storage = getStorage()

      Object.keys(items).forEach(function (name) {
        storage[name] = items[name]
      })

      setStorage(storage)

      setTimeout(callback)
    }
  }
}
<|endoftext|>


<commit_before>'use strict';

var helpers = require('../helpers'),
    capitalize = require('lodash.capitalize'),
    kebabcase = require('lodash.kebabcase');

// Our nestable selector types, separated by type for ease of use with rules
// we replace ident with 'selector' for readability'
var nestableElements = ['selector', 'class', 'id'],
    nestableAttributes = ['attribute'],
    nestablePseudo = ['pseudoClass', 'pseudoElement', 'nth', 'nthSelector'];

/**
 * Formats a string from camelCase to hyphens and capitalizes
 * @param {string} str - The string to be formatted
 * @returns {string} A hyphenated and capitalized string
 */
var formatOutput = function (str) {
  return capitalize(kebabcase(str));
};

module.exports = {
  'name': 'force-element-nesting',
  'defaults': {},
  'detect': function (ast, parser) {
    var result = [],
        elements = nestableElements.concat(nestableAttributes, nestablePseudo);

    ast.traverseByType('ruleset', function (ruleset) {
      var delimOrder = [];

      ruleset.forEach('selector', function (selector) {
        // Where we'll store the previous value
        var previousVal;

        // Keep track of the order of elements & delimeters
        selector.forEach(function (el) {
          var curNode = helpers.mapDelims(el);

          if (curNode !== false) {
            delimOrder.push(curNode);
          }
        });

        selector.forEach('simpleSelector', function (simpleSelector) {
          // check if the next selector is proceeded by a delimiter
          // if it is add it to the curSelector output
          var nextType = delimOrder[0];

          if (nextType === 'd') {
            // Empty the previous value store
            previousVal = null;

            // remove the next delim and selector from our order as we are only
            // looping over selectors
            delimOrder.splice(0, 2);
          }
          else {
            // if no delim then just remove the current selectors marker in the
            // order array
            delimOrder.splice(0, 1);
          }

          // Loop each selector while checking if it should be nested
          simpleSelector.forEach(function (node) {
            // Construct the selector and store the current selector value
            var constructedSelector = helpers.constructSelector(node),
                currentVal = constructedSelector.type;

            if (helpers.isNestable(currentVal, previousVal, elements, nestableElements)) {
              helpers.addUnique(result, {
                'ruleId': parser.rule.name,
                'line': node.start.line,
                'column': node.start.column,
                'message': formatOutput(currentVal) + ' `' + constructedSelector.content + '` should be nested within its parent ' + previousVal,
                'severity': parser.severity
              });
            }

            if (currentVal) {
              // Store current value as previous and continue with the loop
              previousVal = currentVal;
            }
          });
        });
      });
    });

    return result;
  }
};
<commit_msg>Fix force element nesting<commit_after>'use strict';

var helpers = require('../helpers'),
    capitalize = require('lodash.capitalize'),
    kebabcase = require('lodash.kebabcase');

    // Our nestable selector types, separated by type for ease of use with rules
    // we replace ident with 'selector' for readability'
var nestableElements = ['selector', 'class', 'id', 'typeSelector', 'parentSelectorExtension'],
    nestableAttributes = ['attributeSelector'],
    nestablePseudo = ['pseudoClass', 'pseudoElement', 'nth', 'nthSelector'];

/**
 * Formats a string from camelCase to hyphens and capitalizes
 * @param {string} str - The string to be formatted
 * @returns {string} A hyphenated and capitalized string
 */
var formatOutput = function (str) {
  return capitalize(kebabcase(str));
};

module.exports = {
  'name': 'force-element-nesting',
  'defaults': {},
  'detect': function (ast, parser) {
    var result = [],
        elements = nestableElements.concat(nestableAttributes, nestablePseudo);

    ast.traverseByType('ruleset', function (ruleset) {
      ruleset.forEach(function (selector) {

        var previousVal;
        selector.forEach(function (item) {
          if (previousVal) {
            if (helpers.isNestable(item.type, previousVal.type, elements, nestableElements)) {
              helpers.addUnique(result, {
                'ruleId': parser.rule.name,
                'line': selector.start.line,
                'column': selector.start.column,
                'message': formatOutput(item.type) + 'should be nested within its parent ' + previousVal.type,
                'severity': parser.severity
              });
            }
          }
          if (!item.is('space')) {
            previousVal = item;
          }
        });
      });
    });

    return result;
  }
};
<|endoftext|>