<commit_before>module.exports = function(grunt) {
	"use strict";

	var cp = require("child_process")
		, semver = require("semver")
		, Q = require("q");

	function exec(command, args) {
		grunt.log.ok([command, args.join(" ")].join(" "));

		var child = cp.spawn(command, args, { env: process.env, stdio: "inherit" });
		var deferred = Q.defer();

		child.on("exit", function(code) {
			var success = (code === 0);
			var callback = success ? deferred.resolve : deferred.reject;
			callback(success, code);
		});

		return deferred.promise;
	}

	function flow(command, done, args, flags) {
		args = args || [];
		flags = flags || [];

		args = ["flow"].
			concat(command.split(" ")).
			concat(flags).
			concat(args).
			filter(function(value) { return typeof value !== 'undefined' && value !== null; });

		exec("git", args).then(done, done);
	}

	// Start a feature
	grunt.registerTask("feature:start", "Start new feature <name>, optionally basing it on <base> instead of develop.", function(name, base) {
		var flags = this.options({ flags: null }).flags;
		flow("feature start", this.async(), [name, base], flags);
	});

	grunt.registerTask("feature:list", "List existing features", function() {
		var flags = this.options({ flags: null }).flags;
		flow("feature list", this.async(), null, flags);
	});

	grunt.registerTask("feature:finish", "Finish feature <name>", function(name) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("feature finish", this.async(), [name], flags);
	});

	grunt.registerTask("feature:publish", "Start sharing feature <name> on $ORIGIN", function(name) {
		this.requires("test");
		flow("feature publish", this.async(), [name]);
	});

	grunt.registerTask("feature:track", "Start tracking feature <name> that is shared on $ORIGIN", function(name) {
		flow("feature track", this.async(), [name]);
	});

	grunt.registerTask("feature:diff", "Show all changes in <name> that are not in develop", function(name) {
		flow("feature diff", this.async(), [name]);
	});

	grunt.registerTask("feature:rebase", "Rebase <name> on develop.", function(name) {
		var flags = this.options({ flags: null }).flags;
		flow("feature rebase", this.async(), [name], flags);
	});

	grunt.registerTask("feature:checkout", "Switch to feature branch <name>", function(name) {
		flow("feature checkout", this.async(), [name]);
	});

	grunt.registerTask("feature:pull", "Pull feature from <remote> with name <name>", function(remote, name) {
		flow("feature pull", this.async(), [remote, name]);
	});

	grunt.registerTask("release:list", "List existing releases", function() {
		var flags = this.options({ flags: null }).flags;
		flow("release list", this.async(), null, flags);
	});

	["patch", "minor", "major"].forEach(function(bumpType) {
		grunt.registerTask("release:" + bumpType, "Start a new " + bumpType + " release", function() {
			this.requiresConfig("pkg.version");
			this.requires("test");

			var flags = this.options({ flags: null }).flags;
			var version = semver.inc(grunt.config.get("pkg.version"), bumpType);

			flow("release start", this.async(), [version], flags);
			grunt.task.run("bump:" + bumpType);
		});
	});

	grunt.registerTask("release:finish", "Finish release <version>", function(version) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("release finish", this.async(), [version], flags);
	});

	grunt.registerTask("release:publish", "Start sharing release <version> on $ORIGIN", function(version) {
		this.requires("test");
		flow("release publish", this.async(), [version]);
	});

	grunt.registerTask("release:track", "Start tracking release <version> that is shared on $ORIGIN", function(version) {
		flow("release track", this.async(), [version]);
	});

	grunt.registerTask("hotfix:list", "List existing hotfixes", function() {
		var flags = this.options({ flags: null }).flags;
		flow("hotfix list", this.async(), null, flags);
	});

	grunt.registerTask("hotfix:start", "Start a new hotfix named <version>, optionally base it on <base> instead of <master>", function(version, base) {
		var flags = this.options({ flags: null }).flags;
		flow("hotfix start", this.async(), [version, base], flags);
	});

	grunt.registerTask("hotfix:finish", "Finish hotfix <version>", function(version) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("hotfix finish", this.async(), [version], flags);
	});

};
<commit_msg>Comment Git Flow Task<commit_after>module.exports = function(grunt) {
	"use strict";

	var cp = require("child_process"),
		semver = require("semver"),
		Q = require("q");

	/**
	 * Execute the given command
	 * @param  {string} command
	 * @param  {array} args
	 * @return {promise}
	 */
	function exec(command, args) {
		grunt.log.ok([command, args.join(" ")].join(" "));

		var spawnConfig = { env: process.env, stdio: "inherit" },
			child = cp.spawn(command, args, spawnConfig),
			deferred = Q.defer();

		function updatePromise(code) {
			if(code === 0) deferred.resolve(code);
			else deferred.reject(code);
		}

		child.on("exit", updatePromise);
		return deferred.promise;
	}

	/**
	 * Execute a git-flow command
	 * @param  {string}   command
	 * @param  {Function} done
	 * @param  {array}  [args]
	 * @param  {array}  [flags]
	 * @return {promise}
	 */
	function flow(command, done, args, flags) {
		args = args || [];
		flags = flags || [];

		args = ["flow"].
			concat(command.split(" ")).
			concat(flags).
			concat(args).
			filter(function(value) { return typeof value !== 'undefined' && value !== null; });

		exec("git", args).then(done, done);
	}

	// Start a feature
	grunt.registerTask("feature:start", "Start new feature <name>, optionally basing it on <base> instead of develop.", function(name, base) {
		var flags = this.options({ flags: null }).flags;
		flow("feature start", this.async(), [name, base], flags);
	});

	grunt.registerTask("feature:list", "List existing features", function() {
		var flags = this.options({ flags: null }).flags;
		flow("feature list", this.async(), null, flags);
	});

	grunt.registerTask("feature:finish", "Finish feature <name>", function(name) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("feature finish", this.async(), [name], flags);
	});

	grunt.registerTask("feature:publish", "Start sharing feature <name> on $ORIGIN", function(name) {
		this.requires("test");
		flow("feature publish", this.async(), [name]);
	});

	grunt.registerTask("feature:track", "Start tracking feature <name> that is shared on $ORIGIN", function(name) {
		flow("feature track", this.async(), [name]);
	});

	grunt.registerTask("feature:diff", "Show all changes in <name> that are not in develop", function(name) {
		flow("feature diff", this.async(), [name]);
	});

	grunt.registerTask("feature:rebase", "Rebase <name> on develop.", function(name) {
		var flags = this.options({ flags: null }).flags;
		flow("feature rebase", this.async(), [name], flags);
	});

	grunt.registerTask("feature:checkout", "Switch to feature branch <name>", function(name) {
		flow("feature checkout", this.async(), [name]);
	});

	grunt.registerTask("feature:pull", "Pull feature from <remote> with name <name>", function(remote, name) {
		flow("feature pull", this.async(), [remote, name]);
	});

	grunt.registerTask("release:list", "List existing releases", function() {
		var flags = this.options({ flags: null }).flags;
		flow("release list", this.async(), null, flags);
	});

	["patch", "minor", "major"].forEach(function(bumpType) {
		grunt.registerTask("release:" + bumpType, "Start a new " + bumpType + " release", function() {
			this.requiresConfig("pkg.version");
			this.requires("test");

			var flags = this.options({ flags: null }).flags;
			var version = semver.inc(grunt.config.get("pkg.version"), bumpType);

			flow("release start", this.async(), [version], flags);
			grunt.task.run("bump:" + bumpType);
		});
	});

	grunt.registerTask("release:finish", "Finish release <version>", function(version) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("release finish", this.async(), [version], flags);
	});

	grunt.registerTask("release:publish", "Start sharing release <version> on $ORIGIN", function(version) {
		this.requires("test");
		flow("release publish", this.async(), [version]);
	});

	grunt.registerTask("release:track", "Start tracking release <version> that is shared on $ORIGIN", function(version) {
		flow("release track", this.async(), [version]);
	});

	grunt.registerTask("hotfix:list", "List existing hotfixes", function() {
		var flags = this.options({ flags: null }).flags;
		flow("hotfix list", this.async(), null, flags);
	});

	grunt.registerTask("hotfix:start", "Start a new hotfix named <version>, optionally base it on <base> instead of <master>", function(version, base) {
		var flags = this.options({ flags: null }).flags;
		flow("hotfix start", this.async(), [version, base], flags);
	});

	grunt.registerTask("hotfix:finish", "Finish hotfix <version>", function(version) {
		this.requires("test");
		var flags = this.options({ flags: null }).flags;
		flow("hotfix finish", this.async(), [version], flags);
	});

};
<|endoftext|>


<commit_before>exports.up = function (knex, Promise) {
  return knex.schema
             .createTable('oauth_clients', function (table) {
               table.uuid('id').primary()
               table.string('client_secret')
               table.string('redirect_uri')
               table.string('grant_types')
               table.string('scope')
               table.string('user_id')
               table.timestamps(true, true)
             })
             .createTable('oauth_authorization_codes', function (table) {
               table.string('code').primary()
               table.string('client_id')
               table.string('user_id')
               table.string('redirect_uri')
               table.string('scope')
               table.timestamp('expires_at')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('oauth_tokens', function (table) {
               table.increments('id').primary()
               table.string('client_id')
               table.string('user_id')
               table.string('scope')
               table.string('access_token').unique()
               table.string('refresh_token').unique()
               table.timestamp('access_token_expires_at')
               table.timestamp('refresh_token_expires_at')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('users', function (table) {
               table.increments('id').primary()
               table.string('username').unique()
               table.string('password')
               table.string('email').unique()
               table.string('avatar')
               table.string('reset_token')
               table.timestamp('reset_expires').defaultTo(knex.raw('now() + \'2 hours\'::interval'))
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('user2role', function (table) {
               table.integer('user_id')
               table.integer('role_id')
             })
             .createTable('roles', function (table) {
               table.increments('id').primary()
               table.string('name').unique()
               table.string('desc')
               table.jsonb('permissions')
             })
             .createTable('files', function (table) {
               table.increments('id').primary()
               table.string('title')
               table.text('desc')
               table.string('file_name')
               table.string('file_type')
               table.string('file_size')
               table.string('file_path')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('user_accounts', function (table) {
               table.increments('id').primary()
               table.integer('user_id')
               table.string('provider')
               table.string('account_id')
               table.string('access_token')
               table.string('refresh_token')
               table.jsonb('profile')
               table.timestamp('expires_at')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('settings', function (table) {
               table.string('id').primary()
               table.string('name')
               table.text('desc')
               table.json('settings')
             })
             .createTable('posts', function (table) {
               table.increments('id').primary()
               table.string('title')
               table.text('contents')
               table.integer('user_id')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
             .createTable('comments', function (table) {
               table.increments('id').primary()
               table.string('title')
               table.text('contents')
               table.integer('user_id')
               table.integer('post_id')
               table.timestamp('created_at').notNullable().defaultTo(knex.fn.now())
               table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now())
             })
}

exports.down = function (knex, Promise) {
  return knex.schema
             .dropTable('users')
             .dropTable('roles')
             .dropTable('user2role')
             .dropTable('user_accounts')
             .dropTable('oauth_clients')
             .dropTable('oauth_authorization_codes')
             .dropTable('oauth_tokens')
             .dropTable('files')
             .dropTable('settings')
             .dropTable('posts')
             .dropTable('comments')
}
<commit_msg>tidy up<commit_after>exports.up = function (knex, Promise) {
  return knex.schema
    .createTable('oauth_clients', function (table) {
      table.uuid('id').primary()
      table.string('client_secret')
      table.string('redirect_uri')
      table.string('grant_types')
      table.string('scope')
      table.string('user_id')
      table.timestamps(true, true)
    })
    .createTable('oauth_authorization_codes', function (table) {
      table.string('code').primary()
      table.string('client_id')
      table.string('user_id')
      table.string('redirect_uri')
      table.string('scope')
      table.timestamp('expires_at')
      table.timestamps(true, true)
    })
    .createTable('oauth_tokens', function (table) {
      table.increments('id').primary()
      table.string('client_id')
      table.string('user_id')
      table.string('scope')
      table.string('access_token').unique()
      table.string('refresh_token').unique()
      table.timestamp('access_token_expires_at')
      table.timestamp('refresh_token_expires_at')
      table.timestamps(true, true)
    })
    .createTable('users', function (table) {
      table.increments('id').primary()
      table.string('username').unique()
      table.string('password')
      table.string('email').unique()
      table.string('avatar')
      table.string('reset_token')
      table.timestamp('reset_expires').defaultTo(knex.raw('now() + \'2 hours\'::interval'))
      table.timestamps(true, true)
    })
    .createTable('user2role', function (table) {
      table.integer('user_id')
      table.integer('role_id')
    })
    .createTable('roles', function (table) {
      table.increments('id').primary()
      table.string('name').unique()
      table.string('desc')
      table.jsonb('permissions')
    })
    .createTable('files', function (table) {
      table.increments('id').primary()
      table.string('title')
      table.text('desc')
      table.string('file_name')
      table.string('file_type')
      table.string('file_size')
      table.string('file_path')
      table.timestamps(true, true)
    })
    .createTable('user_accounts', function (table) {
      table.increments('id').primary()
      table.integer('user_id')
      table.string('provider')
      table.string('account_id')
      table.string('access_token')
      table.string('refresh_token')
      table.jsonb('profile')
      table.timestamp('expires_at')
      table.timestamps(true, true)
    })
    .createTable('settings', function (table) {
      table.string('id').primary()
      table.string('name')
      table.text('desc')
      table.json('settings')
    })
    .createTable('posts', function (table) {
      table.increments('id').primary()
      table.string('title')
      table.text('contents')
      table.integer('user_id')
      table.timestamps(true, true)
    })
    .createTable('comments', function (table) {
      table.increments('id').primary()
      table.string('title')
      table.text('contents')
      table.integer('user_id')
      table.integer('post_id')
      table.timestamps(true, true)
    })
}

exports.down = function (knex, Promise) {
  return knex.schema
    .dropTable('users')
    .dropTable('roles')
    .dropTable('user2role')
    .dropTable('user_accounts')
    .dropTable('oauth_clients')
    .dropTable('oauth_authorization_codes')
    .dropTable('oauth_tokens')
    .dropTable('files')
    .dropTable('settings')
    .dropTable('posts')
    .dropTable('comments')
}
<|endoftext|>


<commit_before>/*
 * grunt-includes
 * https://github.com/vanetix/grunt-includes
 *
 * Copyright (c) 2013 Matt McFarland
 * Licensed under the MIT license.
 */

module.exports = function(grunt) {

  /**
   * Dependencies
   */

  var path = require('path');

  /**
   * Regex for parsing includes
   */

  var defaultRegexp = /^(\s*)include\s+"(\S+)"\s*$/;

  /**
   * Regex for matching new lines
   */

  var newlineRegexp = /\r?\n/g;

  /**
   * Core `grunt-includes` task
   * Iterates over all source files and calls `recurse(path)` on each
   */

  grunt.registerMultiTask('includes', 'Include other files within files.', function() {
    var banner;

    /**
     * Default options
     */

    var opts = this.options({
      debug: false,
      banner: '',
      silent: false,
      duplicates: true,
      includeRegexp: defaultRegexp,
      includePath: '',
      filenamePrefix: '',
      filenameSuffix: ''
    });

    // Render banner
    banner = grunt.template.process(opts.banner);

    this.files.forEach(function(f) {
      var src, cwd = f.cwd;

      src = f.src.filter(function(p) {
        if(cwd) {
          p = path.join(f.cwd, p);
        }

        if(grunt.file.isFile(p)) {
          return true;
        } else {
          grunt.log.warn('Source file "' + p + '" not found.');
          return false;
        }
      });

      if(src.length > 1 && isFilename(f.dest)) {
        grunt.log.warn('Source file cannot be more than one when dest is a file.');
      }

      src.forEach(function(p) {
        var fileName = f.flatten ? path.basename(p) : p;
        var outFile = isFilename(f.dest) ? f.dest : path.join(f.dest, fileName);

        if(cwd) {
          p = path.join(cwd, p);
        }

        grunt.file.write(outFile, banner + recurse(p, opts));

        if(!opts.silent) {
          grunt.log.oklns('Saved ' + outFile);
        }
      });

    });
  });

  /**
   * Checks if `p` is a filepath, being it has an extension.
   *
   * @param {String} p
   * @return {Boolean}
   */

  function isFilename(p) {
    return !!path.extname(p);
  }

  /**
   * Returns the comment style for file `p`
   *
   * @param {String} p
   * @return {String}
   */

  function commentStyle(p) {
    var comments,
        ext = path.extname(p).slice(1);

    comments = {
      js: "/* %s */",
      css: "/* %s */",
      html: "<!-- %s -->"
    };

    return comments[ext] || '/* %s */';
  }

  /**
   * Returns the new line style for file `p`
   *
   * @param {String} p
   * @return {String}
   */

  function newlineStyle(p) {
    var matches = grunt.file.read(p).match(newlineRegexp);

    return (matches && matches[0]) || grunt.util.linefeed;
  }

  /**
   * Helper for `includes` builds all includes for `p`
   *
   * @param {String} p
   * @return {String}
   */

  function recurse(p, opts, included, indents) {
    var src, next, match, error, comment,
        newline, compiled, indent, fileLocation;

    if(!grunt.file.isFile(p)) {
      grunt.log.warn('Included file "' + p + '" not found.');
      return 'Error including "' + p + '".';
    }

    indents = indents || '';
    comment = commentStyle(p);
    newline = newlineStyle(p);
    included = included || [];

    /**
     * If `opts.duplicates` is false and file has been included,
     * error
     */

    if(!opts.duplicates && ~included.indexOf(p)) {
      error = 'Duplicate include: ' + p + ' skipping.';
      grunt.log.error(error);

      if(opts.debug) {
        return comment.replace(/%s/g, error);
      } else {
        return '';
      }
    }

    /**
     * At this point the file is considered included
     */

    included.push(p);

    /**
     * Split the file on newlines
     */

    src = grunt.file.read(p).split(newline);

    /**
     * Loop through the file calling `recurse` if an include is found
     */

    compiled = src.map(function(line) {
      match = line.match(opts.includeRegexp);

      /**
       * If the line has an include statement, recurse
       */

      if(match) {
        fileLocation = match[2];
        indent = match[1];

        if (!fileLocation) {
          fileLocation = indent;
          indent = '';
        }

        fileLocation = opts.filenamePrefix + fileLocation + opts.filenameSuffix;
        next = path.join((opts.includePath || path.dirname(p)), fileLocation);
        var oldLine = line;
        line = recurse(next, opts, included, indents + indent);

        /**
         * Include debug comments if `opts.debug`
         */

        if(opts.debug) {
          line = comment.replace(/%s/g, 'Begin: ' + next) +
                 newline + line + newline + comment.replace(/%s/g, 'End: ' + next);
        }
        line = oldLine.replace(opts.includeRegexp, line);
      }

      // If there are indents and not a match, add them to the line
      return line && indents && !match ? indents + line : line;
    });

    return  compiled.join(newline);
  }

};
<commit_msg>clean up new feature from @SAPikachu<commit_after>/*
 * grunt-includes
 * https://github.com/vanetix/grunt-includes
 *
 * Copyright (c) 2013 Matt McFarland
 * Licensed under the MIT license.
 */

module.exports = function(grunt) {

  /**
   * Dependencies
   */

  var path = require('path');

  /**
   * Regex for parsing includes
   */

  var defaultRegexp = /^(\s*)include\s+"(\S+)"\s*$/;

  /**
   * Regex for matching new lines
   */

  var newlineRegexp = /\r?\n/g;

  /**
   * Core `grunt-includes` task
   * Iterates over all source files and calls `recurse(path)` on each
   */

  grunt.registerMultiTask('includes', 'Include other files within files.', function() {
    var banner;

    /**
     * Default options
     */

    var opts = this.options({
      debug: false,
      banner: '',
      silent: false,
      duplicates: true,
      includeRegexp: defaultRegexp,
      includePath: '',
      filenamePrefix: '',
      filenameSuffix: ''
    });

    // Render banner
    banner = grunt.template.process(opts.banner);

    this.files.forEach(function(f) {
      var src, cwd = f.cwd;

      src = f.src.filter(function(p) {
        if(cwd) {
          p = path.join(f.cwd, p);
        }

        if(grunt.file.isFile(p)) {
          return true;
        } else {
          grunt.log.warn('Source file "' + p + '" not found.');
          return false;
        }
      });

      if(src.length > 1 && isFilename(f.dest)) {
        grunt.log.warn('Source file cannot be more than one when dest is a file.');
      }

      src.forEach(function(p) {
        var fileName = f.flatten ? path.basename(p) : p;
        var outFile = isFilename(f.dest) ? f.dest : path.join(f.dest, fileName);

        if(cwd) {
          p = path.join(cwd, p);
        }

        grunt.file.write(outFile, banner + recurse(p, opts));

        if(!opts.silent) {
          grunt.log.oklns('Saved ' + outFile);
        }
      });

    });
  });

  /**
   * Checks if `p` is a filepath, being it has an extension.
   *
   * @param {String} p
   * @return {Boolean}
   */

  function isFilename(p) {
    return !!path.extname(p);
  }

  /**
   * Returns the comment style for file `p`
   *
   * @param {String} p
   * @return {String}
   */

  function commentStyle(p) {
    var comments,
        ext = path.extname(p).slice(1);

    comments = {
      js: "/* %s */",
      css: "/* %s */",
      html: "<!-- %s -->"
    };

    return comments[ext] || '/* %s */';
  }

  /**
   * Returns the new line style for file `p`
   *
   * @param {String} p
   * @return {String}
   */

  function newlineStyle(p) {
    var matches = grunt.file.read(p).match(newlineRegexp);

    return (matches && matches[0]) || grunt.util.linefeed;
  }

  /**
   * Helper for `includes` builds all includes for `p`
   *
   * @param {String} p
   * @return {String}
   */

  function recurse(p, opts, included, indents) {
    var src, next, match, error, comment, content,
        newline, compiled, indent, fileLocation;

    if(!grunt.file.isFile(p)) {
      grunt.log.warn('Included file "' + p + '" not found.');
      return 'Error including "' + p + '".';
    }

    indents = indents || '';
    comment = commentStyle(p);
    newline = newlineStyle(p);
    included = included || [];

    /**
     * If `opts.duplicates` is false and file has been included,
     * error
     */

    if(!opts.duplicates && ~included.indexOf(p)) {
      error = 'Duplicate include: ' + p + ' skipping.';
      grunt.log.error(error);

      if(opts.debug) {
        return comment.replace(/%s/g, error);
      } else {
        return '';
      }
    }

    /**
     * At this point the file is considered included
     */

    included.push(p);

    /**
     * Split the file on newlines
     */

    src = grunt.file.read(p).split(newline);

    /**
     * Loop through the file calling `recurse` if an include is found
     */

    compiled = src.map(function(line) {
      match = line.match(opts.includeRegexp);

      /**
       * If the line has an include statement, recurse
       */

      if(match) {
        fileLocation = match[2];
        indent = match[1];

        if (!fileLocation) {
          fileLocation = indent;
          indent = '';
        }

        fileLocation = opts.filenamePrefix + fileLocation + opts.filenameSuffix;
        next = path.join((opts.includePath || path.dirname(p)), fileLocation);
        content = recurse(next, opts, included, indents + indent);
        line = line.replace(opts.includeRegexp, content);

        /**
         * Include debug comments if `opts.debug`
         */

        if(opts.debug) {
          line = comment.replace(/%s/g, 'Begin: ' + next) +
                 newline + line + newline + comment.replace(/%s/g, 'End: ' + next);
        }
      }

      // If there are indents and not a match, add them to the line
      return line && indents && !match ? indents + line : line;
    });

    return  compiled.join(newline);
  }
};<|endoftext|>


<commit_before>/*
 * grunt-injector
 * https://github.com/klei-dev/grunt-injector
 *
 * Copyright (c) 2013 Joakim Bengtson
 * Licensed under the MIT license.
 */

'use strict';

var path = require('path'),
    fs = require('fs'),
    _ = require('lodash'),
    ext = function (file) {
      return path.extname(file).slice(1);
    };

module.exports = function(grunt) {

  grunt.registerMultiTask('injector', 'Inject references to files into other files (think scripts and stylesheets into an html file)', function() {
    // Merge task-specific and/or target-specific options with these defaults.
    var options = this.options({
      min: false,
      template: null,
      bowerPrefix: null,
      relative: false,
      addRootSlash: (function (that) {
        var addRootSlash = true;
        if (that.data.options) {
          addRootSlash = that.data.options.addRootSlash || !that.data.options.relative;
        }
        return addRootSlash;
      })(this),
      starttag: '<!-- injector:{{ext}} -->',
      endtag: '<!-- endinjector -->',
      lineEnding: '\n',
      transform: function (filepath) {
        var e = ext(filepath);
        if (e === 'css') {
          return '<link rel="stylesheet" href="' + filepath + '">';
        } else if (e === 'js') {
          return '<script src="' + filepath + '"></script>';
        } else if (e === 'html') {
          return '<link rel="import" href="' + filepath + '">';
        }
      }
    });

    if (!options.template && !options.templateString) {
      grunt.log.writeln('Missing option `template`, using `dest` as template instead'.grey);
    }

    var filesToInject = {};

    // Iterate over all specified file groups and gather files to inject:

    this.files.forEach(function(f) {
      var template = options.templateString || options.template || options.destFile || f.dest,
          destination = options.destFile || f.dest;

      if (!options.templateString && !grunt.file.exists(template)) {
        grunt.log.error('Could not find template "' + template + '". Injection not possible');
        return false;
      }

      // Group by destination:
      if (!filesToInject[destination]) {
        filesToInject[destination] = {};
      }

      // ...and template:
      if (!filesToInject[destination][template]) {
        filesToInject[destination][template] = [];
      }

      var files = filesToInject[destination][template];

      f.src.forEach(function(filepath) {
        // Warn on and remove invalid source files.
        if (!grunt.file.exists(filepath)) {
          grunt.log.warn('Source file "' + filepath + '" not found.');
          return;
        }

        // Special handling of bower.json:
        if (path.basename(filepath) === 'bower.json') {
          files.push.apply(files, getFilesFromBower(filepath).map(function (fpath) {
            return {path: fpath, ignore: path.dirname(filepath), key: (options.bowerPrefix || '') + ext(fpath)};
          }));
        } else {
          files.push({path: filepath, key: ext(filepath)});
        }
      });

      // Clear existing content between injectors
      var templateContent = options.templateString || grunt.file.read(template),
        templateOriginal = templateContent;

      var re = getInjectorTagsRegExp(options.starttag, options.endtag);
      templateContent = templateContent.replace(re, function (match, indent, starttag, content, endtag) {
        return indent + starttag + options.lineEnding + indent + endtag;
      });

      if (templateContent !== templateOriginal || !grunt.file.exists(destination)) {
        grunt.file.write(destination, templateContent);
      }
    });

    /**
     * Inject all gathered files per destination, template and starttag:
     */
    _.forIn(filesToInject, function (templates, destination) {
      _.forIn(templates, function (files, template) {
        // Remove possible duplicates:
        files = _.uniq(files);

        files.forEach(function (obj) {
          // Get start and end tag for each file:
          obj.starttag = getTag(options.starttag, obj.key);
          obj.endtag = getTag(options.endtag, obj.key);

          // Fix filename (remove ignorepaths and such):
          var file = obj.path;
          if (options.relative) {
            var base =  path.dirname(destination);
            file = path.relative(base, file);
          }
          file = unixify(file);
          file = makeMinifiedIfNeeded(options.min, file);
          if (options.ignorePath || obj.ignore) {
            file = removeBasePath(toArray(options.ignorePath).concat(toArray(obj.ignore)), file);
          }
          if (options.addRootSlash) {
            file = addRootSlash(file);
          } else {
            file = removeRootSlash(file);
          }
          obj.file = file;
        });

        // Read template:
        var templateContent = options.templateString || grunt.file.read(template),
            templateOriginal = templateContent;

        // Inject per start tag:
        _.forIn(_.groupBy(files, 'starttag'), function (sources, starttag) {
          var endtag = sources[0].endtag,
              key = sources[0].key;

          // Transform to injection content:
          sources.forEach(function (obj, i) {
            obj.transformed = options.transform(obj.file, i, sources.length);
          });

          // Sort files if needed:
          if (typeof options.sort === 'function') {
            sources.sort(function (a, b) {
              return options.sort(a.file, b.file);
            });
          }

          // Do the injection:
          var re = getInjectorTagsRegExp(starttag, endtag);
          templateContent = templateContent.replace(re, function (match, indent, starttag, content, endtag) {
            grunt.log.writeln('Injecting ' + key.green + ' files ' + ('(' + sources.length + ' files)').grey);
            return indent + starttag + getIndentedTransformations(sources, indent, options.lineEnding) + endtag;
          });
        });

        // Write the destination file.
        if (templateContent !== templateOriginal || !grunt.file.exists(destination)) {
          grunt.file.write(destination, templateContent);
        } else {
          grunt.log.ok('Nothing changed');
        }
      });
    });

  });
};

function getInjectorTagsRegExp (starttag, endtag) {
  return new RegExp('([\t ]*)(' + escapeForRegExp(starttag) + ')(\\n|\\r|.)*?(' + escapeForRegExp(endtag) + ')', 'gi');
}

function getTag (tag, ext) {
  return tag.replace(new RegExp( escapeForRegExp('{{ext}}'), 'g'), ext);
}

function getFilesFromBower (bowerFile) {

  // Load bower dependencies via `wiredep` programmatic access
  var dependencies = require('wiredep')({
        'bowerJson': JSON.parse(fs.readFileSync(bowerFile, 'utf8')),
        'directory': getBowerComponentsDir(bowerFile)
      }
    );

  // Pluck out just the JS and CSS Dependencies
  var filteredDependencies = _.pick(dependencies,'css','js');

  // Concatenate into a filepaths array
  return Object.keys(filteredDependencies).reduce(function (files, key) {
       return files.concat(filteredDependencies[key]);
    }, []);
}

function getBowerComponentsDir (bowerFile) {
  var bowerBaseDir = path.dirname(bowerFile),
      bowerRcFile = path.join(bowerBaseDir, '.bowerrc'),
      dir = 'bower_components';

  if (fs.existsSync(bowerRcFile)) {
    try {
      dir = JSON.parse(fs.readFileSync(bowerRcFile, 'utf8')).directory;
    } catch (e) {
    }
  }
  return path.join(bowerBaseDir, dir);
}

function unixify (path) {
  return path.replace(/\\/g, '/');
}

function makeMinifiedIfNeeded (doMinify, filepath) {
  if (!doMinify) {
    return filepath;
  }
  var ext = path.extname(filepath);
  var minFile = filepath.slice(0, -ext.length) + '.min' + ext;
  if (fs.existsSync(minFile)) {
    return minFile;
  }
  return filepath;
}

function toArray (arr) {
  if (!Array.isArray(arr)) {
    return arr ? [arr] : [];
  }
  return arr;
}

function addRootSlash (filepath) {
  return filepath.replace(/^\/*([^\/])/, '/$1');
}
function removeRootSlash (filepath) {
  return filepath.replace(/^\/+/, '');
}

function removeBasePath (basedir, filepath) {
  return toArray(basedir).reduce(function (path, remove) {
    if (remove && path.indexOf(remove) === 0) {
      return path.slice(remove.length);
    } else {
      return path;
    }
  }, filepath);
}

function escapeForRegExp (str) {
  return str.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
}

function getIndentedTransformations (sources, indent, lineEnding) {
  var transformations = sources.map(function (s) {
    return s.transformed;
  });
  transformations.unshift('');
  transformations.push('');
  return transformations.join(lineEnding + indent);
}

<commit_msg>Check for minified file before making file relative<commit_after>/*
 * grunt-injector
 * https://github.com/klei-dev/grunt-injector
 *
 * Copyright (c) 2013 Joakim Bengtson
 * Licensed under the MIT license.
 */

'use strict';

var path = require('path'),
    fs = require('fs'),
    _ = require('lodash'),
    ext = function (file) {
      return path.extname(file).slice(1);
    };

module.exports = function(grunt) {

  grunt.registerMultiTask('injector', 'Inject references to files into other files (think scripts and stylesheets into an html file)', function() {
    // Merge task-specific and/or target-specific options with these defaults.
    var options = this.options({
      min: false,
      template: null,
      bowerPrefix: null,
      relative: false,
      addRootSlash: (function (that) {
        var addRootSlash = true;
        if (that.data.options) {
          addRootSlash = that.data.options.addRootSlash || !that.data.options.relative;
        }
        return addRootSlash;
      })(this),
      starttag: '<!-- injector:{{ext}} -->',
      endtag: '<!-- endinjector -->',
      lineEnding: '\n',
      transform: function (filepath) {
        var e = ext(filepath);
        if (e === 'css') {
          return '<link rel="stylesheet" href="' + filepath + '">';
        } else if (e === 'js') {
          return '<script src="' + filepath + '"></script>';
        } else if (e === 'html') {
          return '<link rel="import" href="' + filepath + '">';
        }
      }
    });

    if (!options.template && !options.templateString) {
      grunt.log.writeln('Missing option `template`, using `dest` as template instead'.grey);
    }

    var filesToInject = {};

    // Iterate over all specified file groups and gather files to inject:

    this.files.forEach(function(f) {
      var template = options.templateString || options.template || options.destFile || f.dest,
          destination = options.destFile || f.dest;

      if (!options.templateString && !grunt.file.exists(template)) {
        grunt.log.error('Could not find template "' + template + '". Injection not possible');
        return false;
      }

      // Group by destination:
      if (!filesToInject[destination]) {
        filesToInject[destination] = {};
      }

      // ...and template:
      if (!filesToInject[destination][template]) {
        filesToInject[destination][template] = [];
      }

      var files = filesToInject[destination][template];

      f.src.forEach(function(filepath) {
        // Warn on and remove invalid source files.
        if (!grunt.file.exists(filepath)) {
          grunt.log.warn('Source file "' + filepath + '" not found.');
          return;
        }

        // Special handling of bower.json:
        if (path.basename(filepath) === 'bower.json') {
          files.push.apply(files, getFilesFromBower(filepath).map(function (fpath) {
            return {path: fpath, ignore: path.dirname(filepath), key: (options.bowerPrefix || '') + ext(fpath)};
          }));
        } else {
          files.push({path: filepath, key: ext(filepath)});
        }
      });

      // Clear existing content between injectors
      var templateContent = options.templateString || grunt.file.read(template),
        templateOriginal = templateContent;

      var re = getInjectorTagsRegExp(options.starttag, options.endtag);
      templateContent = templateContent.replace(re, function (match, indent, starttag, content, endtag) {
        return indent + starttag + options.lineEnding + indent + endtag;
      });

      if (templateContent !== templateOriginal || !grunt.file.exists(destination)) {
        grunt.file.write(destination, templateContent);
      }
    });

    /**
     * Inject all gathered files per destination, template and starttag:
     */
    _.forIn(filesToInject, function (templates, destination) {
      _.forIn(templates, function (files, template) {
        // Remove possible duplicates:
        files = _.uniq(files);

        files.forEach(function (obj) {
          // Get start and end tag for each file:
          obj.starttag = getTag(options.starttag, obj.key);
          obj.endtag = getTag(options.endtag, obj.key);

          // Fix filename (remove ignorepaths and such):
          var file = obj.path;
          file = makeMinifiedIfNeeded(options.min, file);
          if (options.relative) {
            var base =  path.dirname(destination);
            file = path.relative(base, file);
          }
          file = unixify(file);
          if (options.ignorePath || obj.ignore) {
            file = removeBasePath(toArray(options.ignorePath).concat(toArray(obj.ignore)), file);
          }
          if (options.addRootSlash) {
            file = addRootSlash(file);
          } else {
            file = removeRootSlash(file);
          }
          obj.file = file;
        });

        // Read template:
        var templateContent = options.templateString || grunt.file.read(template),
            templateOriginal = templateContent;

        // Inject per start tag:
        _.forIn(_.groupBy(files, 'starttag'), function (sources, starttag) {
          var endtag = sources[0].endtag,
              key = sources[0].key;

          // Transform to injection content:
          sources.forEach(function (obj, i) {
            obj.transformed = options.transform(obj.file, i, sources.length);
          });

          // Sort files if needed:
          if (typeof options.sort === 'function') {
            sources.sort(function (a, b) {
              return options.sort(a.file, b.file);
            });
          }

          // Do the injection:
          var re = getInjectorTagsRegExp(starttag, endtag);
          templateContent = templateContent.replace(re, function (match, indent, starttag, content, endtag) {
            grunt.log.writeln('Injecting ' + key.green + ' files ' + ('(' + sources.length + ' files)').grey);
            return indent + starttag + getIndentedTransformations(sources, indent, options.lineEnding) + endtag;
          });
        });

        // Write the destination file.
        if (templateContent !== templateOriginal || !grunt.file.exists(destination)) {
          grunt.file.write(destination, templateContent);
        } else {
          grunt.log.ok('Nothing changed');
        }
      });
    });

  });
};

function getInjectorTagsRegExp (starttag, endtag) {
  return new RegExp('([\t ]*)(' + escapeForRegExp(starttag) + ')(\\n|\\r|.)*?(' + escapeForRegExp(endtag) + ')', 'gi');
}

function getTag (tag, ext) {
  return tag.replace(new RegExp( escapeForRegExp('{{ext}}'), 'g'), ext);
}

function getFilesFromBower (bowerFile) {

  // Load bower dependencies via `wiredep` programmatic access
  var dependencies = require('wiredep')({
        'bowerJson': JSON.parse(fs.readFileSync(bowerFile, 'utf8')),
        'directory': getBowerComponentsDir(bowerFile)
      }
    );

  // Pluck out just the JS and CSS Dependencies
  var filteredDependencies = _.pick(dependencies,'css','js');

  // Concatenate into a filepaths array
  return Object.keys(filteredDependencies).reduce(function (files, key) {
       return files.concat(filteredDependencies[key]);
    }, []);
}

function getBowerComponentsDir (bowerFile) {
  var bowerBaseDir = path.dirname(bowerFile),
      bowerRcFile = path.join(bowerBaseDir, '.bowerrc'),
      dir = 'bower_components';

  if (fs.existsSync(bowerRcFile)) {
    try {
      dir = JSON.parse(fs.readFileSync(bowerRcFile, 'utf8')).directory;
    } catch (e) {
    }
  }
  return path.join(bowerBaseDir, dir);
}

function unixify (path) {
  return path.replace(/\\/g, '/');
}

function makeMinifiedIfNeeded (doMinify, filepath) {
  if (!doMinify) {
    return filepath;
  }
  var ext = path.extname(filepath);
  var minFile = filepath.slice(0, -ext.length) + '.min' + ext;
  if (fs.existsSync(minFile)) {
    return minFile;
  }
  return filepath;
}

function toArray (arr) {
  if (!Array.isArray(arr)) {
    return arr ? [arr] : [];
  }
  return arr;
}

function addRootSlash (filepath) {
  return filepath.replace(/^\/*([^\/])/, '/$1');
}
function removeRootSlash (filepath) {
  return filepath.replace(/^\/+/, '');
}

function removeBasePath (basedir, filepath) {
  return toArray(basedir).reduce(function (path, remove) {
    if (remove && path.indexOf(remove) === 0) {
      return path.slice(remove.length);
    } else {
      return path;
    }
  }, filepath);
}

function escapeForRegExp (str) {
  return str.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
}

function getIndentedTransformations (sources, indent, lineEnding) {
  var transformations = sources.map(function (s) {
    return s.transformed;
  });
  transformations.unshift('');
  transformations.push('');
  return transformations.join(lineEnding + indent);
}

<|endoftext|>


<commit_before>var _ = require('lodash'),
    Environment = require('./Environment'),
    debugLib = require('debug'),
    debug = debugLib('PhantomEnvironment'),
    phantomError = debugLib('Phantom:error'),
    debugParser = debugLib('RedParser'),
    phantom = require('phantom'),
    path = require('path'),
    mkdir = require('mkdir-p'),
    url = require('url'),
    vowNode = require('vow-node'),
    vow = require('vow');

/**
 * @typedef {object} Proxy
 * @property {string} host
 * @property {number} port
 * @property {?string} username
 * @property {?string} password
 */

/**
 * @typedef {object} Resources
 * @property {?Array.<string>} allowed Only `allowed` resources will be loaded. Have higher priority than `denied`
 * @property {?Array.<string>} denied All except `denied` resources will be loaded
 */

/**
 * @typedef {object} Screen
 * @property {number} width
 * @property {number} height
 */

/**
 * @typedef {object} PhantomEnvironmentOptions
 * @property {?number} timeout
 * @property {?boolean} weak
 * @property {?boolean} loadImages
 * @property {?boolean} ignoreSslErrors
 * @property {?string} sslProtocol
 * @property {?string} cookiesFile
 * @property {?boolean} webSecurity
 * @property {?string} phantomPath
 *
 * @property {?string} snapshot perform snapshot during parsing
 * @property {?string} snapshotDir directory for snapshots
 * @property {?Proxy|Array.<Proxy>} proxy single proxy or proxy list
 * @property {?function} proxyRotator proxy rotator function(proxyList, currentProxy) with context of this env. function should return Proxy from the list
 * @property {?string|Array.<string>} userAgent user agent or list of agents for setting to phantom
 * @property {?Screen} screen screen dimensions
 * @property {?Resources} resources white and black lists for loading resources on the page
 */
var defaultOptions = {
    // Phantom options
    timeout: 60000,
    weak: true,
    loadImages: false,
    ignoreSslErrors: true,
    sslProtocol: 'any',
    cookiesFile: null,
    webSecurity: true,
    phantomPath: null,

    // Custom environment options
    snapshot: false,
    snapshotDir: 'snapshots',
    proxy: null,
    proxyRotator: null,
    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12',
    screen: {
        width: 1080,
        height: 768
    },
    resources: {
        allowed: null,
        denied: null
    }
};

var port = 13200;

/**
 * @param {PhantomEnvironmentOptions} options
 * @constructor
 */
function PhantomEnvironment(options) {
    debug('Initializing...');
    Environment.call(this, options);

    this._options = _.defaults(_.clone(options) || {}, defaultOptions);
    this._proxyCurrent = null;
    this._url = options.url;

    if (!this._url) {
        throw new Error('You must pass `url` to PhantomEnvironment');
    }

    this._phantomJS = null;
    this._page = null;
    this._navigationActions = [];
}

PhantomEnvironment.prototype = _.create(Environment.prototype, /**@lends PhantomEnvironment*/{
    prepare: function () {
        debug('Preparing...');
        return Environment.prototype.prepare
            .call(this)
            .then(this._setup, this)
            .then(this._setViewport, this)
            .then(this._setUserAgent, this)
            .then(this._handlePhantomEvents, this)
            .then(this._rotateProxy, this)
            .then(this._navigateTo.bind(this, this._url))
            .then(this._inject, this);
    },

    evaluateJs: function () {
        var deferred = vow.defer(),
            page = this._page,
            args = Array.prototype.slice.call(arguments, 0);

        var evalFunc = args.pop();
        if (typeof evalFunc !== 'function') {
            throw new Error('You must pass function as last argument to PhantomEnvironment.evaluateJs');
        }
        args.unshift(evalFunc, deferred.resolve.bind(deferred));

        page.evaluate.apply(page, args);
        return deferred.promise();
    },

    /**
     * Take screen snapshot
     * @param {string} fileName
     * @returns {Promise}
     */
    snapshot: function (fileName) {
        var options = this._options;
        if (!options.snapshot) {
            return vow.resolve();
        }
        debug('.snapshot()');

        var screenShotFilePath = path.join(options.snapshotDir, this._getHostName(this._url));
        var screenShotFileName = path.join(screenShotFilePath, fileName + '.png');
        return vowNode
            .invoke(mkdir, screenShotFilePath)
            .then(function () {
                this._page.clipRect = {
                    left: 0,
                    top: 0,
                    width: options.screen.width,
                    height: options.screen.height
                };
                this._page.render(screenShotFileName);
            }, this);
    },

    waitForPage: function (timeout) {
        timeout = timeout || 5000;

        var deferred = vow.defer();
        var timeoutId = setTimeout(function () {
            debug('Timeout %s has reached on page load', timeout);
            this._navigationActions = [];
            deferred.reject(new Error('Page navigation timeout'));
        }.bind(this), timeout);

        this._navigationActions.push(function (err) {
            clearTimeout(timeoutId);
            if (err) {
                deferred.reject(err);
            } else {
                deferred.resolve();
            }
        }.bind(this));
        debug('Added page load callback');

        return deferred.promise().then(this._inject, this);
    },

    /**
     * Set up a fresh phantomjs page.
     * @returns {Promise}
     * @private
     */
    _setup: function () {
        return this._createInstance().then(this._createPage, this);
    },

    /**
     * Create a phantomjs instance.
     * @returns {Promise}
     * @private
     */
    _createInstance: function () {
        var options = this._options,
            deferred = vow.defer(),
            flags = [];
        debug('.createInstance() creating Phantom instance with options %o', options);
        flags.push('--load-images=' + options.loadImages);
        flags.push('--ignore-ssl-errors=' + options.ignoreSslErrors);
        flags.push('--ssl-protocol=' + options.sslProtocol);
        flags.push('--web-security=' + options.webSecurity);
        if (options.cookiesFile !== null) {
            flags.push('--cookies-file=' + options.cookiesFile);
        }

        // dnode options for compilation on windows
        var dnodeOpts = {};
        if (options.weak === false) {
            dnodeOpts = {weak: false};
        }

        // combine flags, options and callback into args
        var args = flags;
        args.push({
            port: options.port || this._getPort(),
            dnodeOpts: dnodeOpts,
            path: options.phantomPath,
            onExit: this._handleCrash.bind(this)
        });
        var self = this;
        args.push(function (instance) {
            self._phantomJS = instance;
            deferred.resolve(instance);
        });
        phantom.create.apply(phantom, args);
        return deferred.promise();
    },

    _getPort: function () {
        port++;
        return port;
    },

    /**
     * Creates new page in phantom
     * @returns {Promise}
     */
    _createPage: function () {
        debug('._createPage() has called');
        var deferred = vow.defer();
        this._phantomJS.createPage(function (page) {
            this._page = page;
            debug('._createPage() phantom page created');
            deferred.resolve(page);
        }.bind(this));
        return deferred.promise();
    },

    /**
     * Tear down a phantomjs instance.
     * @private
     */
    tearDown: function () {
        this._phantomJS.exit(0);
        debug('._tearDownInstance() tearing down');
    },

    /**
     * Handles the phantom process ending/crashing unexpectedly.
     * If an `onExit` handler has been bound then that will be called. Otherwise, the error will be re-thrown.
     * @param {Number} code
     * @param {String} [signal]
     */
    _handleCrash: function (code, signal) {
        // if a handler is defined, call it
        if (this._onExit) {
            this._onExit(code, signal);

            // otherwise, if we have a non-zero code we'll throw a better error message
            // than the `phantom` lib would.
        } else if (code !== 0) {
            var err = new Error('The PhantomJS process ended unexpectedly');
            err.code = code;
            err.signal = signal;
            throw err;
        }
    },

    /**
     * Go to url
     * @param url
     * @returns {Promise}
     * @private
     */
    _navigateTo: function (url) {
        var deferred = vow.defer();
        debug('.goto() url: ' + url);
        var self = this;
        this._page.open(url, function (status) {
            debug('.goto() page loaded: ' + status);
            if (status === 'success') {
                return deferred.resolve();
            }

            self._rotateProxy()
                .then(function (proxy) {
                    // cannot set new proxy
                    if (proxy === null) {
                        return deferred.reject(new Error('Page was not loaded'));
                    }

                    // one more attempt to open page through the new proxy
                    return this._navigateTo(url);
                }, self);
        });

        return deferred.promise();
    },

    /**
     * Set the viewport.
     *
     * @returns {Promise}
     * @private
     */
    _setViewport: function () {
        var deferred = vow.defer();
        var screen = this._options.screen;
        if (Array.isArray(screen)) {
            screen = _.sample(screen);
        }
        var width = screen.width;
        var height = screen.height;
        debug('.viewport() to ' + width + ' x ' + height);
        var viewport = {width: width, height: height};
        this._page.set('viewportSize', viewport, function () {
            deferred.resolve();
        });

        return deferred.promise();
    },

    /**
     * Set the user agent.
     *
     * @returns {Promise}
     * @private
     */
    _setUserAgent: function () {
        var deferred = vow.defer();
        var userAgent = this._options.userAgent;
        if (Array.isArray(userAgent)) {
            userAgent = _.sample(this._options.userAgent);
        }
        debug('.userAgent() to ' + userAgent);
        this._page.set('settings.userAgent', userAgent, function () {
            deferred.resolve();
        });

        return deferred.promise();
    },

    /**
     * Set a proxy from the proxy list (unset previous one)
     *
     * @returns {Promise}
     * @private
     */
    _rotateProxy: function () {
        var deferred = vow.defer();
        var proxy = this._options.proxy;
        var currentProxy = this._proxyCurrent;
        if (proxy == undefined) {
            deferred.resolve(null);
            return deferred.promise();
        }
        if (Array.isArray(proxy)) {
            this._removeUnavailableProxy();
            proxy = typeof this._options.proxyRotator === 'function' ?
                this._options.proxyRotator.call(this, proxy, currentProxy) :
                _.sample(proxy);

            if (proxy == undefined) {
                throw new Error('No proxy found');
            }
        }
        debug('._rotateProxy() to %o', proxy);
        var self = this;

        this._phantomJS.setProxy(proxy.host, proxy.port, 'manual', proxy.username, proxy.password, function () {
            self._proxyCurrent = proxy;
            deferred.resolve(proxy);
        });

        return deferred.promise();
    },

    /**
     * Remove from proxy list one which doesn't work
     *
     * @returns {?Proxy}
     * @private
     */
    _removeUnavailableProxy: function () {
        var current = this._proxyCurrent;
        if (!Array.isArray(this._options.proxy) || this._options.proxy.length === 0 || current === null) {
            return null;
        }

        debug('._removeUnavailableProxy()');
        var index = this._options.proxy.findIndex(function (item) {
            return item.host === current.host && item.port === current.port;
        });
        var proxy = null;
        if (index !== -1) {
            // cut off old used proxy from the list
            proxy = this._options.proxy.splice(index, 1);
        }
        return Array.isArray(proxy) ? proxy.pop() : null;
    },

    /**
     * Inject libs which are required for parse process
     *
     * @private
     */
    _inject: function () {
        debug('.inject()-ing parser libs');
        var files = [
            'vendor/sizzle.min.js'
        ];
        files.forEach(function (file) {
            debug('injecting file %s', file);
            this._page.injectJs(path.join(__dirname, '..', file));
        }, this);
        return vow.resolve();
    },

    /**
     * Attach on phantom event
     * @param type
     * @param callback
     * @param {object} context
     * @returns {Promise}
     * @private
     */
    _on: function (type, callback, context) {
        var deferred = vow.defer();

        if (context) {
            callback = callback.bind(context);
        }

        if (type === 'timeout') {
            deferred.resolve();
        }
        // The onExit callback is special-cased here too
        else if (type === 'exit') {
            this._onExit = callback;
            deferred.resolve();
        }
        // resourceRequestStarted has a special function...
        else if (type === 'resourceRequested') {
            var args = [].slice.call(arguments);
            args = args.slice(3);
            this._page.onResourceRequested(callback, /* @covignore */ function (requestData) {
                debug('Resource requested %s', requestData.url);
            }, args[0], args[1]);
            deferred.resolve();
        }
        // All other events handled natively in phantomjs
        else {
            var pageEvent = 'on' + type.charAt(0).toUpperCase() + type.slice(1);
            this._page.set(pageEvent, callback, function () {
                deferred.resolve();
            });
        }

        return deferred.promise();
    },

    _handlePhantomEvents: function () {
        return vow.all([
            this._on('timeout', function (msg) {
                debug(msg);
                this.tearDown();
            }, this),
            this._on('exit', function (code, signal) {
                debug('Phantom exited with code ' + code + ' and signal ' + signal);
            }),
            this._on('error', function (msg, trace) {
                phantomError('%s, trace %o', msg, trace);
            }),
            this._on('consoleMessage', function (msg) {
                var regex = /^(\[GooseParser])(.+)/i;
                var found = msg.match(regex);
                if (found !== null && found[2] !== undefined) {
                    debugParser(found[2].trim());
                } else {
                    debug('Phantom page message: ' + msg);
                }
            }),
            this._on('navigationRequested', function (url) {
                debug('Navigation to %s', url);
            }),
            this._on('resourceRequested', /* @covignore */ function (requestData, request, allowedUrls, blockedUrls) {
                var url = requestData.url;
                var hasAllowedUrls = Array.isArray(allowedUrls) && allowedUrls.length > 0;
                var hasBlockedUrls = Array.isArray(blockedUrls) && blockedUrls.length > 0;
                var allowed = !hasAllowedUrls || allowedUrls.some(function (urlPattern) {
                        return url.match(urlPattern) !== null
                    });

                var blocked = false;
                if (!hasAllowedUrls && hasBlockedUrls) {
                    blocked = blockedUrls.some(function (urlPattern) {
                        return url.match(urlPattern) !== null;
                    });
                }
                if (!allowed || blocked) {
                    console.log('Resource ' + requestData.url + ' was aborted');
                    request.abort();
                }
            }, null, this._options.resources.allowed, this._options.resources.denied),
            this._on('loadFinished', function (status) {
                debug('Page loaded with status %s, fire %s callbacks', status, this._navigationActions.length);
                var callback;
                while (callback = this._navigationActions.shift()) {
                    callback.call(this, status === 'success' ? null : new Error('Page is not loaded'));
                }
            }.bind(this))
        ]);
    },

    /**
     * @param {string} uri
     * @returns {string}
     * @private
     */
    _getHostName: function (uri) {
        var parsed = url.parse(uri);
        return parsed.hostname;
    }
});

module.exports = PhantomEnvironment;
<commit_msg>Fixed PR issues<commit_after>var _ = require('lodash'),
    Environment = require('./Environment'),
    debugLib = require('debug'),
    debug = debugLib('PhantomEnvironment'),
    phantomError = debugLib('Phantom:error'),
    debugParser = debugLib('RedParser'),
    phantom = require('phantom'),
    path = require('path'),
    mkdir = require('mkdir-p'),
    url = require('url'),
    vowNode = require('vow-node'),
    vow = require('vow');

/**
 * @typedef {object} Proxy
 * @property {string} host
 * @property {number} port
 * @property {?string} username
 * @property {?string} password
 */

/**
 * @typedef {object} Resources
 * @property {?Array.<string>} allowed Only `allowed` resources will be loaded. Have higher priority than `denied`
 * @property {?Array.<string>} denied All except `denied` resources will be loaded
 */

/**
 * @typedef {object} Screen
 * @property {number} width
 * @property {number} height
 */

/**
 * @typedef {object} PhantomEnvironmentOptions
 * @property {?number} timeout
 * @property {?boolean} weak
 * @property {?boolean} loadImages
 * @property {?boolean} ignoreSslErrors
 * @property {?string} sslProtocol
 * @property {?string} cookiesFile
 * @property {?boolean} webSecurity
 * @property {?string} phantomPath
 *
 * @property {?string} snapshot perform snapshot during parsing
 * @property {?string} snapshotDir directory for snapshots
 * @property {?Proxy|Array.<Proxy>} proxy single proxy or proxy list
 * @property {?function} proxyRotator proxy rotator function(proxyList, currentProxy) with context of this env. function should return Proxy from the list
 * @property {?string|Array.<string>} userAgent user agent or list of agents for setting to phantom
 * @property {?Screen} screen screen dimensions
 * @property {?Resources} resources white and black lists for loading resources on the page
 */
var defaultOptions = {
    // Phantom options
    timeout: 60000,
    weak: true,
    loadImages: false,
    ignoreSslErrors: true,
    sslProtocol: 'any',
    cookiesFile: null,
    webSecurity: true,
    phantomPath: null,

    // Custom environment options
    snapshot: false,
    snapshotDir: 'snapshots',
    proxy: null,
    proxyRotator: null,
    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/8.0.7 Safari/600.7.12',
    screen: {
        width: 1080,
        height: 768
    },
    resources: {
        allowed: null,
        denied: null
    }
};

var port = 13200;

/**
 * @param {PhantomEnvironmentOptions} options
 * @constructor
 */
function PhantomEnvironment(options) {
    debug('Initializing...');
    Environment.call(this, options);

    this._options = _.defaults(_.clone(options) || {}, defaultOptions);
    this._proxyCurrent = null;
    this._url = options.url;

    if (!this._url) {
        throw new Error('You must pass `url` to PhantomEnvironment');
    }

    this._phantomJS = null;
    this._page = null;
    this._navigationActions = [];
}

PhantomEnvironment.prototype = _.create(Environment.prototype, /**@lends PhantomEnvironment*/{
    prepare: function () {
        debug('Preparing...');
        return Environment.prototype.prepare
            .call(this)
            .then(this._setup, this)
            .then(this._setViewport, this)
            .then(this._setUserAgent, this)
            .then(this._handlePhantomEvents, this)
            .then(this._rotateProxy, this)
            .then(this._navigateTo.bind(this, this._url))
            .then(this._inject, this);
    },

    evaluateJs: function () {
        var deferred = vow.defer(),
            page = this._page,
            args = Array.prototype.slice.call(arguments, 0);

        var evalFunc = args.pop();
        if (typeof evalFunc !== 'function') {
            throw new Error('You must pass function as last argument to PhantomEnvironment.evaluateJs');
        }
        args.unshift(evalFunc, deferred.resolve.bind(deferred));

        page.evaluate.apply(page, args);
        return deferred.promise();
    },

    /**
     * Take screen snapshot
     * @param {string} fileName
     * @returns {Promise}
     */
    snapshot: function (fileName) {
        var options = this._options;
        if (!options.snapshot) {
            return vow.resolve();
        }
        debug('.snapshot()');

        var screenShotFilePath = path.join(options.snapshotDir, this._getHostName(this._url));
        var screenShotFileName = path.join(screenShotFilePath, fileName + '.png');
        return vowNode
            .invoke(mkdir, screenShotFilePath)
            .then(function () {
                this._page.clipRect = {
                    left: 0,
                    top: 0,
                    width: options.screen.width,
                    height: options.screen.height
                };
                this._page.render(screenShotFileName);
            }, this);
    },

    waitForPage: function (timeout) {
        timeout = timeout || 5000;

        var deferred = vow.defer();
        var timeoutId = setTimeout(function () {
            debug('Timeout %s has reached on page load', timeout);
            this._navigationActions = [];
            deferred.reject(new Error('Page navigation timeout'));
        }.bind(this), timeout);

        this._navigationActions.push(function (err) {
            clearTimeout(timeoutId);
            if (err) {
                deferred.reject(err);
            } else {
                deferred.resolve();
            }
        }.bind(this));
        debug('Added page load callback');

        return deferred.promise().then(this._inject, this);
    },

    /**
     * Set up a fresh phantomjs page.
     * @returns {Promise}
     * @private
     */
    _setup: function () {
        return this._createInstance().then(this._createPage, this);
    },

    /**
     * Create a phantomjs instance.
     * @returns {Promise}
     * @private
     */
    _createInstance: function () {
        var options = this._options,
            deferred = vow.defer(),
            flags = [];
        debug('.createInstance() creating Phantom instance with options %o', options);
        flags.push('--load-images=' + options.loadImages);
        flags.push('--ignore-ssl-errors=' + options.ignoreSslErrors);
        flags.push('--ssl-protocol=' + options.sslProtocol);
        flags.push('--web-security=' + options.webSecurity);
        if (options.cookiesFile !== null) {
            flags.push('--cookies-file=' + options.cookiesFile);
        }

        // dnode options for compilation on windows
        var dnodeOpts = {};
        if (options.weak === false) {
            dnodeOpts = {weak: false};
        }

        // combine flags, options and callback into args
        var args = flags;
        args.push({
            port: options.port || this._getPort(),
            dnodeOpts: dnodeOpts,
            path: options.phantomPath,
            onExit: this._handleCrash.bind(this)
        });
        var self = this;
        args.push(function (instance) {
            self._phantomJS = instance;
            deferred.resolve(instance);
        });
        phantom.create.apply(phantom, args);
        return deferred.promise();
    },

    _getPort: function () {
        port++;
        return port;
    },

    /**
     * Creates new page in phantom
     * @returns {Promise}
     */
    _createPage: function () {
        debug('._createPage() has called');
        var deferred = vow.defer();
        this._phantomJS.createPage(function (page) {
            this._page = page;
            debug('._createPage() phantom page created');
            deferred.resolve(page);
        }.bind(this));
        return deferred.promise();
    },

    /**
     * Tear down a phantomjs instance.
     * @private
     */
    tearDown: function () {
        this._phantomJS.exit(0);
        debug('._tearDownInstance() tearing down');
    },

    /**
     * Handles the phantom process ending/crashing unexpectedly.
     * If an `onExit` handler has been bound then that will be called. Otherwise, the error will be re-thrown.
     * @param {Number} code
     * @param {String} [signal]
     */
    _handleCrash: function (code, signal) {
        // if a handler is defined, call it
        if (this._onExit) {
            this._onExit(code, signal);

            // otherwise, if we have a non-zero code we'll throw a better error message
            // than the `phantom` lib would.
        } else if (code !== 0) {
            var err = new Error('The PhantomJS process ended unexpectedly');
            err.code = code;
            err.signal = signal;
            throw err;
        }
    },

    /**
     * Go to url
     * @param url
     * @returns {Promise}
     * @private
     */
    _navigateTo: function (url) {
        var deferred = vow.defer();
        debug('.goto() url: ' + url);
        var self = this;
        this._page.open(url, function (status) {
            debug('.goto() page loaded: ' + status);
            if (status === 'success') {
                return deferred.resolve();
            }

            self._rotateProxy()
                .then(function (proxy) {
                    // cannot set new proxy
                    if (proxy === null) {
                        return deferred.reject(new Error('Page was not loaded'));
                    }

                    // one more attempt to open page through the new proxy
                    return this._navigateTo(url);
                }, self);
        });

        return deferred.promise();
    },

    /**
     * Set the viewport.
     *
     * @returns {Promise}
     * @private
     */
    _setViewport: function () {
        var deferred = vow.defer();
        var screen = this._options.screen;
        if (Array.isArray(screen)) {
            screen = _.sample(screen);
        }
        var width = screen.width;
        var height = screen.height;
        debug('.viewport() to ' + width + ' x ' + height);
        var viewport = {width: width, height: height};
        this._page.set('viewportSize', viewport, function () {
            deferred.resolve();
        });

        return deferred.promise();
    },

    /**
     * Set the user agent.
     *
     * @returns {Promise}
     * @private
     */
    _setUserAgent: function () {
        var deferred = vow.defer();
        var userAgent = this._options.userAgent;
        if (Array.isArray(userAgent)) {
            userAgent = _.sample(this._options.userAgent);
        }
        debug('.userAgent() to ' + userAgent);
        this._page.set('settings.userAgent', userAgent, function () {
            deferred.resolve();
        });

        return deferred.promise();
    },

    /**
     * Set a proxy from the proxy list (unset previous one)
     *
     * @returns {Promise}
     * @private
     */
    _rotateProxy: function () {
        var proxy = this._options.proxy;
        var currentProxy = this._proxyCurrent;
        if (!proxy) {
            return vow.resolve(null);
        }
        if (Array.isArray(proxy)) {
            this._removeUnavailableProxy();
            proxy = typeof this._options.proxyRotator === 'function' ?
                this._options.proxyRotator.call(this, proxy, currentProxy) :
                _.sample(proxy);

            if (!proxy) {
                throw new Error('No proxy found');
            }
        }
        debug('._rotateProxy() to %o', proxy);
        var self = this;
        var deferred = vow.defer();
        this._phantomJS.setProxy(proxy.host, proxy.port, 'manual', proxy.username, proxy.password, function () {
            self._proxyCurrent = proxy;
            deferred.resolve(proxy);
        });

        return deferred.promise();
    },

    /**
     * Remove from proxy list one which doesn't work
     *
     * @returns {?Proxy}
     * @private
     */
    _removeUnavailableProxy: function () {
        var current = this._proxyCurrent;
        if (!Array.isArray(this._options.proxy) || this._options.proxy.length === 0 || current === null) {
            return null;
        }

        debug('._removeUnavailableProxy()');
        var index = this._options.proxy.findIndex(function (item) {
            return item.host === current.host && item.port === current.port;
        });
        var proxy = null;
        if (index !== -1) {
            // cut off old used proxy from the list
            proxy = this._options.proxy.splice(index, 1);
        }
        return Array.isArray(proxy) ? proxy.pop() : null;
    },

    /**
     * Inject libs which are required for parse process
     *
     * @private
     */
    _inject: function () {
        debug('.inject()-ing parser libs');
        var files = [
            'vendor/sizzle.min.js'
        ];
        files.forEach(function (file) {
            debug('injecting file %s', file);
            this._page.injectJs(path.join(__dirname, '..', file));
        }, this);
        return vow.resolve();
    },

    /**
     * Attach on phantom event
     * @param type
     * @param callback
     * @param {object} context
     * @returns {Promise}
     * @private
     */
    _on: function (type, callback, context) {
        var deferred = vow.defer();

        if (context) {
            callback = callback.bind(context);
        }

        if (type === 'timeout') {
            deferred.resolve();
        }
        // The onExit callback is special-cased here too
        else if (type === 'exit') {
            this._onExit = callback;
            deferred.resolve();
        }
        // resourceRequestStarted has a special function...
        else if (type === 'resourceRequested') {
            var args = [].slice.call(arguments);
            args = args.slice(3);
            this._page.onResourceRequested(callback, /* @covignore */ function (requestData) {
                debug('Resource requested %s', requestData.url);
            }, args[0], args[1]);
            deferred.resolve();
        }
        // All other events handled natively in phantomjs
        else {
            var pageEvent = 'on' + type.charAt(0).toUpperCase() + type.slice(1);
            this._page.set(pageEvent, callback, function () {
                deferred.resolve();
            });
        }

        return deferred.promise();
    },

    _handlePhantomEvents: function () {
        return vow.all([
            this._on('timeout', function (msg) {
                debug(msg);
                this.tearDown();
            }, this),
            this._on('exit', function (code, signal) {
                debug('Phantom exited with code ' + code + ' and signal ' + signal);
            }),
            this._on('error', function (msg, trace) {
                phantomError('%s, trace %o', msg, trace);
            }),
            this._on('consoleMessage', function (msg) {
                var regex = /^(\[GooseParser])(.+)/i;
                var found = msg.match(regex);
                if (found !== null && found[2] !== undefined) {
                    debugParser(found[2].trim());
                } else {
                    debug('Phantom page message: ' + msg);
                }
            }),
            this._on('navigationRequested', function (url) {
                debug('Navigation to %s', url);
            }),
            this._on('resourceRequested', /* @covignore */ function (requestData, request, allowedUrls, blockedUrls) {
                var url = requestData.url;
                var hasAllowedUrls = Array.isArray(allowedUrls) && allowedUrls.length > 0;
                var hasBlockedUrls = Array.isArray(blockedUrls) && blockedUrls.length > 0;
                var allowed = !hasAllowedUrls || allowedUrls.some(function (urlPattern) {
                        return url.match(urlPattern) !== null
                    });

                var blocked = false;
                if (!hasAllowedUrls && hasBlockedUrls) {
                    blocked = blockedUrls.some(function (urlPattern) {
                        return url.match(urlPattern) !== null;
                    });
                }
                if (!allowed || blocked) {
                    console.log('Resource ' + requestData.url + ' was aborted');
                    request.abort();
                }
            }, null, this._options.resources.allowed, this._options.resources.denied),
            this._on('loadFinished', function (status) {
                debug('Page loaded with status %s, fire %s callbacks', status, this._navigationActions.length);
                var callback;
                while (callback = this._navigationActions.shift()) {
                    callback.call(this, status === 'success' ? null : new Error('Page is not loaded'));
                }
            }.bind(this))
        ]);
    },

    /**
     * @param {string} uri
     * @returns {string}
     * @private
     */
    _getHostName: function (uri) {
        var parsed = url.parse(uri);
        return parsed.hostname;
    }
});

module.exports = PhantomEnvironment;
<|endoftext|>


<commit_before>var aws = require('aws-sdk');
var _ = require('underscore');
var util = require('util');
var SimpleFileWriter = require('simple-file-writer');
var log4js = require('log4js');
var path = require('path');
var fs = require('fs');
var events = require('events');

var EventEmitter = events.EventEmitter;

var logger = log4js.getLogger('redshift-bulk-insert');

module.exports = RedshiftBulkInsert;

var DELIMITER = '|';
var ESCAPE = '\\';
var NEWLINE = '\n';
var NULL = '\\N';
var SUFFIX = '.log';

util.inherits(RedshiftBulkInsert, EventEmitter);

function RedshiftBulkInsert(options) {

	var pathToLogs = options.pathToLogs;

	this._datastore = options.datastore;
	this._tableName = options.tableName;
	this._fields = options.fields;
	this._pathToLogs = pathToLogs;

	var idleFlushPeriod = options.idleFlushPeriod;
	this._idleFlushPeriod = _.isNumber(idleFlushPeriod) ? idleFlushPeriod : 10000;

	var _threshold = options.threshold;
	this._threshold = _.isNumber(threshold) ? threshold : 1000;

	this._awsBucketName = options.awsBucketName;
	this._awsAccessKeyId = options.awsAccessKeyId;
	this._awsSecretAccessKey = options.awsSecretAccessKey;
	this._awsrRegion = options.awsRegion;

	this._numberOfEventsInFile = false;
	this._activeFlushOps = 0;

	var fileName = this._getLogFileName();
	var pathToFile = path.join(pathToLogs, fileName);

	this._fileName = fileName;
	this._file = new SimpleFileWriter(pathToFile);
	this._s3 = this._createS3();

	this.startIdleFlushMonitor();
}

RedshiftBulkInsert.prototype._getLogFileName = function() {
	return this._tableName + '_' + Date.now() + SUFFIX;
}

RedshiftBulkInsert.prototype._createS3 = function() {
	var options = {
		region: this._awsRegion,
		accessKeyId: this._awsAccessKeyId,
		secretAccessKey: this._awsSecretAccessKey
	};
	return new aws.S3(options);
}

RedshiftBulkInsert.prototype.flush = function() {

	logger.info('flush');

	if (this._numberOfEventsInFile > 0) {

		this.activeFlushOps++;

		var oldFileName = this._fileName;
		var pathToLogs = this._pathToLogs;

		var newFileName = this._getLogFileName();
		this._fileName = newFileName;
		var newPathToFile = path.join(pathToLogs, newFileName);
		this._file.setupFile(newPathToFile);

		var sendToS3 = _.bind(this._sendToS3, this, oldFileName, Date.now());
		var oldPathToFile = path.join(pathToLogs, oldFileName);
		fs.readFile(oldPathToFile, sendToS3);

		this._numberOfEventsInFile = 0;
	}

	this.startIdleFlushMonitor();
};

RedshiftBulkInsert.prototype._sendToS3 = function(fileName, start, err, body) {
	if (err) {
		logger.error(err);
		return;
	}
	var params = {
		Body: body,
		Key: fileName,
		Bucket: this._awsBucketName
	};
	var onSentToS3 = _.bind(this._onSentToS3, this, fileName);
	this._s3.putObject(params, onSentToS3);
};

RedshiftBulkInsert.prototype._onSentToS3 = function(fileName, start, err, data) {
	if (err) {
		logger.error(err);
		return;
	}
	var onSentCopyToRedshift = _.bind(this._onSentCopyToRedshift, this, fileName);
	var query = this._getCopyQuery(fileName);
	this._datastore.query(query, onSentCopyToRedshift);
};

RedshiftBulkInsert.prototype._getCopyQuery = function(fileName) {
	return 'COPY '
		+ this._tableName
		+ ' ('
		+ this._fields.join(', ')
		+ ')'
		+ ' FROM '
		+ "'"
		+ 's3://'
		+ this._awsBucketName
		+ '/'
		+ fileName
		+ "'"
		+ ' CREDENTIALS '
		+ "'aws_access_key_id="
		+ this._awsAccessKeyId
		+ ';'
		+ 'aws_secret_access_key='
		+ this._awsSecretAccessKey
		+ "' ";
		// + ' DELIMITER '
		// + "'"
		// + DELIMITER
		// + "'"
		// + ' NULL AS '
		// + "'"
		// + NULL
		// + "'"
		// + ' ESCAPE '
		// + "'"
		// + ESCAPE
		// + "'";
};

RedshiftBulkInsert.prototype._onSentCopyToRedshift = function(fileName, start, err, result) {
	if (err) {
		logger.error(err);
	}

	var pathToFile = path.join(this._pathToLogs, fileName);
	fs.unlink(pathToFile, _.bind(this._onLogFileRemoved, this, err, result));
};

RedshiftBulkInsert.prototype._onLogFileRemoved = function(start, err, result) {
	this.activeFlushOps--;
	this.emit('flush', err, result, '', Date.now() - start, this);
};

RedshiftBulkInsert.prototype.insert = function(row) {

	var line = this._rowToLine(row);
	this._file.write(line + NEWLINE);
	this._numberOfEventsInFile++;

	if (this._numberOfEventsInFile > this._threshold) {
		clearTimeout(this.ref);
		this.flush();
		return;
	}
};

RedshiftBulkInsert.prototype._rowToLine = function(row) {
	var lineBuff = [];
	for (var i = 0, l = row.length; i < l; i++) {
		var value = row[i];
		value = this._escapeValue(value);
		lineBuff.push(value);
	}
	var line = lineBuff.join(DELIMITER);
	return line;
};

RedshiftBulkInsert.prototype._escapeValue = function(value) {
	if (value === null || value === undefined) {
		return NULL;
	}
	if (_.isString(value)) {
		return value
			.replace(ESCAPE, ESCAPE + ESCAPE)
			.replace(DELIMITER, ESCAPE + DELIMITER);
	}
	return value + '';
};

RedshiftBulkInsert.prototype.close = function() {
	clearTimeout(this.ref);
};

RedshiftBulkInsert.prototype.startIdleFlushMonitor = function() {
	logger.info('startIdleFlushMonitor');

	var flush = _.bind(this.flush, this);
	this.ref = setTimeout(flush, this._idleFlushPeriod);
};<commit_msg>Fix RedshiftBulkInsert.js<commit_after>var aws = require('aws-sdk');
var _ = require('underscore');
var util = require('util');
var SimpleFileWriter = require('simple-file-writer');
var log4js = require('log4js');
var path = require('path');
var fs = require('fs');
var events = require('events');

var EventEmitter = events.EventEmitter;

var logger = log4js.getLogger('redshift-bulk-insert');

module.exports = RedshiftBulkInsert;

var DELIMITER = '|';
var ESCAPE = '\\';
var NEWLINE = '\n';
var NULL = '\\N';
var SUFFIX = '.log';

util.inherits(RedshiftBulkInsert, EventEmitter);

function RedshiftBulkInsert(options) {

	var pathToLogs = options.pathToLogs;

	this._datastore = options.datastore;
	this._tableName = options.tableName;
	this._fields = options.fields;
	this._pathToLogs = pathToLogs;

	var idleFlushPeriod = options.idleFlushPeriod;
	this._idleFlushPeriod = _.isNumber(idleFlushPeriod) ? idleFlushPeriod : 10000;

	var threshold = options.threshold;
	this._threshold = _.isNumber(threshold) ? threshold : 1000;

	this._awsBucketName = options.awsBucketName;
	this._awsAccessKeyId = options.awsAccessKeyId;
	this._awsSecretAccessKey = options.awsSecretAccessKey;
	this._awsrRegion = options.awsRegion;

	this._numberOfEventsInFile = false;
	this._activeFlushOps = 0;

	var fileName = this._getLogFileName();
	var pathToFile = path.join(pathToLogs, fileName);

	this._fileName = fileName;
	this._file = new SimpleFileWriter(pathToFile);
	this._s3 = this._createS3();

	this.startIdleFlushMonitor();
}

RedshiftBulkInsert.prototype._getLogFileName = function() {
	return this._tableName + '_' + Date.now() + SUFFIX;
}

RedshiftBulkInsert.prototype._createS3 = function() {
	var options = {
		region: this._awsRegion,
		accessKeyId: this._awsAccessKeyId,
		secretAccessKey: this._awsSecretAccessKey
	};
	return new aws.S3(options);
}

RedshiftBulkInsert.prototype.flush = function() {

	logger.info('flush');

	if (this._numberOfEventsInFile > 0) {

		this.activeFlushOps++;

		var oldFileName = this._fileName;
		var pathToLogs = this._pathToLogs;

		var newFileName = this._getLogFileName();
		this._fileName = newFileName;
		var newPathToFile = path.join(pathToLogs, newFileName);
		this._file.setupFile(newPathToFile);

		var sendToS3 = _.bind(this._sendToS3, this, oldFileName, Date.now());
		var oldPathToFile = path.join(pathToLogs, oldFileName);
		fs.readFile(oldPathToFile, sendToS3);

		this._numberOfEventsInFile = 0;
	}

	this.startIdleFlushMonitor();
};

RedshiftBulkInsert.prototype._sendToS3 = function(fileName, start, err, body) {
	if (err) {
		logger.error(err);
		return;
	}
	var params = {
		Body: body,
		Key: fileName,
		Bucket: this._awsBucketName
	};
	var onSentToS3 = _.bind(this._onSentToS3, this, fileName);
	this._s3.putObject(params, onSentToS3);
};

RedshiftBulkInsert.prototype._onSentToS3 = function(fileName, start, err, data) {
	if (err) {
		logger.error(err);
		return;
	}
	var onSentCopyToRedshift = _.bind(this._onSentCopyToRedshift, this, fileName);
	var query = this._getCopyQuery(fileName);
	this._datastore.query(query, onSentCopyToRedshift);
};

RedshiftBulkInsert.prototype._getCopyQuery = function(fileName) {
	return 'COPY '
		+ this._tableName
		+ ' ('
		+ this._fields.join(', ')
		+ ')'
		+ ' FROM '
		+ "'"
		+ 's3://'
		+ this._awsBucketName
		+ '/'
		+ fileName
		+ "'"
		+ ' CREDENTIALS '
		+ "'aws_access_key_id="
		+ this._awsAccessKeyId
		+ ';'
		+ 'aws_secret_access_key='
		+ this._awsSecretAccessKey
		+ "' ";
		// + ' DELIMITER '
		// + "'"
		// + DELIMITER
		// + "'"
		// + ' NULL AS '
		// + "'"
		// + NULL
		// + "'"
		// + ' ESCAPE '
		// + "'"
		// + ESCAPE
		// + "'";
};

RedshiftBulkInsert.prototype._onSentCopyToRedshift = function(fileName, start, err, result) {
	if (err) {
		logger.error(err);
	}

	var pathToFile = path.join(this._pathToLogs, fileName);
	fs.unlink(pathToFile, _.bind(this._onLogFileRemoved, this, err, result));
};

RedshiftBulkInsert.prototype._onLogFileRemoved = function(start, err, result) {
	this.activeFlushOps--;
	this.emit('flush', err, result, '', Date.now() - start, this);
};

RedshiftBulkInsert.prototype.insert = function(row) {

	var line = this._rowToLine(row);
	this._file.write(line + NEWLINE);
	this._numberOfEventsInFile++;

	if (this._numberOfEventsInFile > this._threshold) {
		clearTimeout(this.ref);
		this.flush();
		return;
	}
};

RedshiftBulkInsert.prototype._rowToLine = function(row) {
	var lineBuff = [];
	for (var i = 0, l = row.length; i < l; i++) {
		var value = row[i];
		value = this._escapeValue(value);
		lineBuff.push(value);
	}
	var line = lineBuff.join(DELIMITER);
	return line;
};

RedshiftBulkInsert.prototype._escapeValue = function(value) {
	if (value === null || value === undefined) {
		return NULL;
	}
	if (_.isString(value)) {
		return value
			.replace(ESCAPE, ESCAPE + ESCAPE)
			.replace(DELIMITER, ESCAPE + DELIMITER);
	}
	return value + '';
};

RedshiftBulkInsert.prototype.close = function() {
	clearTimeout(this.ref);
};

RedshiftBulkInsert.prototype.startIdleFlushMonitor = function() {
	logger.info('startIdleFlushMonitor');

	var flush = _.bind(this.flush, this);
	this.ref = setTimeout(flush, this._idleFlushPeriod);
};<|endoftext|>


<commit_before>STViewController.subClass('STTabController', {
    init: function()
    {
        this._super();
        this.view = STView.create().setDelegate(this);
        this.tabView = null;
        this.tabControllers = STIndexedArray.create();
        this.activeTab = null;
    },

    activeTab:  ST.$property,
    tabView:    ST.$property,

    destroy: function()
    {
        this.tabControllers.release();
        if (this.tabView) {
            this.tabView.release();
        }
        this._super();
    },

    setActiveTab: function(newTab)
    {
        if (this.activeTab) {
            if (this.activeTab.viewWillHide) this.activeTab.viewWillHide(this.activeTab.view);
            this.view.removeChild(this.activeTab.view);
            if (this.activeTab.viewDidHide) this.activeTab.viewDidHide(this.activeTab.view);
        }

        if (!this.tabControllers.has(newTab)) return;
        $.scrollTo(0,0);

        this.activeTab = newTab;
        if (this.activeTab.viewWillShow) this.activeTab.viewWillShow(this.activeTab.view);
        this.view.addChild(this.activeTab.view);
        if (this.activeTab.viewDidShow) this.activeTab.viewDidShow(this.activeTab.view);
    },

    viewDidLoad: function(view)
    {
        if (view != this.view) return;

        if (!this.tabView) {
            this.tabView = STTabView.create().setDelegate(this).load();
            this.view.addChild(this.tabView);
        }
        this.updateTabView();
        if (this.tabControllers.count()) {
            this.setActiveTab(this.tabControllers.first());
        }
    },

    tabViewSwitchedTab: function(tabView, oldIndex, newIndex)
    {
        this.setActiveTab(this.tabControllers.objectAtIndex(newIndex));
    },

    updateTabView: function()
    {
        if (!this.view.loaded) return;

        var tabs = ST.A();
        this.tabControllers.each(function(controller) {
            tabs.push(controller.tabTitle || '');
        });
        this.tabView.setTabs(tabs);
        this.tabView.setActiveTab(Math.max(this.tabControllers.indexOfObject(this.activeTab), 0));
    },

    emptyTabs: function()
    {
        if (this.activeTab) {
            this.view.removeChild(this.activeTab.view);
            this.activeTab = null;
        }
        this.tabControllers.empty();
    },

    addTab: function(tab)
    {
        var tc = this;

        tab.setTabTitle = function(tabTitle) {
            this.tabTitle = tabTitle;
            tc.updateTabView();
        };

        if (!this.tabControllers.count() && this.view.loaded) {
            this.setActiveTab(tab);
        }

        this.tabControllers.add(tab);
        this.updateTabView();
    },

    insertTabAtIndex: function(tab, index)
    {
        var tc = this;

        tab.setTabTitle = function(tabTitle) {
            this.tabTitle = tabTitle;
            tc.updateTabView();
        };

        this.tabControllers.insertObjectAtIndex(tab, index);

        this.updateTabView();
    },

    insertTabBefore: function(tab, before)
    {
        if (before && before.index != null) {
            this.insertTabAtIndex(tab, before.index)
        }
    },

    insertTabAfter: function(tab, after)
    {
        if (after && after.index >= 0 && after.index < (this.tabControllers.count() - 1)) {
            this.insertTabAtIndex(tab, after.index + 1);
        } else {
            this.addTab(tab);
        }
    },

    removeTab: function(tab)
    {
        var index = this.tabControllers.indexOfObject(tab);
        if (this.activeTab == tab) {
            this.setActiveTab(tab.next || tab.prev || null);
        }
        this.tabControllers.remove(tab);
        this.updateTabView();
    },

end
:0});<commit_msg>Added hideSingleTab option for STTabController<commit_after>STViewController.subClass('STTabController', {
    init: function()
    {
        this._super();
        this.view = STView.create().setDelegate(this);
        this.tabView = null;
        this.tabControllers = STIndexedArray.create();
        this.activeTab = null;
        this.hideSingleTab = false;
    },

    activeTab:      ST.$property,
    tabView:        ST.$property,
    hideSingleTab:  ST.$property,

    destroy: function()
    {
        this.tabControllers.release();
        if (this.tabView) {
            this.tabView.release();
        }
        this._super();
    },

    setActiveTab: function(newTab)
    {
        if (this.activeTab) {
            if (this.activeTab.viewWillHide) this.activeTab.viewWillHide(this.activeTab.view);
            this.view.removeChild(this.activeTab.view);
            if (this.activeTab.viewDidHide) this.activeTab.viewDidHide(this.activeTab.view);
        }

        if (!this.tabControllers.has(newTab)) return;
        $.scrollTo(0,0);

        this.activeTab = newTab;
        if (this.activeTab.viewWillShow) this.activeTab.viewWillShow(this.activeTab.view);
        this.view.addChild(this.activeTab.view);
        if (this.activeTab.viewDidShow) this.activeTab.viewDidShow(this.activeTab.view);
    },

    viewDidLoad: function(view)
    {
        if (view != this.view) return;

        if (!this.tabView) {
            this.tabView = STTabView.create().setDelegate(this);
            this.view.addChild(this.tabView);
        }
        this.updateTabView();
        if (this.tabControllers.count()) {
            this.setActiveTab(this.tabControllers.first());
        }
    },

    tabViewSwitchedTab: function(tabView, oldIndex, newIndex)
    {
        this.setActiveTab(this.tabControllers.objectAtIndex(newIndex));
    },

    updateTabView: function()
    {
        if (!this.view.loaded) return;
        if (this.hideSingleTab && this.tabControllers.count() <= 1) return;

        if (!this.tabView.loaded) this.tabView.load();

        var tabs = ST.A();
        this.tabControllers.each(function(controller) {
            tabs.push(controller.tabTitle || '');
        });
        this.tabView.setTabs(tabs);
        this.tabView.setActiveTab(Math.max(this.tabControllers.indexOfObject(this.activeTab), 0));
    },

    emptyTabs: function()
    {
        if (this.activeTab) {
            this.view.removeChild(this.activeTab.view);
            this.activeTab = null;
        }
        this.tabControllers.empty();
    },

    addTab: function(tab)
    {
        var tc = this;

        tab.setTabTitle = function(tabTitle) {
            this.tabTitle = tabTitle;
            tc.updateTabView();
        };

        if (!this.tabControllers.count() && this.view.loaded) {
            this.setActiveTab(tab);
        }

        this.tabControllers.add(tab);
        this.updateTabView();
    },

    addAndReleaseTab: function(tab)
    {
        this.addTab(tab);
        tab.release();
    },

    insertTabAtIndex: function(tab, index)
    {
        var tc = this;

        tab.setTabTitle = function(tabTitle) {
            this.tabTitle = tabTitle;
            tc.updateTabView();
        };

        this.tabControllers.insertObjectAtIndex(tab, index);

        this.updateTabView();
    },

    insertTabBefore: function(tab, before)
    {
        if (before && before.index != null) {
            this.insertTabAtIndex(tab, before.index)
        }
    },

    insertTabAfter: function(tab, after)
    {
        if (after && after.index >= 0 && after.index < (this.tabControllers.count() - 1)) {
            this.insertTabAtIndex(tab, after.index + 1);
        } else {
            this.addTab(tab);
        }
    },

    removeTab: function(tab)
    {
        var index = this.tabControllers.indexOfObject(tab);
        if (this.activeTab == tab) {
            this.setActiveTab(tab.next || tab.prev || null);
        }
        this.tabControllers.remove(tab);
        this.updateTabView();
    },

end
:0});<|endoftext|>