<commit_before>var tls             = require("tls"),
    fs              = require("fs"),
    http            = require("http"),
    https           = require("https"),
    Connect         = require("connect"),
    EventPublisher  = require("./event-publisher"),
    FSM             = require("./fsm"),
    Semaphore       = require("./semaphore"),
    indexer         = require("./file-indexer"),
    parser          = require("./parser"),
    fileWatcher     = require("./filewatcher"),
    middleware      = require("./middleware"),
    cache           = require("./simple-cache"),
    _               = require("underscore")._,
    restProxy       = require("./restProxy"),
    connectRouter   = require("./router_connect"),
    console         = require("console");

exports.init = function(options, cb) {

  //use a state machine to package/parse required application resources
  var fsm = new FSM({
    states: {
      initial: {
        stateStartup: function() {
          // - index the application's files and directories
          indexer.index(options, function(err, indexedFiles) {
            if (err) fsm.fire("error", err); else {
              cache.setItem("feather-files", indexedFiles);

              // now that feather is cluster-aware, only the master process should parse
              if (options.isMaster) {

                if (indexedFiles.restFiles && options.rest && options.rest.autoGenerateProxy) {
                  fsm.fire("generateRestProxy", indexedFiles);
                } else {
                  fsm.fire("parseFeatherFiles", indexedFiles, null);
                }

              } else {
                fsm.fire("parsingComplete");
              }
            }
          });
        },
        generateRestProxy: function(indexedFiles) {
          restProxy.generateProxy({
            appOptions: options,
            files: indexedFiles.restFiles
          }, function(err, restProxyInfo) {
            if (err) cb(err); else {
              fsm.fire("parseFeatherFiles", indexedFiles);
            }
          });
        },
        parseFeatherFiles: function(indexedFiles) {
          // - pre-parse all feather.html files
          // - move the stateMachine to the next state
          var sem = new Semaphore(function() {
            fsm.fire("parsingComplete");
          });

          //parse the .feather.html files in the app
          _.each(_.keys(indexedFiles.featherFiles), function(_path) {
            sem.increment();
            //guarantee all files get counted in semaphore
            process.nextTick(function() {
              parser.parseFile({
                path: _path,
                request: {page: _path.replace(/.*\/public\/(.*)$/, "$1")} //need a dummy request object for parser since there is no real request at this point
              }, function(err, result) {
                  if (err) throw new Error(JSON.stringify(err)); else {

                    //TODO: figure out gracefully re-publishing changed files w/ watchers... (commenting out for now as a placeholder)
                    // if (!indexedFiles.featherFiles[_path].watchingFile) { //only wire the watcher once
                    //   fileWatcher.watchFileMtime(_path, function(args) {
                    //     // TODO: Trigger a reparse and republish of the page (with clustered processes this could be a messy resource contention issue).
                    //   });
                    //   indexedFiles.featherFiles[_path].watchingFile = true;
                    // }

                    sem.execute();
                  }
              });
            });
          });
        },
        parsingComplete: function() {
          if (!options.isWorker) {
            return fsm.states.complete; //the actual server bindings should only happen in the workers
          } else {
            return fsm.states.createServer;
          }
        }
      },
      createServer: {
        stateStartup: function() {

          //first create the session store
          var sessionStore,
            provider = options.connect.session.provider;

          switch (provider) {
            case "memory":
              if (options.cluster) {
                fsm.fire('error', 'When clustering, you cannot use the default memory store for session.');
              } else {
                sessionStore = new Connect.session.MemoryStore;
                cache.setItem("feather-sessionStore", sessionStore);
                options.connect.session.store = sessionStore;
                getMiddleware(sessionStore);
              }
              break;

            case "redis":
              var RedisStore = require("connect-redis")(Connect);
              var redisOptions = null;
              // If providerOptions is specified, look at the server value.
              // If not, look to see if redis.servers.session exists.  Otherwise just use defaults.
              if (options.connect.session.providerOptions && options.connect.session.providerOptions.server) {
                redisOptions = options.redis.servers[options.connect.session.providerOptions.server];
              } else {
                redisOptions = options.redis.servers.session || {};
              }

              sessionStore = new RedisStore(redisOptions);
              cache.setItem("feather-sessionStore", sessionStore);
              options.connect.session.store = sessionStore;
              getMiddleware(sessionStore);
              break;

            case "custom":
              if (typeof options.getSessionStore === "function") {
                options.getSessionStore(function(err, sessionStore) {
                  if (err) throw err;

                  getMiddleware(sessionStore);
                });
              } else {
                throw new Error('When "custom" is specified for connect.session.provider, you must implement a getSessionStore(cb) method in your app.js file.');
              }
              break;
          }

          //TODO: use getSessionStore or something like that...

          function getMiddleware(sessionStore) {
            middleware.getMiddleware(options, function(err, _middleware, _rest) {
              if (err) fsm.fire("error", err); else {
                var mirror = null,
                  deferCompletion = false,
                  tlsOptions;

                //stash the rest interface
                cache.setItem("feather-rest", _rest);

                //defer final server setup until SSL/mirroring is sorted out below
                var completeServerSetup = function() {
                  //create the underlying Connect server instance
                  var server = Connect();
                  _.each(_middleware, function(ware) {
                    server.use(ware);
                  });

                  // configure session path ignores
                  if (options.connect.session.ignorePaths && server.session) {
                    var si = options.connect.session.ignorePaths.length-1;
                    while (si >= 0) {
                      server.session.ignore.push(options.connect.session.ignorePaths[si]);
                      si -= 1;
                    }
                  }

                  //start listening
                  var port = options.port;
                  if (options.ssl && options.ssl.enabled && options.ssl.port) port = options.ssl.port;

                  if (options.ssl && options.ssl.enabled) {
                    server.httpServer = https.createServer(tlsOptions, server).listen(port);
                  } else {
                    server.httpServer = http.createServer(server).listen(port);
                  }

                  // now that ports have been bound we can change the process user and group
                  if (options.daemon.runAsDaemon == true && options.daemon.runAsUser) {

                    console.debug('setting process and group ids (eval whether this should happen for clustered workers or just group leader - then remove debug log statement)...');

                    if (options.daemon.runAsGroup) {
                      process.setgid(options.daemon.runAsGroup);
                    }

                    process.setuid(options.daemon.runAsUser);
                  }

                  server.sessionStore = sessionStore;
                  fsm.fire("complete", server, mirror);
                };

                //use ssl?
                if (options.ssl && options.ssl.enabled) {
                  if (options.ssl.routes && (!options.ssl.port || options.ssl.port == options.port)) {
                    throw new Error("When explicit SSL routes are defined, you must also specify a value for ssl.port which must be different from the top level (non-SSL) port.");
                  }

                  tlsOptions = {
                    key: fs.readFileSync(options.ssl.key),
                    cert: fs.readFileSync(options.ssl.cert)
                  };
                  if (options.ssl.ca) {
                    tlsOptions.ca = [];
                    _.each(options.ssl.ca, function(ca) {
                      var certs = fs.readFileSync(ca);
                      tlsOptions.ca.push(certs);
                    });
                  }
                  _middleware.unshift(tlsOptions);

                  if (options.ssl.useRedirectServer && !options.ssl.routes) {
                    //ssl is configured as "strict, always on" - i.e. no explicit ssl routes are defined,
                    //therefore, the 'mirror' server on the redirect port need only be a 'throw-away' shim that redirects all requests to SSL
                    var redirectServer = Connect(
                      function(req, res, next) {
                        //do the redirect
                        res.statusCode = 302;
                        var host = options.host;
                        //if ssl port is non-standard (443), make sure it gets included in the redirect url
                        host += options.ssl.port === 443 ? "" : ":" + options.ssl.port;
                        res.setHeader("Location", "https://" + host + req.url);
                        res.end();
                      }
                    );
                    redirectServer.listen(options.ssl.redirectServerPort);
                  } else if (options.ssl.routes) {
                    //ssl is defined as only _enforced_ for a subset of routes (all routes MAY still use https, but the configured routes MUST use it),
                    //therefore we must create a full mirror server that has logic to force-redirect to ssl for specific routes

                    deferCompletion = true; //indicate final server setup needs to be deferred

                    //get another copy of the middleware stack for the mirror (cannot use shared stack as each server needs its own)
                    middleware.getMiddleware(options, function(err, __middleware, __rest) {
                      //stash the mirror's rest interface
                      cache.setItem("feather-rest-mirror", __rest);

                      //add the SSL route enforcement middleware module at the top of the new stack
                      __middleware.unshift(connectRouter(function(app) {
                        _.each(options.ssl.routes, function(route) {

                          //redirect all http verbs
                          _.each(connectRouter.methods, function(verb) {
                            (app[verb])(new RegExp(route), function(req, res, next) {
                              //do the redirect
                              res.statusCode = 302;
                              var host = options.host;
                              //if ssl port is non-standard (443), make sure it gets included in the redirect url
                              host += options.ssl.port === 443 ? "" : ":" + options.ssl.port;
                              res.setHeader("Location", "https://" + host + req.url);
                              res.end();
                            });
                          });
                        });
                      }));

                      //spin up mirror and complete server setup
                      mirror = Connect();
                      _.each(__middleware, function(ware) {
                        mirror.use(ware);
                      });

                      mirror.httpServer = http.createServer(mirror).listen(options.port);

                      completeServerSetup();
                    });
                  }
                }

                if (!deferCompletion) completeServerSetup();
              }
            });
          }

        },
        complete: function() {
          return this.states.complete;
        }
      }, //end createServer state
      complete: {
        stateStartup: function(server, mirror) {
          cb(null, server, mirror);
          fsm.dispose();
        }
      },
      error: {
        stateStartup: function(err) {
          cb(err);
          fsm.dispose();
        }
      }
    }
  });
};<commit_msg>Added file watching capability back in.  This watches feather and template files for changes.  Needs refactoring to remove redundancy with file indexer yet, but it is functional.<commit_after>var tls             = require("tls"),
    fs              = require("fs"),
    http            = require("http"),
    https           = require("https"),
    Connect         = require("connect"),
    EventPublisher  = require("./event-publisher"),
    FSM             = require("./fsm"),
    Semaphore       = require("./semaphore"),
    indexer         = require("./file-indexer"),
    parser          = require("./parser"),
    fileWatcher     = require("./filewatcher"),
    watch           = require("watch"),
    middleware      = require("./middleware"),
    cache           = require("./simple-cache"),
    _               = require("underscore")._,
    restProxy       = require("./restProxy"),
    connectRouter   = require("./router_connect"),
    console         = require("console");

var idFixRegex = /<(([^w\/\s]|w[^i\s]|wi[^d\s]|wid[^g\s]|widg[^e\s]|widge[^t\s])+)([^><]*\sid=['"])([^'"\$]*)(['"][^>]*)>/g;
var idFixReplacement = "<$1 $3${id}_$4$5>";

exports.init = function(options, cb) {

  //use a state machine to package/parse required application resources
  var fsm = new FSM({
    states: {
      initial: {
        stateStartup: function() {
          // - index the application's files and directories
          indexer.index(options, function(err, indexedFiles) {
            if (err) fsm.fire("error", err); else {
              cache.setItem("feather-files", indexedFiles);

              // now that feather is cluster-aware, only the master process should parse
              if (options.isMaster) {

                if (indexedFiles.restFiles && options.rest && options.rest.autoGenerateProxy) {
                  fsm.fire("generateRestProxy", indexedFiles);
                } else {
                  fsm.fire("parseFeatherFiles", indexedFiles, null);
                }

              } else {
                fsm.fire("parsingComplete");
              }
            }
          });
        },
        generateRestProxy: function(indexedFiles) {
          restProxy.generateProxy({
            appOptions: options,
            files: indexedFiles.restFiles
          }, function(err, restProxyInfo) {
            if (err) cb(err); else {
              fsm.fire("parseFeatherFiles", indexedFiles);
            }
          });
        },
        parseFeatherFiles: function(indexedFiles) {
          // - pre-parse all feather.html files
          // - move the stateMachine to the next state
          var sem = new Semaphore(function() {
            fsm.fire("parsingComplete");
          });

          //parse the .feather.html files in the app
          _.each(_.keys(indexedFiles.featherFiles), function(_path) {
            sem.increment();
            //guarantee all files get counted in semaphore
            process.nextTick(function() {
              parser.parseFile({
                path: _path,
                fileMetaData: indexedFiles.featherFiles[_path],
                request: {page: _path.replace(/.*\/public\/(.*)$/, "$1")} //need a dummy request object for parser since there is no real request at this point
              }, function(err, result) {
                  if (err) throw new Error(JSON.stringify(err)); else {

                    //TODO: figure out gracefully re-publishing changed files w/ watchers... (commenting out for now as a placeholder)
                    // if (!indexedFiles.featherFiles[_path].watchingFile) { //only wire the watcher once
                    //   fileWatcher.watchFileMtime(_path, function(args) {
                    //     // TODO: Trigger a reparse and republish of the page (with clustered processes this could be a messy resource contention issue).
                    //   });
                    //   indexedFiles.featherFiles[_path].watchingFile = true;
                    // }

                    sem.execute();
                  }
              });
            });
          });
        },
        parsingComplete: function() {
          if (options.isMaster && options.developer.watchForChanges) {
            // Only the master should watch.
            watch.createMonitor(options.publicRoot, {
              ignoreDotFiles: true,
              filter: function(filePath) {
                var ret = filePath.indexOf('feather-res-cache') === -1 && !!filePath.match(/\.(feather|template)\.html$/); // only watch feather files.
                if (ret) console.log("including " + filePath + " in watch list.");
                return ret;
              }
            }, function(monitor) {
              console.log("Watching feather files and templates for changes.");
              var processChange = function(filename, stat) {
                if (filename.indexOf('feather-res-cache') === -1 && !!filename.match(/\.(feather|template)\.html$/)) {
                  console.log("Processing " + filename + " due to a change");
                  cache.getItemWait("feather-files", function(err, indexedFiles) {
                    if (err) console.log("Couldn't get feather-files from cache: " + err); else {
                      if (filename.match(/\.template.html$/)) { // widget template.  re-parse all pages that use it.

                        // Re-read the template file and replace in the index.
                        fs.readFile(filename, "utf-8", function(fileErr, templateData) {
                          debugger;
                          //parse for id="" attributes and add the "${id}_" tokens
                          //note: regex looks a bit more complex due to the need to exclude <widget> tags from this process
                          templateData = templateData.replace(idFixRegex, idFixReplacement);
                          indexedFiles.templateFiles[filename]  = {
                            stats: stat,
                            data: templateData,
                            needsReload: true
                          };

                          var featherFileMetaData;
                          _.each(_.keys(indexedFiles.featherFiles), function(featherFilePath) {
                            featherFileMetaData = indexedFiles.featherFiles[featherFilePath];
                            if (featherFileMetaData.widgets[filename]) { // If this feather file uses the widget template..
                              process.nextTick(function() {
                                parser.parseFile({
                                  forceReparse: true,
                                  path: featherFilePath,
                                  fileMetaData: featherFileMetaData,
                                  request: {page: filename.replace(/.*\/public\/(.*)$/, "$1")} //need a dummy request object for parser since there is no real request at this point
                                }, function(err, result) {
                                    if (err) console.log(err); else console.log("Re-parsed " + featherFilePath + " due to a change in " + filename);
                                });
                              });
                            }
                          });
                        });
                      } else { // feather file.  re-parse the page.

                        process.nextTick(function() {
                          parser.parseFile({
                            path: filename,
                            fileMetaData: indexedFiles.featherFiles[filename],
                            request: {page: filename.replace(/.*\/public\/(.*)$/, "$1")} //need a dummy request object for parser since there is no real request at this point
                          }, function(err, result) {
                              if (err) console.log(err); else console.log("Re-parsed " + filename);
                          });
                        });
                      }
                    }
                  });
                };
              }
              monitor.on('created', function(f, stat) {
                console.log (f + " created");
                processChange(f, stat);
              });
              monitor.on('changed', function(f, currStat, prevStat) {
                console.log (f + " changed");
                processChange(f, currStat);
              });
            });
          }

          if (!options.isWorker) {
            return fsm.states.complete; //the actual server bindings should only happen in the workers
          } else {
            return fsm.states.createServer;
          }
        }
      },
      createServer: {
        stateStartup: function() {

          //first create the session store
          var sessionStore,
            provider = options.connect.session.provider;

          switch (provider) {
            case "memory":
              if (options.cluster) {
                fsm.fire('error', 'When clustering, you cannot use the default memory store for session.');
              } else {
                sessionStore = new Connect.session.MemoryStore;
                cache.setItem("feather-sessionStore", sessionStore);
                options.connect.session.store = sessionStore;
                getMiddleware(sessionStore);
              }
              break;

            case "redis":
              var RedisStore = require("connect-redis")(Connect);
              var redisOptions = null;
              // If providerOptions is specified, look at the server value.
              // If not, look to see if redis.servers.session exists.  Otherwise just use defaults.
              if (options.connect.session.providerOptions && options.connect.session.providerOptions.server) {
                redisOptions = options.redis.servers[options.connect.session.providerOptions.server];
              } else {
                redisOptions = options.redis.servers.session || {};
              }

              sessionStore = new RedisStore(redisOptions);
              cache.setItem("feather-sessionStore", sessionStore);
              options.connect.session.store = sessionStore;
              getMiddleware(sessionStore);
              break;

            case "custom":
              if (typeof options.getSessionStore === "function") {
                options.getSessionStore(function(err, sessionStore) {
                  if (err) throw err;

                  getMiddleware(sessionStore);
                });
              } else {
                throw new Error('When "custom" is specified for connect.session.provider, you must implement a getSessionStore(cb) method in your app.js file.');
              }
              break;
          }

          //TODO: use getSessionStore or something like that...

          function getMiddleware(sessionStore) {
            middleware.getMiddleware(options, function(err, _middleware, _rest) {
              if (err) fsm.fire("error", err); else {
                var mirror = null,
                  deferCompletion = false,
                  tlsOptions;

                //stash the rest interface
                cache.setItem("feather-rest", _rest);

                //defer final server setup until SSL/mirroring is sorted out below
                var completeServerSetup = function() {
                  //create the underlying Connect server instance
                  var server = Connect();
                  _.each(_middleware, function(ware) {
                    server.use(ware);
                  });

                  // configure session path ignores
                  if (options.connect.session.ignorePaths && server.session) {
                    var si = options.connect.session.ignorePaths.length-1;
                    while (si >= 0) {
                      server.session.ignore.push(options.connect.session.ignorePaths[si]);
                      si -= 1;
                    }
                  }

                  //start listening
                  var port = options.port;
                  if (options.ssl && options.ssl.enabled && options.ssl.port) port = options.ssl.port;

                  if (options.ssl && options.ssl.enabled) {
                    server.httpServer = https.createServer(tlsOptions, server).listen(port);
                  } else {
                    server.httpServer = http.createServer(server).listen(port);
                  }

                  // now that ports have been bound we can change the process user and group
                  if (options.daemon.runAsDaemon == true && options.daemon.runAsUser) {

                    console.debug('setting process and group ids (eval whether this should happen for clustered workers or just group leader - then remove debug log statement)...');

                    if (options.daemon.runAsGroup) {
                      process.setgid(options.daemon.runAsGroup);
                    }

                    process.setuid(options.daemon.runAsUser);
                  }

                  server.sessionStore = sessionStore;
                  fsm.fire("complete", server, mirror);
                };

                //use ssl?
                if (options.ssl && options.ssl.enabled) {
                  if (options.ssl.routes && (!options.ssl.port || options.ssl.port == options.port)) {
                    throw new Error("When explicit SSL routes are defined, you must also specify a value for ssl.port which must be different from the top level (non-SSL) port.");
                  }

                  tlsOptions = {
                    key: fs.readFileSync(options.ssl.key),
                    cert: fs.readFileSync(options.ssl.cert)
                  };
                  if (options.ssl.ca) {
                    tlsOptions.ca = [];
                    _.each(options.ssl.ca, function(ca) {
                      var certs = fs.readFileSync(ca);
                      tlsOptions.ca.push(certs);
                    });
                  }
                  _middleware.unshift(tlsOptions);

                  if (options.ssl.useRedirectServer && !options.ssl.routes) {
                    //ssl is configured as "strict, always on" - i.e. no explicit ssl routes are defined,
                    //therefore, the 'mirror' server on the redirect port need only be a 'throw-away' shim that redirects all requests to SSL
                    var redirectServer = Connect(
                      function(req, res, next) {
                        //do the redirect
                        res.statusCode = 302;
                        var host = options.host;
                        //if ssl port is non-standard (443), make sure it gets included in the redirect url
                        host += options.ssl.port === 443 ? "" : ":" + options.ssl.port;
                        res.setHeader("Location", "https://" + host + req.url);
                        res.end();
                      }
                    );
                    redirectServer.listen(options.ssl.redirectServerPort);
                  } else if (options.ssl.routes) {
                    //ssl is defined as only _enforced_ for a subset of routes (all routes MAY still use https, but the configured routes MUST use it),
                    //therefore we must create a full mirror server that has logic to force-redirect to ssl for specific routes

                    deferCompletion = true; //indicate final server setup needs to be deferred

                    //get another copy of the middleware stack for the mirror (cannot use shared stack as each server needs its own)
                    middleware.getMiddleware(options, function(err, __middleware, __rest) {
                      //stash the mirror's rest interface
                      cache.setItem("feather-rest-mirror", __rest);

                      //add the SSL route enforcement middleware module at the top of the new stack
                      __middleware.unshift(connectRouter(function(app) {
                        _.each(options.ssl.routes, function(route) {

                          //redirect all http verbs
                          _.each(connectRouter.methods, function(verb) {
                            (app[verb])(new RegExp(route), function(req, res, next) {
                              //do the redirect
                              res.statusCode = 302;
                              var host = options.host;
                              //if ssl port is non-standard (443), make sure it gets included in the redirect url
                              host += options.ssl.port === 443 ? "" : ":" + options.ssl.port;
                              res.setHeader("Location", "https://" + host + req.url);
                              res.end();
                            });
                          });
                        });
                      }));

                      //spin up mirror and complete server setup
                      mirror = Connect();
                      _.each(__middleware, function(ware) {
                        mirror.use(ware);
                      });

                      mirror.httpServer = http.createServer(mirror).listen(options.port);

                      completeServerSetup();
                    });
                  }
                }

                if (!deferCompletion) completeServerSetup();
              }
            });
          }

        },
        complete: function() {
          return this.states.complete;
        }
      }, //end createServer state
      complete: {
        stateStartup: function(server, mirror) {
          cb(null, server, mirror);
          fsm.dispose();
        }
      },
      error: {
        stateStartup: function(err) {
          cb(err);
          fsm.dispose();
        }
      }
    }
  });
};<|endoftext|>


<commit_before>/* @flow */

declare type ConfigurationEnvironment = Object & {
  NODE_ENV: string
}
declare type ConfigurationSettings = Object & {
  appNodeModulesDir: string,
  assetsPath: string,
  appSrc: string,
  babelrc: ?string,
  eslintrc: ?string
}

declare type Configuration = {
  env: ConfigurationEnvironment,
  plugins: Array<Plugin>,
  settings: ConfigurationSettings
}

declare interface Environment {
  cwd: string;
  configFileName: string;
  plugins: Array<PluginController>;

  constructor(cwd: string, configFilePath: string): void;
  configFilePath(): string;
  getConfiguration(): Configuration;
  getName(): string;
  start(): void;
  build(): Promise<any>;
  restart(): Promise<any>;
}

declare type Plugin = (env: Environment, runOnce: boolean) => PluginController;

declare type PluginController = {
  terminate(): Promise<any>,
}

declare type ClientWebpackPluginConfiguration = Configuration & {
  settings: ConfigurationSettings & {
    client: {
      index: string,
      bundleDir: string,
    }
  }
}

declare type ServerWebpackPluginConfiguration = Configuration & {
    settings: ConfigurationSettings & {
      server: {
        index: string,
        bundleDir: string,
      }
    }
  }
<commit_msg>add build method to PluginController<commit_after>/* @flow */

declare type ConfigurationEnvironment = Object & {
  NODE_ENV: string
}
declare type ConfigurationSettings = Object & {
  appNodeModulesDir: string,
  assetsPath: string,
  appSrc: string,
  babelrc: ?string,
  eslintrc: ?string
}

declare type Configuration = {
  env: ConfigurationEnvironment,
  plugins: Array<Plugin>,
  settings: ConfigurationSettings
}

declare interface Environment {
  cwd: string;
  configFileName: string;
  plugins: Array<PluginController>;

  constructor(cwd: string, configFilePath: string): void;
  configFilePath(): string;
  getConfiguration(): Configuration;
  getName(): string;
  start(): void;
  build(): Promise<any>;
  restart(): Promise<any>;
}

declare type Plugin = (env: Environment, runOnce: boolean) => PluginController;

declare type PluginController = {
  build(): Promise<any>,
  terminate(): Promise<any>,
}

declare type ClientWebpackPluginConfiguration = Configuration & {
  settings: ConfigurationSettings & {
    client: {
      index: string,
      bundleDir: string,
    }
  }
}

declare type ServerWebpackPluginConfiguration = Configuration & {
    settings: ConfigurationSettings & {
      server: {
        index: string,
        bundleDir: string,
      }
    }
  }
<|endoftext|>


<commit_before>
var http = require('http')
var methods = require('./methods')
var package = require('../package.json')
var response = require('./response')

var Server = module.exports = function() {

  var fn = function (req, res) {
    fn.handle(req, res)
  }

  fn.handle = addMethodHandlers(fn)

  fn.headers = {
    'Accept-Ranges': 'bytes',
    'Server': package.name + '/' + package.version
  }

  fn.listen = function () {
    var server = http.createServer(this)
    return server.listen.apply(server, arguments)
  }

  fn.response = { __proto__: response }

  return fn

}

function addMethodHandlers(to) {

  var methodHandlers = {}
  var methodHandler = function (methodName) {
    return function (route, handler, scope) {
      var a = Array.prototype.slice.call(arguments)
      if (typeof a[0] == 'function') a.unshift('/*')
      methodHandlers[methodName].push({
        handler: a[1],
        route: a[0],
        scope: a[2],
        test: toRouteTest(a[0])
      })
      return this
    }
  }

  to.all = function () {
    var a = Array.prototype.slice.call(arguments)
    methods.forEach(function (METHOD) {
      to[METHOD.toLowerCase()].apply(to, a)
    })
    return this
  }

  methods.forEach(function (METHOD) {
    to[METHOD.toLowerCase()] = methodHandler(METHOD)
    methodHandlers[METHOD] = []
  })

  return function (req, res) {
    res.__proto__ = to.response
    res.req = req
    res.set(to.headers)
    req.path = unescape(req.url).split('?')[0]

    var routes = methodHandlers[req.method]
    .filter(function (route) {
      return route && route.test(req)
    })

    var scope = this
    var next = function () {
      var route = routes.shift()
      if (route) {
        route.handler.call(
          route.scope || scope,
          req, res, next
        )
      }
    }

    next()
  }

}

function is(object, type) {
  return Object.prototype.toString.call(object) === '[object ' + type +']'
}

function toRouteTest(v) {

  var re = v
  var esc = function (s) {
    return s.replace(/[\-\[\]\/\{\}\(\)\+\?\.\\\^\$\|]/g, "\\$&")
            .replace(/\*/g, '.*')
  }

  if (is(v, 'String')) {
    if (v.indexOf('*') === -1) {
      re = new RegExp('^' + esc(v) + '$')
    }
    else {
      re = new RegExp(esc(v))
    }
  }

  if (!is(re, 'RegExp')) {
    re = /^\//
  }

  return function (req) {
    return re.test(unescape(req.url))
  }

}

<commit_msg>Refactored "package" variable name<commit_after>
var http = require('http')
var methods = require('./methods')
var pkg = require('../package.json')
var response = require('./response')

var Server = module.exports = function() {

  var fn = function (req, res) {
    fn.handle(req, res)
  }

  fn.handle = addMethodHandlers(fn)

  fn.headers = {
    'Accept-Ranges': 'bytes',
    'Server': pkg.name + '/' + pkg.version
  }

  fn.listen = function () {
    var server = http.createServer(this)
    return server.listen.apply(server, arguments)
  }

  fn.response = { __proto__: response }

  return fn

}

function addMethodHandlers(to) {

  var methodHandlers = {}
  var methodHandler = function (methodName) {
    return function (route, handler, scope) {
      var a = Array.prototype.slice.call(arguments)
      if (typeof a[0] == 'function') a.unshift('/*')
      methodHandlers[methodName].push({
        handler: a[1],
        route: a[0],
        scope: a[2],
        test: toRouteTest(a[0])
      })
      return this
    }
  }

  to.all = function () {
    var a = Array.prototype.slice.call(arguments)
    methods.forEach(function (METHOD) {
      to[METHOD.toLowerCase()].apply(to, a)
    })
    return this
  }

  methods.forEach(function (METHOD) {
    to[METHOD.toLowerCase()] = methodHandler(METHOD)
    methodHandlers[METHOD] = []
  })

  return function (req, res) {
    res.__proto__ = to.response
    res.req = req
    res.set(to.headers)
    req.path = unescape(req.url).split('?')[0]

    var routes = methodHandlers[req.method]
    .filter(function (route) {
      return route && route.test(req)
    })

    var scope = this
    var next = function () {
      var route = routes.shift()
      if (route) {
        route.handler.call(
          route.scope || scope,
          req, res, next
        )
      }
    }

    next()
  }

}

function is(object, type) {
  return Object.prototype.toString.call(object) === '[object ' + type +']'
}

function toRouteTest(v) {

  var re = v
  var esc = function (s) {
    return s.replace(/[\-\[\]\/\{\}\(\)\+\?\.\\\^\$\|]/g, "\\$&")
            .replace(/\*/g, '.*')
  }

  if (is(v, 'String')) {
    if (v.indexOf('*') === -1) {
      re = new RegExp('^' + esc(v) + '$')
    }
    else {
      re = new RegExp(esc(v))
    }
  }

  if (!is(re, 'RegExp')) {
    re = /^\//
  }

  return function (req) {
    return re.test(unescape(req.url))
  }

}

<|endoftext|>


<commit_before>function ui_ready() {
	var token = getToken();
	if (token) {
		act.update(token).then(replaceUI);
	}
}

function login(pass) {
	act.login(pass).then(function() {
		saveToken();
		replaceUI();
	});
}

function saveToken() {
	// saving the token as a cookie, rather than local storage, under the assumption that people are in the habit of clearing cookies specifically when they want to de-auth a site
	document.cookie = 'chat_token=' + act.token;
}
function getToken() {
	return readCookieValue('chat_token');
}

function readCookieValue(key) {
	return document.cookie.replace(new RegExp('(?:(?:^|.*;\\s*)' + key + '\\s*\\=\\s*([^;]*).*$)|^.*$'), "$1");
}

function replaceUI() {
	$('#chat_pass_login').hide();

	main_div = $("#chat_area");

	main_div.innerHTML = ''

	var user_ul = $('
');
	let tabset = $('
');
	tabset.append(user_ul);
	main_div.append(tabset);

	for (let name in act.users) {
		user = act.users[name];

		let li = $('
');
		li.text(name);
		user_ul.append(li);

		let user_div = $('
');
		main_div.append(user_div);

		li.click(function() {
			$('.user_tab').removeClass('active');
			li.addClass('active');

			$('.user_area').hide();
			user_div.show();
		});

		let chan_ul = $('
');
		let tabset = $('
');
		tabset.append(chan_ul);
		user_div.append(tabset);

		for (let chan in user.channels) {
			let li = $('
');
			li.text(chan);
			chan_ul.append(li);

			let channel_div = $('
');
			user_div.append(channel_div);
			li.click(function() {
				$('.channel_tab').removeClass('active');
				li.addClass('active');

				$('.channel_area').hide();
				channel_div.show();
			});

			let msg_list = $('
');
			channel_div.append(msg_list);

			let list = new MessageList(user.channels[chan], msg_list);
			user.channels[chan].list = list;

			let form = $('<form action="">');
			let input = $('<input type="text" class="chat-input">');
			form.submit(function() {
				try {
					let msg = input.val();
					if (msg[0] == '/') {
						list.handleSlashCommand(msg.slice(1));
					} else {
						if (settings.color_code) {
							msg = '`' + settings.color_code + msg + '`';
						}
						list.channel.send(msg);
					}
					input.val('');
				} catch (e) {
					console.error(e);
				}
				return false;
			})

			input.keydown(function(e) {
				let keycode = e.which;

				if(keycode == 34) { // PgDn
					list.pgDn();
				} else if(keycode == 33) { // PgUp
					list.pgUp();
				}
			});
			form.append(input);
			channel_div.append(form);
		}
	}

	$('.channel_area').hide();
	$('.user_area').hide();

	if (!act.poll_interval) {
		act.poll_interval = setInterval(function() {
			act.poll({after:"last"}).then(function(data) {
				for (user in data.chats) {
					let channels = act.users[user].channels;

					// new messages, in oldest-to-newest order
					// TODO deal with tells
					recent = data.chats[user].filter(m => m.channel && !channels[m.channel].list.messages[m.id]);

					recent.forEach(function(msg) {
						channels[msg.channel].list.recordMessage(msg);
					});
				}
			});
		}, 1200);
	}
}

function colorizeUser(user) {
	let valid_colors = "BEFGHIJLMNQSUVWY";
	let num_colors = valid_colors.length;

	let hash = user.split("").map(e => e.charCodeAt(0)).reduce((a, e) => a+e, 0);
	let colorCode = valid_colors.charAt((user.length + hash) % num_colors);
	let colorized = '`' + colorCode + user + "`";

	return replaceColorCodes(colorized);
}

function replaceColorCodes(string) {
	return escapeHtml(string).replace(/`([0-9a-zA-Z])([^:`\n]{1,2}|[^`\n]{3,}?)`/g, colorCallback);
}

function formatMessage(obj) {
	let date = new Date(obj.t * 1000);
	let timestr = [date.getHours(), date.getMinutes()].map(a => ('0' + a).slice(-2)).join(":");
	let msg = replaceColorCodes(obj.msg).replace(/\n/g, '
');

	return '' + timestr + " " + colorizeUser(obj.from_user) + ' ' + msg + '';
}

function colorCallback(not_used, p1, p2) {
	let css = (p1.match(/[A-Z]/) ? 'col-cap-' : 'col-') + p1;
	return '' + p2 + '';
}

function escapeHtml(str) {
	return str.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
}

<commit_msg>Remove S as a valid username color as it looks 'broken' when chosen<commit_after>function ui_ready() {
	var token = getToken();
	if (token) {
		act.update(token).then(replaceUI);
	}
}

function login(pass) {
	act.login(pass).then(function() {
		saveToken();
		replaceUI();
	});
}

function saveToken() {
	// saving the token as a cookie, rather than local storage, under the assumption that people are in the habit of clearing cookies specifically when they want to de-auth a site
	document.cookie = 'chat_token=' + act.token;
}
function getToken() {
	return readCookieValue('chat_token');
}

function readCookieValue(key) {
	return document.cookie.replace(new RegExp('(?:(?:^|.*;\\s*)' + key + '\\s*\\=\\s*([^;]*).*$)|^.*$'), "$1");
}

function replaceUI() {
	$('#chat_pass_login').hide();

	main_div = $("#chat_area");

	main_div.innerHTML = ''

	var user_ul = $('
');
	let tabset = $('
');
	tabset.append(user_ul);
	main_div.append(tabset);

	for (let name in act.users) {
		user = act.users[name];

		let li = $('
');
		li.text(name);
		user_ul.append(li);

		let user_div = $('
');
		main_div.append(user_div);

		li.click(function() {
			$('.user_tab').removeClass('active');
			li.addClass('active');

			$('.user_area').hide();
			user_div.show();
		});

		let chan_ul = $('
');
		let tabset = $('
');
		tabset.append(chan_ul);
		user_div.append(tabset);

		for (let chan in user.channels) {
			let li = $('
');
			li.text(chan);
			chan_ul.append(li);

			let channel_div = $('
');
			user_div.append(channel_div);
			li.click(function() {
				$('.channel_tab').removeClass('active');
				li.addClass('active');

				$('.channel_area').hide();
				channel_div.show();
			});

			let msg_list = $('
');
			channel_div.append(msg_list);

			let list = new MessageList(user.channels[chan], msg_list);
			user.channels[chan].list = list;

			let form = $('<form action="">');
			let input = $('<input type="text" class="chat-input">');
			form.submit(function() {
				try {
					let msg = input.val();
					if (msg[0] == '/') {
						list.handleSlashCommand(msg.slice(1));
					} else {
						if (settings.color_code) {
							msg = '`' + settings.color_code + msg + '`';
						}
						list.channel.send(msg);
					}
					input.val('');
				} catch (e) {
					console.error(e);
				}
				return false;
			})

			input.keydown(function(e) {
				let keycode = e.which;

				if(keycode == 34) { // PgDn
					list.pgDn();
				} else if(keycode == 33) { // PgUp
					list.pgUp();
				}
			});
			form.append(input);
			channel_div.append(form);
		}
	}

	$('.channel_area').hide();
	$('.user_area').hide();

	if (!act.poll_interval) {
		act.poll_interval = setInterval(function() {
			act.poll({after:"last"}).then(function(data) {
				for (user in data.chats) {
					let channels = act.users[user].channels;

					// new messages, in oldest-to-newest order
					// TODO deal with tells
					recent = data.chats[user].filter(m => m.channel && !channels[m.channel].list.messages[m.id]);

					recent.forEach(function(msg) {
						channels[msg.channel].list.recordMessage(msg);
					});
				}
			});
		}, 1200);
	}
}

function colorizeUser(user) {
	let valid_colors = "BEFGHIJLMNQUVWY";
	let num_colors = valid_colors.length;

	let hash = user.split("").map(e => e.charCodeAt(0)).reduce((a, e) => a+e, 0);
	let colorCode = valid_colors.charAt((user.length + hash) % num_colors);
	let colorized = '`' + colorCode + user + "`";

	return replaceColorCodes(colorized);
}

function replaceColorCodes(string) {
	return escapeHtml(string).replace(/`([0-9a-zA-Z])([^:`\n]{1,2}|[^`\n]{3,}?)`/g, colorCallback);
}

function formatMessage(obj) {
	let date = new Date(obj.t * 1000);
	let timestr = [date.getHours(), date.getMinutes()].map(a => ('0' + a).slice(-2)).join(":");
	let msg = replaceColorCodes(obj.msg).replace(/\n/g, '
');

	return '' + timestr + " " + colorizeUser(obj.from_user) + ' ' + msg + '';
}

function colorCallback(not_used, p1, p2) {
	let css = (p1.match(/[A-Z]/) ? 'col-cap-' : 'col-') + p1;
	return '' + p2 + '';
}

function escapeHtml(str) {
	return str.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
}

<|endoftext|>


<commit_before>// Copyright (c) 2011 Andrew Paprocki

var EventEmitter = require('events').EventEmitter;
var util = require('util');
var dgram = require('dgram');
var parser = require('./parser');

function Server(options) {
    if (options) {
        if (typeof(options) !== 'object')
            throw new TypeError('Server options must be an object');
    } else {
        options = {};
    }

    var self = this;
    EventEmitter.call(this, options);

    this.server = dgram.createSocket('udp4');
    this.server.on('message', function(msg, rinfo) {
      var data = parser.parse( msg, rinfo );
      self.emit('message',data);
    });
    this.server.on('listening', function() {
        var address = self.server.address();
        self.emit('listening', address.address + ':' + address.port);
    });
}
util.inherits(Server, EventEmitter);
module.exports = Server;

Server.prototype.bind = function(host,port) {
    var _this = this;
    if (!port) port = 67;
    this.server.bind(port, host, function() {
    	_this.server.setBroadcast(true);
    });
}
<commit_msg>Update server.js<commit_after>// Copyright (c) 2011 Andrew Paprocki

var EventEmitter = require('events').EventEmitter;
var util = require('util');
var dgram = require('dgram');
var parser = require('./parser');

function Server(options) {
    if (options) {
        if (typeof(options) !== 'object')
            throw new TypeError('Server options must be an object');
    } else {
        options = {};
    }

    var self = this;
    EventEmitter.call(this, options);

    this.server = dgram.createSocket('udp4');
    this.server.on('message', function(msg, rinfo) {
      try {
          var data = parser.parse( msg, rinfo );
          self.emit('message',data);
      } catch (e) {
          self.emit('error',e);
      }
    });
    this.server.on('listening', function() {
        var address = self.server.address();
        self.emit('listening', address.address + ':' + address.port);
    });
}
util.inherits(Server, EventEmitter);
module.exports = Server;

Server.prototype.bind = function(host,port) {
    var _this = this;
    if (!port) port = 67;
    this.server.bind(port, host, function() {
    	_this.server.setBroadcast(true);
    });
}
<|endoftext|>


<commit_before>"use strict";

/*
 * HyperSwitch web service entry point
 *
 * Sets up a HyperSwitch instance by loading and setting up handlers and the
 * storage layer, and then dispatches requests to it.
 */

const P = require('bluebird');
const Busboy = require('busboy');
const qs = require('querystring');
const url = require('url');
const http = require('http');
const zlib = require('zlib');
const stream = require('stream');

const URI = require('swagger-router').URI;

const exporting = require('./exports');
const HyperSwitch = require('./hyperswitch');
const Router = require('./router');
const utils = require('./utils');

// Should make it into 0.12, see https://github.com/joyent/node/pull/7878
const SIMPLE_PATH = /^(\/(?!\/)[^?#\s]*)(?:\?([^#\s]*))?$/;
function parseURL(uri) {
    // Fast path for simple path uris
    const fastMatch = SIMPLE_PATH.exec(uri);
    if (fastMatch) {
        return {
            protocol: null,
            slashes: null,
            auth: null,
            host: null,
            port: null,
            hostname: null,
            hash: null,
            search: fastMatch[2] || '',
            pathname: fastMatch[1],
            path: fastMatch[1],
            query: fastMatch[2] && qs.parse(fastMatch[2]) || {},
            href: uri
        };
    } else {
        return url.parse(uri, true);
    }
}

/**
 * Parse a POST request into request.body with BusBoy
 * Drops file uploads on the floor without creating temporary files
 *
 * @param {Request} req HTTP request
 * @returns {Promise<>}
 */
function read(req) {
    return new P((resolve) => {
        const chunks = [];
        req.on('data', (chunk) => {
            chunks.push(chunk);
        });

        req.on('end', () => {
            resolve(Buffer.concat(chunks));
        });
    });
}

function parsePOST(req) {
    const readIt = (req.method === 'PUT') ||
        (req.method === 'POST' && req.headers &&
            (/^application\/json/i.test(req.headers['content-type'])
            || !req.headers['content-type']));

    if (readIt) {
        return read(req);
    } else if (req.method !== 'POST') {
        return P.resolve();
    } else {
        // Parse the POST
        return new P((resolve) => {
            // Parse POST data
            const bboy = new Busboy({
                headers: req.headers,
                // Increase the form field size limit from the 1M default.
                limits: { fieldSize: 15 * 1024 * 1024 }
            });
            const body = {};
            bboy.on('field', (field, val) => {
                body[field] = val;
            });
            bboy.on('finish', () => {
                resolve(body);
            });
            req.pipe(bboy);
        });
    }
}

function logResponse(opts, response, startTime) {
    const latency = Date.now() - startTime;
    let logLevel = 'trace/request';
    if (latency > 5000) {
        logLevel = 'trace/request/slow';
    }
    opts.log(logLevel, {
        message: response.message || 'Request sample',
        res: {
            status: response.status,
            headers: response.headers,
        },
        stack: response.stack,
        latency,
    });

    if (response.status >= 500) {
        opts.log('error/request', {
            message: response.message,
            res: response,
            stack: response.stack,
            latency,
        });
    }
}

function handleResponse(opts, req, resp, response) {
    if (response && response.status) {
        const rh = response.headers = response.headers || {};

        utils.removeHopToHopHeaders(rh, false);

        // Default to no server-side caching
        rh['cache-control'] = rh['cache-control']
            || 'private, max-age=0, s-maxage=0, must-revalidate';

        // Propagate the request id header
        rh['x-request-id'] = opts.reqId;

        logResponse(opts, response, opts.startTime);

        let body;
        // Prepare error responses for the client
        if (response.status >= 400) {
            rh['content-type'] = 'application/problem+json';

            // Whitelist fields to avoid leaking sensitive info
            const rBody = typeof response.body === 'object' && response.body || {};
            body = {
                type: rBody.type,
                title: rBody.title,
                method: rBody.method || req.method,
                detail: rBody.detail || rBody.description,
                uri: rBody.uri || req.uri
            };
            if (response.status === 404) {
                body.type = body.type || 'not_found';
                body.title = body.title || 'Not found.';
            }
            body.type = body.type || 'unknown_error';

            // Prefix error base URL
            if (!/^https?:\/\//.test(body.type)) {
                body.type = (opts.conf.default_error_uri
                        || 'https://mediawiki.org/wiki/HyperSwitch/errors/')
                    + body.type;
            }
            response.body = body;
        }

        if (req.method === 'head') {
            response.body = null;
        }

        if (opts.metrics) {
            opts.metrics.endTiming([
                'ALL.ALL',
                `${req.method.toUpperCase()}.${response.status.toString()}`
            ], opts.startTime);
        }

        if (response.body) {
            body = response.body;
            const bodyIsStream = body instanceof stream.Readable;
            if (!Buffer.isBuffer(body) && !bodyIsStream) {
                // Convert to a buffer
                if (typeof body === 'object') {
                    rh['content-type'] = rh['content-type'] || 'application/json';
                    body = new Buffer(JSON.stringify(body));
                } else {
                    body = new Buffer(body);
                }
            }

            const cType = rh['content-type'];
            if (/\bgzip\b/.test(req.headers['accept-encoding'])
                    && /^application\/json\b|^text\//.test(cType)
                    && rh['content-encoding'] !== 'gzip') {
                rh['content-encoding'] = 'gzip';
                resp.writeHead(response.status, '', rh);
                const zStream = zlib.createGzip({ level: 3 });
                zStream.pipe(resp);
                if (bodyIsStream) {
                    body.pipe(zStream);
                } else {
                    zStream.end(body);
                }
            } else if (rh['content-encoding'] === 'gzip'
                    && !/\bgzip\b/.test(req.headers['accept-encoding'])) {
                delete rh['content-encoding'];
                resp.writeHead(response.status, '', rh);
                const unzStream = zlib.createGunzip();
                unzStream.pipe(resp);
                if (bodyIsStream) {
                    body.pipe(unzStream);
                } else {
                    unzStream.end(body);
                }
            } else if (bodyIsStream) {
                resp.writeHead(response.status, '', rh);
                body.pipe(resp);
            } else {
                rh['content-length'] = body.length;
                resp.writeHead(response.status, '', rh);
                resp.end(body);
            }
        } else {
            resp.writeHead(response.status, '', rh);
            resp.end();
        }
    } else {
        opts.log('error/request', {
            root_req: req,
            msg: "No content returned"
        });

        response = response || {};
        response.headers = response.headers || {};
        response.headers['content-type'] = 'application/problem+json';

        resp.writeHead(response.status || 500, '', response.headers);
        resp.end(req.method === 'head' ? undefined : JSON.stringify({
            type: 'server_error#empty_response',
            title: 'HyperSwitch error: No content returned by backend.',
            uri: req.url,
            method: req.method
        }));
    }
}

// Handle a single request
function handleRequest(opts, req, resp) {
    // Set the request ID early on for external requests
    req.headers = req.headers || {};
    req.headers['x-request-id'] = req.headers['x-request-id'] || utils.generateRequestId();

    const remoteAddr = req.headers['x-client-ip'] || req.socket.remoteAddress;
    req.headers['x-client-ip'] = remoteAddr;
    const xff = req.headers['x-forwarded-for'];
    if (xff) {
        // Prepend current client to XFF, so that it is passed on in
        // sub-requests.
        req.headers['x-forwarded-for'] = `${xff}, ${req.socket.remoteAddress}`;
    } else {
        req.headers['x-forwarded-for'] = req.socket.remoteAddress;
    }

    const reqOpts = {
        conf: opts.conf,
        log: null,
        reqId: req.headers['x-request-id'],
        metrics: opts.metrics,
        ratelimiter: opts.ratelimiter,
        startTime: Date.now()
    };

    /**
     * We use separate metric trees for requests from private networks, so
     * that we can separately study performance for those different use cases.
     * We implement this by passing a specialized prefixed metric producer to
     * the handler. All derived metrics associated with a private / update
     * request are thus recorded in .internal or .internal_update prefixed
     * metric trees.
     */
    if (/^(?:::ffff:)?(?:10|127)\./.test(remoteAddr)) {
        if (req.headers['cache-control'] && /no-cache/i.test(req.headers['cache-control'])) {
            reqOpts.metrics = opts.child_metrics.internal_update;
            req.headers['x-request-class'] = 'internal_update';
        } else {
            reqOpts.metrics = opts.child_metrics.internal;
            req.headers['x-request-class'] = 'internal';
        }
    } else {
        reqOpts.metrics = opts.child_metrics.external;
        req.headers['x-request-class'] = 'external';
    }

    // Create a child logger with selected request information.
    reqOpts.logger = opts.logger && opts.logger.child({
        root_req: {
            method: req.method.toLowerCase(),
            uri: req.url,
            headers: {
                'cache-control': req.headers['cache-control'],
                'content-length': req.headers['content-length'],
                'content-type': req.headers['content-type'],
                'if-match': req.headers['if-match'],
                'user-agent': req.headers['user-agent'],
                'api-user-agent': req.headers['api-user-agent'],
                'x-client-ip': req.headers['x-client-ip'],
                'x-forwarded-for': req.headers['x-forwarded-for'],
                'x-request-id': req.headers['x-request-id'],
                'x-request-class': req.headers['x-request-class'],
                'x-triggered-by': req.headers['x-triggered-by']
            },
        },
        request_id: req.headers['x-request-id']
    });
    reqOpts.log = reqOpts.logger && reqOpts.logger.log.bind(reqOpts.logger) || (() => {});

    // Create a new, clean request object
    const urlData = parseURL(req.url);

    const newReq = {
        uri: new URI(urlData.pathname),
        query: urlData.query,
        method: req.method.toLowerCase(),
        headers: req.headers
    };

    // Start off by parsing any POST data with BusBoy
    return parsePOST(req)
    .catchThrow(new exporting.HTTPError({
        status: 400,
        body: {
            type: 'invalid_request'
        }
    }))

    // Then process the request
    .then((body) => {

        if (body && /^application\/json/i.test(req.headers['content-type'])) {
            const bodyStr = body.toString();
            if (bodyStr) {
                try {
                    body = JSON.parse(bodyStr);
                } catch (e) {
                    reqOpts.log('error/request/json-parsing', e);
                }
            }
        }

        newReq.body = body;

        return opts.hyper.request(newReq, reqOpts);
    })

    // And finally handle the response
    .then((result) => {
        return handleResponse(reqOpts, newReq, resp, result);
    })
    .catch((e) => {
        if (!e || e.name !== 'HTTPError') {
            const originalError = e;
            const stack = e && e.stack;
            e = new exporting.HTTPError({
                status: 500,
                body: {
                    type: 'internal_error',
                    description: `${e}`
                    // Probably better to keep this private for now
                    // stack: e.stack
                }
            });
            // Log this internally
            e.stack = stack;
            e.innerError = originalError;
        }
        if (!e.status) {
            e.status = 500;
        }
        return handleResponse(reqOpts, newReq, resp, e);
    });
}

// Main app setup
function main(options) {
    const conf = options.config || {};
    // Set up the global options object with a logger
    conf.service_name = options.name;
    const opts = {
        appBasePath: options.appBasePath,
        conf,
        logger: options.logger,
        log: options.logger && options.logger.log.bind(options.logger) || (() => {}),
        metrics: options.metrics,
        ratelimiter: options.ratelimiter,
        child_metrics: {
            external: options.metrics && options.metrics.makeChild('external'),
            internal: options.metrics && options.metrics.makeChild('internal'),
            internal_update: options.metrics && options.metrics.makeChild('internal_update'),
        }
    };

    main.server = http.createServer(handleRequest.bind(null, opts));
    main.server.maxConnections = 500;

    opts.router = new Router(opts);
    opts.hyper = new HyperSwitch(opts);
    // Use a child HyperSwitch instance to sidestep the security protection for
    // direct requests to /sys
    const childHyperSwitch = opts.hyper.makeChild({ uri: '#internal-startup' }, {
        metrics: options.metrics && options.metrics.makeChild('internal_startup')
    });

    // Main app startup happens during async spec loading:
    return opts.router.loadSpec(conf.spec, childHyperSwitch)
    .then(() => {
        // Use a large listen queue
        // Also, echo 1024 | sudo tee /proc/sys/net/core/somaxconn
        // (from 128 default)
        const port = conf.port || 7231;
        const host = conf.host;
        // Apply some back-pressure.
        main.server.listen(port, host);
        opts.log('warn/startup', `listening on ${host || '*'}:${port}`);
        // Don't delay incomplete packets for 40ms (Linux default) on
        // pipelined HTTP sockets. We write in large chunks or buffers, so
        // lack of coalescing should not be an issue here.
        main.server.on("connection", (socket) => {
            socket.setNoDelay(true);
        });
        return main.server;
    })
    .catch((e) => {
        opts.log('fatal/startup', {
            status: e.status,
            err: e,
            stack: e.body && e.body.stack || e.stack
        });
        // Delay exiting to avoid heavy restart load & let the logger finish its business
        setTimeout(() => {
            process.exit(1);
        }, 2000);
    });
}

if (module.parent === null) {
    main();
} else {
    module.exports = main;
}
<commit_msg>Increase default HTTP socket timeout from 2 to 6 minutes<commit_after>"use strict";

/*
 * HyperSwitch web service entry point
 *
 * Sets up a HyperSwitch instance by loading and setting up handlers and the
 * storage layer, and then dispatches requests to it.
 */

const P = require('bluebird');
const Busboy = require('busboy');
const qs = require('querystring');
const url = require('url');
const http = require('http');
const zlib = require('zlib');
const stream = require('stream');

const URI = require('swagger-router').URI;

const exporting = require('./exports');
const HyperSwitch = require('./hyperswitch');
const Router = require('./router');
const utils = require('./utils');

// Should make it into 0.12, see https://github.com/joyent/node/pull/7878
const SIMPLE_PATH = /^(\/(?!\/)[^?#\s]*)(?:\?([^#\s]*))?$/;
function parseURL(uri) {
    // Fast path for simple path uris
    const fastMatch = SIMPLE_PATH.exec(uri);
    if (fastMatch) {
        return {
            protocol: null,
            slashes: null,
            auth: null,
            host: null,
            port: null,
            hostname: null,
            hash: null,
            search: fastMatch[2] || '',
            pathname: fastMatch[1],
            path: fastMatch[1],
            query: fastMatch[2] && qs.parse(fastMatch[2]) || {},
            href: uri
        };
    } else {
        return url.parse(uri, true);
    }
}

/**
 * Parse a POST request into request.body with BusBoy
 * Drops file uploads on the floor without creating temporary files
 *
 * @param {Request} req HTTP request
 * @returns {Promise<>}
 */
function read(req) {
    return new P((resolve) => {
        const chunks = [];
        req.on('data', (chunk) => {
            chunks.push(chunk);
        });

        req.on('end', () => {
            resolve(Buffer.concat(chunks));
        });
    });
}

function parsePOST(req) {
    const readIt = (req.method === 'PUT') ||
        (req.method === 'POST' && req.headers &&
            (/^application\/json/i.test(req.headers['content-type'])
            || !req.headers['content-type']));

    if (readIt) {
        return read(req);
    } else if (req.method !== 'POST') {
        return P.resolve();
    } else {
        // Parse the POST
        return new P((resolve) => {
            // Parse POST data
            const bboy = new Busboy({
                headers: req.headers,
                // Increase the form field size limit from the 1M default.
                limits: { fieldSize: 15 * 1024 * 1024 }
            });
            const body = {};
            bboy.on('field', (field, val) => {
                body[field] = val;
            });
            bboy.on('finish', () => {
                resolve(body);
            });
            req.pipe(bboy);
        });
    }
}

function logResponse(opts, response, startTime) {
    const latency = Date.now() - startTime;
    let logLevel = 'trace/request';
    if (latency > 5000) {
        logLevel = 'trace/request/slow';
    }
    opts.log(logLevel, {
        message: response.message || 'Request sample',
        res: {
            status: response.status,
            headers: response.headers,
        },
        stack: response.stack,
        latency,
    });

    if (response.status >= 500) {
        opts.log('error/request', {
            message: response.message,
            res: response,
            stack: response.stack,
            latency,
        });
    }
}

function handleResponse(opts, req, resp, response) {
    if (response && response.status) {
        const rh = response.headers = response.headers || {};

        utils.removeHopToHopHeaders(rh, false);

        // Default to no server-side caching
        rh['cache-control'] = rh['cache-control']
            || 'private, max-age=0, s-maxage=0, must-revalidate';

        // Propagate the request id header
        rh['x-request-id'] = opts.reqId;

        logResponse(opts, response, opts.startTime);

        let body;
        // Prepare error responses for the client
        if (response.status >= 400) {
            rh['content-type'] = 'application/problem+json';

            // Whitelist fields to avoid leaking sensitive info
            const rBody = typeof response.body === 'object' && response.body || {};
            body = {
                type: rBody.type,
                title: rBody.title,
                method: rBody.method || req.method,
                detail: rBody.detail || rBody.description,
                uri: rBody.uri || req.uri
            };
            if (response.status === 404) {
                body.type = body.type || 'not_found';
                body.title = body.title || 'Not found.';
            }
            body.type = body.type || 'unknown_error';

            // Prefix error base URL
            if (!/^https?:\/\//.test(body.type)) {
                body.type = (opts.conf.default_error_uri
                        || 'https://mediawiki.org/wiki/HyperSwitch/errors/')
                    + body.type;
            }
            response.body = body;
        }

        if (req.method === 'head') {
            response.body = null;
        }

        if (opts.metrics) {
            opts.metrics.endTiming([
                'ALL.ALL',
                `${req.method.toUpperCase()}.${response.status.toString()}`
            ], opts.startTime);
        }

        if (response.body) {
            body = response.body;
            const bodyIsStream = body instanceof stream.Readable;
            if (!Buffer.isBuffer(body) && !bodyIsStream) {
                // Convert to a buffer
                if (typeof body === 'object') {
                    rh['content-type'] = rh['content-type'] || 'application/json';
                    body = new Buffer(JSON.stringify(body));
                } else {
                    body = new Buffer(body);
                }
            }

            const cType = rh['content-type'];
            if (/\bgzip\b/.test(req.headers['accept-encoding'])
                    && /^application\/json\b|^text\//.test(cType)
                    && rh['content-encoding'] !== 'gzip') {
                rh['content-encoding'] = 'gzip';
                resp.writeHead(response.status, '', rh);
                const zStream = zlib.createGzip({ level: 3 });
                zStream.pipe(resp);
                if (bodyIsStream) {
                    body.pipe(zStream);
                } else {
                    zStream.end(body);
                }
            } else if (rh['content-encoding'] === 'gzip'
                    && !/\bgzip\b/.test(req.headers['accept-encoding'])) {
                delete rh['content-encoding'];
                resp.writeHead(response.status, '', rh);
                const unzStream = zlib.createGunzip();
                unzStream.pipe(resp);
                if (bodyIsStream) {
                    body.pipe(unzStream);
                } else {
                    unzStream.end(body);
                }
            } else if (bodyIsStream) {
                resp.writeHead(response.status, '', rh);
                body.pipe(resp);
            } else {
                rh['content-length'] = body.length;
                resp.writeHead(response.status, '', rh);
                resp.end(body);
            }
        } else {
            resp.writeHead(response.status, '', rh);
            resp.end();
        }
    } else {
        opts.log('error/request', {
            root_req: req,
            msg: "No content returned"
        });

        response = response || {};
        response.headers = response.headers || {};
        response.headers['content-type'] = 'application/problem+json';

        resp.writeHead(response.status || 500, '', response.headers);
        resp.end(req.method === 'head' ? undefined : JSON.stringify({
            type: 'server_error#empty_response',
            title: 'HyperSwitch error: No content returned by backend.',
            uri: req.url,
            method: req.method
        }));
    }
}

// Handle a single request
function handleRequest(opts, req, resp) {
    // Set the request ID early on for external requests
    req.headers = req.headers || {};
    req.headers['x-request-id'] = req.headers['x-request-id'] || utils.generateRequestId();

    const remoteAddr = req.headers['x-client-ip'] || req.socket.remoteAddress;
    req.headers['x-client-ip'] = remoteAddr;
    const xff = req.headers['x-forwarded-for'];
    if (xff) {
        // Prepend current client to XFF, so that it is passed on in
        // sub-requests.
        req.headers['x-forwarded-for'] = `${xff}, ${req.socket.remoteAddress}`;
    } else {
        req.headers['x-forwarded-for'] = req.socket.remoteAddress;
    }

    const reqOpts = {
        conf: opts.conf,
        log: null,
        reqId: req.headers['x-request-id'],
        metrics: opts.metrics,
        ratelimiter: opts.ratelimiter,
        startTime: Date.now()
    };

    /**
     * We use separate metric trees for requests from private networks, so
     * that we can separately study performance for those different use cases.
     * We implement this by passing a specialized prefixed metric producer to
     * the handler. All derived metrics associated with a private / update
     * request are thus recorded in .internal or .internal_update prefixed
     * metric trees.
     */
    if (/^(?:::ffff:)?(?:10|127)\./.test(remoteAddr)) {
        if (req.headers['cache-control'] && /no-cache/i.test(req.headers['cache-control'])) {
            reqOpts.metrics = opts.child_metrics.internal_update;
            req.headers['x-request-class'] = 'internal_update';
        } else {
            reqOpts.metrics = opts.child_metrics.internal;
            req.headers['x-request-class'] = 'internal';
        }
    } else {
        reqOpts.metrics = opts.child_metrics.external;
        req.headers['x-request-class'] = 'external';
    }

    // Create a child logger with selected request information.
    reqOpts.logger = opts.logger && opts.logger.child({
        root_req: {
            method: req.method.toLowerCase(),
            uri: req.url,
            headers: {
                'cache-control': req.headers['cache-control'],
                'content-length': req.headers['content-length'],
                'content-type': req.headers['content-type'],
                'if-match': req.headers['if-match'],
                'user-agent': req.headers['user-agent'],
                'api-user-agent': req.headers['api-user-agent'],
                'x-client-ip': req.headers['x-client-ip'],
                'x-forwarded-for': req.headers['x-forwarded-for'],
                'x-request-id': req.headers['x-request-id'],
                'x-request-class': req.headers['x-request-class'],
                'x-triggered-by': req.headers['x-triggered-by']
            },
        },
        request_id: req.headers['x-request-id']
    });
    reqOpts.log = reqOpts.logger && reqOpts.logger.log.bind(reqOpts.logger) || (() => {});

    // Create a new, clean request object
    const urlData = parseURL(req.url);

    const newReq = {
        uri: new URI(urlData.pathname),
        query: urlData.query,
        method: req.method.toLowerCase(),
        headers: req.headers
    };

    // Start off by parsing any POST data with BusBoy
    return parsePOST(req)
    .catchThrow(new exporting.HTTPError({
        status: 400,
        body: {
            type: 'invalid_request'
        }
    }))

    // Then process the request
    .then((body) => {

        if (body && /^application\/json/i.test(req.headers['content-type'])) {
            const bodyStr = body.toString();
            if (bodyStr) {
                try {
                    body = JSON.parse(bodyStr);
                } catch (e) {
                    reqOpts.log('error/request/json-parsing', e);
                }
            }
        }

        newReq.body = body;

        return opts.hyper.request(newReq, reqOpts);
    })

    // And finally handle the response
    .then((result) => {
        return handleResponse(reqOpts, newReq, resp, result);
    })
    .catch((e) => {
        if (!e || e.name !== 'HTTPError') {
            const originalError = e;
            const stack = e && e.stack;
            e = new exporting.HTTPError({
                status: 500,
                body: {
                    type: 'internal_error',
                    description: `${e}`
                    // Probably better to keep this private for now
                    // stack: e.stack
                }
            });
            // Log this internally
            e.stack = stack;
            e.innerError = originalError;
        }
        if (!e.status) {
            e.status = 500;
        }
        return handleResponse(reqOpts, newReq, resp, e);
    });
}

// Main app setup
function main(options) {
    const conf = options.config || {};
    // Set up the global options object with a logger
    conf.service_name = options.name;
    const opts = {
        appBasePath: options.appBasePath,
        conf,
        logger: options.logger,
        log: options.logger && options.logger.log.bind(options.logger) || (() => {}),
        metrics: options.metrics,
        ratelimiter: options.ratelimiter,
        child_metrics: {
            external: options.metrics && options.metrics.makeChild('external'),
            internal: options.metrics && options.metrics.makeChild('internal'),
            internal_update: options.metrics && options.metrics.makeChild('internal_update'),
        }
    };

    main.server = http.createServer(handleRequest.bind(null, opts));
    main.server.maxConnections = 500;

    opts.router = new Router(opts);
    opts.hyper = new HyperSwitch(opts);
    // Use a child HyperSwitch instance to sidestep the security protection for
    // direct requests to /sys
    const childHyperSwitch = opts.hyper.makeChild({ uri: '#internal-startup' }, {
        metrics: options.metrics && options.metrics.makeChild('internal_startup')
    });

    // Main app startup happens during async spec loading:
    return opts.router.loadSpec(conf.spec, childHyperSwitch)
    .then(() => {
        // Use a large listen queue
        // Also, echo 1024 | sudo tee /proc/sys/net/core/somaxconn
        // (from 128 default)
        const port = conf.port || 7231;
        const host = conf.host;
        // Apply some back-pressure.
        main.server.listen(port, host);
        opts.log('warn/startup', `listening on ${host || '*'}:${port}`);
        // Don't delay incomplete packets for 40ms (Linux default) on
        // pipelined HTTP sockets. We write in large chunks or buffers, so
        // lack of coalescing should not be an issue here.
        main.server.on("connection", (socket) => {
            socket.setNoDelay(true);
        });

        // Bump up the default socket timeout from 2 minutes to 6 minutes, so
        // that timeout responses at the HTTP level are actually returned
        // before the socket is closed. 6 minutes is the default http request
        // timeout in mainstream browsers, so there is no point in waiting
        // beyond that in browser-targeted services. Which is all of the ones
        // we care about, really.
        main.server.setTimeout(6 * 60 * 1000);
        return main.server;
    })
    .catch((e) => {
        opts.log('fatal/startup', {
            status: e.status,
            err: e,
            stack: e.body && e.body.stack || e.stack
        });
        // Delay exiting to avoid heavy restart load & let the logger finish its business
        setTimeout(() => {
            process.exit(1);
        }, 2000);
    });
}

if (module.parent === null) {
    main();
} else {
    module.exports = main;
}
<|endoftext|>