<commit_before>/*
 * Licensed to Cloudkick, Inc ('Cloudkick') under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * Cloudkick licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var http = require('http');
var url = require('url');
var log = require('util/log');
var ps = require('util/pubsub');
var async = require('extern/async');
var version = require('util/version');
var sprintf = require('extern/sprintf').sprintf;
var category_map = require('categories');
var sys = require('sys');

var config = require('util/config');
var db = require('util/db');

var NPM_BASE = 'http://registry.npmjs.org';
var GITHUB_BASE = 'http://github.com/api/v2/json';
var INDEXER_WAIT_TIME = 10 * 60 * 1000;
var INDEXER_PACKAGE_DELAY = 1 * 1000;

/**
 * A -really- primitive HTTP GETter
 */
function http_get(uri, cb) {
  var parsed = url.parse(uri);
  var port = parsed.port || 80;
  var host = parsed.host;
  var path = parsed.pathname;

  var client = http.createClient(port, host);
  var request = client.request('GET', path, {host: host, 'User-Agent': version.toString()});
  request.end();

  request.on('response', function(response) {
    log.info(uri + " returned " + response.statusCode);
    var chunks = [];
    response.on('data', function(data) {
      chunks.push(data);
    });
    response.on('end', function() {
      var data = JSON.parse(chunks.join(''));
      return cb(response, data);
    });
  });
}

function npm_get(path, cb) {
  http_get(NPM_BASE + path, cb);
}

function github_get(path, cb) {
  http_get(GITHUB_BASE + path, cb);
}

/**
 * Index packages.
 */
function PackageIndexer() {
  this.timeout_id = null;
}

/**
 * Schedule an indexer run. Will publish events on the app-wide pubsub as it
 * finishes indexing individual packages, and when the full run is complete.
 *
 * @param {Number} ms How many milliseconds from now to kick off the run
 */
PackageIndexer.prototype.schedule = function(ms) {
  var self = this;
  var time = (ms !== undefined) ? ms : INDEXER_WAIT_TIME;

  self.cancel();

  self.timeout_id = setTimeout(function() {
    self.run();
  }, time);
};

/**
 * Schedule a pending indexer run.
 */
PackageIndexer.prototype.cancel = function() {
  var self = this;
  if (self.timeout_id) {
    clearTimeout(self.timeout_id);
    self.timeout_id = null;
  }
};

/**
 * Do useful stuff.
 */
PackageIndexer.prototype.run = function() {
  var self = this;
  self.timeout_id = null;
  ps.emit('indexer.indexing.begin');

  npm_get('/', function(response, data) {
    self.handle_root(data);
  });
};

/**
 * Grab details about each package.
 */
PackageIndexer.prototype.handle_root = function(root) {
  var self = this;
  var names = Object.getOwnPropertyNames(root);

  // Grab details on each package in series - we're in no hurry
  async.forEachSeries(names, function(name, callback) {
    if (!root[name] || !root[name].url) {
      log.err("Skipping package, no URL: " + name);
      return callback();
    }

    var categories = category_map[name] || ["Uncategorized"];
    var pkg = {basics: root[name], categories: categories};

    async.parallel([
      // Get details from NPM
      function(callback) {
        http_get(root[name].url, function(response, data) {
          if (response.statusCode !== 200) {
            log.err("Error fetching package data from NPM: " + name);
            return callback();
          }
          else {
            pkg.details = data;
            callback();
          }
        });
      },

      // Get details from Github
      function(callback) {
        var repo;
        if (pkg.basics.repository && pkg.basics.repository.url && pkg.basics.repository.url.match(/^(http|git)\:\/\/github\.com/)) {
          // Figure out the GH url
          try {
            repo = pkg.basics.repository.url.match(/^(http|git)\:\/\/github\.com\/(.+)\.git$/)[2];
          }
          catch (err) {
            log.err("Error matching github URL (" + pkg.basics.repository.url + "): " + err);
            return callback();
          }

          log.debug("Github Repo: " + repo);

          // Fetch commits from GH master
          github_get('/commits/show/' + repo + '/master', function(response, commits) {
            if (response.statusCode !== 200) {
              log.debug("Unable to fetch github data for " + repo);
              callback();
            }
            else {
              pkg.github = {commits: commits};
              callback();
            }
          });
        }
        else {
          callback();
        }
      }
    ],

    // Process whatever we got
    function(err) {
      self.index_package(pkg, callback);
    });
  },

  // When we're done let everyone know
  function(err) {
    ps.emit('indexer.indexing.completed');
    self.schedule();
  });
};

PackageIndexer.prototype.index_package = function(pkg, cb) {
  var self = this;
  var package_data;

  function finish() {
    setTimeout(function() {
      cb();
    }, INDEXER_PACKAGE_DELAY);
  }

  try {
    package_data = db.format_package_data_for_storage(pkg);
  }
  catch (exception) {
    log.err("Error formatting package for storage");
    return finish();
  }

  db.collection.findOne({'_id': package_data._id, 'latest_version': package_data.latest_version}, function(err, doc) {
    if (!doc) {
      // Data for this version is not yet saved
      db.collection.update({'_id': package_data._id}, package_data, {upsert: true}, function(err, doc) {
        if (err) {
          log.err("Error saving package data: " + err.message);
        }
        else {
          log.debug(sprintf('Saved package %s, version: %s', package_data._id, package_data.latest_version));
          ps.emit('indexer.indexed.package', pkg.basics.name);
        }
        return finish();
      });
    }
    else {
      return finish();
    }
  });
};

exports.indexer = new PackageIndexer();

exports.load = function() {
  ps.on(ps.STATE_START, function() {
    exports.indexer.schedule(0);
  });

  ps.on(ps.STATE_STOP, function() {
    exports.indexer.cancel();
  });
};
<commit_msg>Match more github URLs<commit_after>/*
 * Licensed to Cloudkick, Inc ('Cloudkick') under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * Cloudkick licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

var http = require('http');
var url = require('url');
var log = require('util/log');
var ps = require('util/pubsub');
var async = require('extern/async');
var version = require('util/version');
var sprintf = require('extern/sprintf').sprintf;
var category_map = require('categories');
var sys = require('sys');

var config = require('util/config');
var db = require('util/db');

var NPM_BASE = 'http://registry.npmjs.org';
var GITHUB_BASE = 'http://github.com/api/v2/json';
var INDEXER_WAIT_TIME = 10 * 60 * 1000;
var INDEXER_PACKAGE_DELAY = 1 * 1000;

/**
 * A -really- primitive HTTP GETter
 */
function http_get(uri, cb) {
  var parsed = url.parse(uri);
  var port = parsed.port || 80;
  var host = parsed.host;
  var path = parsed.pathname;

  var client = http.createClient(port, host);
  var request = client.request('GET', path, {host: host, 'User-Agent': version.toString()});
  request.end();

  request.on('response', function(response) {
    log.info(uri + " returned " + response.statusCode);
    var chunks = [];
    response.on('data', function(data) {
      chunks.push(data);
    });
    response.on('end', function() {
      var data = JSON.parse(chunks.join(''));
      return cb(response, data);
    });
  });
}

function npm_get(path, cb) {
  http_get(NPM_BASE + path, cb);
}

function github_get(path, cb) {
  http_get(GITHUB_BASE + path, cb);
}

/**
 * Index packages.
 */
function PackageIndexer() {
  this.timeout_id = null;
}

/**
 * Schedule an indexer run. Will publish events on the app-wide pubsub as it
 * finishes indexing individual packages, and when the full run is complete.
 *
 * @param {Number} ms How many milliseconds from now to kick off the run
 */
PackageIndexer.prototype.schedule = function(ms) {
  var self = this;
  var time = (ms !== undefined) ? ms : INDEXER_WAIT_TIME;

  self.cancel();

  self.timeout_id = setTimeout(function() {
    self.run();
  }, time);
};

/**
 * Schedule a pending indexer run.
 */
PackageIndexer.prototype.cancel = function() {
  var self = this;
  if (self.timeout_id) {
    clearTimeout(self.timeout_id);
    self.timeout_id = null;
  }
};

/**
 * Do useful stuff.
 */
PackageIndexer.prototype.run = function() {
  var self = this;
  self.timeout_id = null;
  ps.emit('indexer.indexing.begin');

  npm_get('/', function(response, data) {
    self.handle_root(data);
  });
};

/**
 * Grab details about each package.
 */
PackageIndexer.prototype.handle_root = function(root) {
  var self = this;
  var names = Object.getOwnPropertyNames(root);

  // Grab details on each package in series - we're in no hurry
  async.forEachSeries(names, function(name, callback) {
    if (!root[name] || !root[name].url) {
      log.err("Skipping package, no URL: " + name);
      return callback();
    }

    var categories = category_map[name] || ["Uncategorized"];
    var pkg = {basics: root[name], categories: categories};

    async.parallel([
      // Get details from NPM
      function(callback) {
        http_get(root[name].url, function(response, data) {
          if (response.statusCode !== 200) {
            log.err("Error fetching package data from NPM: " + name);
            return callback();
          }
          else {
            pkg.details = data;
            callback();
          }
        });
      },

      // Get details from Github
      function(callback) {
        var repo;
        if (pkg.basics.repository && pkg.basics.repository.url && pkg.basics.repository.url.match(/^(http|git)\:\/\/github\.com/)) {
          // Figure out the GH url
          try {
            repo = pkg.basics.repository.url.match(/^(http|git)\:\/\/github\.com\/(.+)\.git$/)[2];
          }
          catch (err) {
            log.err("Error matching github URL (" + pkg.basics.repository.url + "): " + err);
            try {
              repo = pkg.basics.repository.url.match(/^(http|git)\:\/\/github\.com\/(.+)$/)[2];
            }
            catch (err) {
              log.err("Error matching github URL (" + pkg.basics.repository.url + "): " + err);
              return callback();
            }
          }

          log.debug("Github Repo: " + repo);

          // Fetch commits from GH master
          github_get('/commits/show/' + repo + '/master', function(response, commits) {
            if (response.statusCode !== 200) {
              log.debug("Unable to fetch github data for " + repo);
              callback();
            }
            else {
              pkg.github = {commits: commits};
              callback();
            }
          });
        }
        else {
          callback();
        }
      }
    ],

    // Process whatever we got
    function(err) {
      self.index_package(pkg, callback);
    });
  },

  // When we're done let everyone know
  function(err) {
    ps.emit('indexer.indexing.completed');
    self.schedule();
  });
};

PackageIndexer.prototype.index_package = function(pkg, cb) {
  var self = this;
  var package_data;

  function finish() {
    setTimeout(function() {
      cb();
    }, INDEXER_PACKAGE_DELAY);
  }

  try {
    package_data = db.format_package_data_for_storage(pkg);
  }
  catch (exception) {
    log.err("Error formatting package for storage");
    return finish();
  }

  db.collection.findOne({'_id': package_data._id, 'latest_version': package_data.latest_version}, function(err, doc) {
    if (!doc) {
      // Data for this version is not yet saved
      db.collection.update({'_id': package_data._id}, package_data, {upsert: true}, function(err, doc) {
        if (err) {
          log.err("Error saving package data: " + err.message);
        }
        else {
          log.debug(sprintf('Saved package %s, version: %s', package_data._id, package_data.latest_version));
          ps.emit('indexer.indexed.package', pkg.basics.name);
        }
        return finish();
      });
    }
    else {
      return finish();
    }
  });
};

exports.indexer = new PackageIndexer();

exports.load = function() {
  ps.on(ps.STATE_START, function() {
    exports.indexer.schedule(0);
  });

  ps.on(ps.STATE_STOP, function() {
    exports.indexer.cancel();
  });
};
<|endoftext|>


<commit_before>'use strict';

var archy = require('archy');
var chalk = require('chalk');
var gutil = require('gulp-util');

function logTasks(tree, getDescription) {
  var padding = 0;
  archy(tree)
    .split('\n')
    .filter(function(v, i) {
      // Log first line as is
      if (i === 0) {
        gutil.log(v);
        return false;
      }
      // Search for longest line
      if (v.length > padding) {
        padding = v.length;
      }
      return v.trim().length !== 0;
    }).forEach(function(v) {
      var line = v.split(' ');
      var task = line.slice(1).join(' ');

      if (/.└/.test(v)) {
        // Log dependencies as is
        gutil.log(v);
      } else {
        // Pretty task with optionnal description
        gutil.log(
          line[0] + ' ' + chalk.cyan(task) +
          Array(padding + 3 - v.length).join(' ') +
          (getDescription(task) || '')
        );
      }
    });
}

module.exports = logTasks;
<commit_msg>Fix dependency regex in logTasks<commit_after>'use strict';

var archy = require('archy');
var chalk = require('chalk');
var gutil = require('gulp-util');

function logTasks(tree, getDescription) {
  var padding = 0;
  var rdependency = /[ │] [├└]/;
  archy(tree)
    .split('\n')
    .filter(function(v, i) {
      // Log first line as is
      if (i === 0) {
        gutil.log(v);
        return false;
      }
      // Search for longest line
      if (v.length > padding) {
        padding = v.length;
      }
      return v.trim().length !== 0;
    }).forEach(function(v) {
      var line = v.split(' ');
      var task = line.slice(1).join(' ');

      // Log dependencies as is
      if (rdependency.test(v)) {
        gutil.log(v);
        return;
      }

      // Pretty task with optionnal description
      gutil.log(
        line[0] + ' ' + chalk.cyan(task) +
        Array(padding + 3 - v.length).join(' ') +
        (getDescription(task) || '')
      );
    });
}

module.exports = logTasks;
<|endoftext|>


<commit_before>import {Buffer} from 'node:buffer';
import crypto from 'node:crypto';
import fs from 'node:fs';
import {findSourceMap} from 'node:module';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import zlib from 'node:zlib';

import cbor from 'cbor';
import concordance from 'concordance';
import indentString from 'indent-string';
import mem from 'mem';
import slash from 'slash';
import writeFileAtomic from 'write-file-atomic';

import {snapshotManager as concordanceOptions} from './concordance-options.js';

// Increment if encoding layout or Concordance serialization versions change. Previous AVA versions will not be able to
// decode buffers generated by a newer version, so changing this value will require a major version bump of AVA itself.
// The version is encoded as an unsigned 16 bit integer.
const VERSION = 3;

const VERSION_HEADER = Buffer.alloc(2);
VERSION_HEADER.writeUInt16LE(VERSION);

// The decoder matches on the trailing newline byte (0x0A).
const READABLE_PREFIX = Buffer.from(`AVA Snapshot v${VERSION}\n`, 'ascii');
const REPORT_SEPARATOR = Buffer.from('\n\n', 'ascii');
const REPORT_TRAILING_NEWLINE = Buffer.from('\n', 'ascii');

const SHA_256_HASH_LENGTH = 32;

export class SnapshotError extends Error {
	constructor(message, snapPath) {
		super(message);
		this.name = 'SnapshotError';
		this.snapPath = snapPath;
	}
}

export class ChecksumError extends SnapshotError {
	constructor(snapPath) {
		super('Checksum mismatch', snapPath);
		this.name = 'ChecksumError';
	}
}

export class VersionMismatchError extends SnapshotError {
	constructor(snapPath, version) {
		super('Unexpected snapshot version', snapPath);
		this.name = 'VersionMismatchError';
		this.snapVersion = version;
		this.expectedVersion = VERSION;
	}
}

export class InvalidSnapshotError extends SnapshotError {
	constructor(snapPath) {
		super('Invalid snapshot file', snapPath);
		this.name = 'InvalidSnapshotError';
	}
}

const LEGACY_SNAPSHOT_HEADER = Buffer.from('// Jest Snapshot v1');
function isLegacySnapshot(buffer) {
	return LEGACY_SNAPSHOT_HEADER.equals(buffer.slice(0, LEGACY_SNAPSHOT_HEADER.byteLength));
}

export class LegacyError extends SnapshotError {
	constructor(snapPath) {
		super('Legacy snapshot file', snapPath);
		this.name = 'LegacyError';
	}
}

function tryRead(file) {
	try {
		return fs.readFileSync(file);
	} catch (error) {
		if (error.code === 'ENOENT') {
			return null;
		}

		throw error;
	}
}

function formatEntry(snapshot, index) {
	const {
		data,
		label = `Snapshot ${index + 1}`, // Human-readable labels start counting at 1.
	} = snapshot;

	const description = data
		? concordance.formatDescriptor(concordance.deserialize(data), concordanceOptions)
		: '<No Data>';

	const blockquote = label.split(/\n/).map(line => '> ' + line).join('\n');

	return `${blockquote}\n\n${indentString(description, 4)}`;
}

function combineEntries({blocks}) {
	const combined = new BufferBuilder();

	for (const {title, snapshots} of blocks) {
		const last = snapshots[snapshots.length - 1];
		combined.write(`\n\n## ${title}\n\n`);

		for (const [index, snapshot] of snapshots.entries()) {
			combined.write(formatEntry(snapshot, index));

			if (snapshot !== last) {
				combined.write(REPORT_SEPARATOR);
			}
		}
	}

	return combined;
}

function generateReport(relFile, snapFile, snapshots) {
	return new BufferBuilder()
		.write(`# Snapshot report for \`${slash(relFile)}\`

The actual snapshot is saved in \`${snapFile}\`.

Generated by [AVA](https://avajs.dev).`)
		.append(combineEntries(snapshots))
		.write(REPORT_TRAILING_NEWLINE)
		.toBuffer();
}

class BufferBuilder {
	constructor() {
		this.buffers = [];
		this.byteOffset = 0;
	}

	append(builder) {
		this.buffers.push(...builder.buffers);
		this.byteOffset += builder.byteOffset;
		return this;
	}

	write(data) {
		if (typeof data === 'string') {
			this.write(Buffer.from(data, 'utf8'));
		} else {
			this.buffers.push(data);
			this.byteOffset += data.byteLength;
		}

		return this;
	}

	toBuffer() {
		return Buffer.concat(this.buffers, this.byteOffset);
	}
}

function sortBlocks(blocksByTitle, blockIndices) {
	return [...blocksByTitle].sort(
		([aTitle], [bTitle]) => {
			const a = blockIndices.get(aTitle);
			const b = blockIndices.get(bTitle);

			if (a === undefined) {
				if (b === undefined) {
					return 0;
				}

				return 1;
			}

			if (b === undefined) {
				return -1;
			}

			return a - b;
		},
	);
}

async function encodeSnapshots(snapshotData) {
	const encoded = await cbor.encodeAsync(snapshotData, {
		omitUndefinedProperties: true,
		canonical: true,
	});
	const compressed = zlib.gzipSync(encoded);
	compressed[9] = 0x03; // Override the GZip header containing the OS to always be Linux
	const sha256sum = crypto.createHash('sha256').update(compressed).digest();
	return Buffer.concat([
		READABLE_PREFIX,
		VERSION_HEADER,
		sha256sum,
		compressed,
	], READABLE_PREFIX.byteLength + VERSION_HEADER.byteLength + SHA_256_HASH_LENGTH + compressed.byteLength);
}

function decodeSnapshots(buffer, snapPath) {
	if (isLegacySnapshot(buffer)) {
		throw new LegacyError(snapPath);
	}

	// The version starts after the readable prefix, which is ended by a newline
	// byte (0x0A).
	const newline = buffer.indexOf(0x0A);
	if (newline === -1) {
		throw new InvalidSnapshotError(snapPath);
	}

	const versionOffset = newline + 1;
	const version = buffer.readUInt16LE(versionOffset);
	if (version !== VERSION) {
		throw new VersionMismatchError(snapPath, version);
	}

	const sha256sumOffset = versionOffset + 2;
	const compressedOffset = sha256sumOffset + SHA_256_HASH_LENGTH;
	const compressed = buffer.slice(compressedOffset);

	const sha256sum = crypto.createHash('sha256').update(compressed).digest();
	const expectedSum = buffer.slice(sha256sumOffset, compressedOffset);
	if (!sha256sum.equals(expectedSum)) {
		throw new ChecksumError(snapPath);
	}

	const decompressed = zlib.gunzipSync(compressed);
	return cbor.decode(decompressed);
}

class Manager {
	constructor(options) {
		this.dir = options.dir;
		this.recordNewSnapshots = options.recordNewSnapshots;
		this.updating = options.updating;
		this.relFile = options.relFile;
		this.reportFile = options.reportFile;
		this.reportPath = options.reportPath;
		this.snapFile = options.snapFile;
		this.snapPath = options.snapPath;
		this.oldBlocksByTitle = options.oldBlocksByTitle;
		this.newBlocksByTitle = options.newBlocksByTitle;
		this.blockIndices = new Map();
		this.error = options.error;

		this.hasChanges = false;
	}

	touch(title, taskIndex) {
		this.blockIndices.set(title, taskIndex);
	}

	compare(options) {
		if (this.error) {
			throw this.error;
		}

		const block = this.newBlocksByTitle.get(options.belongsTo);

		const snapshot = block && block.snapshots[options.index];
		const data = snapshot && snapshot.data;

		if (!data) {
			if (!this.recordNewSnapshots) {
				return {pass: false};
			}

			if (options.deferRecording) {
				const record = this.deferRecord(options);
				return {pass: true, record};
			}

			this.record(options);
			return {pass: true};
		}

		const actual = concordance.deserialize(data, concordanceOptions);
		const expected = concordance.describe(options.expected, concordanceOptions);
		const pass = concordance.compareDescriptors(actual, expected);

		return {actual, expected, pass};
	}

	recordSerialized({data, label, belongsTo, index}) {
		let block = this.newBlocksByTitle.get(belongsTo);
		if (!block) {
			block = {snapshots: []};
		}

		const {snapshots} = block;

		if (index > snapshots.length) {
			throw new RangeError(`Cannot record snapshot ${index} for ${JSON.stringify(belongsTo)}, exceeds expected index of ${snapshots.length}`);
		} else if (index < snapshots.length) {
			if (snapshots[index].data) {
				throw new RangeError(`Cannot record snapshot ${index} for ${JSON.stringify(belongsTo)}, already exists`);
			}

			snapshots[index] = {data, label};
		} else {
			snapshots.push({data, label});
		}

		this.newBlocksByTitle.set(belongsTo, block);
	}

	deferRecord(options) {
		const {expected, belongsTo, label, index} = options;
		const descriptor = concordance.describe(expected, concordanceOptions);
		const data = concordance.serialize(descriptor);

		return () => { // Must be called in order!
			this.hasChanges = true;
			this.recordSerialized({data, label, belongsTo, index});
		};
	}

	record(options) {
		const record = this.deferRecord(options);
		record();
	}

	skipBlock(title) {
		const block = this.oldBlocksByTitle.get(title);

		if (block) {
			this.newBlocksByTitle.set(title, block);
		}
	}

	skipSnapshot({belongsTo, index, deferRecording}) {
		const oldBlock = this.oldBlocksByTitle.get(belongsTo);
		let snapshot = oldBlock && oldBlock.snapshots[index];

		if (!snapshot) {
			snapshot = {};
		}

		// Retain the label from the old snapshot, so as not to assume that the
		// snapshot.skip() arguments are well-formed.

		// Defer recording if called in a try().
		if (deferRecording) {
			return () => { // Must be called in order!
				this.recordSerialized({belongsTo, index, ...snapshot});
			};
		}

		this.recordSerialized({belongsTo, index, ...snapshot});
	}

	async save() {
		const {dir, relFile, snapFile, snapPath, reportPath} = this;

		if (this.updating && this.newBlocksByTitle.size === 0) {
			return {
				changedFiles: [cleanFile(snapPath), cleanFile(reportPath)].flat(),
				temporaryFiles: [],
			};
		}

		if (!this.hasChanges) {
			return null;
		}

		const snapshots = {
			blocks: sortBlocks(this.newBlocksByTitle, this.blockIndices).map(
				([title, block]) => ({title, ...block}),
			),
		};

		const buffer = await encodeSnapshots(snapshots);
		const reportBuffer = generateReport(relFile, snapFile, snapshots);

		await fs.promises.mkdir(dir, {recursive: true});

		const temporaryFiles = [];
		const tmpfileCreated = file => temporaryFiles.push(file);
		await Promise.all([
			writeFileAtomic(snapPath, buffer, {tmpfileCreated}),
			writeFileAtomic(reportPath, reportBuffer, {tmpfileCreated}),
		]);
		return {
			changedFiles: [snapPath, reportPath],
			temporaryFiles,
		};
	}
}

const resolveSourceFile = mem(file => {
	const sourceMap = findSourceMap(file);
	if (sourceMap === undefined) {
		return file;
	}

	const {payload} = sourceMap;
	if (payload.sources.length === 0) { // Hypothetical?
		return file;
	}

	return payload.sources[0].startsWith('file://')
		? fileURLToPath(payload.sources[0])
		: payload.sources[0];
});

export const determineSnapshotDir = mem(({file, fixedLocation, projectDir}) => {
	const testDir = path.dirname(resolveSourceFile(file));
	if (fixedLocation) {
		const relativeTestLocation = path.relative(projectDir, testDir);
		return path.join(fixedLocation, relativeTestLocation);
	}

	const parts = new Set(path.relative(projectDir, testDir).split(path.sep));
	if (parts.has('__tests__')) {
		return path.join(testDir, '__snapshots__');
	}

	if (parts.has('test') || parts.has('tests')) { // Accept tests, even though it's not in the default test patterns
		return path.join(testDir, 'snapshots');
	}

	return testDir;
}, {cacheKey: ([{file}]) => file});

function determineSnapshotPaths({file, fixedLocation, projectDir}) {
	const dir = determineSnapshotDir({file, fixedLocation, projectDir});
	const relFile = path.relative(projectDir, resolveSourceFile(file));
	const name = path.basename(relFile);
	const reportFile = `${name}.md`;
	const snapFile = `${name}.snap`;

	return {
		dir,
		relFile,
		snapFile,
		reportFile,
		snapPath: path.join(dir, snapFile),
		reportPath: path.join(dir, reportFile),
	};
}

function cleanFile(file) {
	try {
		fs.unlinkSync(file);
		return [file];
	} catch (error) {
		if (error.code === 'ENOENT') {
			return [];
		}

		throw error;
	}
}

export function load({file, fixedLocation, projectDir, recordNewSnapshots, updating}) {
	// Keep runner unit tests that use `new Runner()` happy
	if (file === undefined || projectDir === undefined) {
		return new Manager({
			recordNewSnapshots,
			updating,
			oldBlocksByTitle: new Map(),
			newBlocksByTitle: new Map(),
		});
	}

	const paths = determineSnapshotPaths({file, fixedLocation, projectDir});
	const buffer = tryRead(paths.snapPath);

	if (!buffer) {
		return new Manager({
			recordNewSnapshots,
			updating,
			...paths,
			oldBlocksByTitle: new Map(),
			newBlocksByTitle: new Map(),
		});
	}

	let blocksByTitle;
	let snapshotError;

	try {
		const data = decodeSnapshots(buffer, paths.snapPath);
		blocksByTitle = new Map(data.blocks.map(({title, ...block}) => [title, block]));
	} catch (error) {
		blocksByTitle = new Map();

		if (!updating) { // Discard all decoding errors when updating snapshots
			snapshotError = error instanceof SnapshotError
				? error
				: new InvalidSnapshotError(paths.snapPath);
		}
	}

	return new Manager({
		recordNewSnapshots,
		updating,
		...paths,
		oldBlocksByTitle: blocksByTitle,
		newBlocksByTitle: updating ? new Map() : blocksByTitle,
		error: snapshotError,
	});
}
<commit_msg>Handle findSourceMap returning null since Node.js 18.8.0<commit_after>import {Buffer} from 'node:buffer';
import crypto from 'node:crypto';
import fs from 'node:fs';
import {findSourceMap} from 'node:module';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import zlib from 'node:zlib';

import cbor from 'cbor';
import concordance from 'concordance';
import indentString from 'indent-string';
import mem from 'mem';
import slash from 'slash';
import writeFileAtomic from 'write-file-atomic';

import {snapshotManager as concordanceOptions} from './concordance-options.js';

// Increment if encoding layout or Concordance serialization versions change. Previous AVA versions will not be able to
// decode buffers generated by a newer version, so changing this value will require a major version bump of AVA itself.
// The version is encoded as an unsigned 16 bit integer.
const VERSION = 3;

const VERSION_HEADER = Buffer.alloc(2);
VERSION_HEADER.writeUInt16LE(VERSION);

// The decoder matches on the trailing newline byte (0x0A).
const READABLE_PREFIX = Buffer.from(`AVA Snapshot v${VERSION}\n`, 'ascii');
const REPORT_SEPARATOR = Buffer.from('\n\n', 'ascii');
const REPORT_TRAILING_NEWLINE = Buffer.from('\n', 'ascii');

const SHA_256_HASH_LENGTH = 32;

export class SnapshotError extends Error {
	constructor(message, snapPath) {
		super(message);
		this.name = 'SnapshotError';
		this.snapPath = snapPath;
	}
}

export class ChecksumError extends SnapshotError {
	constructor(snapPath) {
		super('Checksum mismatch', snapPath);
		this.name = 'ChecksumError';
	}
}

export class VersionMismatchError extends SnapshotError {
	constructor(snapPath, version) {
		super('Unexpected snapshot version', snapPath);
		this.name = 'VersionMismatchError';
		this.snapVersion = version;
		this.expectedVersion = VERSION;
	}
}

export class InvalidSnapshotError extends SnapshotError {
	constructor(snapPath) {
		super('Invalid snapshot file', snapPath);
		this.name = 'InvalidSnapshotError';
	}
}

const LEGACY_SNAPSHOT_HEADER = Buffer.from('// Jest Snapshot v1');
function isLegacySnapshot(buffer) {
	return LEGACY_SNAPSHOT_HEADER.equals(buffer.slice(0, LEGACY_SNAPSHOT_HEADER.byteLength));
}

export class LegacyError extends SnapshotError {
	constructor(snapPath) {
		super('Legacy snapshot file', snapPath);
		this.name = 'LegacyError';
	}
}

function tryRead(file) {
	try {
		return fs.readFileSync(file);
	} catch (error) {
		if (error.code === 'ENOENT') {
			return null;
		}

		throw error;
	}
}

function formatEntry(snapshot, index) {
	const {
		data,
		label = `Snapshot ${index + 1}`, // Human-readable labels start counting at 1.
	} = snapshot;

	const description = data
		? concordance.formatDescriptor(concordance.deserialize(data), concordanceOptions)
		: '<No Data>';

	const blockquote = label.split(/\n/).map(line => '> ' + line).join('\n');

	return `${blockquote}\n\n${indentString(description, 4)}`;
}

function combineEntries({blocks}) {
	const combined = new BufferBuilder();

	for (const {title, snapshots} of blocks) {
		const last = snapshots[snapshots.length - 1];
		combined.write(`\n\n## ${title}\n\n`);

		for (const [index, snapshot] of snapshots.entries()) {
			combined.write(formatEntry(snapshot, index));

			if (snapshot !== last) {
				combined.write(REPORT_SEPARATOR);
			}
		}
	}

	return combined;
}

function generateReport(relFile, snapFile, snapshots) {
	return new BufferBuilder()
		.write(`# Snapshot report for \`${slash(relFile)}\`

The actual snapshot is saved in \`${snapFile}\`.

Generated by [AVA](https://avajs.dev).`)
		.append(combineEntries(snapshots))
		.write(REPORT_TRAILING_NEWLINE)
		.toBuffer();
}

class BufferBuilder {
	constructor() {
		this.buffers = [];
		this.byteOffset = 0;
	}

	append(builder) {
		this.buffers.push(...builder.buffers);
		this.byteOffset += builder.byteOffset;
		return this;
	}

	write(data) {
		if (typeof data === 'string') {
			this.write(Buffer.from(data, 'utf8'));
		} else {
			this.buffers.push(data);
			this.byteOffset += data.byteLength;
		}

		return this;
	}

	toBuffer() {
		return Buffer.concat(this.buffers, this.byteOffset);
	}
}

function sortBlocks(blocksByTitle, blockIndices) {
	return [...blocksByTitle].sort(
		([aTitle], [bTitle]) => {
			const a = blockIndices.get(aTitle);
			const b = blockIndices.get(bTitle);

			if (a === undefined) {
				if (b === undefined) {
					return 0;
				}

				return 1;
			}

			if (b === undefined) {
				return -1;
			}

			return a - b;
		},
	);
}

async function encodeSnapshots(snapshotData) {
	const encoded = await cbor.encodeAsync(snapshotData, {
		omitUndefinedProperties: true,
		canonical: true,
	});
	const compressed = zlib.gzipSync(encoded);
	compressed[9] = 0x03; // Override the GZip header containing the OS to always be Linux
	const sha256sum = crypto.createHash('sha256').update(compressed).digest();
	return Buffer.concat([
		READABLE_PREFIX,
		VERSION_HEADER,
		sha256sum,
		compressed,
	], READABLE_PREFIX.byteLength + VERSION_HEADER.byteLength + SHA_256_HASH_LENGTH + compressed.byteLength);
}

function decodeSnapshots(buffer, snapPath) {
	if (isLegacySnapshot(buffer)) {
		throw new LegacyError(snapPath);
	}

	// The version starts after the readable prefix, which is ended by a newline
	// byte (0x0A).
	const newline = buffer.indexOf(0x0A);
	if (newline === -1) {
		throw new InvalidSnapshotError(snapPath);
	}

	const versionOffset = newline + 1;
	const version = buffer.readUInt16LE(versionOffset);
	if (version !== VERSION) {
		throw new VersionMismatchError(snapPath, version);
	}

	const sha256sumOffset = versionOffset + 2;
	const compressedOffset = sha256sumOffset + SHA_256_HASH_LENGTH;
	const compressed = buffer.slice(compressedOffset);

	const sha256sum = crypto.createHash('sha256').update(compressed).digest();
	const expectedSum = buffer.slice(sha256sumOffset, compressedOffset);
	if (!sha256sum.equals(expectedSum)) {
		throw new ChecksumError(snapPath);
	}

	const decompressed = zlib.gunzipSync(compressed);
	return cbor.decode(decompressed);
}

class Manager {
	constructor(options) {
		this.dir = options.dir;
		this.recordNewSnapshots = options.recordNewSnapshots;
		this.updating = options.updating;
		this.relFile = options.relFile;
		this.reportFile = options.reportFile;
		this.reportPath = options.reportPath;
		this.snapFile = options.snapFile;
		this.snapPath = options.snapPath;
		this.oldBlocksByTitle = options.oldBlocksByTitle;
		this.newBlocksByTitle = options.newBlocksByTitle;
		this.blockIndices = new Map();
		this.error = options.error;

		this.hasChanges = false;
	}

	touch(title, taskIndex) {
		this.blockIndices.set(title, taskIndex);
	}

	compare(options) {
		if (this.error) {
			throw this.error;
		}

		const block = this.newBlocksByTitle.get(options.belongsTo);

		const snapshot = block && block.snapshots[options.index];
		const data = snapshot && snapshot.data;

		if (!data) {
			if (!this.recordNewSnapshots) {
				return {pass: false};
			}

			if (options.deferRecording) {
				const record = this.deferRecord(options);
				return {pass: true, record};
			}

			this.record(options);
			return {pass: true};
		}

		const actual = concordance.deserialize(data, concordanceOptions);
		const expected = concordance.describe(options.expected, concordanceOptions);
		const pass = concordance.compareDescriptors(actual, expected);

		return {actual, expected, pass};
	}

	recordSerialized({data, label, belongsTo, index}) {
		let block = this.newBlocksByTitle.get(belongsTo);
		if (!block) {
			block = {snapshots: []};
		}

		const {snapshots} = block;

		if (index > snapshots.length) {
			throw new RangeError(`Cannot record snapshot ${index} for ${JSON.stringify(belongsTo)}, exceeds expected index of ${snapshots.length}`);
		} else if (index < snapshots.length) {
			if (snapshots[index].data) {
				throw new RangeError(`Cannot record snapshot ${index} for ${JSON.stringify(belongsTo)}, already exists`);
			}

			snapshots[index] = {data, label};
		} else {
			snapshots.push({data, label});
		}

		this.newBlocksByTitle.set(belongsTo, block);
	}

	deferRecord(options) {
		const {expected, belongsTo, label, index} = options;
		const descriptor = concordance.describe(expected, concordanceOptions);
		const data = concordance.serialize(descriptor);

		return () => { // Must be called in order!
			this.hasChanges = true;
			this.recordSerialized({data, label, belongsTo, index});
		};
	}

	record(options) {
		const record = this.deferRecord(options);
		record();
	}

	skipBlock(title) {
		const block = this.oldBlocksByTitle.get(title);

		if (block) {
			this.newBlocksByTitle.set(title, block);
		}
	}

	skipSnapshot({belongsTo, index, deferRecording}) {
		const oldBlock = this.oldBlocksByTitle.get(belongsTo);
		let snapshot = oldBlock && oldBlock.snapshots[index];

		if (!snapshot) {
			snapshot = {};
		}

		// Retain the label from the old snapshot, so as not to assume that the
		// snapshot.skip() arguments are well-formed.

		// Defer recording if called in a try().
		if (deferRecording) {
			return () => { // Must be called in order!
				this.recordSerialized({belongsTo, index, ...snapshot});
			};
		}

		this.recordSerialized({belongsTo, index, ...snapshot});
	}

	async save() {
		const {dir, relFile, snapFile, snapPath, reportPath} = this;

		if (this.updating && this.newBlocksByTitle.size === 0) {
			return {
				changedFiles: [cleanFile(snapPath), cleanFile(reportPath)].flat(),
				temporaryFiles: [],
			};
		}

		if (!this.hasChanges) {
			return null;
		}

		const snapshots = {
			blocks: sortBlocks(this.newBlocksByTitle, this.blockIndices).map(
				([title, block]) => ({title, ...block}),
			),
		};

		const buffer = await encodeSnapshots(snapshots);
		const reportBuffer = generateReport(relFile, snapFile, snapshots);

		await fs.promises.mkdir(dir, {recursive: true});

		const temporaryFiles = [];
		const tmpfileCreated = file => temporaryFiles.push(file);
		await Promise.all([
			writeFileAtomic(snapPath, buffer, {tmpfileCreated}),
			writeFileAtomic(reportPath, reportBuffer, {tmpfileCreated}),
		]);
		return {
			changedFiles: [snapPath, reportPath],
			temporaryFiles,
		};
	}
}

const resolveSourceFile = mem(file => {
	const sourceMap = findSourceMap(file);
	// Prior to Node.js 18.8.0, the value when a source map could not be found was `undefined`.
	// This changed to `null` in <https://github.com/nodejs/node/pull/43875>. Check both.
	if (sourceMap === undefined || sourceMap === null) {
		return file;
	}

	const {payload} = sourceMap;
	if (payload.sources.length === 0) { // Hypothetical?
		return file;
	}

	return payload.sources[0].startsWith('file://')
		? fileURLToPath(payload.sources[0])
		: payload.sources[0];
});

export const determineSnapshotDir = mem(({file, fixedLocation, projectDir}) => {
	const testDir = path.dirname(resolveSourceFile(file));
	if (fixedLocation) {
		const relativeTestLocation = path.relative(projectDir, testDir);
		return path.join(fixedLocation, relativeTestLocation);
	}

	const parts = new Set(path.relative(projectDir, testDir).split(path.sep));
	if (parts.has('__tests__')) {
		return path.join(testDir, '__snapshots__');
	}

	if (parts.has('test') || parts.has('tests')) { // Accept tests, even though it's not in the default test patterns
		return path.join(testDir, 'snapshots');
	}

	return testDir;
}, {cacheKey: ([{file}]) => file});

function determineSnapshotPaths({file, fixedLocation, projectDir}) {
	const dir = determineSnapshotDir({file, fixedLocation, projectDir});
	const relFile = path.relative(projectDir, resolveSourceFile(file));
	const name = path.basename(relFile);
	const reportFile = `${name}.md`;
	const snapFile = `${name}.snap`;

	return {
		dir,
		relFile,
		snapFile,
		reportFile,
		snapPath: path.join(dir, snapFile),
		reportPath: path.join(dir, reportFile),
	};
}

function cleanFile(file) {
	try {
		fs.unlinkSync(file);
		return [file];
	} catch (error) {
		if (error.code === 'ENOENT') {
			return [];
		}

		throw error;
	}
}

export function load({file, fixedLocation, projectDir, recordNewSnapshots, updating}) {
	// Keep runner unit tests that use `new Runner()` happy
	if (file === undefined || projectDir === undefined) {
		return new Manager({
			recordNewSnapshots,
			updating,
			oldBlocksByTitle: new Map(),
			newBlocksByTitle: new Map(),
		});
	}

	const paths = determineSnapshotPaths({file, fixedLocation, projectDir});
	const buffer = tryRead(paths.snapPath);

	if (!buffer) {
		return new Manager({
			recordNewSnapshots,
			updating,
			...paths,
			oldBlocksByTitle: new Map(),
			newBlocksByTitle: new Map(),
		});
	}

	let blocksByTitle;
	let snapshotError;

	try {
		const data = decodeSnapshots(buffer, paths.snapPath);
		blocksByTitle = new Map(data.blocks.map(({title, ...block}) => [title, block]));
	} catch (error) {
		blocksByTitle = new Map();

		if (!updating) { // Discard all decoding errors when updating snapshots
			snapshotError = error instanceof SnapshotError
				? error
				: new InvalidSnapshotError(paths.snapPath);
		}
	}

	return new Manager({
		recordNewSnapshots,
		updating,
		...paths,
		oldBlocksByTitle: blocksByTitle,
		newBlocksByTitle: updating ? new Map() : blocksByTitle,
		error: snapshotError,
	});
}
<|endoftext|>


<commit_before>var VerticesToVerticesStream = require('./verticesToVerticesStream');

module.exports = function (sourceStreamPrototype) {
  sourceStreamPrototype.out = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'out'));
  };

  sourceStreamPrototype.in = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'in'));
  };

  sourceStreamPrototype.both = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'both'));
  };
}
<commit_msg>Added comments<commit_after>var VerticesToVerticesStream = require('./verticesToVerticesStream');

module.exports = function (sourceStreamPrototype) {
  /**
   * Pipes source stream of vertices into stream which
   * produces adjacent vertices, connected by head of an edge.
   *
   * Example:
   *   A -> B -> C
   *   g.V('B').out() - emits C
   */
  sourceStreamPrototype.out = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'out'));
  };

  /**
   * Pipes source stream of vertices into stream which
   * produces adjacent vertices, connected by tail of an edge.
   *
   * Example:
   *   A -> B -> C
   *   g.V('B').in() - emits A
   */
  sourceStreamPrototype.in = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'in'));
  };

  /**
   * Pipes source stream of vertices into stream which
   * produces adjacent vertices from both sides of an edge
   *
   * Example:
   *   A -> B -> C
   *   g.V('B').both() - emits A and C
   */
  sourceStreamPrototype.both = function () {
    return this.pipe(new VerticesToVerticesStream(this._graph, 'both'));
  };
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
'use strict';

var captureHost = require('../util.js').captureHost;
var errors = require('../errors.js');
var isEmptyArray = require('../util.js').isEmptyArray;
var numOrDefault = require('../util.js').numOrDefault;
var safeParse = require('../util.js').safeParse;
var TypedError = require('error/typed');

var JoinAbortedError = TypedError({
    type: 'ringpop.join-aborted',
    message: 'Join aborted because `{reason}`',
    reason: null
});

var JoinDurationExceededError = TypedError({
    type: 'ringpop.join-duration-exceeded',
    message: 'Join duration of `{duration}` exceeded max `{max}`',
    duration: null,
    max: null
});

var JOIN_RETRY_DELAY = 100;
var JOIN_SIZE = 3;
var JOIN_TIMEOUT = 1000;
// If a node cannot complete a join within MAX_JOIN_DURATION
// there is likely something very wrong. The aim is for the join
// operation to take no more than 1s, under normal conditions.
//
// The duration assigned below is very high for the following
// purposes:
//   - Gives an application developer some time to diagnose
//   what could be wrong.
//   - Gives an operator some time to bootstrap a newly
//   provisioned cluster
//   - Trying forever is futile
var MAX_JOIN_DURATION = 120000;
var PARALLELISM_FACTOR = 2;

function isSingleNodeCluster(ringpop) {
    return Array.isArray(ringpop.bootstrapHosts) &&
        ringpop.bootstrapHosts.length === 1 &&
        ringpop.bootstrapHosts[0] === ringpop.hostPort;
}

// Note that this function mutates the array passed in.
function takeNode(hosts) {
    var index = Math.floor(Math.random() * hosts.length);
    var host = hosts[index];
    hosts.splice(index, 1);
    return host;
}

function JoinCluster(opts) {
    opts = opts || {};

    if (!opts.ringpop) {
        throw errors.OptionRequiredError({
            option: 'ringpop'
        });
    }

    if (isEmptyArray(opts.ringpop.bootstrapHosts)) {
        throw errors.InvalidOptionError({
            option: 'ringpop',
            reason: '`bootstrapHosts` is expected to be an array of size 1 or more'
        });
    }

    this.ringpop = opts.ringpop;
    this.host = captureHost(this.ringpop.hostPort);
    this.joinTimeout = numOrDefault(opts.joinTimeout, JOIN_TIMEOUT);

    // This is used as a multiple of the required nodes left
    // to join to satisfy `joinSize`. Additional parallelism
    // can be applied in order for `joinSize` to be satisified
    // faster.
    this.parallelismFactor = numOrDefault(opts.parallelismFactor,
        PARALLELISM_FACTOR);

    // We eventually want to give up if the join process cannot
    // succeed. `maxJoinDuration` is used to restrict that process
    // to a certain time limit.
    this.maxJoinDuration = numOrDefault(opts.maxJoinDuration,
        MAX_JOIN_DURATION);

    // We do not want to retry joining as hard as we can. We
    // want to have some fixed backoff applied before we try
    // to join again
    this.joinRetryDelay = numOrDefault(opts.joinRetryDelay,
        JOIN_RETRY_DELAY);

    // Potential nodes are nodes in the ringpop bootstrap
    // list that can be joined. Upon instantiation, this step
    // simply filters out a node from attempting to join itself.
    this.potentialNodes = this.collectPotentialNodes();
    this.preferredNodes = null;
    this.nonPreferredNodes = null;

    // We either join the number of nodes defined by `joinSize`
    // or limit it to the number of `potentialNodes`. After all,
    // we can't join more than there are to join in the first place.
    var joinSize = numOrDefault(opts.joinSize, JOIN_SIZE);
    this.joinSize = Math.min(joinSize, this.potentialNodes.length);

    // A round is defined as a complete cycle through all
    // potential join targets. Once a round is completed,
    // we start all over again. A full-cycle should be pretty
    // darned rare. We will try and try to join other nodes
    // until `joinSize` is reached or `maxJoinDuration` is
    // exceeded.
    this.roundPotentialNodes = null;
    this.roundPreferredNodes = null;
}

// Potential nodes are those that are not this instance of ringpop.
JoinCluster.prototype.collectPotentialNodes = function collectPotentialNodes(nodesJoined) {
    nodesJoined = nodesJoined || [];

    var self = this;
    return this.ringpop.bootstrapHosts.filter(function filterHost(hostPort) {
        return self.ringpop.hostPort !== hostPort && nodesJoined.indexOf(hostPort) === -1;
    });
};

// Preferred nodes are those that are not on the same host as this instance of ringpop.
JoinCluster.prototype.collectPreferredNodes = function collectPreferredNodes() {
    var self = this;
    return this.potentialNodes.filter(function filterHost(hostPort) {
        return self.host !== captureHost(hostPort);
    });
};

// Non-preferred nodes are everyone else.
JoinCluster.prototype.collectNonPreferredNodes = function collectNonPreferredNodes() {
    var self = this;

    if (isEmptyArray(this.preferredNodes)) {
        return this.potentialNodes;
    } else {
        return this.potentialNodes.filter(function filterHost(hostPort) {
            return self.preferredNodes.indexOf(hostPort) === -1;
        });
    }
};

JoinCluster.prototype.init = function init(nodesJoined) {
    // TODO The "collect" operations are fairly inefficient. This
    // can be improved by indexing by host/port values.
    this.potentialNodes = this.collectPotentialNodes(nodesJoined);
    this.preferredNodes = this.collectPreferredNodes();
    this.nonPreferredNodes = this.collectNonPreferredNodes();

    // Note that these are copies. During the process of joining
    // the "round"-prefixed collections are mutated.
    this.roundPotentialNodes = this.potentialNodes.slice(0);
    this.roundPreferredNodes = this.preferredNodes.slice(0);
    this.roundNonPreferredNodes = this.nonPreferredNodes.slice(0);
};

JoinCluster.prototype.join = function join(callback) {
    var self = this;

    if (this.ringpop.destroyed) {
        process.nextTick(function onTick() {
            callback(JoinAbortedError({
                reason: 'joiner was destroyed'
            }));
        });
        return;
    }

    // No need to go through the join process if you're the only one in the cluster.
    if (isSingleNodeCluster(this.ringpop)) {
        this.ringpop.logger.info('ringpop received a single node cluster join', {
            local: this.ringpop.whoami()
        });

        process.nextTick(function onTick() {
            callback();
        });
        return;
    }

    var nodesJoined = [];
    var numGroups = 0;
    var numJoined = 0;
    var numFailed = 0;
    var startTime = Date.now();
    var calledBack = false;

    function onJoin(err, nodes) {
        if (calledBack) {
            return;
        }

        if (self.ringpop.destroyed) {
            calledBack = true;
            callback(JoinAbortedError({
                reason: 'joiner was destroyed',
            }));
            return;
        }

        if (err) {
            calledBack = true;
            callback(err);
            return;
        }

        nodesJoined = nodesJoined.concat(nodes.successes);
        numJoined += nodes.successes.length;
        numFailed += nodes.failures.length;
        numGroups++;

        if (numJoined >= self.joinSize) {
            self.ringpop.stat('increment', 'join.complete');
            self.ringpop.logger.info('ringpop join complete', {
                local: self.ringpop.whoami(),
                joinSize: self.joinSize,
                joinTime: Date.now() - startTime,
                numJoined: numJoined,
                numGroups: numGroups,
                numFailed: numFailed
            });

            calledBack = true;
            callback(null, nodesJoined);
        } else {
            var joinDuration = Date.now() - startTime;
            if (joinDuration > self.maxJoinDuration) {
                self.ringpop.logger.warn('ringpop max join duration exceeded', {
                    local: self.ringpop.whoami(),
                    joinDuration: joinDuration,
                    maxJoinDuration: self.maxJoinDuration,
                    numJoined: numJoined,
                    numFailed: numFailed,
                    startTime: startTime
                });

                calledBack = true;
                callback(JoinDurationExceededError({
                    joinDuration: joinDuration,
                    maxJoinDuration: self.maxJoinDuration
                }));
                return;
            }

            self.ringpop.logger.debug('ringpop join not yet complete', {
                local: self.ringpop.whoami(),
                joinSize: self.joinSize,
                numJoined: numJoined,
                numFailed: numFailed,
                numNodesLeft: self.joinSize - numJoined
            });

            setTimeout(reJoin, self.joinRetryDelay);
        }

        function reJoin() {
            self.joinGroup(nodesJoined, onJoin);
        }
    }

    this.joinGroup(nodesJoined, onJoin);
};

JoinCluster.prototype.joinGroup = function joinGroup(totalNodesJoined, callback) {
    var self = this;
    var group = this.selectGroup(totalNodesJoined);

    this.ringpop.logger.debug('ringpop selected join group', {
        local: self.ringpop.whoami(),
        group: group,
        numNodes: group.length
    });

    var nodesJoined = [];
    var nodesFailed = [];
    var numNodesLeft = this.joinSize - totalNodesJoined.length;
    var calledBack = false;
    var startTime = Date.now();

    function onJoin(err, node) {
        if (calledBack) {
            return;
        }

        if (err) {
            nodesFailed.push(node);
        } else {
            nodesJoined.push(node);
        }

        var numCompleted = nodesJoined.length + nodesFailed.length;

        // Finished when either all joins have completed or enough to satisfy
        // the join requirements as defined by `joinSize`.
        if (nodesJoined.length >= numNodesLeft || numCompleted >= group.length) {
            self.ringpop.logger.debug('ringpop join group complete', {
                local: self.ringpop.whoami(),
                groupSize: group.length,
                joinSize: self.joinSize,
                joinTime: Date.now() - startTime,
                numFailures: nodesFailed.length,
                numSuccesses: nodesJoined.length,
                numNodesLeft: numNodesLeft,
                failures: nodesFailed,
                successes: nodesJoined
            });

            calledBack = true;
            callback(null, {
                successes: nodesJoined,
                failures: nodesFailed
            });
        }
    }

    for (var i = 0; i < group.length; i++) {
        this.joinNode(group[i], onJoin);
    }
};

JoinCluster.prototype.joinNode = function joinNode(node, callback) {
    var self = this;
    var joinOpts = {
        host: node,
        timeout: this.joinTimeout
    };
    var joinBody = JSON.stringify({
        app: this.ringpop.app,
        source: this.ringpop.whoami(),
        incarnationNumber: this.ringpop.membership.localMember.incarnationNumber
    });

    this.ringpop.channel.send(joinOpts, '/protocol/join', null, joinBody, function onSend(err, head, body) {
        if (err) {
            return callback(err, node);
        }

        var bodyObj = safeParse(body.toString());

        if (bodyObj) {
            self.ringpop.membership.update(bodyObj.membership);
        }

        callback(null, node);
    });
};

JoinCluster.prototype.selectGroup = function selectGroup(nodesJoined) {
    nodesJoined = nodesJoined || [];

    var self = this;

    // If fully exhausted or first round, initialize this rounds' nodes.
    if (isEmptyArray(this.roundPreferredNodes) && isEmptyArray(this.roundNonPreferredNodes)) {
        this.init(nodesJoined);
    }

    var preferredNodes = this.roundPreferredNodes;
    var nonPreferredNodes = this.roundNonPreferredNodes;
    var numNodesLeft = self.joinSize - nodesJoined.length;
    var group = [];

    function continueSelect() {
        var numNodesSelected = group.length;
        if (numNodesSelected === numNodesLeft * self.parallelismFactor) {
            return false;
        }

        var numNodesAvailable = preferredNodes.length + nonPreferredNodes.length;
        if (numNodesAvailable === 0) {
            return false;
        }

        return true;
    }

    while (continueSelect()) {
        if (preferredNodes.length > 0) {
            group.push(takeNode(preferredNodes));
        } else if (nonPreferredNodes.length > 0) {
            group.push(takeNode(nonPreferredNodes));
        }
    }

    return group;
};

function createJoiner(opts) {
    return new JoinCluster(opts);
}

function joinCluster(opts, callback) {
    var joiner = createJoiner(opts);
    joiner.join(callback);
    return joiner;
}

module.exports = {
    createJoiner: createJoiner,
    joinCluster: joinCluster
};
<commit_msg>swim/join: fail the join if max attempts exceeded<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
'use strict';

var captureHost = require('../util.js').captureHost;
var errors = require('../errors.js');
var isEmptyArray = require('../util.js').isEmptyArray;
var numOrDefault = require('../util.js').numOrDefault;
var safeParse = require('../util.js').safeParse;
var TypedError = require('error/typed');

var JoinAbortedError = TypedError({
    type: 'ringpop.join-aborted',
    message: 'Join aborted because `{reason}`',
    reason: null
});

var JoinDurationExceededError = TypedError({
    type: 'ringpop.join-duration-exceeded',
    message: 'Join duration of `{duration}` exceeded max `{max}`',
    duration: null,
    max: null
});

var JoinAttemptsExceededError = TypedError({
    type: 'ringpop.join-attempts-exceeded',
    message: 'Join attempts of `{joinAttempts}` ' +
        'exceeded max `{maxJoinAttempts}`.\n',
    joinAttempts: null,
    maxJoinAttempts: null
});

var JOIN_RETRY_DELAY = 100;
var JOIN_SIZE = 3;
var JOIN_TIMEOUT = 1000;
// If a node cannot complete a join within MAX_JOIN_DURATION
// there is likely something very wrong. The aim is for the join
// operation to take no more than 1s, under normal conditions.
//
// The duration assigned below is very high for the following
// purposes:
//   - Gives an application developer some time to diagnose
//   what could be wrong.
//   - Gives an operator some time to bootstrap a newly
//   provisioned cluster
//   - Trying forever is futile
var MAX_JOIN_DURATION = 120000;
var MAX_JOIN_ATTEMPTS = 50;
var PARALLELISM_FACTOR = 2;

function isSingleNodeCluster(ringpop) {
    return Array.isArray(ringpop.bootstrapHosts) &&
        ringpop.bootstrapHosts.length === 1 &&
        ringpop.bootstrapHosts[0] === ringpop.hostPort;
}

// Note that this function mutates the array passed in.
function takeNode(hosts) {
    var index = Math.floor(Math.random() * hosts.length);
    var host = hosts[index];
    hosts.splice(index, 1);
    return host;
}

function JoinCluster(opts) {
    opts = opts || {};

    if (!opts.ringpop) {
        throw errors.OptionRequiredError({
            option: 'ringpop'
        });
    }

    if (isEmptyArray(opts.ringpop.bootstrapHosts)) {
        throw errors.InvalidOptionError({
            option: 'ringpop',
            reason: '`bootstrapHosts` is expected to be an array of size 1 or more'
        });
    }

    this.ringpop = opts.ringpop;
    this.host = captureHost(this.ringpop.hostPort);
    this.joinTimeout = numOrDefault(opts.joinTimeout, JOIN_TIMEOUT);

    // This is used as a multiple of the required nodes left
    // to join to satisfy `joinSize`. Additional parallelism
    // can be applied in order for `joinSize` to be satisified
    // faster.
    this.parallelismFactor = numOrDefault(opts.parallelismFactor,
        PARALLELISM_FACTOR);

    // We eventually want to give up if the join process cannot
    // succeed. `maxJoinDuration` is used to restrict that process
    // to a certain time limit.
    this.maxJoinDuration = numOrDefault(opts.maxJoinDuration,
        MAX_JOIN_DURATION);

    // We want to abort the joining process if we have failed
    // a certain number of times.
    this.maxJoinAttempts = numOrDefault(opts.maxJoinAttempts,
        MAX_JOIN_ATTEMPTS);

    // We do not want to retry joining as hard as we can. We
    // want to have some fixed backoff applied before we try
    // to join again
    this.joinRetryDelay = numOrDefault(opts.joinRetryDelay,
        JOIN_RETRY_DELAY);

    // Potential nodes are nodes in the ringpop bootstrap
    // list that can be joined. Upon instantiation, this step
    // simply filters out a node from attempting to join itself.
    this.potentialNodes = this.collectPotentialNodes();
    this.preferredNodes = null;
    this.nonPreferredNodes = null;

    // We either join the number of nodes defined by `joinSize`
    // or limit it to the number of `potentialNodes`. After all,
    // we can't join more than there are to join in the first place.
    var joinSize = numOrDefault(opts.joinSize, JOIN_SIZE);
    this.joinSize = Math.min(joinSize, this.potentialNodes.length);

    // A round is defined as a complete cycle through all
    // potential join targets. Once a round is completed,
    // we start all over again. A full-cycle should be pretty
    // darned rare. We will try and try to join other nodes
    // until `joinSize` is reached or `maxJoinDuration` is
    // exceeded.
    this.roundPotentialNodes = null;
    this.roundPreferredNodes = null;
}

// Potential nodes are those that are not this instance of ringpop.
JoinCluster.prototype.collectPotentialNodes = function collectPotentialNodes(nodesJoined) {
    nodesJoined = nodesJoined || [];

    var self = this;
    return this.ringpop.bootstrapHosts.filter(function filterHost(hostPort) {
        return self.ringpop.hostPort !== hostPort && nodesJoined.indexOf(hostPort) === -1;
    });
};

// Preferred nodes are those that are not on the same host as this instance of ringpop.
JoinCluster.prototype.collectPreferredNodes = function collectPreferredNodes() {
    var self = this;
    return this.potentialNodes.filter(function filterHost(hostPort) {
        return self.host !== captureHost(hostPort);
    });
};

// Non-preferred nodes are everyone else.
JoinCluster.prototype.collectNonPreferredNodes = function collectNonPreferredNodes() {
    var self = this;

    if (isEmptyArray(this.preferredNodes)) {
        return this.potentialNodes;
    } else {
        return this.potentialNodes.filter(function filterHost(hostPort) {
            return self.preferredNodes.indexOf(hostPort) === -1;
        });
    }
};

JoinCluster.prototype.init = function init(nodesJoined) {
    // TODO The "collect" operations are fairly inefficient. This
    // can be improved by indexing by host/port values.
    this.potentialNodes = this.collectPotentialNodes(nodesJoined);
    this.preferredNodes = this.collectPreferredNodes();
    this.nonPreferredNodes = this.collectNonPreferredNodes();

    // Note that these are copies. During the process of joining
    // the "round"-prefixed collections are mutated.
    this.roundPotentialNodes = this.potentialNodes.slice(0);
    this.roundPreferredNodes = this.preferredNodes.slice(0);
    this.roundNonPreferredNodes = this.nonPreferredNodes.slice(0);
};

JoinCluster.prototype.join = function join(callback) {
    var self = this;

    if (this.ringpop.destroyed) {
        process.nextTick(function onTick() {
            callback(JoinAbortedError({
                reason: 'joiner was destroyed'
            }));
        });
        return;
    }

    // No need to go through the join process if you're the only one in the cluster.
    if (isSingleNodeCluster(this.ringpop)) {
        this.ringpop.logger.info('ringpop received a single node cluster join', {
            local: this.ringpop.whoami()
        });

        process.nextTick(function onTick() {
            callback();
        });
        return;
    }

    var nodesJoined = [];
    var numGroups = 0;
    var numJoined = 0;
    var numFailed = 0;
    var startTime = Date.now();
    var calledBack = false;

    function onJoin(err, nodes) {
        if (calledBack) {
            return;
        }

        if (self.ringpop.destroyed) {
            calledBack = true;
            callback(JoinAbortedError({
                reason: 'joiner was destroyed',
            }));
            return;
        }

        if (err) {
            calledBack = true;
            callback(err);
            return;
        }

        nodesJoined = nodesJoined.concat(nodes.successes);
        numJoined += nodes.successes.length;
        numFailed += nodes.failures.length;
        numGroups++;

        if (numJoined >= self.joinSize) {
            self.ringpop.stat('increment', 'join.complete');
            self.ringpop.logger.info('ringpop join complete', {
                local: self.ringpop.whoami(),
                joinSize: self.joinSize,
                joinTime: Date.now() - startTime,
                numJoined: numJoined,
                numGroups: numGroups,
                numFailed: numFailed
            });

            calledBack = true;
            callback(null, nodesJoined);
        } else if (numFailed >= self.maxJoinAttempts) {
            self.ringpop.logger.warn('ringpop max join attempts exceeded', {
                local: self.ringpop.whoami(),
                joinAttempts: numFailed,
                maxJoinAttempts: self.maxJoinAttempts,
                numJoined: numJoined,
                numFailed: numFailed,
                startTime: startTime
            });

            calledBack = true;
            callback(JoinAttemptsExceededError({
                joinAttempts: numFailed,
                maxJoinAttempts: self.maxJoinAttempts
            }));
            return;
        } else {
            var joinDuration = Date.now() - startTime;
            if (joinDuration > self.maxJoinDuration) {
                self.ringpop.logger.warn('ringpop max join duration exceeded', {
                    local: self.ringpop.whoami(),
                    joinDuration: joinDuration,
                    maxJoinDuration: self.maxJoinDuration,
                    numJoined: numJoined,
                    numFailed: numFailed,
                    startTime: startTime
                });

                calledBack = true;
                callback(JoinDurationExceededError({
                    joinDuration: joinDuration,
                    maxJoinDuration: self.maxJoinDuration
                }));
                return;
            }

            self.ringpop.logger.debug('ringpop join not yet complete', {
                local: self.ringpop.whoami(),
                joinSize: self.joinSize,
                numJoined: numJoined,
                numFailed: numFailed,
                numNodesLeft: self.joinSize - numJoined
            });

            setTimeout(reJoin, self.joinRetryDelay);
        }

        function reJoin() {
            self.joinGroup(nodesJoined, onJoin);
        }
    }

    this.joinGroup(nodesJoined, onJoin);
};

JoinCluster.prototype.joinGroup = function joinGroup(totalNodesJoined, callback) {
    var self = this;
    var group = this.selectGroup(totalNodesJoined);

    this.ringpop.logger.debug('ringpop selected join group', {
        local: self.ringpop.whoami(),
        group: group,
        numNodes: group.length
    });

    var nodesJoined = [];
    var nodesFailed = [];
    var numNodesLeft = this.joinSize - totalNodesJoined.length;
    var calledBack = false;
    var startTime = Date.now();

    function onJoin(err, node) {
        if (calledBack) {
            return;
        }

        if (err) {
            nodesFailed.push(node);
        } else {
            nodesJoined.push(node);
        }

        var numCompleted = nodesJoined.length + nodesFailed.length;

        // Finished when either all joins have completed or enough to satisfy
        // the join requirements as defined by `joinSize`.
        if (nodesJoined.length >= numNodesLeft || numCompleted >= group.length) {
            self.ringpop.logger.debug('ringpop join group complete', {
                local: self.ringpop.whoami(),
                groupSize: group.length,
                joinSize: self.joinSize,
                joinTime: Date.now() - startTime,
                numFailures: nodesFailed.length,
                numSuccesses: nodesJoined.length,
                numNodesLeft: numNodesLeft,
                failures: nodesFailed,
                successes: nodesJoined
            });

            calledBack = true;
            callback(null, {
                successes: nodesJoined,
                failures: nodesFailed
            });
        }
    }

    for (var i = 0; i < group.length; i++) {
        this.joinNode(group[i], onJoin);
    }
};

JoinCluster.prototype.joinNode = function joinNode(node, callback) {
    var self = this;
    var joinOpts = {
        host: node,
        timeout: this.joinTimeout
    };
    var joinBody = JSON.stringify({
        app: this.ringpop.app,
        source: this.ringpop.whoami(),
        incarnationNumber: this.ringpop.membership.localMember.incarnationNumber
    });

    this.ringpop.channel.send(joinOpts, '/protocol/join', null, joinBody, function onSend(err, head, body) {
        if (err) {
            return callback(err, node);
        }

        var bodyObj = safeParse(body.toString());

        if (bodyObj) {
            self.ringpop.membership.update(bodyObj.membership);
        }

        callback(null, node);
    });
};

JoinCluster.prototype.selectGroup = function selectGroup(nodesJoined) {
    nodesJoined = nodesJoined || [];

    var self = this;

    // If fully exhausted or first round, initialize this rounds' nodes.
    if (isEmptyArray(this.roundPreferredNodes) && isEmptyArray(this.roundNonPreferredNodes)) {
        this.init(nodesJoined);
    }

    var preferredNodes = this.roundPreferredNodes;
    var nonPreferredNodes = this.roundNonPreferredNodes;
    var numNodesLeft = self.joinSize - nodesJoined.length;
    var group = [];

    function continueSelect() {
        var numNodesSelected = group.length;
        if (numNodesSelected === numNodesLeft * self.parallelismFactor) {
            return false;
        }

        var numNodesAvailable = preferredNodes.length + nonPreferredNodes.length;
        if (numNodesAvailable === 0) {
            return false;
        }

        return true;
    }

    while (continueSelect()) {
        if (preferredNodes.length > 0) {
            group.push(takeNode(preferredNodes));
        } else if (nonPreferredNodes.length > 0) {
            group.push(takeNode(nonPreferredNodes));
        }
    }

    return group;
};

function createJoiner(opts) {
    return new JoinCluster(opts);
}

function joinCluster(opts, callback) {
    var joiner = createJoiner(opts);
    joiner.join(callback);
    return joiner;
}

module.exports = {
    createJoiner: createJoiner,
    joinCluster: joinCluster
};
<|endoftext|>