<commit_before>'use strict'

/**
 * Native Dependencies
 */
var path = require('path')

/**
 * External Dependencies
 */
var glob = require('glob')

/**
 * Local Dependencies
 */
var Plugin = require('./plugin')
var Injector = require('./injector')

/**
 * Symbols
 */
const plugins = Symbol()
const injector = Symbol()

/**
 * Sunstone
 */
class Sunstone {

  /**
   * Constructor
   */
  constructor () {
    this[injector] = new Injector()
    this[plugins] = {}
  }

  /**
   * Plugin
   */
  plugin (name, manifest) {
    if (manifest) {
      let plugin = new Plugin(name, manifest, this[injector])
      this[plugins][name] = plugin
      return plugin
    } else {
      return this[plugins][name]
    }
  }

  /**
   * Require
   */
  require() {
    let segments = Array.prototype.slice.call(arguments, this.require.length)
    let directory = path.join.apply(null, segments)
    let pattern = path.join(directory, '**/*.js')
    let plugins = glob.sync(path.resolve(pattern))

    plugins.forEach((plugin) => {
      require(plugin)(this)
    })
  }

  /**
   * Bootstrap
   */
  bootstrap () {
    let cwd = process.cwd()

    // load plugins from source code
    this.require(__dirname, 'plugins')

    // load plugins from node_modules
    // ???

    // load plugins from project
    if (cwd !== __dirname) {
      this.require(cwd, 'plugins')
    }

    // assemble the server and create a reference
    this.server = this[injector].get('server')
    this.settings = this[injector].get('settings')

    // register routes
    this[injector].find({
      type: 'router'
    }).forEach((router) => {
      router.mount()
    })
  }

  /**
   * Start
   */
  static start () {
    let sunstone = new Sunstone()
    sunstone.bootstrap()

    let host = sunstone.settings.host
    let port = sunstone.settings.port

    sunstone.server.listen(port, function () {
      console.log(`Listening on ${port}`)
    })
  }

}

/**
 * Exports
 */
module.exports = Sunstone

<commit_msg>refactor(require): enforce plugin entry point needs to be called index.js<commit_after>'use strict'

/**
 * Native Dependencies
 */
var path = require('path')

/**
 * External Dependencies
 */
var glob = require('glob')

/**
 * Local Dependencies
 */
var Plugin = require('./plugin')
var Injector = require('./injector')

/**
 * Symbols
 */
const plugins = Symbol()
const injector = Symbol()

/**
 * Sunstone
 */
class Sunstone {

  /**
   * Constructor
   */
  constructor () {
    this[injector] = new Injector()
    this[plugins] = {}
  }

  /**
   * Plugin
   */
  plugin (name, manifest) {
    if (manifest) {
      let plugin = new Plugin(name, manifest, this[injector])
      this[plugins][name] = plugin
      return plugin
    } else {
      return this[plugins][name]
    }
  }

  /**
   * Require
   */
  require () {
    let segments = Array.prototype.slice.call(arguments, this.require.length)
    let directory = path.join.apply(null, segments)
    let pattern = path.join(directory, '**/index.js')
    let plugins = glob.sync(path.resolve(pattern))

    plugins.forEach((plugin) => {
      require(plugin)(this)
    })
  }

  /**
   * Bootstrap
   */
  bootstrap () {
    let cwd = process.cwd()

    // load plugins from source code
    this.require(__dirname, 'plugins')

    // load plugins from node_modules
    // ???

    // load plugins from project
    if (cwd !== __dirname) {
      this.require(cwd, 'plugins')
    }

    // assemble the server and create a reference
    this.server = this[injector].get('server')
    this.settings = this[injector].get('settings')

    // register routes
    this[injector].find({
      type: 'router'
    }).forEach((router) => {
      router.mount()
    })
  }

  /**
   * Start
   */
  static start () {
    let sunstone = new Sunstone()
    sunstone.bootstrap()

    let host = sunstone.settings.host
    let port = sunstone.settings.port

    sunstone.server.listen(port, function () {
      console.log(`Listening on ${port}`)
    })
  }

}

/**
 * Exports
 */
module.exports = Sunstone

<|endoftext|>


<commit_before>#!/usr/bin/env node
'use strict'
var Bluebird = require('bluebird')
var simpleFetch = require('./simple-fetch')
var fs = require('fs')
var TOML = require('./toml.js')
var getFic = require('./get-fic.js')
var ficToEpub = require('./fic-to-epub.js')
var Gauge = require('gauge')
var TrackerGroup = require('are-we-there-yet').TrackerGroup
var Tracker = require('are-we-there-yet').Tracker
var spinWith = require('./spin-with.js')
var filenameize = require('./filenameize.js')
var ms = require('mississippi')
var pipe = Bluebird.promisify(ms.pipe)
var argv = require('yargs')
  .usage('Usage: $0 <fic> [--xf_session=<sessionid>] [--xf_user=<userid>]')
  .demand(1, '<fic> - A fic metadata file to fetch a fic for. Typically ends in .fic.toml')
  .describe('xf_session', 'value of your xf_session variable')
  .describe('xf_user', 'value of your xf_session variable')
  .argv

main()

function main () {
  var cookie = argv.xf_session
  var user = argv.xf_user
  var fetchOpts = {cacheBreak: false}
  if (cookie) {
    if (!fetchOpts.headers) fetchOpts.headers = {}
    fetchOpts.headers.Cookie = 'xf_session=' + cookie
  }
  if (user) {
    if (!fetchOpts.headers) fetchOpts.headers = {}
    fetchOpts.headers.Cookie += '; xf_user=' + user
  }
  var fetchWithCache = simpleFetch(fetchOpts)
  var fetch = simpleFetch(fetchOpts)
  var gauge = new Gauge()
  var trackerGroup = new TrackerGroup()
  trackerGroup.on('change', function (name, completed) {
    gauge.show({completed: completed})
  })
  var trackers = argv._.map(function () { return trackerGroup.newItem(1) })
  var spin = spinWith(gauge)

  return Bluebird.each(argv._, fetchFic)

  function fetchFic (ficFile, ii) {
    var fic = TOML.parse(fs.readFileSync(ficFile, 'utf8'))
    fic.chapters.forEach(function (chapter, ii) {
      chapter.order = ii
    })

    var tracker = trackers[ii]
    gauge.show(fic.title + ': Fetching fic')
    var fetchWithOpts = function (url) {
      return spin(fetchWithCache(url)).tap(function () {
        tracker.completeWork(1)
      })
    }
    var filename = filenameize(fic.title) + '.epub'
    tracker.addWork(fic.chapters.length)

    return pipe(
      getFic(fetchWithOpts, fic.chapters, 1),
      ficToEpub(fic),
      fs.createWriteStream(filename)
    ).tap(function () {
      tracker.finish()
      gauge.hide()
      console.log(filename)
    })
  }
}
<commit_msg>Work around a bug in our TOML emitter =/<commit_after>#!/usr/bin/env node
'use strict'
var Bluebird = require('bluebird')
var simpleFetch = require('./simple-fetch')
var fs = require('fs')
var TOML = require('./toml.js')
var getFic = require('./get-fic.js')
var ficToEpub = require('./fic-to-epub.js')
var Gauge = require('gauge')
var TrackerGroup = require('are-we-there-yet').TrackerGroup
var Tracker = require('are-we-there-yet').Tracker
var spinWith = require('./spin-with.js')
var filenameize = require('./filenameize.js')
var ms = require('mississippi')
var pipe = Bluebird.promisify(ms.pipe)
var argv = require('yargs')
  .usage('Usage: $0 <fic> [--xf_session=<sessionid>] [--xf_user=<userid>]')
  .demand(1, '<fic> - A fic metadata file to fetch a fic for. Typically ends in .fic.toml')
  .describe('xf_session', 'value of your xf_session variable')
  .describe('xf_user', 'value of your xf_session variable')
  .argv

main()

function main () {
  var cookie = argv.xf_session
  var user = argv.xf_user
  var fetchOpts = {cacheBreak: false}
  if (cookie) {
    if (!fetchOpts.headers) fetchOpts.headers = {}
    fetchOpts.headers.Cookie = 'xf_session=' + cookie
  }
  if (user) {
    if (!fetchOpts.headers) fetchOpts.headers = {}
    fetchOpts.headers.Cookie += '; xf_user=' + user
  }
  var fetchWithCache = simpleFetch(fetchOpts)
  var fetch = simpleFetch(fetchOpts)
  var gauge = new Gauge()
  var trackerGroup = new TrackerGroup()
  trackerGroup.on('change', function (name, completed) {
    gauge.show({completed: completed})
  })
  var trackers = argv._.map(function () { return trackerGroup.newItem(1) })
  var spin = spinWith(gauge)

  return Bluebird.each(argv._, fetchFic)

  function fetchFic (ficFile, ii) {
    var fic = TOML.parse(fs.readFileSync(ficFile, 'utf8'))
    fic.description = fic.description.replace(/\\n/, String.fromCharCode(10))
    fic.chapters.forEach(function (chapter, ii) {
      chapter.order = ii
    })

    var tracker = trackers[ii]
    gauge.show(fic.title + ': Fetching fic')
    var fetchWithOpts = function (url) {
      return spin(fetchWithCache(url)).tap(function () {
        tracker.completeWork(1)
      })
    }
    var filename = filenameize(fic.title) + '.epub'
    tracker.addWork(fic.chapters.length)

    return pipe(
      getFic(fetchWithOpts, fic.chapters, 1),
      ficToEpub(fic),
      fs.createWriteStream(filename)
    ).tap(function () {
      tracker.finish()
      gauge.hide()
      console.log(filename)
    })
  }
}
<|endoftext|>


<commit_before>/* @flow */

const path = require('path')
const _ = require('lodash')

const { getInode } = require('./event')
const localChange = require('./change')
const logger = require('../logger')
const metadata = require('../metadata')
const measureTime = require('../perftools')

/*::
import type { LocalEvent } from './event'
import type {
  LocalChange,
  LocalDirAddition,
  LocalDirDeletion,
  LocalDirMove,
  LocalFileAddition,
  LocalFileDeletion,
  LocalFileMove
} from './change'
*/

const log = logger({
  component: 'LocalAnalysis'
})

module.exports = function analysis (events /*: LocalEvent[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  const changes /*: LocalChange[] */ = analyseEvents(events, pendingChanges)
  sortBeforeSquash(changes)
  squashMoves(changes)
  finalSort(changes)
  return separatePendingChanges(changes, pendingChanges)
}

const panic = (context, description) => {
  log.error(_.merge({sentry: true}, context), description)
  throw new Error(description)
}

function analyseEvents (events /*: LocalEvent[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  const stopMeasure = measureTime('LocalWatcher#analyseEvents')
  // OPTIMIZE: new Array(events.length)
  const changes /*: LocalChange[] */ = []
  const changesByInode /*: Map<number, LocalChange> */ = new Map()
  const changesByPath /*: Map<string, LocalChange> */ = new Map()
  const getChangeByInode = (e) => {
    const ino = getInode(e)
    if (ino) return changesByInode.get(ino)
    else return null
  }
  const getChangeByPath = (e) => {
    return changesByPath.get(e.path)
  }
  const changeFound = (c /*: LocalChange | true */) => {
    if (c === true) {
      // A previous change was transformed. Nothing more to index.
      return
    }
    changesByPath.set(c.path, c)
    if (typeof c.ino === 'number') changesByInode.set(c.ino, c)
    else changes.push(c)
  }

  if (pendingChanges.length > 0) {
    log.warn({changes: pendingChanges}, `Prepend ${pendingChanges.length} pending change(s)`)
    for (const a of pendingChanges) { changeFound(a) }
    pendingChanges.length = 0
  }

  log.trace('Analyze events...')

  for (let e/*: LocalEvent */ of events) {
    if (process.env.DEBUG) log.trace({currentEvent: e, path: e.path})
    try {
      // chokidar make mistakes
      if (e.type === 'unlinkDir' && e.old && e.old.docType === 'file') {
        log.warn({event: e, old: e.old, path: e.path}, 'chokidar miscategorized event (was file, event unlinkDir)')
        // $FlowFixMe
        e.type = 'unlink'
      }

      if (e.type === 'unlink' && e.old && e.old.docType === 'folder') {
        log.warn({event: e, old: e.old, path: e.path}, 'chokidar miscategorized event (was folder, event unlink)')
        // $FlowFixMe
        e.type = 'unlinkDir'
      }

      switch (e.type) {
        case 'add':
          {
            const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
            if (moveChange) {
              localChange.includeAddEventInFileMove(moveChange, e)
              break
            }

            const unlinkChange /*: ?LocalFileDeletion */ = localChange.maybeDeleteFile(getChangeByInode(e))
            if (unlinkChange) {
              changeFound(localChange.fileMoveFromUnlinkAdd(unlinkChange, e))
              break
            }

            changeFound(
              localChange.fileMoveIdenticalOffline(e) ||
              localChange.fileAddition(e)
            )
          }
          break
        case 'addDir':
          {
            const moveChange /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByInode(e))
            if (moveChange) {
              localChange.includeAddDirEventInDirMove(moveChange, e)
              break
            }
            const unlinkChange /*: ?LocalDirDeletion */ = localChange.maybeDeleteFolder(getChangeByInode(e))
            if (unlinkChange) {
              changeFound(localChange.dirMoveFromUnlinkAdd(unlinkChange, e))
              break
            }
            const addChange /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByInode(e))
            if (addChange && metadata.id(addChange.path) === metadata.id(e.path) && addChange.path !== e.path) {
              changeFound(localChange.dirRenamingCaseOnlyFromAddAdd(addChange, e))
              break
            }
            changeFound(
              localChange.dirMoveIdenticalOffline(e) ||
              localChange.dirAddition(e)
            )
          }
          break
        case 'change':
          const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
          if (moveChange) {
            localChange.includeChangeEventIntoFileMove(moveChange, e)
            break
          }

          // There was an unlink on the same file, this is most probably a move and replace
          const unlinkChange /*: ?LocalFileDeletion */ = localChange.maybeDeleteFile(getChangeByInode(e))
          if (unlinkChange) {
            const moveChange = localChange.fileMoveFromFileDeletionChange(unlinkChange, e)
            changeFound(moveChange)
            break
          }

          const addChange /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByInode(e))
          if (addChange && metadata.id(addChange.path) === metadata.id(e.path) && addChange.path !== e.path) {
            changeFound(localChange.fileMoveIdentical(addChange, e))
            break
          }

          changeFound(localChange.fileUpdate(e))
          break
        case 'unlink':
          {
            const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
            /* istanbul ignore next */
            if (moveChange) {
              // TODO: Pending move
              panic({path: e.path, moveChange, event: e},
                'We should not have both move and unlink changes since ' +
                'checksumless adds and inode-less unlink events are dropped')
            }

            const addChange /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByInode(e))
            if (addChange) {
              // TODO: pending move
              changeFound(localChange.fileMoveFromAddUnlink(addChange, e))
              break
            } else if (getInode(e)) {
              changeFound(localChange.fileDeletion(e))
              break
            }
            const moveChangeSamePath /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByPath(e))
            if (moveChangeSamePath && moveChangeSamePath.md5sum == null) { // FIXME: if change && change.wip?
              localChange.convertFileMoveToDeletion(moveChangeSamePath)
              break
            }
            const addChangeSamePath /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByPath(e))
            if (addChangeSamePath && addChangeSamePath.wip) {
              // $FlowFixMe
              addChangeSamePath.type = 'Ignored'
              delete addChangeSamePath.wip
              delete addChangeSamePath.md5sum
            }
            // Otherwise, skip unlink event by multiple moves
          }
          break
        case 'unlinkDir':
          {
            const moveChange /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByInode(e))
            /* istanbul ignore next */
            if (moveChange) {
              // TODO: pending move
              panic({path: e.path, moveChange, event: e},
                'We should not have both move and unlinkDir changes since ' +
                'non-existing addDir and inode-less unlinkDir events are dropped')
            }

            const addChange /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByInode(e))
            if (addChange) {
              changeFound(localChange.dirMoveFromAddUnlink(addChange, e))
              break
            }
            if (getInode(e)) {
              changeFound(localChange.dirDeletion(e))
              break
            }
            const addChangeSamePath /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByPath(e))
            if (addChangeSamePath && addChangeSamePath.wip) {
              log.debug({path: addChangeSamePath.path, ino: addChangeSamePath.ino},
                'Folder was added then deleted. Ignoring add.')
              // $FlowFixMe
              addChangeSamePath.type = 'Ignored'
              break
            }

            const moveChangeSamePath /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByPath(e))
            if (moveChangeSamePath && moveChangeSamePath.wip) {
              localChange.convertDirMoveToDeletion(moveChangeSamePath)
            }
          }
          break
        default:
          throw new TypeError(`Unknown event type: ${e.type}`)
      }
    } catch (err) {
      const sentry = err.name === 'InvalidLocalMoveEvent'
      log.error({err, path: e.path, sentry})
      throw err
    }
  }

  log.trace('Flatten changes map...')
  for (let a of changesByInode.values()) changes.push(a)

  stopMeasure()
  return changes
}

// TODO: Rename according to the sort logic
function sortBeforeSquash (changes /*: LocalChange[] */) {
  log.trace('Sort changes before squash...')
  const stopMeasure = measureTime('LocalWatcher#sortBeforeSquash')
  changes.sort((a, b) => {
    if (a.type === 'DirMove' || a.type === 'FileMove') {
      if (b.type === 'DirMove' || b.type === 'FileMove') {
        if (a.path < b.path) return -1
        else if (a.path > b.path) return 1
        else return 0
      } else return -1
    } else if (b.type === 'DirMove' || b.type === 'FileMove') {
      return 1
    } else {
      return 0
    }
  })
  stopMeasure()
}

function squashMoves (changes /*: LocalChange[] */) {
  log.trace('Squash moves...')
  const stopMeasure = measureTime('LocalWatcher#squashMoves')

  for (let i = 0; i < changes.length; i++) {
    let a = changes[i]

    if (a.type !== 'DirMove' && a.type !== 'FileMove') break
    for (let j = i + 1; j < changes.length; j++) {
      let b = changes[j]
      if (b.type !== 'DirMove' && b.type !== 'FileMove') break

      // inline of LocalChange.isChildMove
      if (a.type === 'DirMove' &&
      b.path.indexOf(a.path + path.sep) === 0 &&
      a.old && b.old &&
      b.old.path.indexOf(a.old.path + path.sep) === 0) {
        log.debug({oldpath: b.old.path, path: b.path}, 'descendant move')
        a.wip = a.wip || b.wip
        if (b.path.substr(a.path.length) === b.old.path.substr(a.old.path.length)) {
          log.debug({oldpath: b.old.path, path: b.path}, 'ignoring explicit child move')
          changes.splice(j--, 1)
        } else {
          log.debug({oldpath: b.old.path, path: b.path}, 'move inside move')
          b.old.path = b.old.path.replace(a.old.path, a.path)
          b.needRefetch = true
        }
      }
    }
  }

  stopMeasure()
}

function separatePendingChanges (changes /*: LocalChange[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  log.trace('Reserve changes in progress for next flush...')

  for (let i = 0; i < changes.length; i++) {
    const change = changes[i]
    if (change.wip) {
      if (change.type === 'DirMove' || change.type === 'FileMove') {
        log.debug({
          change: change.type,
          oldpath: change.old.path,
          path: change.path,
          ino: change.ino
        }, 'incomplete change')
      } else {
        log.debug({change: change.type, path: change.path}, 'incomplete change')
      }
      pendingChanges.push(changes[i])
    } else {
      log.debug(`Identified ${changes.length} change(s).`)
      log.debug(`${pendingChanges.length} of them are still pending.`)
      return changes.slice(i)
    }
  }
  // All actions are WIP
  return []
}

// TODO: Rename according to the sort logic
const finalSorter = (a /*: LocalChange */, b /*: LocalChange */) => {
  if (a.wip && !b.wip) return -1
  if (b.wip && !a.wip) return 1

  // b is deleting something which is a children of what a adds
  if (!localChange.addPath(b) && localChange.childOf(localChange.addPath(a), localChange.delPath(b))) return 1
  // a is deleting something which is a children of what b adds
  if (!localChange.addPath(a) && localChange.childOf(localChange.addPath(b), localChange.delPath(a))) return -1

  // b is moving something which is a children of what a adds
  if (localChange.childOf(localChange.addPath(a), localChange.delPath(b))) return -1
  // a is deleting or moving something which is a children of what b adds
  if (localChange.childOf(localChange.addPath(b), localChange.delPath(a))) return 1

  // if one change is a child of another, it takes priority
  if (localChange.isChildAdd(a, b)) return -1
  if (localChange.isChildDelete(b, a)) return -1
  if (localChange.isChildAdd(b, a)) return 1
  if (localChange.isChildDelete(a, b)) return 1

  // a is deleted what b added
  if (localChange.delPath(a) === localChange.addPath(b)) return -1
  // b is deleting what a added
  if (localChange.delPath(b) === localChange.addPath(a)) return 1

  // both adds at same path (seen with move + add)
  if (localChange.addPath(a) && localChange.addPath(a) === localChange.addPath(b)) return -1
  // both deletes at same path (seen with delete + move)
  if (localChange.delPath(a) && (localChange.delPath(a) === localChange.delPath(b))) return 1

  // otherwise, order by add path
  if (localChange.lower(localChange.addPath(a), localChange.addPath(b))) return -1
  if (localChange.lower(localChange.addPath(b), localChange.addPath(a))) return 1

  // if there isnt 2 add paths, sort by del path
  if (localChange.lower(localChange.delPath(b), localChange.delPath(a))) return -1

  return 1
}

// TODO: Rename according to the sort logic
function finalSort (changes /*: LocalChange[] */) {
  log.trace('Final sort...')
  const stopMeasure = measureTime('LocalWatcher#finalSort')
  changes.sort(finalSorter)
  stopMeasure()
}
<commit_msg>core/local/analysis: Add changeFound(undefined) support<commit_after>/* @flow */

const path = require('path')
const _ = require('lodash')

const { getInode } = require('./event')
const localChange = require('./change')
const logger = require('../logger')
const metadata = require('../metadata')
const measureTime = require('../perftools')

/*::
import type { LocalEvent } from './event'
import type {
  LocalChange,
  LocalDirAddition,
  LocalDirDeletion,
  LocalDirMove,
  LocalFileAddition,
  LocalFileDeletion,
  LocalFileMove
} from './change'
*/

const log = logger({
  component: 'LocalAnalysis'
})

module.exports = function analysis (events /*: LocalEvent[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  const changes /*: LocalChange[] */ = analyseEvents(events, pendingChanges)
  sortBeforeSquash(changes)
  squashMoves(changes)
  finalSort(changes)
  return separatePendingChanges(changes, pendingChanges)
}

const panic = (context, description) => {
  log.error(_.merge({sentry: true}, context), description)
  throw new Error(description)
}

function analyseEvents (events /*: LocalEvent[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  const stopMeasure = measureTime('LocalWatcher#analyseEvents')
  // OPTIMIZE: new Array(events.length)
  const changes /*: LocalChange[] */ = []
  const changesByInode /*: Map<number, LocalChange> */ = new Map()
  const changesByPath /*: Map<string, LocalChange> */ = new Map()
  const getChangeByInode = (e) => {
    const ino = getInode(e)
    if (ino) return changesByInode.get(ino)
    else return null
  }
  const getChangeByPath = (e) => {
    return changesByPath.get(e.path)
  }
  const changeFound = (c /*: ?LocalChange | true */) => {
    if (c == null) {
      // No change was found. Nothing to do.
      return
    }
    if (c === true) {
      // A previous change was transformed. Nothing more to index.
      return
    }
    changesByPath.set(c.path, c)
    if (typeof c.ino === 'number') changesByInode.set(c.ino, c)
    else changes.push(c)
  }

  if (pendingChanges.length > 0) {
    log.warn({changes: pendingChanges}, `Prepend ${pendingChanges.length} pending change(s)`)
    for (const a of pendingChanges) { changeFound(a) }
    pendingChanges.length = 0
  }

  log.trace('Analyze events...')

  for (let e/*: LocalEvent */ of events) {
    if (process.env.DEBUG) log.trace({currentEvent: e, path: e.path})
    try {
      // chokidar make mistakes
      if (e.type === 'unlinkDir' && e.old && e.old.docType === 'file') {
        log.warn({event: e, old: e.old, path: e.path}, 'chokidar miscategorized event (was file, event unlinkDir)')
        // $FlowFixMe
        e.type = 'unlink'
      }

      if (e.type === 'unlink' && e.old && e.old.docType === 'folder') {
        log.warn({event: e, old: e.old, path: e.path}, 'chokidar miscategorized event (was folder, event unlink)')
        // $FlowFixMe
        e.type = 'unlinkDir'
      }

      switch (e.type) {
        case 'add':
          {
            const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
            if (moveChange) {
              localChange.includeAddEventInFileMove(moveChange, e)
              break
            }

            const unlinkChange /*: ?LocalFileDeletion */ = localChange.maybeDeleteFile(getChangeByInode(e))
            if (unlinkChange) {
              changeFound(localChange.fileMoveFromUnlinkAdd(unlinkChange, e))
              break
            }

            changeFound(
              localChange.fileMoveIdenticalOffline(e) ||
              localChange.fileAddition(e)
            )
          }
          break
        case 'addDir':
          {
            const moveChange /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByInode(e))
            if (moveChange) {
              localChange.includeAddDirEventInDirMove(moveChange, e)
              break
            }
            const unlinkChange /*: ?LocalDirDeletion */ = localChange.maybeDeleteFolder(getChangeByInode(e))
            if (unlinkChange) {
              changeFound(localChange.dirMoveFromUnlinkAdd(unlinkChange, e))
              break
            }
            const addChange /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByInode(e))
            if (addChange && metadata.id(addChange.path) === metadata.id(e.path) && addChange.path !== e.path) {
              changeFound(localChange.dirRenamingCaseOnlyFromAddAdd(addChange, e))
              break
            }
            changeFound(
              localChange.dirMoveIdenticalOffline(e) ||
              localChange.dirAddition(e)
            )
          }
          break
        case 'change':
          const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
          if (moveChange) {
            localChange.includeChangeEventIntoFileMove(moveChange, e)
            break
          }

          // There was an unlink on the same file, this is most probably a move and replace
          const unlinkChange /*: ?LocalFileDeletion */ = localChange.maybeDeleteFile(getChangeByInode(e))
          if (unlinkChange) {
            const moveChange = localChange.fileMoveFromFileDeletionChange(unlinkChange, e)
            changeFound(moveChange)
            break
          }

          const addChange /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByInode(e))
          if (addChange && metadata.id(addChange.path) === metadata.id(e.path) && addChange.path !== e.path) {
            changeFound(localChange.fileMoveIdentical(addChange, e))
            break
          }

          changeFound(localChange.fileUpdate(e))
          break
        case 'unlink':
          {
            const moveChange /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByInode(e))
            /* istanbul ignore next */
            if (moveChange) {
              // TODO: Pending move
              panic({path: e.path, moveChange, event: e},
                'We should not have both move and unlink changes since ' +
                'checksumless adds and inode-less unlink events are dropped')
            }

            const addChange /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByInode(e))
            if (addChange) {
              // TODO: pending move
              changeFound(localChange.fileMoveFromAddUnlink(addChange, e))
              break
            } else if (getInode(e)) {
              changeFound(localChange.fileDeletion(e))
              break
            }
            const moveChangeSamePath /*: ?LocalFileMove */ = localChange.maybeMoveFile(getChangeByPath(e))
            if (moveChangeSamePath && moveChangeSamePath.md5sum == null) { // FIXME: if change && change.wip?
              localChange.convertFileMoveToDeletion(moveChangeSamePath)
              break
            }
            const addChangeSamePath /*: ?LocalFileAddition */ = localChange.maybeAddFile(getChangeByPath(e))
            if (addChangeSamePath && addChangeSamePath.wip) {
              // $FlowFixMe
              addChangeSamePath.type = 'Ignored'
              delete addChangeSamePath.wip
              delete addChangeSamePath.md5sum
            }
            // Otherwise, skip unlink event by multiple moves
          }
          break
        case 'unlinkDir':
          {
            const moveChange /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByInode(e))
            /* istanbul ignore next */
            if (moveChange) {
              // TODO: pending move
              panic({path: e.path, moveChange, event: e},
                'We should not have both move and unlinkDir changes since ' +
                'non-existing addDir and inode-less unlinkDir events are dropped')
            }

            const addChange /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByInode(e))
            if (addChange) {
              changeFound(localChange.dirMoveFromAddUnlink(addChange, e))
              break
            }
            if (getInode(e)) {
              changeFound(localChange.dirDeletion(e))
              break
            }
            const addChangeSamePath /*: ?LocalDirAddition */ = localChange.maybePutFolder(getChangeByPath(e))
            if (addChangeSamePath && addChangeSamePath.wip) {
              log.debug({path: addChangeSamePath.path, ino: addChangeSamePath.ino},
                'Folder was added then deleted. Ignoring add.')
              // $FlowFixMe
              addChangeSamePath.type = 'Ignored'
              break
            }

            const moveChangeSamePath /*: ?LocalDirMove */ = localChange.maybeMoveFolder(getChangeByPath(e))
            if (moveChangeSamePath && moveChangeSamePath.wip) {
              localChange.convertDirMoveToDeletion(moveChangeSamePath)
            }
          }
          break
        default:
          throw new TypeError(`Unknown event type: ${e.type}`)
      }
    } catch (err) {
      const sentry = err.name === 'InvalidLocalMoveEvent'
      log.error({err, path: e.path, sentry})
      throw err
    }
  }

  log.trace('Flatten changes map...')
  for (let a of changesByInode.values()) changes.push(a)

  stopMeasure()
  return changes
}

// TODO: Rename according to the sort logic
function sortBeforeSquash (changes /*: LocalChange[] */) {
  log.trace('Sort changes before squash...')
  const stopMeasure = measureTime('LocalWatcher#sortBeforeSquash')
  changes.sort((a, b) => {
    if (a.type === 'DirMove' || a.type === 'FileMove') {
      if (b.type === 'DirMove' || b.type === 'FileMove') {
        if (a.path < b.path) return -1
        else if (a.path > b.path) return 1
        else return 0
      } else return -1
    } else if (b.type === 'DirMove' || b.type === 'FileMove') {
      return 1
    } else {
      return 0
    }
  })
  stopMeasure()
}

function squashMoves (changes /*: LocalChange[] */) {
  log.trace('Squash moves...')
  const stopMeasure = measureTime('LocalWatcher#squashMoves')

  for (let i = 0; i < changes.length; i++) {
    let a = changes[i]

    if (a.type !== 'DirMove' && a.type !== 'FileMove') break
    for (let j = i + 1; j < changes.length; j++) {
      let b = changes[j]
      if (b.type !== 'DirMove' && b.type !== 'FileMove') break

      // inline of LocalChange.isChildMove
      if (a.type === 'DirMove' &&
      b.path.indexOf(a.path + path.sep) === 0 &&
      a.old && b.old &&
      b.old.path.indexOf(a.old.path + path.sep) === 0) {
        log.debug({oldpath: b.old.path, path: b.path}, 'descendant move')
        a.wip = a.wip || b.wip
        if (b.path.substr(a.path.length) === b.old.path.substr(a.old.path.length)) {
          log.debug({oldpath: b.old.path, path: b.path}, 'ignoring explicit child move')
          changes.splice(j--, 1)
        } else {
          log.debug({oldpath: b.old.path, path: b.path}, 'move inside move')
          b.old.path = b.old.path.replace(a.old.path, a.path)
          b.needRefetch = true
        }
      }
    }
  }

  stopMeasure()
}

function separatePendingChanges (changes /*: LocalChange[] */, pendingChanges /*: LocalChange[] */) /*: LocalChange[] */ {
  log.trace('Reserve changes in progress for next flush...')

  for (let i = 0; i < changes.length; i++) {
    const change = changes[i]
    if (change.wip) {
      if (change.type === 'DirMove' || change.type === 'FileMove') {
        log.debug({
          change: change.type,
          oldpath: change.old.path,
          path: change.path,
          ino: change.ino
        }, 'incomplete change')
      } else {
        log.debug({change: change.type, path: change.path}, 'incomplete change')
      }
      pendingChanges.push(changes[i])
    } else {
      log.debug(`Identified ${changes.length} change(s).`)
      log.debug(`${pendingChanges.length} of them are still pending.`)
      return changes.slice(i)
    }
  }
  // All actions are WIP
  return []
}

// TODO: Rename according to the sort logic
const finalSorter = (a /*: LocalChange */, b /*: LocalChange */) => {
  if (a.wip && !b.wip) return -1
  if (b.wip && !a.wip) return 1

  // b is deleting something which is a children of what a adds
  if (!localChange.addPath(b) && localChange.childOf(localChange.addPath(a), localChange.delPath(b))) return 1
  // a is deleting something which is a children of what b adds
  if (!localChange.addPath(a) && localChange.childOf(localChange.addPath(b), localChange.delPath(a))) return -1

  // b is moving something which is a children of what a adds
  if (localChange.childOf(localChange.addPath(a), localChange.delPath(b))) return -1
  // a is deleting or moving something which is a children of what b adds
  if (localChange.childOf(localChange.addPath(b), localChange.delPath(a))) return 1

  // if one change is a child of another, it takes priority
  if (localChange.isChildAdd(a, b)) return -1
  if (localChange.isChildDelete(b, a)) return -1
  if (localChange.isChildAdd(b, a)) return 1
  if (localChange.isChildDelete(a, b)) return 1

  // a is deleted what b added
  if (localChange.delPath(a) === localChange.addPath(b)) return -1
  // b is deleting what a added
  if (localChange.delPath(b) === localChange.addPath(a)) return 1

  // both adds at same path (seen with move + add)
  if (localChange.addPath(a) && localChange.addPath(a) === localChange.addPath(b)) return -1
  // both deletes at same path (seen with delete + move)
  if (localChange.delPath(a) && (localChange.delPath(a) === localChange.delPath(b))) return 1

  // otherwise, order by add path
  if (localChange.lower(localChange.addPath(a), localChange.addPath(b))) return -1
  if (localChange.lower(localChange.addPath(b), localChange.addPath(a))) return 1

  // if there isnt 2 add paths, sort by del path
  if (localChange.lower(localChange.delPath(b), localChange.delPath(a))) return -1

  return 1
}

// TODO: Rename according to the sort logic
function finalSort (changes /*: LocalChange[] */) {
  log.trace('Final sort...')
  const stopMeasure = measureTime('LocalWatcher#finalSort')
  changes.sort(finalSorter)
  stopMeasure()
}
<|endoftext|>


<commit_before>'use strict'
module.exports = update

const Bluebird = require('bluebird')
const qw = require('qw')

const fetch = use('fetch')
const Fic = use('fic')
const ficInflate = use('fic-inflate')
const filenameize = use('filenameize')
const fs = use('fs-promises')
const progress = use('progress')
const promisify = use('promisify')
const TOML = use('toml')
const moment = require('moment')
const uniq = require('lodash.uniq')
const tagmap = use('tagmap')

function detagTitle (title) {
  const tags = []
  const tagExp = /[(](.*?)[)]|[\[](.*?)[\]]|[{](.*?)[}]/g
  const tagMatch = title.match(tagExp)
  if (tagMatch) {
    title = title.replace(tagExp, '').trim()
    tagMatch.map(t =>
      t.slice(1,-1)
       .split(/[/,|]/)
       .map(st => 'freeform:' + st.trim())
       .forEach(st => tags.push(st)))
  }
  return {title, tags}
}

function update (args) {
  const fetchOpts = {
    cacheBreak: !args.cache,
    noNetwork: !args.network,
    maxConcurrency: 6,
    requestsPerSecond: 10,
    perSite: {
      "www.alternatehistory.com": {
        maxConcurrency: 1,
        requestsPerSecond: 1,
      },
      "forums.spacebattles.com": {
        maxConcurrency: 2,
        requestsPerSecond: 1,
      },
      "forums.sufficientvelocity.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
      "forum.questionablequesting.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
      "questionablequesting.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
    },
    timeout: 3500
  }
  const fetchAndSpin = fetch.withOpts(fetchOpts).wrapWith(progress.spinWhileAnd)
  if (args.xf_user) fetchAndSpin.setGlobalCookie(`xf_user=${args.xf_user}`)

  return Bluebird.map(args.fic, updateFic(fetchAndSpin, args), {concurrency: 10})
    .reduce((exit, result) => result != null ? result : exit)
    .then(exit => exit > 0 ? 0 : 1) // error if nothing updated
}

function readFic (fic) {
  return fs.readFile(fic).then(toml => Fic.fromJSON(TOML.sync.parse(toml)))
}

function updateFic (fetch, args) {
  const addNone = args['add-none']
  const addAll = args['add-all']
  const add = addNone ? 'none' : addAll ? 'all' : 'new'
  let fromThreadmarks = !args.scrape || args['and-fetch']
  let fromScrape = args.scrape || args['and-scrape']
  const refresh = args['refresh']
  let fast = args.fast

  return ficFile => {
    return Bluebird.try(() => {
      const existingFic = readFic(ficFile)
      let newFic = fetchLatestVersionWithoutInflate(fetch, existingFic, fromThreadmarks, fromScrape)
      if (fast) {
        return Bluebird.join(existingFic, newFic, (existingFic, newFic) => {
          const lastExisting = existingFic.chapters.slice(-1)[0] || existingFic
          const lastNew = newFic.chapters.slice(-1)[0] || newFic
          if (lastExisting === lastNew || (lastExisting && lastNew && createdDate(lastExisting).isAfter(createdDate(lastNew)))) return
          newFic = ficInflate(newFic, fetch.withOpts({cacheBreak: false}))
          return doMerge()
        })
      } else {
        return doMerge()
      }
      function doMerge () {
        return mergeFic(existingFic, newFic, add).then(changes => {
          const inflatedFic = ficInflate(existingFic, fetch.withOpts({cacheBreak: false}))
          return writeUpdatedFic(ficFile, inflatedFic, refreshMetadata(inflatedFic, changes), refresh)
        })
      }
    }).catch(ex => {
      process.emit('warn', `Skipping ${ficFile}:`, ex)
    })
  }
}

function writeUpdatedFic (ficFile, existingFic, changes, forceSave) {
  return Bluebird.resolve(changes).then(changes => {
    if (!changes.length && !forceSave) return null
    return fs.writeFile(ficFile, TOML.stringify(existingFic)).then(() => {
      progress.output(`${ficFile}\n`)
      if (changes.length) progress.output(`    ${changes.join('\n    ')} \n`)
      return changes.refresh ? 1 : null
    })
  })
}

var fetchLatestVersion = promisify.args((fetch, existingFic, fromThreadmarks, fromScrape) => {
  const newFic = fetchLatestVersionWithoutInflate(fetch, existingFic, fromThreadmarks, fromScrape)

  return ficInflate(newFic, fetch.withOpts({cacheBreak: false}))
})

var fetchLatestVersionWithoutInflate = promisify.args((fetch, existingFic, fromThreadmarks, fromScrape) => {
  const updateFrom = existingFic.updateWith()
  let thisFromThreadmarks = (!existingFic.scrapeMeta && fromThreadmarks) || existingFic.fetchMeta
  let thisFromScrape = fromScrape || existingFic.scrapeMeta

  function getFic (fetch) {
    if (thisFromThreadmarks && thisFromScrape) {
      return Fic.fromUrlAndScrape(fetch, updateFrom)
    } else if (thisFromThreadmarks) {
      return Fic.fromOnlyUrl(fetch, updateFrom)
    } else {
      return Fic.scrapeFromUrl(fetch, updateFrom)
    }
  }

  // Fetch the fic from cache first, which ensures we get any cookies
  // associated with it, THEN fetch it w/o the cache to get updates.
  return getFic(fetch.withOpts({cacheBreak: false})).then(()=> getFic(fetch))
})

function createdDate (chapOrFic) {
  return moment(chapOrFic.created || chapOrFic.modified)
}

var mergeFic = promisify.args(function mergeFic (existingFic, newFic, add) {
  const changes = []
  const toAdd = []

  if (add !== 'none') {
    const types = uniq(newFic.chapters.map(ch => ch.type))
    types.forEach(type => {
      const latestExisting = existingFic
        .chapters
        .filter(ch => ch.type === type)
        .concat(existingFic.fics)
        .filter(c => c.created || c.modified)
        .map(createdDate)
        .reduce((aa, bb) => aa > bb ? aa : bb, new Date(0))
      const chapters = newFic.chapters.filter(ch => ch.type === type)
      const newestIndex = chapters.length - 1
      if (chapters[newestIndex].created || chapters[newestIndex].modified) {
        for (let newChapter of chapters) {
          if (existingFic.chapterExists(newChapter.link) || existingFic.chapterExists(newChapter.fetchFrom)) {
            continue
          }
          const created = newChapter.created || newChapter.modified
          if (add === 'all' || createdDate(newChapter).isAfter(latestExisting)) {
            toAdd.push(newChapter)
          }
        }
      } else {
        for (let ii = newestIndex; ii >= 0; --ii) {
          const newChapter = chapters[ii]
          if (existingFic.chapterExists(newChapter.link) || existingFic.chapterExists(newChapter.fetchFrom)) {
            if (add === 'all') { continue } else { break }
          }
          toAdd.unshift(newChapter)
        }
      }
    })
  }

  if (existingFic.description == null && newFic.description != null) {
    existingFic.description = newFic.description
    changes.push(`${existingFic.title}: Set fic description to ${newFic.description}`)
  }
  if (existingFic.tags == null && newFic.tags != null && newFic.tags.length) {
    existingFic.tags = newFic.tags
    changes.push(`${existingFic.title}: Set fic tags to ${newFic.tags.join(', ')}`)
  }
  for (let prop of qw`publisher author authorUrl updateFrom link title`) {
    if (existingFic[prop] == null && newFic[prop] != null && (!existingFic.parent || existingFic.parent[prop] !== newFic[prop])) {
      existingFic[prop] = newFic[prop]
      changes.push(`${existingFic.title}: Set fic ${prop} to ${existingFic[prop]}`)
    }
  }

  if (existingFic.tags.some(_ => _ === 'Snippets')) {
    toAdd.forEach(ch => {
      const subFic = new Fic.SubFic(existingFic)
      const {title, tags} = detagTitle(ch.name)
      subFic.title = existingFic.author + ' Snip: ' + title
      if (tags.length) {
        subFic.tags = tagmap(existingFic.site.type)(existingFic.tags.concat(tags))
      }
      subFic.chapters.push(ch)
      existingFic.fics.push(subFic)
    })
  } else {
    existingFic.chapters.push.apply(existingFic.chapters, toAdd)
    existingFic.chapters.sort()
  }

  if (toAdd.length) {
    changes.push(`${existingFic.title}: Added ${toAdd.length} new chapters`)
    changes.refresh = true
  }

  const fics = [existingFic].concat(existingFic.fics)
  for (let fic of fics) {
    // Find any chapters with created dates and update them if need be.
    for (let chapter of fic.chapters) {
      const match = newFic.chapters.filter(andChapterEquals(chapter))
      for (let newChapter of match) {
        if (chapter.type === 'chapter' && newChapter.type !== chapter.type) {
          changes.push(`${fic.title}: Updated type for chapter "${newChapter.name}" from ${chapter.type} to ${newChapter.type}`)
          chapter.type = newChapter.type
        }
        if (isDate(newChapter.created) && !dateEqual(newChapter.created, chapter.created)) {
          changes.push(`${fic.title}: Updated creation date for chapter "${newChapter.name}" from ${chapter.created} to ${newChapter.created}`)
          chapter.created = newChapter.created
        }
        if (isDate(newChapter.modified) && !dateEqual(newChapter.modified, chapter.modified)) {
          changes.push(`${fic.title}: Updated modification date for chapter "${newChapter.name}" from ${chapter.modified} to ${newChapter.modified}`)
          chapter.modified = newChapter.modified
        }
        for (let prop of qw`name link fetchFrom author authorUrl tags words`) {
          if (chapter[prop] == null && newChapter[prop] != null) {
            if (newChapter[prop] != fic[prop]) {
              chapter[prop] = newChapter[prop]
              changes.push(`${fic.title}: Set ${prop} for chapter "${newChapter.name}" to ${chapter[prop]}`)
            }
          }
        }
      }
    }
  }
  return changes
})

var refreshMetadata = promisify.args(function mergeFic (existingFic, changes) {
  const fics = [existingFic].concat(existingFic.fics)

  for (let fic of fics) {
    let now = moment()
    let then = moment(0)
    let created = fic.chapters.filter(c => c.type === 'chapter' && (c.created || c.modified)).reduce((ficCreated, chapter) => {
      return ficCreated < moment(chapter.created || chapter.modified) ? ficCreated : moment(chapter.created || chapter.modified)
    }, now)
    if (isDate(created) && (created !== now && (!isDate(fic.created) || !dateEqual(created, fic.created)))) {
      changes.push(`${fic.title}: Updated fic publish time from ${fic.created} to ${created} (from earliest chapter)`)
      fic.created = created
    }

    let modified = fic.chapters.filter(c => c.type === 'chapter' && (c.modified || c.created)).reduce((ficModified, chapter) => ficModified > moment(chapter.modified||chapter.created) ? ficModified : moment(chapter.modified||chapter.created), then)
    if (isDate(modified) && (modified !== then && (!isDate(fic.modified) || !dateEqual(modified, fic.modified)))) {
      changes.push(`${fic.title}: Updated fic last update time from ${fic.modified} to ${modified} (from latest chapter)`)
      fic.modified = modified
    }
  }
  if (existingFic.chapters.length === 0) {
    let created = existingFic.fics.filter(f => f.created).reduce((ficCreated, subfic) => ficCreated < moment(subfic.created) ? ficCreated : moment(subfic.created), moment(existingFic.created))
    if (isDate(created) && (!dateEqual(existingFic.created, created))) {
      changes.push(`${existingFic.title}: Updated fic publish time from ${existingFic.created} to ${created} (from earliest subfic)`)
      existingFic.created = created
    }
    let modified = existingFic.fics.filter(f => f.modified || f.created).reduce((ficModified, subfic) => ficModified > moment(subfic.modified||subfic.created) ? ficModified : moment(subfic.modified||subfic.created), moment(existingFic.modified))
    if (isDate(modified) && (!dateEqual(existingFic.modified, modified))) {
      changes.push(`${existingFic.title}: Updated fic last update time from ${existingFic.modified} to ${modified} (from latest subfic)`)
      existingFic.modified = modified
    }
  }
  return changes
})

function andChapterEquals (chapterA) {
  return chapterB => chapterEqual(chapterA, chapterB)
}

function chapterEqual (chapterA, chapterB) {
  return (chapterA.link && chapterB.link && chapterA.link === chapterB.link) ||
         (chapterA.fetchFrom && chapterB.fetchFrom && chapterA.fetchFrom === chapterB.fetchFrom)
}

function dateEqual (dateA, dateB) {
  return moment(dateA).isSame(dateB)
}

function isDate (date) {
  if (date == null) return false
  if (isNaN(date)) return false
  return date instanceof Date || date instanceof moment
}

<commit_msg>ff-update: Improve tag from title extraction<commit_after>'use strict'
module.exports = update

const Bluebird = require('bluebird')
const qw = require('qw')
const qr = require('@perl/qr')

const fetch = use('fetch')
const Fic = use('fic')
const ficInflate = use('fic-inflate')
const filenameize = use('filenameize')
const fs = use('fs-promises')
const progress = use('progress')
const promisify = use('promisify')
const TOML = use('toml')
const moment = require('moment')
const uniq = require('lodash.uniq')
const tagmap = use('tagmap')
const deepEqual = require('fast-deep-equal')
const notDeepEqual = (a, b) => !deepEqual(a, b)

function detagTitle (titleAndTags) {
  let [title, tags] = titleAndTags || [undefined, []]
  const tagMatchers = [ qr`[{](?:.{2,})[}]`, qr`(?!^)[\[](?:.{2,})[\]]`, qr`[(](?:.{2,})[)]` ]
  for (let tagExp of tagMatchers) {
    const tagMatch = title.match(tagExp)
    if (tagMatch) {
      const [,tagBit] = title.match(qr`(${tagExp}.*)$`)
      title = title.replace(qr`(${tagExp}.*)$`, '').trim()
      tagBit.split(/[{}\[\]()/,;|]/)
       .map(_ => _.trim())
       .filter(_ => _)
       .map(_ => `freeform:title:${_}`)
       .forEach(st => tags.push(st))
      break
    }
  }
  return {title, tags}
}

function update (args) {
  const fetchOpts = {
    cacheBreak: !args.cache,
    noNetwork: !args.network,
    maxConcurrency: 6,
    requestsPerSecond: 10,
    perSite: {
      "www.alternatehistory.com": {
        maxConcurrency: 1,
        requestsPerSecond: 1,
      },
      "forums.spacebattles.com": {
        maxConcurrency: 2,
        requestsPerSecond: 1,
      },
      "forums.sufficientvelocity.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
      "forum.questionablequesting.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
      "questionablequesting.com": {
        maxConcurrency: 6,
        requestsPerSecond: 4,
      },
    },
    timeout: 3500
  }
  const fetchAndSpin = fetch.withOpts(fetchOpts).wrapWith(progress.spinWhileAnd)
  if (args.xf_user) fetchAndSpin.setGlobalCookie(`xf_user=${args.xf_user}`)

  return Bluebird.map(args.fic, updateFic(fetchAndSpin, args), {concurrency: 10})
    .reduce((exit, result) => result != null ? result : exit)
    .then(exit => exit > 0 ? 0 : 1) // error if nothing updated
}

function readFic (fic) {
  return fs.readFile(fic).then(toml => Fic.fromJSON(TOML.sync.parse(toml)))
}

function updateFic (fetch, args) {
  const addNone = args['add-none']
  const addAll = args['add-all']
  const add = addNone ? 'none' : addAll ? 'all' : 'new'
  let fromThreadmarks = !args.scrape || args['and-fetch']
  let fromScrape = args.scrape || args['and-scrape']
  const refresh = args['refresh']
  let fast = args.fast

  return ficFile => {
    return Bluebird.try(() => {
      const existingFic = readFic(ficFile)
      let newFic = fetchLatestVersionWithoutInflate(fetch, existingFic, fromThreadmarks, fromScrape)
      if (fast) {
        return Bluebird.join(existingFic, newFic, (existingFic, newFic) => {
          const lastExisting = existingFic.chapters.slice(-1)[0] || existingFic
          const lastNew = newFic.chapters.slice(-1)[0] || newFic
          if (lastExisting === lastNew || (lastExisting && lastNew && createdDate(lastExisting).isAfter(createdDate(lastNew)))) return
          newFic = ficInflate(newFic, fetch.withOpts({cacheBreak: false}))
          return doMerge()
        })
      } else {
        return doMerge()
      }
      function doMerge () {
        return mergeFic(existingFic, newFic, add).then(changes => {
          const inflatedFic = ficInflate(existingFic, fetch.withOpts({cacheBreak: false}))
          return writeUpdatedFic(ficFile, inflatedFic, refreshMetadata(inflatedFic, changes), refresh)
        })
      }
    }).catch(ex => {
      process.emit('warn', `Skipping ${ficFile}:`, ex)
    })
  }
}

function writeUpdatedFic (ficFile, existingFic, changes, forceSave) {
  return Bluebird.resolve(changes).then(changes => {
    if (!changes.length && !forceSave) return null
    return fs.writeFile(ficFile, TOML.stringify(existingFic)).then(() => {
      progress.output(`${ficFile}\n`)
      if (changes.length) progress.output(`    ${changes.join('\n    ')} \n`)
      return changes.refresh ? 1 : null
    })
  })
}

var fetchLatestVersion = promisify.args((fetch, existingFic, fromThreadmarks, fromScrape) => {
  const newFic = fetchLatestVersionWithoutInflate(fetch, existingFic, fromThreadmarks, fromScrape)

  return ficInflate(newFic, fetch.withOpts({cacheBreak: false}))
})

var fetchLatestVersionWithoutInflate = promisify.args((fetch, existingFic, fromThreadmarks, fromScrape) => {
  const updateFrom = existingFic.updateWith()
  let thisFromThreadmarks = (!existingFic.scrapeMeta && fromThreadmarks) || existingFic.fetchMeta
  let thisFromScrape = fromScrape || existingFic.scrapeMeta

  function getFic (fetch) {
    if (thisFromThreadmarks && thisFromScrape) {
      return Fic.fromUrlAndScrape(fetch, updateFrom)
    } else if (thisFromThreadmarks) {
      return Fic.fromOnlyUrl(fetch, updateFrom)
    } else {
      return Fic.scrapeFromUrl(fetch, updateFrom)
    }
  }

  // Fetch the fic from cache first, which ensures we get any cookies
  // associated with it, THEN fetch it w/o the cache to get updates.
  return getFic(fetch.withOpts({cacheBreak: false})).then(()=> getFic(fetch))
})

function createdDate (chapOrFic) {
  return moment(chapOrFic.created || chapOrFic.modified)
}

var mergeFic = promisify.args(function mergeFic (existingFic, newFic, add) {
  const changes = []
  const toAdd = []

  if (add !== 'none') {
    const types = uniq(newFic.chapters.map(ch => ch.type))
    types.forEach(type => {
      const latestExisting = existingFic
        .chapters
        .filter(ch => ch.type === type)
        .concat(existingFic.fics)
        .filter(c => c.created || c.modified)
        .map(createdDate)
        .reduce((aa, bb) => aa > bb ? aa : bb, new Date(0))
      const chapters = newFic.chapters.filter(ch => ch.type === type)
      const newestIndex = chapters.length - 1
      if (chapters[newestIndex].created || chapters[newestIndex].modified) {
        for (let newChapter of chapters) {
          if (existingFic.chapterExists(newChapter.link) || existingFic.chapterExists(newChapter.fetchFrom)) {
            continue
          }
          const created = newChapter.created || newChapter.modified
          if (add === 'all' || createdDate(newChapter).isAfter(latestExisting)) {
            toAdd.push(newChapter)
          }
        }
      } else {
        for (let ii = newestIndex; ii >= 0; --ii) {
          const newChapter = chapters[ii]
          if (existingFic.chapterExists(newChapter.link) || existingFic.chapterExists(newChapter.fetchFrom)) {
            if (add === 'all') { continue } else { break }
          }
          toAdd.unshift(newChapter)
        }
      }
    })
  }

  if (existingFic.description == null && newFic.description != null) {
    existingFic.description = newFic.description
    changes.push(`${existingFic.title}: Set fic description to ${newFic.description}`)
  }
  if (existingFic.tags == null && newFic.tags != null && newFic.tags.length) {
    existingFic.tags = newFic.tags
    changes.push(`${existingFic.title}: Set fic tags to ${newFic.tags.join(', ')}`)
  }
  for (let prop of qw`publisher author authorUrl updateFrom link title`) {
    if (existingFic[prop] == null && newFic[prop] != null && (!existingFic.parent || existingFic.parent[prop] !== newFic[prop])) {
      existingFic[prop] = newFic[prop]
      changes.push(`${existingFic.title}: Set fic ${prop} to ${existingFic[prop]}`)
    }
  }

  if (existingFic.tags.some(_ => _ === 'Snippets')) {
    toAdd.forEach(ch => {
      const subFic = new Fic.SubFic(existingFic)
      const {title, tags} = detagTitle(ch.name)
      subFic.title = existingFic.author + ' Snip: ' + title
      if (tags.length) {
        subFic.tags = tagmap(existingFic.site.type)(existingFic.tags.concat(tags))
      }
      subFic.chapters.push(ch)
      existingFic.fics.push(subFic)
    })
  } else {
    existingFic.chapters.push.apply(existingFic.chapters, toAdd)
    existingFic.chapters.sort()
  }

  if (toAdd.length) {
    changes.push(`${existingFic.title}: Added ${toAdd.length} new chapters`)
    changes.refresh = true
  }

  const fics = [existingFic].concat(existingFic.fics)
  for (let fic of fics) {
    // Find any chapters with created dates and update them if need be.
    for (let chapter of fic.chapters) {
      const match = newFic.chapters.filter(andChapterEquals(chapter))
      for (let newChapter of match) {
        if (chapter.type === 'chapter' && newChapter.type !== chapter.type) {
          changes.push(`${fic.title}: Updated type for chapter "${newChapter.name}" from ${chapter.type} to ${newChapter.type}`)
          chapter.type = newChapter.type
        }
        if (isDate(newChapter.created) && !dateEqual(newChapter.created, chapter.created)) {
          changes.push(`${fic.title}: Updated creation date for chapter "${newChapter.name}" from ${chapter.created} to ${newChapter.created}`)
          chapter.created = newChapter.created
        }
        if (isDate(newChapter.modified) && !dateEqual(newChapter.modified, chapter.modified)) {
          changes.push(`${fic.title}: Updated modification date for chapter "${newChapter.name}" from ${chapter.modified} to ${newChapter.modified}`)
          chapter.modified = newChapter.modified
        }
        for (let prop of qw`name link fetchFrom author authorUrl tags words`) {
          if (chapter[prop] == null && newChapter[prop] != null) {
            if (newChapter[prop] != fic[prop]) {
              chapter[prop] = newChapter[prop]
              changes.push(`${fic.title}: Set ${prop} for chapter "${newChapter.name}" to ${chapter[prop]}`)
            }
          }
        }
      }
    }
  }
  return changes
})

var refreshMetadata = promisify.args(function mergeFic (existingFic, changes) {
  const fics = [existingFic].concat(existingFic.fics)

  for (let fic of fics) {
    let now = moment()
    let then = moment(0)
    let created = fic.chapters.filter(c => c.type === 'chapter' && (c.created || c.modified)).reduce((ficCreated, chapter) => {
      return ficCreated < moment(chapter.created || chapter.modified) ? ficCreated : moment(chapter.created || chapter.modified)
    }, now)
    if (isDate(created) && (created !== now && (!isDate(fic.created) || !dateEqual(created, fic.created)))) {
      changes.push(`${fic.title}: Updated fic publish time from ${fic.created} to ${created} (from earliest chapter)`)
      fic.created = created
    }

    let modified = fic.chapters.filter(c => c.type === 'chapter' && (c.modified || c.created)).reduce((ficModified, chapter) => ficModified > moment(chapter.modified||chapter.created) ? ficModified : moment(chapter.modified||chapter.created), then)
    if (isDate(modified) && (modified !== then && (!isDate(fic.modified) || !dateEqual(modified, fic.modified)))) {
      changes.push(`${fic.title}: Updated fic last update time from ${fic.modified} to ${modified} (from latest chapter)`)
      fic.modified = modified
    }
  }
  if (existingFic.chapters.length === 0) {
    let created = existingFic.fics.filter(f => f.created).reduce((ficCreated, subfic) => ficCreated < moment(subfic.created) ? ficCreated : moment(subfic.created), moment(existingFic.created))
    if (isDate(created) && (!dateEqual(existingFic.created, created))) {
      changes.push(`${existingFic.title}: Updated fic publish time from ${existingFic.created} to ${created} (from earliest subfic)`)
      existingFic.created = created
    }
    let modified = existingFic.fics.filter(f => f.modified || f.created).reduce((ficModified, subfic) => ficModified > moment(subfic.modified||subfic.created) ? ficModified : moment(subfic.modified||subfic.created), moment(existingFic.modified))
    if (isDate(modified) && (!dateEqual(existingFic.modified, modified))) {
      changes.push(`${existingFic.title}: Updated fic last update time from ${existingFic.modified} to ${modified} (from latest subfic)`)
      existingFic.modified = modified
    }
  }
  return changes
})

function andChapterEquals (chapterA) {
  return chapterB => chapterEqual(chapterA, chapterB)
}

function chapterEqual (chapterA, chapterB) {
  return (chapterA.link && chapterB.link && chapterA.link === chapterB.link) ||
         (chapterA.fetchFrom && chapterB.fetchFrom && chapterA.fetchFrom === chapterB.fetchFrom)
}

function dateEqual (dateA, dateB) {
  return moment(dateA).isSame(dateB)
}

function isDate (date) {
  if (date == null) return false
  if (isNaN(date)) return false
  return date instanceof Date || date instanceof moment
}

<|endoftext|>


<commit_before>var codeMelon = codeMelon || {};
codeMelon.games = codeMelon.games || {};

(function(_cg) {
    _cg.ArchOrigView = Backbone.View.extend({
        events: {
            'mousedown': 'handleMouseDown',
            'mousemove': 'handleMouseMove',
            'mouseup': 'handleMouseUp'
        },

        initialize: function(options) {
            _.bindAll(this,
                'render',
                'setConstants',
                'initMembers',
                'setImageData',
                'setTransparentImageData',
                'handleMouseDown',
                'handleMouseMove',
                'handleMouseUp'
            );

            this.setConstants(options);
            this.initMembers(options);
            this.render();
        },

        render: function() {
        },

        setConstants: function(options) {
            this.CONTEXT = this.el.getContext('2d');
        },

        initMembers: function(options) {
            var _this = this;

            _this.image = new Image();
            _this.image.src = '../common/arch.png';
            _this.image.onload = function(event) {
                _this.CONTEXT.drawImage(_this.image, 0, 0);
                _this.setImageData();
                _this.setTransparentImageData();
                _this.archModifiedView = new _cg.ArchModifiedView({
                    el: '#arch-modified',
                    imageData: _this.transparentImageData
                });
            };
            _this.dragging = false;
        },

        setImageData: function() {
            this.imageData = this.CONTEXT.getImageData(0, 0, this.el.width, this.el.height);
        },

        setTransparentImageData: function() {
            var max = this.imageData.data.length - 3,
                i;

            this.transparentImageData = this.CONTEXT.createImageData(this.el.width, this.el.height);
            for (i = 0; i < max; i += 4) {
                this.transparentImageData.data[i] = this.imageData.data[i];
                this.transparentImageData.data[i + 1] = this.imageData.data[i + 1];
                this.transparentImageData.data[i + 2] = this.imageData.data[i + 2];
                this.transparentImageData.data[i + 3] = this.imageData.data[i + 3] / 2;
            }
        },

        handleMouseDown: function(event) {
            this.dragging = true;
            this.rectangle = new _c.draw.Rectangle({
                corner: _c.draw.windowToCanvas(this.el, event.clientX, event.clientY)
            });
        },

        handleMouseMove: function(event) {
            var loc = _c.draw.windowToCanvas(this.el, event.clientX, event.clientY);

            if (!this.dragging) { return; }
            this.rectangle.width = loc.x - this.rectangle.corner.x;
            this.rectangle.height = loc.y - this.rectangle.corner.y;
            this.CONTEXT.drawImage(this.image, 0, 0);
            this.rectangle.stroke(this.CONTEXT);
        },

        handleMouseUp: function(event) {
            this.dragging = false;
            this.rectangle = null;
            this.CONTEXT.drawImage(this.image, 0, 0);
        }
    });
})(codeMelon.games);
<commit_msg>Transfer transparent rectangle into selected area.<commit_after>var codeMelon = codeMelon || {};
codeMelon.games = codeMelon.games || {};

(function(_cg) {
    _cg.ArchOrigView = Backbone.View.extend({
        events: {
            'mousedown': 'handleMouseDown',
            'mousemove': 'handleMouseMove',
            'mouseup': 'handleMouseUp'
        },

        initialize: function(options) {
            _.bindAll(this,
                'render',
                'setConstants',
                'initMembers',
                'setImageData',
                'setTransparentImageData',
                'handleMouseDown',
                'handleMouseMove',
                'handleMouseUp'
            );

            this.setConstants(options);
            this.initMembers(options);
            this.render();
        },

        render: function() {
        },

        setConstants: function(options) {
            this.CONTEXT = this.el.getContext('2d');
        },

        initMembers: function(options) {
            var _this = this;

            _this.image = new Image();
            _this.image.src = '../common/arch.png';
            _this.image.onload = function(event) {
                _this.CONTEXT.drawImage(_this.image, 0, 0);
                _this.setImageData();
                _this.setTransparentImageData();
                _this.archModifiedView = new _cg.ArchModifiedView({
                    el: '#arch-modified',
                    imageData: _this.transparentImageData
                });
                _this.deviceWidthOverCSSPixels = _this.imageData.width / _this.el.width;
                _this.deviceHeightOverCSSPixels = _this.imageData.height / _this.el.height;
            };
            _this.dragging = false;
        },

        setImageData: function() {
            this.imageData = this.CONTEXT.getImageData(0, 0, this.el.width, this.el.height);
        },

        setTransparentImageData: function() {
            var max = this.imageData.data.length - 3,
                i;

            this.transparentImageData = this.CONTEXT.createImageData(this.el.width, this.el.height);
            for (i = 0; i < max; i += 4) {
                this.transparentImageData.data[i] = this.imageData.data[i];
                this.transparentImageData.data[i + 1] = this.imageData.data[i + 1];
                this.transparentImageData.data[i + 2] = this.imageData.data[i + 2];
                this.transparentImageData.data[i + 3] = this.imageData.data[i + 3] / 2;
            }
        },

        handleMouseDown: function(event) {
            this.dragging = true;
            this.rectangle = new _c.draw.Rectangle({
                corner: _c.draw.windowToCanvas(this.el, event.clientX, event.clientY)
            });
        },

        handleMouseMove: function(event) {
            var loc = _c.draw.windowToCanvas(this.el, event.clientX, event.clientY);

            if (!this.dragging) { return; }
            this.rectangle.width = loc.x - this.rectangle.corner.x;
            this.rectangle.height = loc.y - this.rectangle.corner.y;
            this.CONTEXT.drawImage(this.image, 0, 0);
            this.CONTEXT.putImageData(this.transparentImageData, 0, 0,
                this.rectangle.left() * this.deviceWidthOverCSSPixels,
                this.rectangle.top() * this.deviceHeightOverCSSPixels,
                this.rectangle.absoluteWidth() * this.deviceWidthOverCSSPixels,
                this.rectangle.absoluteHeight() * this.deviceHeightOverCSSPixels);
            this.rectangle.stroke(this.CONTEXT);
        },

        handleMouseUp: function(event) {
            this.dragging = false;
            this.rectangle = null;
            this.CONTEXT.drawImage(this.image, 0, 0);
        }
    });
})(codeMelon.games);
<|endoftext|>


<commit_before>// These are the primary log levels.
// Your logger object has a method for each of these.
var LOG_LEVELS = {
	emergency : 7,
	alert     : 6,
	critical  : 5,
	error     : 4,
	warning   : 3,
	notice    : 2,
	info      : 1,
	debug     : 0,
}

var config = {
	main: {
		baseLevel: 'debug',
		levels: LOG_LEVELS,
		colors: {
			emergency : 'red',
			alert     : 'yellow',
			critical  : 'red',
			error     : 'red',
			warning   : 'red',
			notice    : 'yellow',
			info      : 'green',
			debug     : 'blue',
		},
	},

	// This config is for an internal logger used by the `time` method.
	stats: {
		baseLevel: 'fast',
		levels: {
			slow: 2,
			fine: 1,
			fast: 0,
		},
		colors: {
			slow: 'red',
			fine: 'yellow',
			fast: 'green',
		},
	},
};

// This is global so that we don't wind up with different copies for triton
// and corvair.
//
// The best reference I've found for the `xterm-256color` palette is:
//
//   http://www.pixelbeat.org/docs/terminal_colours/#256
//
global._getTritonLoggingNameColor || (global._getTritonLoggingNameColor = (function(){

	var colors = [];

	var makeColor = SERVER_SIDE
		?(r,g,b) => 16 + r*36 + g*6 + b
		:(r,g,b) => `rgb(${(r*42.5)|0},${(g*42.5)|0},${(b*42.5)|0})`

	for (var r = 1; r < 6; r+=2)
		for (var g = 1; g < 6; g+=2)
			for (var b = 1; b < 6; b+=2)
				if (r != g || g != b)
					colors.push(makeColor(r,g,b));

	var colorMap = {}
	return function(name){
		if (!colorMap[name])
			colors.push(colorMap[name] = colors.shift());
		return colorMap[name];
	}
})());

var getNameColor = _getTritonLoggingNameColor;

module.exports = { config, getNameColor };
<commit_msg>Update log level colors based on feedback from Nav<commit_after>// These are the primary log levels.
// Your logger object has a method for each of these.
var LOG_LEVELS = {
	emergency : 7,
	alert     : 6,
	critical  : 5,
	error     : 4,
	warning   : 3,
	notice    : 2,
	info      : 1,
	debug     : 0,
}

var config = {
	main: {
		baseLevel: 'debug',
		levels: LOG_LEVELS,
		colors: {
			emergency : 'red',
			alert     : 'red',
			critical  : 'red',
			error     : 'red',
			warning   : 'yellow',
			notice    : 'yellow',
			info      : 'green',
			debug     : 'blue',
		},
	},

	// This config is for an internal logger used by the `time` method.
	stats: {
		baseLevel: 'fast',
		levels: {
			slow: 2,
			fine: 1,
			fast: 0,
		},
		colors: {
			slow: 'red',
			fine: 'yellow',
			fast: 'green',
		},
	},
};

// This is global so that we don't wind up with different copies for triton
// and corvair.
//
// The best reference I've found for the `xterm-256color` palette is:
//
//   http://www.pixelbeat.org/docs/terminal_colours/#256
//
global._getTritonLoggingNameColor || (global._getTritonLoggingNameColor = (function(){

	var colors = [];

	var makeColor = SERVER_SIDE
		?(r,g,b) => 16 + r*36 + g*6 + b
		:(r,g,b) => `rgb(${(r*42.5)|0},${(g*42.5)|0},${(b*42.5)|0})`

	for (var r = 1; r < 6; r+=2)
		for (var g = 1; g < 6; g+=2)
			for (var b = 1; b < 6; b+=2)
				if (r != g || g != b)
					colors.push(makeColor(r,g,b));

	var colorMap = {}
	return function(name){
		if (!colorMap[name])
			colors.push(colorMap[name] = colors.shift());
		return colorMap[name];
	}
})());

var getNameColor = _getTritonLoggingNameColor;

module.exports = { config, getNameColor };
<|endoftext|>


<commit_before>FCKConfig.ToolbarSets["IndicoMinimal"] = [
    ['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Indent', 'Outdent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],
    ['SpecialChar']
] ;

FCKConfig.ToolbarSets["IndicoFull"] = [
    ['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Indent', 'Outdent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],
    ['Style','FontFormat','FontName'],
    ['FontSize', 'TextColor','BGColor'],
    ['SpecialChar'], ['FitWindow']
] ;

FCKConfig.ToolbarSets["IndicoPages"] = [
    ['Source'],['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Indent', 'Outdent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],['Image'],
    ['Style','FontFormat','FontName'],
    ['FontSize', 'TextColor','BGColor'],
    ['SpecialChar'], ['FitWindow']
] ;

FCKConfig.ImageUpload = false ;
FCKConfig.ImageBrowser = false ;<commit_msg>[FIX] 'Outdent' and 'Indent' icons<commit_after>FCKConfig.ToolbarSets["IndicoMinimal"] = [
    ['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Outdent', 'Indent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],
    ['SpecialChar']
] ;

FCKConfig.ToolbarSets["IndicoFull"] = [
    ['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Outdent', 'Indent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],
    ['Style','FontFormat','FontName'],
    ['FontSize', 'TextColor','BGColor'],
    ['SpecialChar'], ['FitWindow']
] ;

FCKConfig.ToolbarSets["IndicoPages"] = [
    ['Source'],['Bold','Italic','Underline','StrikeThrough','-','Subscript','Superscript','-'],
    ['Outdent', 'Indent', 'OrderedList','UnorderedList','Blockquote','-','Link','Unlink','Anchor'],
    ['JustifyLeft','JustifyCenter','JustifyRight','JustifyFull'],['Image'],
    ['Style','FontFormat','FontName'],
    ['FontSize', 'TextColor','BGColor'],
    ['SpecialChar'], ['FitWindow']
] ;

FCKConfig.ImageUpload = false ;
FCKConfig.ImageBrowser = false ;<|endoftext|>


<commit_before>var endTimestamp = [
	[new Date(1504612800000), "It begins..."],
	[new Date(1514005200000), "Have a break."],
	[new Date(1514178000000), "Oh Christmas tree,
oh Christmas tree..."],
	[new Date(1516971600000), "Your examination is ready."],
	[new Date(1520658000000), "Have another break."],
	[new Date(1529668800000), "Your other examination
is ready."],
	[new Date(1530209100000), "It is finished."],
];

var daysInMs = 1000 * 60 * 60 * 24;
var hoursInMs = 1000 * 60 * 60;
var minutesInMs = 1000 * 60;

var timerIndex = 0;

function updateTime() {
	var timeUntil = endTimestamp[timerIndex][0].getTime() - new Date().getTime();
	var until = document.getElementById("until");

	if(timeUntil <= 0) {
		until.innerHTML = 'Since '+endTimestamp[timerIndex][0].toLocaleString()+'
'+endTimestamp[timerIndex][1];
		timeUntil = new Date().getTime() - endTimestamp[timerIndex][0].getTime();
	}
	else {
		until.innerHTML = 'Until '+endTimestamp[timerIndex][0].toLocaleString()+'
'+endTimestamp[timerIndex][1];
	}

	var days = document.getElementById("days");
	var hours = document.getElementById("hours");
	var minutes = document.getElementById("minutes");
	var seconds = document.getElementById("seconds");
	var millis = document.getElementById("millis");

	var daysUntil = timeUntil / daysInMs;
	var hoursUntil = (timeUntil / hoursInMs) % 24;
	var minutesUntil = (timeUntil / minutesInMs) % 60;
	var secondsUntil = (timeUntil / 1000) % 60;
	var millisUntil = timeUntil % 1000;

	days.textContent = Math.floor(daysUntil) + ' day';
	hours.textContent = Math.floor(hoursUntil) + ' hour';
	minutes.textContent = Math.floor(minutesUntil) + ' minute';
	seconds.textContent = Math.floor(secondsUntil) + ' second';
	millis.textContent = 'and ' + Math.floor(millisUntil) + ' millisecond';

	if(Math.floor(daysUntil) != 1) days.textContent = days.textContent + 's';
	if(Math.floor(hoursUntil) != 1) hours.textContent = hours.textContent + 's';
	if(Math.floor(minutesUntil) != 1) minutes.textContent = minutes.textContent + 's';
	if(Math.floor(secondsUntil) != 1) seconds.textContent = seconds.textContent + 's';
	if(Math.floor(millisUntil) != 1) millis.textContent = millis.textContent + 's';
}

function changeTimerSelect(change) {
	timerIndex += change;

	if(timerIndex < 0) timerIndex = endTimestamp.length - 1;
	else if(timerIndex >= endTimestamp.length) timerIndex = 0;

	var timerCount = document.getElementById("timerCount");
	if(timerCount != null)
	timerCount.innerText = "Timer {0}/{1}".replace("{0}", (timerIndex+1).toString()).replace("{1}", endTimestamp.length.toString());
}

window.setInterval(updateTime, 1);
<commit_msg>Add Christmas Eve countdown<commit_after>var endTimestamp = [
	[new Date(1504612800000), "It begins..."],
	[new Date(1514005200000), "Have a break."],
   [new Date(1514178000000), "On Chistmas Eve..."],
	[new Date(1514178000000), "Oh Christmas tree,
oh Christmas tree..."],
	[new Date(1516971600000), "Your examination is ready."],
	[new Date(1520658000000), "Have another break."],
	[new Date(1529668800000), "Your other examination
is ready."],
	[new Date(1530209100000), "It is finished."],
];

var daysInMs = 1000 * 60 * 60 * 24;
var hoursInMs = 1000 * 60 * 60;
var minutesInMs = 1000 * 60;

var timerIndex = 0;

function updateTime() {
	var timeUntil = endTimestamp[timerIndex][0].getTime() - new Date().getTime();
	var until = document.getElementById("until");

	if(timeUntil <= 0) {
		until.innerHTML = 'Since '+endTimestamp[timerIndex][0].toLocaleString()+'
'+endTimestamp[timerIndex][1];
		timeUntil = new Date().getTime() - endTimestamp[timerIndex][0].getTime();
	}
	else {
		until.innerHTML = 'Until '+endTimestamp[timerIndex][0].toLocaleString()+'
'+endTimestamp[timerIndex][1];
	}

	var days = document.getElementById("days");
	var hours = document.getElementById("hours");
	var minutes = document.getElementById("minutes");
	var seconds = document.getElementById("seconds");
	var millis = document.getElementById("millis");

	var daysUntil = timeUntil / daysInMs;
	var hoursUntil = (timeUntil / hoursInMs) % 24;
	var minutesUntil = (timeUntil / minutesInMs) % 60;
	var secondsUntil = (timeUntil / 1000) % 60;
	var millisUntil = timeUntil % 1000;

	days.textContent = Math.floor(daysUntil) + ' day';
	hours.textContent = Math.floor(hoursUntil) + ' hour';
	minutes.textContent = Math.floor(minutesUntil) + ' minute';
	seconds.textContent = Math.floor(secondsUntil) + ' second';
	millis.textContent = 'and ' + Math.floor(millisUntil) + ' millisecond';

	if(Math.floor(daysUntil) != 1) days.textContent = days.textContent + 's';
	if(Math.floor(hoursUntil) != 1) hours.textContent = hours.textContent + 's';
	if(Math.floor(minutesUntil) != 1) minutes.textContent = minutes.textContent + 's';
	if(Math.floor(secondsUntil) != 1) seconds.textContent = seconds.textContent + 's';
	if(Math.floor(millisUntil) != 1) millis.textContent = millis.textContent + 's';
}

function changeTimerSelect(change) {
	timerIndex += change;

	if(timerIndex < 0) timerIndex = endTimestamp.length - 1;
	else if(timerIndex >= endTimestamp.length) timerIndex = 0;

	var timerCount = document.getElementById("timerCount");
	if(timerCount != null)
	timerCount.innerText = "Timer {0}/{1}".replace("{0}", (timerIndex+1).toString()).replace("{1}", endTimestamp.length.toString());
}

window.setInterval(updateTime, 1);
<|endoftext|>


<commit_before>/**
 * TODO(developer):
 * Add your service key to the current folder.
 * Uncomment and fill in these variables.
 */
// const projectId = 'my-project';
// const locationId = 'global';
// const agentId = 'my-agent';
// const languageCode = 'en'

// const twitterAPIKey = "Place your consumer key here";
// const twitterSecretAPIKey = "Place you secret consumer key here";
// const twitterAccessToken = "Place your access token here";
// const twitterSecretAccessToken = "Place your secret access token here";
// const environmentName = "Place your twitter environment name here";

// const targetUrl = "Place your server's url here";

const express = require('express');
const request = require('request');
const app = express();
const {SessionsClient} = require('@google-cloud/dialogflow-cx');
const crypto = require('crypto');
const twitterText = require('twitter-text');

app.use(express.json());

const twitterOAuth = {
  consumer_key: twitterAPIKey,
  consumer_secret: twitterSecretAPIKey,
  token: twitterAccessToken,
  token_secret: twitterSecretAccessToken
};

const twitterId =
    twitterAccessToken.substring(0, twitterAccessToken.indexOf('-'));
const client = new SessionsClient(
    {apiEndpoint: locationId + '-dialogflow.googleapis.com'});

const listener = app.listen(process.env.PORT, function() {
  console.log(
      'Your Twitter integration server is listening on port ' +
      listener.address().port);
  init();
});

process.on('SIGTERM', () => {
  listener.close(async () => {
    deleteSubscription();
    await deleteWebhooksByUrl(targetUrl);
    console.log('Closing http server.');
    process.exit(0);
  });
});

// Converts Twitter request to a detectIntent request.
function twitterToDetectIntent(message, sessionPath) {
  const request = {
    session: sessionPath,
    queryInput: {
      text: {
        text: message,
      },
      languageCode,
    }
  };

  return request;
}

/**
 * Takes as input a request from twitter and converts the request to
 * detectIntent request which is used to call the detectIntent() function
 * and finally output the response given by detectIntent().
 */
async function detectIntentResponse(message, userId) {
  const sessionId = userId;
  const sessionPath = client.projectLocationAgentSessionPath(
      projectId, locationId, agentId, sessionId);
  console.info(sessionPath);

  var request = twitterToDetectIntent(message, sessionPath);
  var [response] = await client.detectIntent(request);
  console.log(JSON.stringify(response))
  return response;
};

function sendMessage(text, recipientId) {
  request.post(
      'https://api.twitter.com/1.1/direct_messages/events/new.json', {
        oauth: twitterOAuth,
        json: {
          event: {
            type: 'message_create',
            message_create: {
              target: {recipient_id: recipientId},
              message_data: {
                text: text,
              }
            }
          }
        }
      },
      function(err, resp, body) {
        if (err) {
          console.eror('Failed to send message: ' + err);
        }
      });
}

function sendStatus(text, tweetId) {
  // Tweets must be less than 280 characters. See documentation below.
  // https://developer.twitter.com/en/docs/developer-utilities/twitter-text.html
  const parsedTweet = twitterText.parseTweet(text);
  if (parsedTweet.valid) {
    request.post(
        'https://api.twitter.com/1.1/statuses/update.json', {
          oauth: twitterOAuth,
          form: {status: text, in_reply_to_status_id: tweetId},
        },
        function(err, resp, body) {
          if (err) {
            console.error(err);
          }
        });
  } else {
    console.error(
        'Failed to send status because it is longer than 280 characters: ' +
        err);
  }
}

async function init() {
  deleteSubscription();
  await deleteWebhooksByUrl(targetUrl);
  await registerWebhook();
  registerSubscription();
}

function registerWebhook() {
  return new Promise((resolve, reject) => {
    request.post(
        'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
            '/webhooks.json',
        {oauth: twitterOAuth, form: {url: targetUrl}},
        function(err, resp, body) {
          if (err) {
            console.error('Failed to register webhook: ' + err);
            reject();
          } else {
            resolve();
          }
        });
  });
}

function registerSubscription() {
  request.post(
      'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
          '/subscriptions.json',
      {
        oauth: twitterOAuth,
      },
      function(err, resp, body) {
        if (err) {
          console.error('Failed to register subscription: ' + err);
        }
      });
}

function listWebhooks(url) {
  return new Promise((resolve, reject) => {
    request.get(
        'https://api.twitter.com/1.1/account_activity/all/webhooks.json',
        {oauth: twitterOAuth}, function(err, resp, body) {
          if (err) {
            console.error('Failed to check webhooks: ' + err);
            reject(err);
          }
          const environments = JSON.parse(body).environments;
          if (Array.isArray(environments)) {
            const environment = environments.find((element) => {
              return element.environment_name === environmentName;
            });
            if (environment) {
              const webhooks =
                  environment.webhooks.filter((value, index, arr) => {
                    return value.url === url;
                  });
              resolve(webhooks);
            }
            resolve([]);
          }
        });
  });
}

function deleteWebhookById(webhookId) {
  return new Promise((resolve, reject) =>{
    request.delete(
        'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
            '/webhooks/' + webhookId + '.json',
        {oauth: twitterOAuth}, function(err, resp, body) {
          if (err) {
            console.error('Failed to delete webhook: ' + err);
            reject(err);
          }
          resolve();
        });
  });
}

async function deleteWebhooksByUrl(targetUrl) {
  const webhooks = await listWebhooks(targetUrl);
  for (webhook of webhooks) {
    if (webhook.id) {
      await deleteWebhookById(webhook.id);
    }
  }
}

function deleteSubscription() {
  request.delete(
      'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
          '/subscriptions.json',
      {oauth: twitterOAuth}, function(err, resp, body) {
        if (err) {
          console.error('Failed to delete subscriptions: ' + err);
        }
      });
}

app.post('/', async function(req, res) {
  if (Array.isArray(req.body.direct_message_events) &&
      req.body.direct_message_events.length > 0) {
    const message = req.body.direct_message_events[0];
    const text = message.message_create.message_data.text;
    const senderId = message.message_create.sender_id;
    if (senderId !== twitterId) {
      var responses = await detectIntentResponse(text, senderId);

      for (let response of responses.queryResult.responseMessages) {
        if (response.hasOwnProperty('text')) {
          sendMessage(response.text.text.join(), senderId);
          res.sendStatus(200);
        };
      };
    }
  } else if (
      Array.isArray(req.body.tweet_create_events) &&
      req.body.tweet_create_events.length > 0) {
    const message = req.body.tweet_create_events[0];
    const text = message.text;
    const senderId = message.user.id_str;
    if (senderId !== twitterId && message.in_reply_to_user_id_str) {
      var responses = await detectIntentResponse(
          text.replace('@' + message.in_reply_to_screen_name, ''), senderId);
      const screenName = message.user.screen_name;

      for (let response of responses.queryResult.responseMessages) {
        if (response.hasOwnProperty('text')) {
          sendStatus(
              '@' + screenName + ' ' + response.text.text.join(), senderId);
          res.sendStatus(200);
        };
      };
    }
  }
});

// responds to challenge response check verification requests
app.get('/', function(req, res) {
  const crc = req.query.crc_token;
  const hmac = 'sha256=' +
      crypto.createHmac('sha256', twitterSecretAPIKey)
          .update(crc)
          .digest('base64');
  res.json({'response_token': hmac});
});
<commit_msg>Update server.js<commit_after>/**
 * TODO(developer):
 * Add your service key to the current folder.
 * Uncomment and fill in these variables.
 */
// const projectId = 'my-project';
// const locationId = 'global';
// const agentId = 'my-agent';
// const languageCode = 'en'

// const twitterAPIKey = "Place your consumer key here";
// const twitterSecretAPIKey = "Place you secret consumer key here";
// const twitterAccessToken = "Place your access token here";
// const twitterSecretAccessToken = "Place your secret access token here";
// const environmentName = "Place your twitter environment name here";

// const targetUrl = "Place your server's url here";

const express = require('express');
const request = require('request');
const app = express();
const {SessionsClient} = require('@google-cloud/dialogflow-cx');
const crypto = require('crypto');
const twitterText = require('twitter-text');

app.use(express.json());

const twitterOAuth = {
  consumer_key: twitterAPIKey,
  consumer_secret: twitterSecretAPIKey,
  token: twitterAccessToken,
  token_secret: twitterSecretAccessToken
};

const twitterId =
    twitterAccessToken.substring(0, twitterAccessToken.indexOf('-'));
const client = new SessionsClient(
    {apiEndpoint: locationId + '-dialogflow.googleapis.com'});

const listener = app.listen(process.env.PORT, function() {
  console.log(
      'Your Twitter integration server is listening on port ' +
      listener.address().port);
  init();
});

process.on('SIGTERM', () => {
  listener.close(async () => {
    deleteSubscription();
    await deleteWebhooksByUrl(targetUrl);
    console.log('Closing http server.');
    process.exit(0);
  });
});

// Converts Twitter request to a detectIntent request.
function twitterToDetectIntent(message, sessionPath) {
  const request = {
    session: sessionPath,
    queryInput: {
      text: {
        text: message,
      },
      languageCode,
    }
  };

  return request;
}

/**
 * Takes as input a request from twitter and converts the request to
 * detectIntent request which is used to call the detectIntent() function
 * and finally output the response given by detectIntent().
 */
async function detectIntentResponse(message, userId) {
  const sessionId = userId;
  const sessionPath = client.projectLocationAgentSessionPath(
      projectId, locationId, agentId, sessionId);
  console.info(sessionPath);

  var request = twitterToDetectIntent(message, sessionPath);
  var [response] = await client.detectIntent(request);
  console.log(JSON.stringify(response))
  return response;
};

function sendMessage(text, recipientId) {
  request.post(
      'https://api.twitter.com/1.1/direct_messages/events/new.json', {
        oauth: twitterOAuth,
        json: {
          event: {
            type: 'message_create',
            message_create: {
              target: {recipient_id: recipientId},
              message_data: {
                text: text,
              }
            }
          }
        }
      },
      function(err, resp, body) {
        if (err) {
          console.eror('Failed to send message: ' + err);
        }
      });
}

function sendStatus(text, tweetId) {
  // Tweets must be less than 280 characters. See documentation below.
  // https://developer.twitter.com/en/docs/developer-utilities/twitter-text.html
  const parsedTweet = twitterText.parseTweet(text);
  if (parsedTweet.valid) {
    request.post(
        'https://api.twitter.com/1.1/statuses/update.json', {
          oauth: twitterOAuth,
          form: {status: text, in_reply_to_status_id: tweetId},
        },
        function(err, resp, body) {
          if (err) {
            console.error(err);
          }
        });
  } else {
    console.error(
        'Failed to send status because it is longer than 280 characters: ' +
        err);
  }
}

async function init() {
  deleteSubscription();
  await deleteWebhooksByUrl(targetUrl);
  await registerWebhook();
  registerSubscription();
}

function registerWebhook() {
  return new Promise((resolve, reject) => {
    request.post(
        'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
            '/webhooks.json',
        {oauth: twitterOAuth, form: {url: targetUrl}},
        function(err, resp, body) {
          if (err) {
            console.error('Failed to register webhook: ' + err);
            reject();
          } else {
            resolve();
          }
        });
  });
}

function registerSubscription() {
  request.post(
      'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
          '/subscriptions.json',
      {
        oauth: twitterOAuth,
      },
      function(err, resp, body) {
        if (err) {
          console.error('Failed to register subscription: ' + err);
        }
      });
}

function listWebhooks(url) {
  return new Promise((resolve, reject) => {
    request.get(
        'https://api.twitter.com/1.1/account_activity/all/webhooks.json',
        {oauth: twitterOAuth}, function(err, resp, body) {
          if (err) {
            console.error('Failed to check webhooks: ' + err);
            reject(err);
          }
          const environments = JSON.parse(body).environments;
          if (Array.isArray(environments)) {
            const environment = environments.find((element) => {
              return element.environment_name === environmentName;
            });
            if (environment) {
              const webhooks =
                  environment.webhooks.filter((value, index, arr) => {
                    return value.url === url;
                  });
              resolve(webhooks);
            }
            resolve([]);
          }
        });
  });
}

function deleteWebhookById(webhookId) {
  return new Promise((resolve, reject) =>{
    request.delete(
        'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
            '/webhooks/' + webhookId + '.json',
        {oauth: twitterOAuth}, function(err, resp, body) {
          if (err) {
            console.error('Failed to delete webhook: ' + err);
            reject(err);
          }
          resolve();
        });
  });
}

async function deleteWebhooksByUrl(targetUrl) {
  const webhooks = await listWebhooks(targetUrl);
  for (webhook of webhooks) {
    if (webhook.id) {
      await deleteWebhookById(webhook.id);
    }
  }
}

function deleteSubscription() {
  request.delete(
      'https://api.twitter.com/1.1/account_activity/all/' + environmentName +
          '/subscriptions.json',
      {oauth: twitterOAuth}, function(err, resp, body) {
        if (err) {
          console.error('Failed to delete subscriptions: ' + err);
        }
      });
}

app.post('/', async function(req, res) {
  if (Array.isArray(req.body.direct_message_events) &&
      req.body.direct_message_events.length > 0) {
    const message = req.body.direct_message_events[0];
    const text = message.message_create.message_data.text;
    const senderId = message.message_create.sender_id;
    if (senderId !== twitterId) {
      var responses = await detectIntentResponse(text, senderId);

      for (let response of responses.queryResult.responseMessages) {
        if (response.hasOwnProperty('text')) {
          sendMessage(response.text.text.join(), senderId);
          ;
        };
      };
      res.sendStatus(200)
    }
  } else if (
      Array.isArray(req.body.tweet_create_events) &&
      req.body.tweet_create_events.length > 0) {
    const message = req.body.tweet_create_events[0];
    const text = message.text;
    const senderId = message.user.id_str;
    if (senderId !== twitterId && message.in_reply_to_user_id_str) {
      var responses = await detectIntentResponse(
          text.replace('@' + message.in_reply_to_screen_name, ''), senderId);
      const screenName = message.user.screen_name;

      for (let response of responses.queryResult.responseMessages) {
        if (response.hasOwnProperty('text')) {
          sendStatus(
              '@' + screenName + ' ' + response.text.text.join(), senderId);
        };
      };
      res.sendStatus(200);
    }
  }
});

// responds to challenge response check verification requests
app.get('/', function(req, res) {
  const crc = req.query.crc_token;
  const hmac = 'sha256=' +
      crypto.createHmac('sha256', twitterSecretAPIKey)
          .update(crc)
          .digest('base64');
  res.json({'response_token': hmac});
});
<|endoftext|>