<commit_before>
/*!
 * socket.io-node
 * Copyright(c) 2011 LearnBoost <dev@learnboost.com>
 * MIT Licensed
 */

/**
 * Module requirements.
 */

var Transport = require('../transport')
  , parser = require('../parser')
  , qs = require('querystring');

/**
 * Export the constructor.
 */

exports = module.exports = HTTPTransport;

/**
 * HTTP interface constructor. For all non-websocket transports.
 *
 * @api public
 */

function HTTPTransport (mng, data, req) {
  Transport.call(this, mng, data, req);
};

/**
 * Inherits from Transport.
 */

HTTPTransport.prototype.__proto__ = Transport.prototype;

/**
 * Handles a request.
 *
 * @api private
 */

HTTPTransport.prototype.handleRequest = function (req) {

  // Always set the response in case an error is returned to the client
  this.response = req.res;

  if (req.method == 'POST') {
    var buffer = ''
      , res = req.res
      , origin = req.headers.origin
      , headers = { 'Content-Length': 1, 'Content-Type': 'text/plain; charset=UTF-8' }
      , self = this;

    req.on('data', function (data) {
      buffer += data;

      if (Buffer.byteLength(buffer) >= self.manager.get('destroy buffer size')) {
        buffer = '';
        req.connection.destroy();
      }
    });

    req.on('end', function () {
      res.writeHead(200, headers);
      res.end('1');

      self.onData(self.postEncoded ? qs.parse(buffer).d : buffer);
    });

    // prevent memory leaks for uncompleted requests
    req.on('close', function () {
      buffer = '';
      self.onClose();
    });

    if (origin) {
      // https://developer.mozilla.org/En/HTTP_Access_Control
      headers['Access-Control-Allow-Origin'] = origin;
      headers['Access-Control-Allow-Credentials'] = 'true';
    }
  } else {
    Transport.prototype.handleRequest.call(this, req);
  }
};

/**
 * Handles data payload.
 *
 * @api private
 */

HTTPTransport.prototype.onData = function (data) {
  var messages = parser.decodePayload(data);
  this.log.debug(this.name + ' received data packet', data);

  for (var i = 0, l = messages.length; i < l; i++) {
    this.onMessage(messages[i]);
  }
};

/**
 * Closes the request-response cycle
 *
 * @api private
 */

HTTPTransport.prototype.doClose = function () {
  this.response.end();
};

/**
 * Writes a payload of messages
 *
 * @api private
 */

HTTPTransport.prototype.payload = function (msgs) {
  this.write(parser.encodePayload(msgs));
};
<commit_msg>http-polling : adding 'X-XSS-Protection : 0;' to headers necessary not only to jsonp-polling but http-polling<commit_after>
/*!
 * socket.io-node
 * Copyright(c) 2011 LearnBoost <dev@learnboost.com>
 * MIT Licensed
 */

/**
 * Module requirements.
 */

var Transport = require('../transport')
  , parser = require('../parser')
  , qs = require('querystring');

/**
 * Export the constructor.
 */

exports = module.exports = HTTPTransport;

/**
 * HTTP interface constructor. For all non-websocket transports.
 *
 * @api public
 */

function HTTPTransport (mng, data, req) {
  Transport.call(this, mng, data, req);
};

/**
 * Inherits from Transport.
 */

HTTPTransport.prototype.__proto__ = Transport.prototype;

/**
 * Handles a request.
 *
 * @api private
 */

HTTPTransport.prototype.handleRequest = function (req) {

  // Always set the response in case an error is returned to the client
  this.response = req.res;

  if (req.method == 'POST') {
    var buffer = ''
      , res = req.res
      , origin = req.headers.origin
      , headers = { 'Content-Length': 1, 'Content-Type': 'text/plain; charset=UTF-8' }
      , self = this;

    req.on('data', function (data) {
      buffer += data;

      if (Buffer.byteLength(buffer) >= self.manager.get('destroy buffer size')) {
        buffer = '';
        req.connection.destroy();
      }
    });

    req.on('end', function () {
      res.writeHead(200, headers);
      res.end('1');

      self.onData(self.postEncoded ? qs.parse(buffer).d : buffer);
    });

    // prevent memory leaks for uncompleted requests
    req.on('close', function () {
      buffer = '';
      self.onClose();
    });

    if (origin) {
      // https://developer.mozilla.org/En/HTTP_Access_Control
      headers['Access-Control-Allow-Origin'] = origin;
      headers['Access-Control-Allow-Credentials'] = 'true';
      headers['X-XSS-Protection'] = '0';
    }
  } else {
    Transport.prototype.handleRequest.call(this, req);
  }
};

/**
 * Handles data payload.
 *
 * @api private
 */

HTTPTransport.prototype.onData = function (data) {
  var messages = parser.decodePayload(data);
  this.log.debug(this.name + ' received data packet', data);

  for (var i = 0, l = messages.length; i < l; i++) {
    this.onMessage(messages[i]);
  }
};

/**
 * Closes the request-response cycle
 *
 * @api private
 */

HTTPTransport.prototype.doClose = function () {
  this.response.end();
};

/**
 * Writes a payload of messages
 *
 * @api private
 */

HTTPTransport.prototype.payload = function (msgs) {
  this.write(parser.encodePayload(msgs));
};
<|endoftext|>


<commit_before>const {CONTEXT, GRAPH, ID, TYPE, VALUE, LANGUAGE, LIST, VOCAB, BASE} =
  require('../jsonld/keywords')

module.exports = {serialize}

function serialize (data, out = process.stdout, {
  context = null,
  baseIri = null,
  settings = {
    indentChars: '  ',
    upcaseKeywords: false,
    useGraphKeyword: false,
    spaceBeforeSemicolon: true,
    predicateRepeatNewLine: true,
    bracketStartNewLine: false,
    bracketEndNewLine: false,
    listItemNewLine: true,
    prologueEndLine: 1,
    separatorLines: 1
  }
} = {}) {
  let state = new SerializerState(context, out, settings)
  state.serialize(data)
}

class SerializerState {
  constructor (context, outstream, settings, parent = null) {
    this.outstream = outstream || parent.outstream
    this.initContext(context)
    this.parent = parent
    this.settings = parent ? parent.settings : settings
    let kw = it => this.settings.upcaseKeywords ? it.toUpperCase() : it
    this.prefixKeyword = kw('prefix')
    this.baseKeyword = kw('base')
    if (this.settings.useGraphKeyword) {
      this.graphKeyword = kw('graph')
    }
    this.uniqueBNodeSuffix = ''
  }

  initContext (context) {
    this.keys = {}
    this.prefixes = {}

    if (context == null) return
    if (this.context != null) {
      throw 'context already initialized'
    }
    let ctx = {}
    // TODO: merge arrays, deref uri:s ...
    if (Array.isArray(context)) {
      for (let item of context) {
        Object.assign(ctx, item)
      }
    } else if (typeof context === 'object') {
        Object.assign(ctx, context)
    }
    this.context = ctx
    this.prefixes = collectPrefixes(context)
    this.keys = {
      id: "@id",
      value: '@value',
      type: '@type',
      lang: '@language',
      graph: '@graph',
      graph: '@list'
    }
  }

  serialize (data) {
    if (data[CONTEXT]) {
      this.initContext(data[CONTEXT])
    }
    this.prelude()
    if (data[GRAPH]) {
      for (let node of data[GRAPH]) {
        this.objectToTurtle(node)
      }
    } else {
      this.objectToTurtle(data)
    }
  }

  prelude () {
    for (let k in this.prefixes) {
      let v = this.prefixes[k]
      if (k === BASE) {
        this.writeBase(v)
      } else {
        this.writeln(`${this.prefixKeyword} ${k}: <${v}>`)
      }
    }
    if (this.base) {
      this.writeBase(this.base)
    }
    if (this.settings.prologueEndLine > 1) {
      this.writeln()
    }
  }

  writeBase (iri) {
    this.writeln(`${this.baseKeyword} <${iri}>`)
  }

  isListContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@list'
  }

  isLangContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@language'
  }

  objectToTrig (iri, obj) {
    this.writeln()
    if (this.graphKeyword) {
        this.write(`${this.graphKeyword} `)
    }
    this.writeln("<$iri> {")
    this.writeln()
    this.objectToTurtle(obj)
    this.writeln("}")
    this.writeln()
  }

  objectToTurtle (obj, depth = 0, viaKey = null) {
    if (depth > 0 && obj[CONTEXT]) {
      // TODO: use new SerializerState(node[CONTEXT], null, null, state)
      throw 'Nested context not supported yet'
    }

    if (this.isLangContainer(viaKey) && typeof obj === 'object') {
      let first = true
      for (let lang in obj) {
        let value = obj[lang]
          if (!first) this.write(' , ')
        this.toLiteral(
          { [this.keys.value]: value, [this.keys.lang]: lang },
          viaKey)
        first = false
      }
      return []
    }

    if (!(typeof obj === 'object') || obj[this.keys.value]) {
      this.toLiteral(obj, viaKey)
      return []
    }

    let explicitList = obj[LIST]

    if (this.isListContainer(viaKey)) {
      obj = { [this.keys.list]: obj }
    }

    let s = obj[this.keys.id]

    let isList = obj[LIST]
    let startedList = isList

    if (explicitList) {
      this.write('( ')
    }
    if (s != null && this.hasKeys(obj, 2)) {
      if (depth === 0) {
        this.writeln()
      }
      this.write(this.refRepr(s))
    } else if (depth > 0) {
      if (!isList) {
        depth++
        this.write("[")
      }
    } else {
      if (obj[this.keys.graph]) {
        // TODO: this is the default graph, described...
      } else {
        return []
      }
    }

    let indent = this.getIndent(depth)

    let topObjects = []

    let first = true
    let endedList = false

    for (let key in obj) {
      let vs = obj[key]
      let term = this.termFor(key)

      let revKey = (term == null)? this.revKeyFor(key) : null
      if (term == null && revKey == null)
        continue

      if (term == this.keys.id || term == "@context")
        continue

      vs = Array.isArray(vs) ? vs : vs != null ? [vs] : []
      if (!vs.length) // TODO: && not @list
        continue

      if (term == this.keys.graph) {
        for (let it of vs) {
          topObjects.push(it)
        }
        continue
      }

      let inList = isList || this.isListContainer(key)

      if (revKey) {
        for (let it of vs) {
          let node = it.clone()
          node[revKey] = { [this.keys.id]: s }
          topObjects.push(node)
        }
      } else {
        let useIndent = indent
        if (first) {
          useIndent = ' '
          first = false
        } else {
          if (startedList && !inList && !endedList) {
            endedList = true
            this.write(" )")
          }
          this.writeln(" ;")
        }

        if (term == "@type") {
          term = "a"
          this.write(useIndent + term + " " + vs.map(it =>
            this.toValidTerm(this.termFor(it))
          ).join(", "))
          continue
        }

        if (term != '@list') {
          term = this.toValidTerm(term)
          this.write(useIndent + term + " ")
        }

        for (let i = 0; i < vs.length; i++) {
          let v = vs[i]
            if (inList) {
              if (!startedList) {
                this.write("(")
                startedList = true
              }
              this.write(" ")
            } else if (i > 0) {
              if (this.settings.predicateRepeatNewLine) {
                this.writeln(' ,')
                this.write(this.getIndent(depth + 1))
              } else {
                this.write(' , ')
              }
            }
          if (this.bnodeSkolemBase && typeof v === 'object' && !v[this.keys.id]) {
            v[this.keys.id] = s = this.genSkolemId()
          }
          if (typeof v === 'object' && this.keys.id in v) {
            topObjects.push(v)
            this.write(this.refRepr(v[this.keys.id]))
          } else {
            for (let it of this.objectToTurtle(v, depth + 1, key)) {
              topObjects.push(it)
            }
          }
        }
      }
    }

    if (explicitList || (!isList && startedList) && !endedList) {
      this.write(" )")
    }

    if (depth == 0) {
      if (!first) {
        this.writeln(" .")
      }
      //this.writeln()
      for (let it of topObjects) {
        this.objectToTurtle(it)
      }
      return []
    } else {
      indent = this.getIndent(depth - 1)
      if (this.settings.bracketEndNewLine) {
        this.writeln()
        this.write(indent)
      } else {
        this.write(' ')
      }
      if (!isList) {
        // NOTE: hack for e.g. BlazeGraph
        if (!this.hasKeys(obj) && markEmptyBnode) {
          this.writeln("a $emptyMarker")
          this.write(indent)
        }
        this.write("]")
      }
      return topObjects
    }
  }

  toLiteral (obj, viaKey=null) {
    let value = obj
    let lang = this.context["@language"]
    let datatype = null
    if (typeof obj === 'object') {
      value = obj[this.keys.value]
      datatype = obj[this.keys.type]
      lang = obj[this.keys.lang]
    } else {
      let kdef = this.context[viaKey]
      let coerceTo = (typeof kdef === 'object')? kdef["@type"] : null
      if (coerceTo == "@vocab") {
        this.write(typeof value === 'string' ?
          this.refRepr(value, true): value)
        return
      } else if (coerceTo == "@id") {
        this.write(this.refRepr(value))
        return
      } else if (coerceTo) {
        datatype = coerceTo
      } else {
        let termLang = (typeof kdef === 'object')? kdef["@language"] : null
        if (typeof kdef === 'object') {
          lang = termLang
        }
      }
    }
    if (typeof value === 'string') {
      let escaped = value.replace(/"/g, '\\"')
      this.write('"')
      this.write(escaped)
      this.write('"')
      if (datatype)
        this.write("^^" + this.termFor(datatype))
      else if (lang)
        this.write("@" + lang)
    } else { // boolean or number
      this.write(value.toString())
    }
  }

  termFor (key) {
    if (key.startsWith("@")) {
      return key
    } else if (key.indexOf(":") > -1) {
      return key
    } else if (this.context[key]) {
      let kdef = this.context[key]
      if (kdef == null)
        return null
      let term = null
      if (typeof kdef === 'object') {
        term = kdef["@id"] || key
      } else {
        term = kdef
      }
      if (term == null)
        return null
      return (term.indexOf(":") == -1)? ":" + term : term
    } else {
      return ":" + key
    }
  }

  revKeyFor (key) {
    let kdef = this.context[key]
    if (!(typeof kdef === 'object'))
      return null
    return kdef["@reverse"]
  }

  refRepr (ref, useVocab = false) {
    let cI = ref.indexOf(":")
    if (cI > -1) {
      let pfx = ref.substring(0, cI)
      if (pfx == "_") {
        let nodeId = ref + this.uniqueBNodeSuffix
        if (this.bnodeSkolemBase) {
          ref = this.bnodeSkolemBase + nodeId.substring(2)
        } else {
          return this.toValidTerm(nodeId)
        }
      } else if (this.context[pfx]) {
        return ref
      }
    } else if (useVocab && ref.indexOf("/") == -1) {
      return ":" + ref
    }
    ref = this.cleanValue(ref)
    return `<${ref}>`
  }

  toValidTerm (term) {
    term = this.cleanValue(term)
    if (term.indexOf('://') > -1) {
      return `<${term}>`
    }
    // TODO: hack to pseudo-fix problematic pnames...
    return term
  }

  hasKeys (obj, atLeast = 1) {
    let seen = 0
    for (let k in obj) if (++seen === atLeast) return true
    return false
  }

  cleanValue (v) {
    return v
  }

  genSkolemId () {
    return this.bnodeSkolemBase + UUID.randomUUID()
  }

  getIndent (depth) {
    let chunks = []
    for (let i = -1; i++ < depth;) {
      chunks.push(this.settings.indentChars)
    }
    return chunks.join('')
  }

  write (s) {
    this.outstream.write(s || '')
  }

  writeln (s) {
    this.outstream.write((s || '') + '\n')
  }
}

function collectPrefixes (context) {
  if (context == null) return

  let prefixes = {}
  for (let key in context) {
    let value = context[key]
    // TODO: verify JSON-LD 1.1 prefix rules
    if (typeof value === 'string' && value.match(/[#\/:]$/)) {
      prefixes[key === VOCAB ? '' : key] = value
    } else if (value && value['@prefix'] === true) {
      prefixes[key] = value[ID]
    }
  }
  return prefixes
}
<commit_msg>Output PName if IRI starts with @vocab<commit_after>const {CONTEXT, GRAPH, ID, TYPE, VALUE, LANGUAGE, LIST, VOCAB, BASE} =
  require('../jsonld/keywords')

module.exports = {serialize}

function serialize (data, out = process.stdout, {
  context = null,
  baseIri = null,
  settings = {
    indentChars: '  ',
    upcaseKeywords: false,
    useGraphKeyword: false,
    spaceBeforeSemicolon: true,
    predicateRepeatNewLine: true,
    bracketStartNewLine: false,
    bracketEndNewLine: false,
    listItemNewLine: true,
    prologueEndLine: 1,
    separatorLines: 1
  }
} = {}) {
  let state = new SerializerState(context, out, settings)
  state.serialize(data)
}

class SerializerState {
  constructor (context, outstream, settings, parent = null) {
    this.outstream = outstream || parent.outstream
    this.initContext(context)
    this.parent = parent
    this.settings = parent ? parent.settings : settings
    let kw = it => this.settings.upcaseKeywords ? it.toUpperCase() : it
    this.prefixKeyword = kw('prefix')
    this.baseKeyword = kw('base')
    if (this.settings.useGraphKeyword) {
      this.graphKeyword = kw('graph')
    }
    this.uniqueBNodeSuffix = ''
  }

  initContext (context) {
    this.keys = {}
    this.prefixes = {}

    if (context == null) return
    if (this.context != null) {
      throw 'context already initialized'
    }
    let ctx = {}
    // TODO: merge arrays, deref uri:s ...
    if (Array.isArray(context)) {
      for (let item of context) {
        Object.assign(ctx, item)
      }
    } else if (typeof context === 'object') {
        Object.assign(ctx, context)
    }
    this.context = ctx
    this.prefixes = collectPrefixes(context)
    this.keys = {
      id: "@id",
      value: '@value',
      type: '@type',
      lang: '@language',
      graph: '@graph',
      graph: '@list'
    }
  }

  serialize (data) {
    if (data[CONTEXT]) {
      this.initContext(data[CONTEXT])
    }
    this.prelude()
    if (data[GRAPH]) {
      for (let node of data[GRAPH]) {
        this.objectToTurtle(node)
      }
    } else {
      this.objectToTurtle(data)
    }
  }

  prelude () {
    for (let k in this.prefixes) {
      let v = this.prefixes[k]
      if (k === BASE) {
        this.writeBase(v)
      } else {
        this.writeln(`${this.prefixKeyword} ${k}: <${v}>`)
      }
    }
    if (this.base) {
      this.writeBase(this.base)
    }
    if (this.settings.prologueEndLine > 1) {
      this.writeln()
    }
  }

  writeBase (iri) {
    this.writeln(`${this.baseKeyword} <${iri}>`)
  }

  isListContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@list'
  }

  isLangContainer (term) {
    if (this.context == null) return false
    return typeof this.context[term] === 'object' &&
      this.context[term]['@container'] == '@language'
  }

  objectToTrig (iri, obj) {
    this.writeln()
    if (this.graphKeyword) {
        this.write(`${this.graphKeyword} `)
    }
    this.writeln("<$iri> {")
    this.writeln()
    this.objectToTurtle(obj)
    this.writeln("}")
    this.writeln()
  }

  objectToTurtle (obj, depth = 0, viaKey = null) {
    if (depth > 0 && obj[CONTEXT]) {
      // TODO: use new SerializerState(node[CONTEXT], null, null, state)
      throw 'Nested context not supported yet'
    }

    if (this.isLangContainer(viaKey) && typeof obj === 'object') {
      let first = true
      for (let lang in obj) {
        let value = obj[lang]
          if (!first) this.write(' , ')
        this.toLiteral(
          { [this.keys.value]: value, [this.keys.lang]: lang },
          viaKey)
        first = false
      }
      return []
    }

    if (!(typeof obj === 'object') || obj[this.keys.value]) {
      this.toLiteral(obj, viaKey)
      return []
    }

    let explicitList = obj[LIST]

    if (this.isListContainer(viaKey)) {
      obj = { [this.keys.list]: obj }
    }

    let s = obj[this.keys.id]

    let isList = obj[LIST]
    let startedList = isList

    if (explicitList) {
      this.write('( ')
    }
    if (s != null && this.hasKeys(obj, 2)) {
      if (depth === 0) {
        this.writeln()
      }
      this.write(this.refRepr(s))
    } else if (depth > 0) {
      if (!isList) {
        depth++
        this.write("[")
      }
    } else {
      if (obj[this.keys.graph]) {
        // TODO: this is the default graph, described...
      } else {
        return []
      }
    }

    let indent = this.getIndent(depth)

    let topObjects = []

    let first = true
    let endedList = false

    for (let key in obj) {
      let vs = obj[key]
      let term = this.termFor(key)

      let revKey = (term == null)? this.revKeyFor(key) : null
      if (term == null && revKey == null)
        continue

      if (term == this.keys.id || term == "@context")
        continue

      vs = Array.isArray(vs) ? vs : vs != null ? [vs] : []
      if (!vs.length) // TODO: && not @list
        continue

      if (term == this.keys.graph) {
        for (let it of vs) {
          topObjects.push(it)
        }
        continue
      }

      let inList = isList || this.isListContainer(key)

      if (revKey) {
        for (let it of vs) {
          let node = it.clone()
          node[revKey] = { [this.keys.id]: s }
          topObjects.push(node)
        }
      } else {
        let useIndent = indent
        if (first) {
          useIndent = ' '
          first = false
        } else {
          if (startedList && !inList && !endedList) {
            endedList = true
            this.write(" )")
          }
          this.writeln(" ;")
        }

        if (term == "@type") {
          term = "a"
          this.write(useIndent + term + " " + vs.map(it =>
            this.toValidTerm(this.termFor(it))
          ).join(", "))
          continue
        }

        if (term != '@list') {
          term = this.toValidTerm(term)
          this.write(useIndent + term + " ")
        }

        for (let i = 0; i < vs.length; i++) {
          let v = vs[i]
            if (inList) {
              if (!startedList) {
                this.write("(")
                startedList = true
              }
              this.write(" ")
            } else if (i > 0) {
              if (this.settings.predicateRepeatNewLine) {
                this.writeln(' ,')
                this.write(this.getIndent(depth + 1))
              } else {
                this.write(' , ')
              }
            }
          if (this.bnodeSkolemBase && typeof v === 'object' && !v[this.keys.id]) {
            v[this.keys.id] = s = this.genSkolemId()
          }
          if (typeof v === 'object' && this.keys.id in v) {
            topObjects.push(v)
            this.write(this.refRepr(v[this.keys.id]))
          } else {
            for (let it of this.objectToTurtle(v, depth + 1, key)) {
              topObjects.push(it)
            }
          }
        }
      }
    }

    if (explicitList || (!isList && startedList) && !endedList) {
      this.write(" )")
    }

    if (depth == 0) {
      if (!first) {
        this.writeln(" .")
      }
      //this.writeln()
      for (let it of topObjects) {
        this.objectToTurtle(it)
      }
      return []
    } else {
      indent = this.getIndent(depth - 1)
      if (this.settings.bracketEndNewLine) {
        this.writeln()
        this.write(indent)
      } else {
        this.write(' ')
      }
      if (!isList) {
        // NOTE: hack for e.g. BlazeGraph
        if (!this.hasKeys(obj) && markEmptyBnode) {
          this.writeln("a $emptyMarker")
          this.write(indent)
        }
        this.write("]")
      }
      return topObjects
    }
  }

  toLiteral (obj, viaKey=null) {
    let value = obj
    let lang = this.context["@language"]
    let datatype = null
    if (typeof obj === 'object') {
      value = obj[this.keys.value]
      datatype = obj[this.keys.type]
      lang = obj[this.keys.lang]
    } else {
      let kdef = this.context[viaKey]
      let coerceTo = (typeof kdef === 'object')? kdef["@type"] : null
      if (coerceTo == "@vocab") {
        this.write(typeof value === 'string' ?
          this.refRepr(value, true): value)
        return
      } else if (coerceTo == "@id") {
        this.write(this.refRepr(value))
        return
      } else if (coerceTo) {
        datatype = coerceTo
      } else {
        let termLang = (typeof kdef === 'object')? kdef["@language"] : null
        if (typeof kdef === 'object') {
          lang = termLang
        }
      }
    }
    if (typeof value === 'string') {
      let escaped = value.replace(/"/g, '\\"')
      this.write('"')
      this.write(escaped)
      this.write('"')
      if (datatype)
        this.write("^^" + this.termFor(datatype))
      else if (lang)
        this.write("@" + lang)
    } else { // boolean or number
      this.write(value.toString())
    }
  }

  termFor (key) {
    if (key.startsWith("@")) {
      return key
    } else if (key.indexOf(":") > -1) {
      return key
    } else if (this.context[key]) {
      let kdef = this.context[key]
      if (kdef == null)
        return null
      let term = null
      if (typeof kdef === 'object') {
        term = kdef["@id"] || key
      } else {
        term = kdef
      }
      if (term == null)
        return null
      return (term.indexOf(":") == -1)? ":" + term : term
    } else {
      return ":" + key
    }
  }

  revKeyFor (key) {
    let kdef = this.context[key]
    if (!(typeof kdef === 'object'))
      return null
    return kdef["@reverse"]
  }

  refRepr (ref, useVocab = false) {
    let cI = ref.indexOf(":")
    if (cI > -1) {
      let pfx = ref.substring(0, cI)
      if (pfx == "_") {
        let nodeId = ref + this.uniqueBNodeSuffix
        if (this.bnodeSkolemBase) {
          ref = this.bnodeSkolemBase + nodeId.substring(2)
        } else {
          return this.toValidTerm(nodeId)
        }
      } else if (this.context[pfx]) {
        return ref
      }
    } else if (useVocab && ref.indexOf("/") == -1) {
      return ":" + ref
    }

    if (ref.startsWith(this.context[VOCAB])) {
      return ":" + ref.substring(this.context[VOCAB].length)
    }

    ref = this.cleanValue(ref)

    return `<${ref}>`
  }

  toValidTerm (term) {
    term = this.cleanValue(term)
    if (term.indexOf('://') > -1) {
      return `<${term}>`
    }
    // TODO: hack to pseudo-fix problematic pnames...
    return term
  }

  hasKeys (obj, atLeast = 1) {
    let seen = 0
    for (let k in obj) if (++seen === atLeast) return true
    return false
  }

  cleanValue (v) {
    return v
  }

  genSkolemId () {
    return this.bnodeSkolemBase + UUID.randomUUID()
  }

  getIndent (depth) {
    let chunks = []
    for (let i = -1; i++ < depth;) {
      chunks.push(this.settings.indentChars)
    }
    return chunks.join('')
  }

  write (s) {
    this.outstream.write(s || '')
  }

  writeln (s) {
    this.outstream.write((s || '') + '\n')
  }
}

function collectPrefixes (context) {
  if (context == null) return

  let prefixes = {}
  for (let key in context) {
    let value = context[key]
    // TODO: verify JSON-LD 1.1 prefix rules
    if (typeof value === 'string' && value.match(/[#\/:]$/)) {
      prefixes[key === VOCAB ? '' : key] = value
    } else if (value && value['@prefix'] === true) {
      prefixes[key] = value[ID]
    }
  }
  return prefixes
}
<|endoftext|>


<commit_before>const fetch = require('node-fetch');
const fs = require("fs");
const handlebars = require('handlebars');

/* eslint-disable-next-line no-extend-native */
String.prototype.replaceAll = function (search, replacement) {
    const target = this;
    return target.split(search).join(replacement);
};

const generateSlug = (title) => {
    let slug = '';
    const allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ -';
    for (let i = 0; i < title.length; i++) {
        const ch = title[i];
        if (allowed.indexOf(ch) >= 0) {
            slug += ch;
        }
    }
    slug = slug.toLowerCase().replaceAll(' ', '-');
    return slug;
};

// Get tagged projects from the public API
const getTaggedProjects = (done) => {
    const tagged = {};
    let skip = 0;
    const get = (cb) => {
        fetch(`https://playcanvas.com/api/apps?limit=32&tags=tutorial&skip=${skip}`)
            .then(res => res.json())
            .then((json) => {
                const samples = json.result;

                for (let i = 0; i < samples.length; i++) {
                    const sample = samples[i];
                    const appUrl = sample.url;
                    const editorUrl = `https://playcanvas.com/editor/project/${sample.project_id}`;
                    const projectUrl = `https://playcanvas.com/project/${sample.project_id}/overview`;
                    const thumbUrl = sample.thumbnails.m;
                    const name = sample.name;
                    const tags = sample.tags || [];
                    const slug = generateSlug(name);
                    const description = sample.description;

                    const url = '/tutorials/' + slug;
                    const path = 'en' + url;

                    tagged[path] = {
                        title: name,
                        url: url,
                        path: path,
                        tags: tags,
                        slug: slug,
                        appUrl: appUrl,
                        editorUrl: editorUrl,
                        projectUrl: projectUrl,
                        thumbUrl: thumbUrl,
                        description: description
                    };
                }

                if (samples.length === 32) {
                    skip += 32;
                    get(function () {
                        cb();
                    });
                } else {
                    cb();
                }
            })
            .catch((err) => {
                console.log(`failed to download tagged projects (${err})`);
                cb();
            });
    };

    get(() => {
        done(tagged);
    });
};

const getTags = (done) => {
    fetch('https://playcanvas.com/api/tags')
        .then(res => res.json())
        .then((json) => {
            let tags = json.result;
            tags = tags.filter((tag) => {
                if (['skype', 'tutorial', 'challenge'].indexOf(tag.id) >= 0) return false;
                return tag.kind === 'app';
            });
            done(tags);
        })
        .catch((err) => {
            console.log(`failed to download tags (${err})`);
            done([]);
        });
};

module.exports = function makePlugin(dir) {
    // compile the template used to generate pages for tagged projects
    handlebars.registerPartial("template-tagged-project", fs.readFileSync("templates/partials/tutorial-tagged-project.tmpl.html", "utf-8"));
    const taggedProjectTemplate = handlebars.compile(handlebars.partials["template-tagged-project"]);

    return function (opts) {
        return function (files, metalsmith, done) {
            const langs = ['en', 'ru', 'ja', 'zh'];

            const tutorialsIndex = {}; // index object of tutorials
            const projectDemosIndex = {}; // index object of project demos

            for (let i = 0; i < langs.length; i++) {
                const lang = langs[i];
                tutorialsIndex[lang] = {};
                // get all tutorials and insert into index
                for (const filepath in files) {
                    if (filepath.indexOf('.DS_Store') >= 0) continue;

                    const file = files[filepath];

                    if (filepath.startsWith(lang + '/' + dir)) {
                        if (filepath.endsWith('tutorials/index.html')) continue; // skip main index page

                        // separate tags
                        const tags = file.tags || '';
                        file.tags = tags.split(',').filter(function (tag) {
                            return !!tag;
                        }).map((tag) => {
                            return tag.trim();
                        }) || [];

                        tutorialsIndex[lang][filepath] = {
                            title: file.title,
                            url: filepath.replace(lang, '').replace('index.html', ''),
                            tags: file.tags,
                            thumbUrl: file.thumb
                        };

                        // add unique tag for tutorials
                        tutorialsIndex[lang][filepath].tags.push('_tutorial');
                    }
                }

                projectDemosIndex[lang] = {};
            }

            getTags(function (tags) {
                getTaggedProjects(function (tagged) {
                    // add tagged projects into main index
                    for (const key in tagged) {
                        // Add the demo tag so we can tell later what is a demo
                        // and what is a full tutorial
                        tagged[key].tags.push('_demo');

                        for (let i = 0; i < langs.length; i++) {
                            const lang = langs[i];
                            const project = tagged[key];

                            // insert tagged project into index
                            projectDemosIndex[lang][key] = project;

                            // create a new output file for each tagged project
                            const file = {
                                title: project.title,
                                template: 'tutorial-page.tmpl.html',
                                tags: project.tags,
                                mode: '0644',
                                path: lang + project.url,
                                locale: lang,
                                en: true,
                                description: project.description,
                                contents: Buffer.from(taggedProjectTemplate(project))
                            };

                            files[file.path + '/index.html'] = file;
                        }
                    }

                    const context = {
                        pages: {},
                        tags: tags
                    };

                    for (const lang in tutorialsIndex) {
                        context.pages[lang] = {
                            tutorials: [],
                            projectDemos: []
                        };
                        // Add all the tutorial pages to the language
                        for (const key in tutorialsIndex[lang]) {
                            context.pages[lang].tutorials.push(tutorialsIndex[lang][key]);
                        }

                        // Add all the project demos to the language
                        for (const key in projectDemosIndex[lang]) {
                            context.pages[lang].projectDemos.push(projectDemosIndex[lang][key]);
                        }

                        const comparePages = (a, b) => {
                            if (a.title && b.title) {
                                const _a = a.title.toLowerCase();
                                const _b = b.title.toLowerCase();
                                if (_a < _b) return -1;
                                if (_a > _b) return 1;
                            }
                            return 0;
                        };

                        context.pages[lang].tutorials.sort(comparePages);
                        context.pages[lang].projectDemos.sort(comparePages);
                    }

                    // tutorial main page filters
                    // render a new template using the template's template(!) and the content from the file tree
                    const contentsTemplatePartialName = 'template-tutorial-contents';
                    handlebars.registerPartial(contentsTemplatePartialName, fs.readFileSync("templates/partials/tutorial-contents.tmpl.html", "utf-8"));
                    let contentsTemplate = handlebars.compile(handlebars.partials[contentsTemplatePartialName]);
                    let contents = contentsTemplate(context);

                    // registry the generated contents template as the partial used in the tutorial pages
                    handlebars.registerPartial('tutorials-navigation', contents);

                    // render a new template used to template the tag selection on a tutorial page
                    const contentsLinksTemplatePartialName = 'template-tutorial-contents-links';
                    handlebars.registerPartial(contentsLinksTemplatePartialName, fs.readFileSync("templates/partials/tutorial-contents-links.tmpl.html", "utf-8"));
                    contentsTemplate = handlebars.compile(handlebars.partials[contentsLinksTemplatePartialName]);
                    contents = contentsTemplate(context);
                    handlebars.registerPartial('tutorials-navigation-links', contents);

                    // tutorial navigation
                    // render a new template using the template's template(!) and the content from the file tree
                    const listTemplatePartialName = 'template-tutorial-list';
                    handlebars.registerPartial(listTemplatePartialName, fs.readFileSync("templates/partials/tutorial-list.tmpl.html", "utf-8"));
                    const listTemplate = handlebars.compile(handlebars.partials[listTemplatePartialName]);
                    contents = listTemplate(context);

                    // generate pages for tagged projects

                    // registry the generated list template as the partial used in the tutorial pages
                    handlebars.registerPartial('tutorials-list', contents);

                    done();
                });
            });
        };
    };
};
<commit_msg>Fix building tutorials page on Windows<commit_after>const fetch = require('node-fetch');
const fs = require("fs");
const handlebars = require('handlebars');
const path = require('path');

/* eslint-disable-next-line no-extend-native */
String.prototype.replaceAll = function (search, replacement) {
    const target = this;
    return target.split(search).join(replacement);
};

const generateSlug = (title) => {
    let slug = '';
    const allowed = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ -';
    for (let i = 0; i < title.length; i++) {
        const ch = title[i];
        if (allowed.indexOf(ch) >= 0) {
            slug += ch;
        }
    }
    slug = slug.toLowerCase().replaceAll(' ', '-');
    return slug;
};

// Get tagged projects from the public API
const getTaggedProjects = (done) => {
    const tagged = {};
    let skip = 0;
    const get = (cb) => {
        fetch(`https://playcanvas.com/api/apps?limit=32&tags=tutorial&skip=${skip}`)
            .then(res => res.json())
            .then((json) => {
                const samples = json.result;

                for (let i = 0; i < samples.length; i++) {
                    const sample = samples[i];
                    const appUrl = sample.url;
                    const editorUrl = `https://playcanvas.com/editor/project/${sample.project_id}`;
                    const projectUrl = `https://playcanvas.com/project/${sample.project_id}/overview`;
                    const thumbUrl = sample.thumbnails.m;
                    const name = sample.name;
                    const tags = sample.tags || [];
                    const slug = generateSlug(name);
                    const description = sample.description;

                    const url = '/tutorials/' + slug;
                    const path = 'en' + url;

                    tagged[path] = {
                        title: name,
                        url: url,
                        path: path,
                        tags: tags,
                        slug: slug,
                        appUrl: appUrl,
                        editorUrl: editorUrl,
                        projectUrl: projectUrl,
                        thumbUrl: thumbUrl,
                        description: description
                    };
                }

                if (samples.length === 32) {
                    skip += 32;
                    get(function () {
                        cb();
                    });
                } else {
                    cb();
                }
            })
            .catch((err) => {
                console.log(`failed to download tagged projects (${err})`);
                cb();
            });
    };

    get(() => {
        done(tagged);
    });
};

const getTags = (done) => {
    fetch('https://playcanvas.com/api/tags')
        .then(res => res.json())
        .then((json) => {
            let tags = json.result;
            tags = tags.filter((tag) => {
                if (['skype', 'tutorial', 'challenge'].indexOf(tag.id) >= 0) return false;
                return tag.kind === 'app';
            });
            done(tags);
        })
        .catch((err) => {
            console.log(`failed to download tags (${err})`);
            done([]);
        });
};

module.exports = function makePlugin(dir) {
    // compile the template used to generate pages for tagged projects
    handlebars.registerPartial("template-tagged-project", fs.readFileSync("templates/partials/tutorial-tagged-project.tmpl.html", "utf-8"));
    const taggedProjectTemplate = handlebars.compile(handlebars.partials["template-tagged-project"]);

    return function (opts) {
        return function (files, metalsmith, done) {
            const langs = ['en', 'ru', 'ja', 'zh'];

            const tutorialsIndex = {}; // index object of tutorials
            const projectDemosIndex = {}; // index object of project demos

            for (let i = 0; i < langs.length; i++) {
                const lang = langs[i];
                tutorialsIndex[lang] = {};
                // get all tutorials and insert into index
                for (const filepath in files) {
                    if (filepath.indexOf('.DS_Store') >= 0) continue;

                    const file = files[filepath];

                    if (filepath.startsWith(path.join(lang, dir))) {
                        // skip main tutorials index page
                        if (filepath.endsWith(path.join('tutorials', 'index.html'))) continue;

                        // separate tags
                        const tags = file.tags || '';
                        file.tags = tags.split(',').filter(function (tag) {
                            return !!tag;
                        }).map((tag) => {
                            return tag.trim();
                        }) || [];

                        const url = filepath.split(path.sep).join('/');

                        tutorialsIndex[lang][filepath] = {
                            title: file.title,
                            url: url.replace(lang, '').replace('index.html', ''),
                            tags: file.tags,
                            thumbUrl: file.thumb
                        };

                        // add unique tag for tutorials
                        tutorialsIndex[lang][filepath].tags.push('_tutorial');
                    }
                }

                projectDemosIndex[lang] = {};
            }

            getTags(function (tags) {
                getTaggedProjects(function (tagged) {
                    // add tagged projects into main index
                    for (const key in tagged) {
                        // Add the demo tag so we can tell later what is a demo
                        // and what is a full tutorial
                        tagged[key].tags.push('_demo');

                        for (let i = 0; i < langs.length; i++) {
                            const lang = langs[i];
                            const project = tagged[key];

                            // insert tagged project into index
                            projectDemosIndex[lang][key] = project;

                            // create a new output file for each tagged project
                            const file = {
                                title: project.title,
                                template: 'tutorial-page.tmpl.html',
                                tags: project.tags,
                                mode: '0644',
                                path: lang + project.url,
                                locale: lang,
                                en: true,
                                description: project.description,
                                contents: Buffer.from(taggedProjectTemplate(project))
                            };

                            files[file.path + '/index.html'] = file;
                        }
                    }

                    const context = {
                        pages: {},
                        tags: tags
                    };

                    for (const lang in tutorialsIndex) {
                        context.pages[lang] = {
                            tutorials: [],
                            projectDemos: []
                        };
                        // Add all the tutorial pages to the language
                        for (const key in tutorialsIndex[lang]) {
                            context.pages[lang].tutorials.push(tutorialsIndex[lang][key]);
                        }

                        // Add all the project demos to the language
                        for (const key in projectDemosIndex[lang]) {
                            context.pages[lang].projectDemos.push(projectDemosIndex[lang][key]);
                        }

                        const comparePages = (a, b) => {
                            if (a.title && b.title) {
                                const _a = a.title.toLowerCase();
                                const _b = b.title.toLowerCase();
                                if (_a < _b) return -1;
                                if (_a > _b) return 1;
                            }
                            return 0;
                        };

                        context.pages[lang].tutorials.sort(comparePages);
                        context.pages[lang].projectDemos.sort(comparePages);
                    }

                    // tutorial main page filters
                    // render a new template using the template's template(!) and the content from the file tree
                    const contentsTemplatePartialName = 'template-tutorial-contents';
                    handlebars.registerPartial(contentsTemplatePartialName, fs.readFileSync("templates/partials/tutorial-contents.tmpl.html", "utf-8"));
                    let contentsTemplate = handlebars.compile(handlebars.partials[contentsTemplatePartialName]);
                    let contents = contentsTemplate(context);

                    // registry the generated contents template as the partial used in the tutorial pages
                    handlebars.registerPartial('tutorials-navigation', contents);

                    // render a new template used to template the tag selection on a tutorial page
                    const contentsLinksTemplatePartialName = 'template-tutorial-contents-links';
                    handlebars.registerPartial(contentsLinksTemplatePartialName, fs.readFileSync("templates/partials/tutorial-contents-links.tmpl.html", "utf-8"));
                    contentsTemplate = handlebars.compile(handlebars.partials[contentsLinksTemplatePartialName]);
                    contents = contentsTemplate(context);
                    handlebars.registerPartial('tutorials-navigation-links', contents);

                    // tutorial navigation
                    // render a new template using the template's template(!) and the content from the file tree
                    const listTemplatePartialName = 'template-tutorial-list';
                    handlebars.registerPartial(listTemplatePartialName, fs.readFileSync("templates/partials/tutorial-list.tmpl.html", "utf-8"));
                    const listTemplate = handlebars.compile(handlebars.partials[listTemplatePartialName]);
                    contents = listTemplate(context);

                    // generate pages for tagged projects

                    // registry the generated list template as the partial used in the tutorial pages
                    handlebars.registerPartial('tutorials-list', contents);

                    done();
                });
            });
        };
    };
};
<|endoftext|>


<commit_before>"use strict"

const request = require("request");
const {EventEmitter} = require("events");

class Twitter {
	constructor(auth) {
		this.version = require("../package.json").version;
    this.events = new EventEmitter();
    this._backoff = {};
		this._endpoint = "https://api.twitter.com/1.1/";
    this._stream = "https://stream.twitter.com/1.1/statuses/filter.json";
    this._user = "https://userstream.twitter.com/1.1/user.json";
		this._request_options = {
			oauth: {
        consumer_key: auth.consumer_key,
        consumer_secret: auth.consumer_secret,
        token: auth.token,
        token_secret: auth.token_secret
      },
			headers: {
				Accept: "*/*",
				Connection: "close",
				"User-Agent": "twitter-promise-" + this.version,
			}
		};
		this._request = request.defaults(this._request_options);
	}

  // check back off satus for a specific endpoint.
	_shouldBackoff(opts) {
    if (!this._backoff.hasOwnProperty(opts.path)) return false;
    const _stop = this._backoff[opts.path] > Date.now();
    if (!_stop) delete this._backoff[opts.path];
    return _stop;
	}

  // helper function to set back off status for an end point based on the api response from the headers.
	_checkBackoff(path, headers) {
    let api = {
      limit: headers["x-rate-limit-limit"] || 15,
      remain: headers["x-rate-limit-remaining"] || 1,
      reset: headers["x-rate-limit-reset"] || Date.now(),
    };

    return new Promise((resolve, reject) => {
      if (headers.status !== "200 OK") {
        reject({
          type: "HTTP_STATUS",
          headers,
          path,
          api,
        });
      }

      if (!api.remain) {
        this._backoff[path] = api.reset;
        reject({
          type: "RATE_LIMIT",
          headers,
          path,
          api
        });
      }
      else {
        resolve({
          type: "SUCCESS",
          headers,
          path,
          api
        });
      }
    })
  }

  // return a formatted endpoint string
	endpoint(path, stream=false) {
    const _endpoint = !stream ? this._endpoint : this._stream;
		return `${_endpoint}${path}.json`;
	}

  // make a proper request to twitter using the this._request local
  // returns a promise
  // request is used as a helper function to get and post
	request(opts) {
    opts.params = opts.params || {};

		// build request options
		const _options = {
			method: opts.method ? opts.method.toLowerCase() : "get",
			uri: this.endpoint(opts.path)
		}

   if (_options.method === "get") {
     _options.qs = opts.params;
   }
   else {
     _options.form = opts.params;
   }

    // return new promise based on back off api status
    return new Promise((resolve,reject) => {
      // make the request
      if (this._shouldBackoff(opts.path)) {
        reject({msg: `${opts.path} is locked right now. Check back at: ${this._backoff[opts.path]}`, api: {}});
      }

      // make request to twitter
      this._request(_options, (err, res, data) => {
        if (err) reject(err);

        data = JSON.parse(data);
        // check backoff status of path and resolve or reject based on the end point api status
        this._checkBackoff(opts.path, res.headers)
          .then(msg => {
            resolve({ api:msg.api, data });
          })
          .catch(err => {
            reject({err,res,data});
          });
      });

    });
	}

  // makes a get request
	get(opts) {
		opts.method = "get";
		return this.request(opts);
	}

  // makes a post request
	post(opts) {
    opts.method = "post";
		return this.request(opts);
	}

  // open a twitter stream based on a passed in track parameter
  // uses the node event emitter to send events to itself that can be watched for.
  // this is useful when extending twitter into other components.
  stream(track) {
    const _opts = {
      method: "post",
      url: this._stream,
      qs: {
        track,
        language: "en",
        stall_warnings: true
      }
    };
    // create initial request with auth & options
    const _request = this._request(_opts);
    _request
      .on("response", response => {
        let buffer = "";

        if(response.statusCode !== 200) {
          this.events.emit("twitter:stream:error", { type: "STATUS", err: response.statusCode });
          this.events.emit("twitter:stream:abort", { type: "ABORT", response });
          _request.abort(); // abort reqeust
        }

        response.on("data", data => {
          let index, json;
          buffer += data.toString("utf8");

          while ((index = buffer.indexOf("\r\n")) > -1) {
            json = buffer.slice(0, index);
            buffer = buffer.slice(index + 2);
            if (json.length) {
              try {
                json = JSON.parse(json);
                this.events.emit("twitter:stream:success", json);
              }
              catch(err) {
                this.events.emit("twitter:stream:error", {type: "PARSE", err})
              }
            }
          }
        })
        // response error
        .on("error", err => {
          this.events.emit("twitter:stream:error", {type: "RESPONSE", err});
        })
        // response end
        .on("end", () => {
          this.events.emit("twitter:stream:end");
        });
      })
      // request error
      .on("error", err => {
        this.events.emit("twitter:stream:error", {type: "REQUEST", err})
      });
    _request.end();
  }

  userStream(track) {
    const _opts = {
      method: "post",
      url: this._user,
      qs: {
        replies: true,
        with: false,
        stall_warnings: true
      }
    };
    // create initial request with auth & options
    const _request = this._request(_opts);
    _request
      .on("response", response => {
        let buffer = "";

        if(response.statusCode !== 200) {
          this.events.emit("twitter:stream:user:error", { type: "STATUS", err: response.statusCode });
          this.events.emit("twitter:stream:user:abort", { type: "ABORT", response });
          _request.abort(); // abort reqeust
        }

        response.on("data", data => {
          let index, json;
          buffer += data.toString("utf8");

          while ((index = buffer.indexOf("\r\n")) > -1) {
            json = buffer.slice(0, index);
            buffer = buffer.slice(index + 2);
            if (json.length) {
              try {
                json = JSON.parse(json);
                this.events.emit("twitter:stream:user:success", json);
              }
              catch(err) {
                this.events.emit("twitter:stream:user:error", {type: "PARSE", err})
              }
            }
          }
        })
        // response error
        .on("error", err => {
          this.events.emit("twitter:stream:error", {type: "RESPONSE", err});
        })
        // response end
        .on("end", () => {
          this.events.emit("twitter:stream:end");
        });
      })
      // request error
      .on("error", err => {
        this.events.emit("twitter:stream:user:error", {type: "REQUEST", err})
      });
    _request.end();
  }

}

module.exports = Twitter;
<commit_msg>update error reporting in stream<commit_after>"use strict"

const request = require("request");
const {EventEmitter} = require("events");

class Twitter {
	constructor(auth) {
		this.version = require("../package.json").version;
    this.events = new EventEmitter();
    this._backoff = {};
		this._endpoint = "https://api.twitter.com/1.1/";
    this._stream = "https://stream.twitter.com/1.1/statuses/filter.json";
    this._user = "https://userstream.twitter.com/1.1/user.json";
		this._request_options = {
			oauth: {
        consumer_key: auth.consumer_key,
        consumer_secret: auth.consumer_secret,
        token: auth.token,
        token_secret: auth.token_secret
      },
			headers: {
				Accept: "*/*",
				Connection: "close",
				"User-Agent": "twitter-promise-" + this.version,
			}
		};
		this._request = request.defaults(this._request_options);
	}

  // check back off satus for a specific endpoint.
	_shouldBackoff(opts) {
    if (!this._backoff.hasOwnProperty(opts.path)) return false;
    const _stop = this._backoff[opts.path] > Date.now();
    if (!_stop) delete this._backoff[opts.path];
    return _stop;
	}

  // helper function to set back off status for an end point based on the api response from the headers.
	_checkBackoff(path, headers) {
    let api = {
      limit: headers["x-rate-limit-limit"] || 15,
      remain: headers["x-rate-limit-remaining"] || 1,
      reset: headers["x-rate-limit-reset"] || Date.now(),
    };

    return new Promise((resolve, reject) => {
      if (headers.status !== "200 OK") {
        reject({
          type: "HTTP_STATUS",
          headers,
          path,
          api,
        });
      }

      if (!api.remain) {
        this._backoff[path] = api.reset;
        reject({
          type: "RATE_LIMIT",
          headers,
          path,
          api
        });
      }
      else {
        resolve({
          type: "SUCCESS",
          headers,
          path,
          api
        });
      }
    })
  }

  // return a formatted endpoint string
	endpoint(path, stream=false) {
    const _endpoint = !stream ? this._endpoint : this._stream;
		return `${_endpoint}${path}.json`;
	}

  // make a proper request to twitter using the this._request local
  // returns a promise
  // request is used as a helper function to get and post
	request(opts) {
    opts.params = opts.params || {};

		// build request options
		const _options = {
			method: opts.method ? opts.method.toLowerCase() : "get",
			uri: this.endpoint(opts.path)
		}

   if (_options.method === "get") {
     _options.qs = opts.params;
   }
   else {
     _options.form = opts.params;
   }

    // return new promise based on back off api status
    return new Promise((resolve,reject) => {
      // make the request
      if (this._shouldBackoff(opts.path)) {
        reject({msg: `${opts.path} is locked right now. Check back at: ${this._backoff[opts.path]}`, api: {}});
      }

      // make request to twitter
      this._request(_options, (err, res, data) => {
        if (err) reject(err);

        data = JSON.parse(data);
        // check backoff status of path and resolve or reject based on the end point api status
        this._checkBackoff(opts.path, res.headers)
          .then(msg => {
            resolve({ api:msg.api, data });
          })
          .catch(err => {
            reject({err,res,data});
          });
      });

    });
	}

  // makes a get request
	get(opts) {
		opts.method = "get";
		return this.request(opts);
	}

  // makes a post request
	post(opts) {
    opts.method = "post";
		return this.request(opts);
	}

  // open a twitter stream based on a passed in track parameter
  // uses the node event emitter to send events to itself that can be watched for.
  // this is useful when extending twitter into other components.
  stream(track) {
    const _opts = {
      method: "post",
      url: this._stream,
      qs: {
        track,
        language: "en",
        stall_warnings: true
      }
    };
    // create initial request with auth & options
    const _request = this._request(_opts);
    _request
      .on("response", response => {
        let buffer = "";

        if(response.statusCode !== 200) {
          this.events.emit("twitter:stream:error", { type: "STATUS", err: response });
          this.events.emit("twitter:stream:abort", { type: "ABORT", response });
          _request.abort(); // abort reqeust
        }

        response.on("data", data => {
          let index, json;
          buffer += data.toString("utf8");

          while ((index = buffer.indexOf("\r\n")) > -1) {
            json = buffer.slice(0, index);
            buffer = buffer.slice(index + 2);
            if (json.length) {
              try {
                json = JSON.parse(json);
                this.events.emit("twitter:stream:success", json);
              }
              catch(err) {
                this.events.emit("twitter:stream:error", {type: "PARSE", err})
              }
            }
          }
        })
        // response error
        .on("error", err => {
          this.events.emit("twitter:stream:error", {type: "RESPONSE", err});
        })
        // response end
        .on("end", () => {
          this.events.emit("twitter:stream:end");
        });
      })
      // request error
      .on("error", err => {
        this.events.emit("twitter:stream:error", {type: "REQUEST", err})
      });
    _request.end();
  }

  userStream(track) {
    const _opts = {
      method: "post",
      url: this._user,
      qs: {
        replies: true,
        with: false,
        stall_warnings: true
      }
    };
    // create initial request with auth & options
    const _request = this._request(_opts);
    _request
      .on("response", response => {
        let buffer = "";

        if(response.statusCode !== 200) {
          this.events.emit("twitter:stream:user:error", { type: "STATUS", err: response.statusCode });
          this.events.emit("twitter:stream:user:abort", { type: "ABORT", response });
          _request.abort(); // abort reqeust
        }

        response.on("data", data => {
          let index, json;
          buffer += data.toString("utf8");

          while ((index = buffer.indexOf("\r\n")) > -1) {
            json = buffer.slice(0, index);
            buffer = buffer.slice(index + 2);
            if (json.length) {
              try {
                json = JSON.parse(json);
                this.events.emit("twitter:stream:user:success", json);
              }
              catch(err) {
                this.events.emit("twitter:stream:user:error", {type: "PARSE", err})
              }
            }
          }
        })
        // response error
        .on("error", err => {
          this.events.emit("twitter:stream:error", {type: "RESPONSE", err});
        })
        // response end
        .on("end", () => {
          this.events.emit("twitter:stream:end");
        });
      })
      // request error
      .on("error", err => {
        this.events.emit("twitter:stream:user:error", {type: "REQUEST", err})
      });
    _request.end();
  }

}

module.exports = Twitter;
<|endoftext|>


<commit_before>'use strict';
const fs = require('fs');
const path = require('path');
const async = require('async');
const jsdiff = require('diff');
const _ = require('lodash');
const typedError = require('error/typed');
const binaryDiff = require('./binary-diff');

const AbortedError = typedError({
  type: 'AbortedError',
  message: 'Process aborted by user'
});

const ConflictError = typedError({
  type: 'ConflicterConflict',
  message: 'Process aborted by conflict'
});

/**
 * The Conflicter is a module that can be used to detect conflict between files. Each
 * Generator file system helpers pass files through this module to make sure they don't
 * break a user file.
 *
 * When a potential conflict is detected, we prompt the user and ask them for
 * confirmation before proceeding with the actual write.
 *
 * @constructor
 * @property {Boolean} force - same as the constructor argument
 *
 * @param  {TerminalAdapter} adapter - The generator adapter
 * @param  {Boolean} force - When set to true, we won't check for conflict. (the
 *                           conflicter become a passthrough)
 * @param  {Boolean} bail - When set to true, we will abort on first conflict. (used for
 *                           testing reproducibility)
 */
class Conflicter {
  constructor(adapter, force, options = {}) {
    if (typeof options === 'boolean') {
      this.bail = options;
    } else {
      this.bail = options.bail;
      this.ignoreWhitespace = options.ignoreWhitespace;
      this.skipRegenerate = options.skipRegenerate;
      this.dryRun = options.dryRun;
    }

    this.force = force === true;
    this.adapter = adapter;
    this.conflicts = [];

    this.diffOptions = options.diffOptions;

    if (this.bail) {
      // Set ignoreWhitespace as true by default for bail.
      // Probably just testing, so don't override.
      this.ignoreWhitespace = true;
      this.skipRegenerate = true;
    }

    if (this.dryRun) {
      // Ignore whitespace changes with "ignoreWhitespace === true" option
      this.skipRegenerate = true;
    }
  }

  /**
   * Add a file to conflicter queue
   *
   * @param {String} filepath - File destination path
   * @param {String} contents - File new contents
   * @param {Function} callback - callback to be called once we know if the user want to
   *                              proceed or not.
   */
  checkForCollision(filepath, contents, callback) {
    if (typeof contents === 'function') {
      const status = filepath.conflicter;
      callback = contents;
      contents = filepath.contents;
      filepath = filepath.path;
      if (status) {
        const log = this.adapter.log[status];
        if (log) {
          const rfilepath = path.relative(process.cwd(), filepath);
          log(rfilepath);
        }

        callback(null, status);
        return;
      }
    }

    this.conflicts.push({
      file: {
        path: path.resolve(filepath),
        contents
      },
      callback
    });
  }

  /**
   * Process the _potential conflict_ queue and ask the user to resolve conflict when they
   * occur
   *
   * The user is presented with the following options:
   *
   *   - `Y` Yes, overwrite
   *   - `n` No, do not overwrite
   *   - `a` All, overwrite this and all others
   *   - `x` Exit, abort
   *   - `d` Diff, show the differences between the old and the new
   *   - `h` Help, show this help
   *
   * @param  {Function} cb Callback once every conflict are resolved. (note that each
   *                       file can specify it's own callback. See `#checkForCollision()`)
   */
  resolve(cb) {
    cb = cb || (() => {});

    const resolveConflicts = conflict => {
      return next => {
        if (!conflict) {
          next();
          return;
        }

        this.collision(conflict.file, status => {
          // Remove the resolved conflict from the queue
          _.pull(this.conflicts, conflict);
          conflict.callback(null, status);
          next();
        });
      };
    };

    async.series(this.conflicts.map(resolveConflicts), cb.bind(this));
  }

  /**
   * Print the file differences to console
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   */
  _printDiff(file) {
    if (file.binary === undefined) {
      file.binary = binaryDiff.isBinary(file.path, file.contents);
    }

    if (file.binary) {
      this.adapter.log.writeln(binaryDiff.diff(file.path, file.contents));
    } else {
      const existing = fs.readFileSync(file.path);
      this.adapter.diff(
        existing.toString(),
        (file.contents || '').toString(),
        file.changes
      );
    }
  }

  /**
   * Detect conflicts between file contents at `filepath` with the `contents` passed to the
   * function
   *
   * If `filepath` points to a folder, we'll always return true.
   *
   * Based on detect-conflict module
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   * @return {Boolean} `true` if there's a conflict, `false` otherwise.
   */
  _detectConflict(file) {
    let contents = file.contents;
    const filepath = path.resolve(file.path);

    // If file path point to a directory, then it's not safe to write
    if (fs.statSync(filepath).isDirectory()) return true;

    if (file.binary === undefined) {
      file.binary = binaryDiff.isBinary(file.path, file.contents);
    }

    const actual = fs.readFileSync(path.resolve(filepath));

    if (!(contents instanceof Buffer)) {
      contents = Buffer.from(contents || '', 'utf8');
    }

    if (file.binary) {
      return actual.toString('hex') !== contents.toString('hex');
    }

    if (this.ignoreWhitespace) {
      file.changes = jsdiff.diffWords(
        actual.toString(),
        contents.toString(),
        this.diffOptions
      );
    } else {
      file.changes = jsdiff.diffLines(
        actual.toString(),
        contents.toString(),
        this.diffOptions
      );
    }

    const changes = file.changes;
    return changes.length > 1 || changes[0].added || changes[0].removed;
  }

  /**
   * Check if a file conflict with the current version on the user disk
   *
   * A basic check is done to see if the file exists, if it does:
   *
   *   1. Read its content from  `fs`
   *   2. Compare it with the provided content
   *   3. If identical, mark it as is and skip the check
   *   4. If diverged, prepare and show up the file collision menu
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   * @param  {Function} cb Callback receiving a status string ('identical', 'create',
   *                       'skip', 'force')
   */
  collision(file, cb) {
    const rfilepath = path.relative(process.cwd(), file.path);

    if (!fs.existsSync(file.path)) {
      this.adapter.log.create(rfilepath);
      if (this.bail) {
        this.adapter.log.writeln('Aborting ...');
        throw new ConflictError();
      }

      if (this.dryRun) {
        cb('skip');
        return;
      }

      cb('create');
      return;
    }

    if (this.force) {
      this.adapter.log.force(rfilepath);
      cb('force');
      return;
    }

    if (this._detectConflict(file)) {
      this.adapter.log.conflict(rfilepath);
      if (this.bail) {
        this._printDiff(file);
        this.adapter.log.writeln('Aborting ...');
        throw new ConflictError();
      }

      if (this.dryRun) {
        this._printDiff(file);
        cb('skip');
        return;
      }

      this._ask(file, cb, 1);
    } else {
      this.adapter.log.identical(rfilepath);
      if (this.skipRegenerate) {
        cb('skip');
        return;
      }

      cb('identical');
    }
  }

  /**
   * Actual prompting logic
   * @private
   * @param {Object} file vinyl file object
   * @param {Function} cb callback receiving the next action
   * @param {Number} counter prompts
   */
  _ask(file, cb, counter) {
    const rfilepath = path.relative(process.cwd(), file.path);
    const prompt = {
      name: 'action',
      type: 'expand',
      message: `Overwrite ${rfilepath}?`,
      choices: [
        {
          key: 'y',
          name: 'overwrite',
          value: 'write'
        },
        {
          key: 'n',
          name: 'do not overwrite',
          value: 'skip'
        },
        {
          key: 'a',
          name: 'overwrite this and all others',
          value: 'force'
        },
        {
          key: 'x',
          name: 'abort',
          value: 'abort'
        }
      ]
    };

    // Only offer diff option for files
    if (fs.statSync(file.path).isFile()) {
      prompt.choices.push({
        key: 'd',
        name: 'show the differences between the old and the new',
        value: 'diff'
      });
    }

    this.adapter.prompt([prompt]).then(result => {
      if (result.action === 'abort') {
        this.adapter.log.writeln('Aborting ...');
        throw new AbortedError();
      }

      if (result.action === 'diff') {
        this._printDiff(file);

        counter++;
        if (counter === 5) {
          throw new Error(`Recursive error ${prompt.message}`);
        }

        return this._ask(file, cb, counter);
      }

      if (result.action === 'force') {
        this.force = true;
      }

      if (result.action === 'write') {
        result.action = 'force';
      }

      this.adapter.log[result.action || 'force'](rfilepath);
      return cb(result.action);
    });
  }
}

module.exports = Conflicter;
<commit_msg>Fix log call (#1211)<commit_after>'use strict';
const fs = require('fs');
const path = require('path');
const async = require('async');
const jsdiff = require('diff');
const _ = require('lodash');
const typedError = require('error/typed');
const binaryDiff = require('./binary-diff');

const AbortedError = typedError({
  type: 'AbortedError',
  message: 'Process aborted by user'
});

const ConflictError = typedError({
  type: 'ConflicterConflict',
  message: 'Process aborted by conflict'
});

/**
 * The Conflicter is a module that can be used to detect conflict between files. Each
 * Generator file system helpers pass files through this module to make sure they don't
 * break a user file.
 *
 * When a potential conflict is detected, we prompt the user and ask them for
 * confirmation before proceeding with the actual write.
 *
 * @constructor
 * @property {Boolean} force - same as the constructor argument
 *
 * @param  {TerminalAdapter} adapter - The generator adapter
 * @param  {Boolean} force - When set to true, we won't check for conflict. (the
 *                           conflicter become a passthrough)
 * @param  {Boolean} bail - When set to true, we will abort on first conflict. (used for
 *                           testing reproducibility)
 */
class Conflicter {
  constructor(adapter, force, options = {}) {
    if (typeof options === 'boolean') {
      this.bail = options;
    } else {
      this.bail = options.bail;
      this.ignoreWhitespace = options.ignoreWhitespace;
      this.skipRegenerate = options.skipRegenerate;
      this.dryRun = options.dryRun;
    }

    this.force = force === true;
    this.adapter = adapter;
    this.conflicts = [];

    this.diffOptions = options.diffOptions;

    if (this.bail) {
      // Set ignoreWhitespace as true by default for bail.
      // Probably just testing, so don't override.
      this.ignoreWhitespace = true;
      this.skipRegenerate = true;
    }

    if (this.dryRun) {
      // Ignore whitespace changes with "ignoreWhitespace === true" option
      this.skipRegenerate = true;
    }
  }

  /**
   * Add a file to conflicter queue
   *
   * @param {String} filepath - File destination path
   * @param {String} contents - File new contents
   * @param {Function} callback - callback to be called once we know if the user want to
   *                              proceed or not.
   */
  checkForCollision(filepath, contents, callback) {
    if (typeof contents === 'function') {
      const status = filepath.conflicter;
      callback = contents;
      contents = filepath.contents;
      filepath = filepath.path;
      if (status) {
        const log = this.adapter.log[status];
        if (log) {
          const rfilepath = path.relative(process.cwd(), filepath);
          log.call(this.adapter.log, rfilepath);
        }

        callback(null, status);
        return;
      }
    }

    this.conflicts.push({
      file: {
        path: path.resolve(filepath),
        contents
      },
      callback
    });
  }

  /**
   * Process the _potential conflict_ queue and ask the user to resolve conflict when they
   * occur
   *
   * The user is presented with the following options:
   *
   *   - `Y` Yes, overwrite
   *   - `n` No, do not overwrite
   *   - `a` All, overwrite this and all others
   *   - `x` Exit, abort
   *   - `d` Diff, show the differences between the old and the new
   *   - `h` Help, show this help
   *
   * @param  {Function} cb Callback once every conflict are resolved. (note that each
   *                       file can specify it's own callback. See `#checkForCollision()`)
   */
  resolve(cb) {
    cb = cb || (() => {});

    const resolveConflicts = conflict => {
      return next => {
        if (!conflict) {
          next();
          return;
        }

        this.collision(conflict.file, status => {
          // Remove the resolved conflict from the queue
          _.pull(this.conflicts, conflict);
          conflict.callback(null, status);
          next();
        });
      };
    };

    async.series(this.conflicts.map(resolveConflicts), cb.bind(this));
  }

  /**
   * Print the file differences to console
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   */
  _printDiff(file) {
    if (file.binary === undefined) {
      file.binary = binaryDiff.isBinary(file.path, file.contents);
    }

    if (file.binary) {
      this.adapter.log.writeln(binaryDiff.diff(file.path, file.contents));
    } else {
      const existing = fs.readFileSync(file.path);
      this.adapter.diff(
        existing.toString(),
        (file.contents || '').toString(),
        file.changes
      );
    }
  }

  /**
   * Detect conflicts between file contents at `filepath` with the `contents` passed to the
   * function
   *
   * If `filepath` points to a folder, we'll always return true.
   *
   * Based on detect-conflict module
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   * @return {Boolean} `true` if there's a conflict, `false` otherwise.
   */
  _detectConflict(file) {
    let contents = file.contents;
    const filepath = path.resolve(file.path);

    // If file path point to a directory, then it's not safe to write
    if (fs.statSync(filepath).isDirectory()) return true;

    if (file.binary === undefined) {
      file.binary = binaryDiff.isBinary(file.path, file.contents);
    }

    const actual = fs.readFileSync(path.resolve(filepath));

    if (!(contents instanceof Buffer)) {
      contents = Buffer.from(contents || '', 'utf8');
    }

    if (file.binary) {
      return actual.toString('hex') !== contents.toString('hex');
    }

    if (this.ignoreWhitespace) {
      file.changes = jsdiff.diffWords(
        actual.toString(),
        contents.toString(),
        this.diffOptions
      );
    } else {
      file.changes = jsdiff.diffLines(
        actual.toString(),
        contents.toString(),
        this.diffOptions
      );
    }

    const changes = file.changes;
    return changes.length > 1 || changes[0].added || changes[0].removed;
  }

  /**
   * Check if a file conflict with the current version on the user disk
   *
   * A basic check is done to see if the file exists, if it does:
   *
   *   1. Read its content from  `fs`
   *   2. Compare it with the provided content
   *   3. If identical, mark it as is and skip the check
   *   4. If diverged, prepare and show up the file collision menu
   *
   * @param  {Object}   file File object respecting this interface: { path, contents }
   * @param  {Function} cb Callback receiving a status string ('identical', 'create',
   *                       'skip', 'force')
   */
  collision(file, cb) {
    const rfilepath = path.relative(process.cwd(), file.path);

    if (!fs.existsSync(file.path)) {
      this.adapter.log.create(rfilepath);
      if (this.bail) {
        this.adapter.log.writeln('Aborting ...');
        throw new ConflictError();
      }

      if (this.dryRun) {
        cb('skip');
        return;
      }

      cb('create');
      return;
    }

    if (this.force) {
      this.adapter.log.force(rfilepath);
      cb('force');
      return;
    }

    if (this._detectConflict(file)) {
      this.adapter.log.conflict(rfilepath);
      if (this.bail) {
        this._printDiff(file);
        this.adapter.log.writeln('Aborting ...');
        throw new ConflictError();
      }

      if (this.dryRun) {
        this._printDiff(file);
        cb('skip');
        return;
      }

      this._ask(file, cb, 1);
    } else {
      this.adapter.log.identical(rfilepath);
      if (this.skipRegenerate) {
        cb('skip');
        return;
      }

      cb('identical');
    }
  }

  /**
   * Actual prompting logic
   * @private
   * @param {Object} file vinyl file object
   * @param {Function} cb callback receiving the next action
   * @param {Number} counter prompts
   */
  _ask(file, cb, counter) {
    const rfilepath = path.relative(process.cwd(), file.path);
    const prompt = {
      name: 'action',
      type: 'expand',
      message: `Overwrite ${rfilepath}?`,
      choices: [
        {
          key: 'y',
          name: 'overwrite',
          value: 'write'
        },
        {
          key: 'n',
          name: 'do not overwrite',
          value: 'skip'
        },
        {
          key: 'a',
          name: 'overwrite this and all others',
          value: 'force'
        },
        {
          key: 'x',
          name: 'abort',
          value: 'abort'
        }
      ]
    };

    // Only offer diff option for files
    if (fs.statSync(file.path).isFile()) {
      prompt.choices.push({
        key: 'd',
        name: 'show the differences between the old and the new',
        value: 'diff'
      });
    }

    this.adapter.prompt([prompt]).then(result => {
      if (result.action === 'abort') {
        this.adapter.log.writeln('Aborting ...');
        throw new AbortedError();
      }

      if (result.action === 'diff') {
        this._printDiff(file);

        counter++;
        if (counter === 5) {
          throw new Error(`Recursive error ${prompt.message}`);
        }

        return this._ask(file, cb, counter);
      }

      if (result.action === 'force') {
        this.force = true;
      }

      if (result.action === 'write') {
        result.action = 'force';
      }

      this.adapter.log[result.action || 'force'](rfilepath);
      return cb(result.action);
    });
  }
}

module.exports = Conflicter;
<|endoftext|>