<commit_before>var helpers = require('./helpers'),
  filters = require('./filters'),
  _ = require('underscore');

var variableRegexp  = /^\{\{.*?\}\}$/,
  logicRegexp   = /^\{%[^\r]*?%\}$/,
  commentRegexp   = /^\{#[^\r]*?#\}$/,

  TEMPLATE = exports.TEMPLATE = 0,
  LOGIC_TOKEN = 1,
  VAR_TOKEN   = 2;

exports.TOKEN_TYPES = {
  TEMPLATE: TEMPLATE,
  LOGIC: LOGIC_TOKEN,
  VAR: VAR_TOKEN
};

function getMethod(input) {
  return input.match(/^[\w\.]+/)[0];
}

function doubleEscape(input) {
  return input.replace(/\\/g, '\\\\');
}

function getArgs(input) {
  return doubleEscape(input.replace(/^[\w\.]+\(|\)$/g, ''));
}

function getContextVar(varName, context) {
  var a = varName.split(".");
  while (a.length) {
    context = context[a.splice(0, 1)[0]];
  }
  return context;
}

function getTokenArgs(token, parts) {
  parts = _.map(parts, doubleEscape);

  var i = 0,
    l = parts.length,
    arg,
    ender,
    out = [];

  function concat(from, ending) {
    var end = new RegExp('\\' + ending + '$'),
      i = from,
      out = '';

    while (!(end).test(out) && i < parts.length) {
      out += ' ' + parts[i];
      parts[i] = null;
      i += 1;
    }

    if (!end.test(out)) {
      throw new Error('Malformed arguments ' + out + ' sent to tag.');
    }

    return out.replace(/^ /, '');
  }

  for (i; i < l; i += 1) {
    arg = parts[i];
    if (arg === null || (/^\s+$/).test(arg)) {
      continue;
    }

    if (
      ((/^\"/).test(arg) && !(/\"[\]\}]?$/).test(arg))
        || ((/^\'/).test(arg) && !(/\'[\]\}]?$/).test(arg))
        || ((/^\{/).test(arg) && !(/\}$/).test(arg))
        || ((/^\[/).test(arg) && !(/\]$/).test(arg))
    ) {
      switch (arg.substr(0, 1)) {
      case "'":
        ender = "'";
        break;
      case '"':
        ender = '"';
        break;
      case '[':
        ender = ']';
        break;
      case '{':
        ender = '}';
        break;
      }
      out.push(concat(i, ender));
      continue;
    }

    out.push(arg);
  }

  return out;
}

function findSubBlocks(topToken, blocks) {
  _.each(topToken.tokens, function (token, index) {
    if (token.name === 'block') {
      blocks[token.args[0]] = token;
      findSubBlocks(token, blocks);
    }
  });
}

function getParentBlock(token) {
  var block;

  if (token.parentBlock) {
    block = token.parentBlock;
  } else if (token.parent) {
    block = getParentBlock(_.last(token.parent));
  }

  return block;
}

exports.parseVariable = function (token, escape) {
  if (!token) {
    return {
      type: null,
      name: '',
      filters: [],
      escape: escape
    };
  }

  var filters = [],
    parts = token.replace(/^\{\{ *| *\}\}$/g, '').split('|'),
    varname = parts.shift(),
    i = 0,
    l = parts.length,
    args = null,
    filter_name,
    part;

  if ((/\(/).test(varname)) {
    args = getArgs(varname.replace(/^\w+\./, ''));
    varname = getMethod(varname);
  }

  for (i; i < l; i += 1) {
    part = parts[i];
    if (part && ((/^[\w\.]+\(/).test(part) || (/\)$/).test(part)) && !(/^[\w\.]+\([^\)]*\)$/).test(part)) {
      parts[i] += '|' + parts[i + 1];
      parts[i + 1] = false;
    }
  }
  parts = _.without(parts, false);

  i = 0;
  l = parts.length;
  for (i; i < l; i += 1) {
    part = parts[i];
    filter_name = getMethod(part);
    if ((/\(/).test(part)) {
      filters.push({
        name: filter_name,
        args: getArgs(part)
      });
    } else {
      filters.push({ name: filter_name, args: '' });
    }
  }

  return {
    type: VAR_TOKEN,
    name: varname,
    args: args,
    filters: filters,
    escape: escape
  };
};

exports.parse = function (data, tags, autoescape) {
  var rawtokens = data.replace(/(^\s+)|(\s+$)/g, '').split(/(\{%[^\r]*?%\}|\{\{.*?\}\}|\{#[^\r]*?#\})/),
    escape = !!autoescape,
    last_escape = escape,
    stack = [[]],
    index = 0,
    i = 0,
    j = rawtokens.length,
    filters = [],
    filter_name,
    varname,
    token,
    parts,
    part,
    names,
    matches,
    tagname,
    lines = 1,
    curline = 1,
    newlines = null,
    lastToken,
    rawStart = /^\{\% *raw *\%\}/,
    rawEnd = /\{\% *endraw *\%\}$/,
    inRaw = false,
    stripAfter = false,
    stripBefore = false,
    stripStart = false,
    stripEnd = false;

  for (i; i < j; i += 1) {
    token = rawtokens[i];
    curline = lines;
    newlines = token.match(/\n/g);
    stripAfter = false;
    stripBefore = false;
    stripStart = false;
    stripEnd = false;

    if (newlines) {
      lines += newlines.length;
    }

    if (inRaw !== false && !rawEnd.test(token)) {
      inRaw += token;
      continue;
    }

    // Ignore empty strings and comments
    if (token.length === 0 || commentRegexp.test(token)) {
      continue;
    } else if (/^(\s|\n)+$/.test(token)) {
      token = token.replace(/ +/, ' ').replace(/\n+/, '\n');
    } else if (variableRegexp.test(token)) {
      token = exports.parseVariable(token, escape);
    } else if (logicRegexp.test(token)) {
      if (rawEnd.test(token)) {
        // Don't care about the content in a raw tag, so end tag may not start correctly
        token = inRaw + token.replace(rawEnd, '');
        inRaw = false;
        stack[index].push(token);
        continue;
      }

      if (rawStart.test(token)) {
        // Have to check the whole token directly, not just parts, as the tag may not end correctly while in raw
        inRaw = token.replace(rawStart, '');
        continue;
      }

      parts = token.replace(/^\{%\s*|\s*%\}$/g, '').split(' ');
      if (parts[0] === '-') {
        stripBefore = true;
        parts.shift();
      }
      tagname = parts.shift();
      if (_.last(parts) === '-') {
        stripAfter = true;
        parts.pop();
      }

      if (index > 0 && (/^end/).test(tagname)) {
        lastToken = _.last(stack[stack.length - 2]);
        if ('end' + lastToken.name === tagname) {
          if (lastToken.name === 'autoescape') {
            escape = last_escape;
          }
          lastToken.strip.end = stripBefore;
          lastToken.strip.after = stripAfter;
          stack.pop();
          index -= 1;
          continue;
        }

        throw new Error('Expected end tag for "' + lastToken.name + '", but found "' + tagname + '" at line ' + lines + '.');
      }

      if (!tags.hasOwnProperty(tagname)) {
        throw new Error('Unknown logic tag at line ' + lines + ': "' + tagname + '".');
      }

      if (tagname === 'autoescape') {
        last_escape = escape;
        escape = (!parts.length || parts[0] === 'true') ? ((parts.length >= 2) ? parts[1] : true) : false;
      }

      token = {
        type: LOGIC_TOKEN,
        line: curline,
        name: tagname,
        compile: tags[tagname],
        parent: _.uniq(stack[stack.length - 2] || []),
        strip: {
          before: stripBefore,
          after: stripAfter,
          start: false,
          end: false
        }
      };
      token.args = getTokenArgs(token, parts);

      if (tags[tagname].ends) {
        token.strip.after = false;
        token.strip.start = stripAfter;
        stack[index].push(token);
        stack.push(token.tokens = []);
        index += 1;
        continue;
      }
    }

    // Everything else is treated as a string
    stack[index].push(token);
  }

  if (inRaw !== false) {
    throw new Error('Missing expected end tag for "raw" on line ' + curline + '.');
  }

  if (index !== 0) {
    lastToken = _.last(stack[stack.length - 2]);
    throw new Error('Missing end tag for "' + lastToken.name + '" that was opened on line ' + lastToken.line + '.');
  }

  return stack[index];
};

function precompile(indent, context) {
  var filepath,
    extendsHasVar,
    parentTokens,
    preservedTokens = [];

  // Precompile - extract blocks and create hierarchy based on 'extends' tags
  // TODO: make block and extends tags accept context variables

  // Only precompile at the template level
  if (this.type === TEMPLATE) {

    _.each(this.tokens, function (token, index) {

      if (!extendsHasVar) {
        // Load the parent template
        if (token.name === 'extends') {
          filepath = token.args[0];

          if (!helpers.isStringLiteral(filepath)) {

            if (!context) {
              extendsHasVar = true;
              return;
            }
            filepath = "\"" + getContextVar(filepath, context) + "\"";
          }

          if (!helpers.isStringLiteral(filepath) || token.args.length > 1) {
            throw new Error('Extends tag on line ' + token.line + ' accepts exactly one string literal as an argument.');
          }
          if (index > 0) {
            throw new Error('Extends tag must be the first tag in the template, but "extends" found on line ' + token.line + '.');
          }
          token.template = this.compileFile(filepath.replace(/['"]/g, ''), true);
          this.parent = token.template;

          // inherit tokens/blocks from parent.
          this.blocks = _.extend({}, this.parent.blocks, this.blocks);

        } else if (token.name === 'block') { // Make a list of blocks
          var blockname = token.args[0],
            parentBlockIndex;

          if (!helpers.isValidBlockName(blockname) || token.args.length !== 1) {
            throw new Error('Invalid block tag name "' + blockname + '" on line ' + token.line + '.');
          }

          // store blocks as flat reference list on top-level
          // template object
          this.blocks[blockname] = token;

          // child tokens may contain more blocks at this template
          // level - apply to flat this.blocks object
          findSubBlocks(token, this.blocks);

          // search parent list for a matching block, replacing the
          // parent template block tokens with the derived token.
          if (this.parent) {

            // Store parent token object on a derived block
            token.parentBlock = this.parent.blocks[blockname];

            // this will return -1 for a nested block
            parentBlockIndex = _.indexOf(this.parent.tokens,
                this.parent.blocks[blockname]);
            if (parentBlockIndex >= 0) {
              this.parent.tokens[parentBlockIndex] = token;
            }

          }
        } else if (token.type === LOGIC_TOKEN || token.type === VAR_TOKEN) {
          // Preserve any template logic from the extended template.
          preservedTokens.push(token);
        }
        // else, discard any tokens that are not under a LOGIC_TOKEN
        // or VAR_TOKEN (for example, static strings).

      }
    }, this);

    // If extendsHasVar == true, then we know {% extends %} is not using a string literal, thus we can't
    // compile until render is called, so we return false.
    if (extendsHasVar) {
      return false;
    }

    if (this.parent && this.parent.tokens) {
      this.tokens = preservedTokens.concat(this.parent.tokens);
    }
  }
}

exports.compile = function compile(indent, context, template) {
  var code = '',
    wrappedInMethod,
    blockname,
    parentBlock;

  indent = indent || '';

  // Template parameter is optional (not used at the top-level), initialize
  if (this.type === TEMPLATE) {
    template = this;
  }

  // Initialize blocks
  if (!this.blocks) {
    this.blocks = {};
  }

  // Precompile step - process block inheritence into true token hierarchy
  if (precompile.call(this, indent, context) === false) {
    return false;
  }

  // If this is not a template then just iterate through its tokens
  _.each(this.tokens, function (token, index) {
    var name, key, args, prev, next;
    if (typeof token === 'string') {
      prev = this.tokens[index - 1];
      next = this.tokens[index + 1];
      if (prev && prev.strip && prev.strip.after) {
        token = token.replace(/^\s+/, '');
      }
      if (next && next.strip && next.strip.before) {
        token = token.replace(/\s+$/, '');
      }
      code += '_output += "' + doubleEscape(token).replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/"/g, '\\"') + '";\n';
      return code;
    }

    if (typeof token !== 'object') {
      return; // Tokens can be either strings or objects
    }

    if (token.type === VAR_TOKEN) {
      name = token.name.replace(/\W/g, '_');
      key = (helpers.isLiteral(name)) ? '["' + name + '"]' : '.' + name;
      args = (token.args && token.args.length) ? token.args : '';

      code += 'if (typeof _context !== "undefined" && typeof _context' + key + ' === "function") {\n';
      wrappedInMethod = helpers.wrapMethod('', { name: name, args: args }, '_context');
      code += '  _output = (typeof _output === "undefined") ? ' + wrappedInMethod + ': _output + ' + wrappedInMethod + ';\n';
      if (helpers.isValidName(name)) {
        code += '} else if (typeof ' + name + ' === "function") {\n';
        wrappedInMethod = helpers.wrapMethod('', { name: name, args: args });
        code += '  _output = (typeof _output === "undefined") ? ' + wrappedInMethod + ': _output + ' + wrappedInMethod + ';\n';
      }
      code += '} else {\n';
      code += helpers.setVar('__' + name, token);
      code += '  _output = (typeof _output === "undefined") ? __' + name + ': _output + __' + name + ';\n';
      code += '}\n';
    }

    if (token.type !== LOGIC_TOKEN) {
      return; // Tokens can be either VAR_TOKEN or LOGIC_TOKEN
    }

    if (token.name === 'block') {
      blockname = token.args[0];

      // Sanity check - the template should be in the flat block list.
      if (!template.blocks.hasOwnProperty(blockname)) {
        throw new Error('Unrecognized nested block.  Block \"' + blockname +
                '\" at line ' + token.line + ' of \"' + template.id +
                '\" is not in template block list.');
      }

      code += compile.call(template.blocks[token.args[0]], indent + '  ', context, template);
    } else if (token.name === 'parent') {
      parentBlock = getParentBlock(token);
      if (!parentBlock) {
        throw new Error('No parent block found for parent tag at line ' +
                token.line + '.');
      }

      code += compile.call(parentBlock, indent + '  ', context);
    } else if (token.hasOwnProperty("compile")) {
      if (token.strip.start && token.tokens.length && typeof token.tokens[0] === 'string') {
        token.tokens[0] = token.tokens[0].replace(/^\s+/, '');
      }
      if (token.strip.end && token.tokens.length && typeof _.last(token.tokens) === 'string') {
        token.tokens[token.tokens.length - 1] = _.last(token.tokens).replace(/\s+$/, '');
      }
      code += token.compile(indent + '  ', exports);
    } else {
      code += compile.call(token, indent + '  ', context);
    }

  }, this);

  return code;
};

<commit_msg>Removing unnecessary inclusion of VAR_TOKEN from inherited files.<commit_after>var helpers = require('./helpers'),
  filters = require('./filters'),
  _ = require('underscore');

var variableRegexp  = /^\{\{.*?\}\}$/,
  logicRegexp   = /^\{%[^\r]*?%\}$/,
  commentRegexp   = /^\{#[^\r]*?#\}$/,

  TEMPLATE = exports.TEMPLATE = 0,
  LOGIC_TOKEN = 1,
  VAR_TOKEN   = 2;

exports.TOKEN_TYPES = {
  TEMPLATE: TEMPLATE,
  LOGIC: LOGIC_TOKEN,
  VAR: VAR_TOKEN
};

function getMethod(input) {
  return input.match(/^[\w\.]+/)[0];
}

function doubleEscape(input) {
  return input.replace(/\\/g, '\\\\');
}

function getArgs(input) {
  return doubleEscape(input.replace(/^[\w\.]+\(|\)$/g, ''));
}

function getContextVar(varName, context) {
  var a = varName.split(".");
  while (a.length) {
    context = context[a.splice(0, 1)[0]];
  }
  return context;
}

function getTokenArgs(token, parts) {
  parts = _.map(parts, doubleEscape);

  var i = 0,
    l = parts.length,
    arg,
    ender,
    out = [];

  function concat(from, ending) {
    var end = new RegExp('\\' + ending + '$'),
      i = from,
      out = '';

    while (!(end).test(out) && i < parts.length) {
      out += ' ' + parts[i];
      parts[i] = null;
      i += 1;
    }

    if (!end.test(out)) {
      throw new Error('Malformed arguments ' + out + ' sent to tag.');
    }

    return out.replace(/^ /, '');
  }

  for (i; i < l; i += 1) {
    arg = parts[i];
    if (arg === null || (/^\s+$/).test(arg)) {
      continue;
    }

    if (
      ((/^\"/).test(arg) && !(/\"[\]\}]?$/).test(arg))
        || ((/^\'/).test(arg) && !(/\'[\]\}]?$/).test(arg))
        || ((/^\{/).test(arg) && !(/\}$/).test(arg))
        || ((/^\[/).test(arg) && !(/\]$/).test(arg))
    ) {
      switch (arg.substr(0, 1)) {
      case "'":
        ender = "'";
        break;
      case '"':
        ender = '"';
        break;
      case '[':
        ender = ']';
        break;
      case '{':
        ender = '}';
        break;
      }
      out.push(concat(i, ender));
      continue;
    }

    out.push(arg);
  }

  return out;
}

function findSubBlocks(topToken, blocks) {
  _.each(topToken.tokens, function (token, index) {
    if (token.name === 'block') {
      blocks[token.args[0]] = token;
      findSubBlocks(token, blocks);
    }
  });
}

function getParentBlock(token) {
  var block;

  if (token.parentBlock) {
    block = token.parentBlock;
  } else if (token.parent) {
    block = getParentBlock(_.last(token.parent));
  }

  return block;
}

exports.parseVariable = function (token, escape) {
  if (!token) {
    return {
      type: null,
      name: '',
      filters: [],
      escape: escape
    };
  }

  var filters = [],
    parts = token.replace(/^\{\{ *| *\}\}$/g, '').split('|'),
    varname = parts.shift(),
    i = 0,
    l = parts.length,
    args = null,
    filter_name,
    part;

  if ((/\(/).test(varname)) {
    args = getArgs(varname.replace(/^\w+\./, ''));
    varname = getMethod(varname);
  }

  for (i; i < l; i += 1) {
    part = parts[i];
    if (part && ((/^[\w\.]+\(/).test(part) || (/\)$/).test(part)) && !(/^[\w\.]+\([^\)]*\)$/).test(part)) {
      parts[i] += '|' + parts[i + 1];
      parts[i + 1] = false;
    }
  }
  parts = _.without(parts, false);

  i = 0;
  l = parts.length;
  for (i; i < l; i += 1) {
    part = parts[i];
    filter_name = getMethod(part);
    if ((/\(/).test(part)) {
      filters.push({
        name: filter_name,
        args: getArgs(part)
      });
    } else {
      filters.push({ name: filter_name, args: '' });
    }
  }

  return {
    type: VAR_TOKEN,
    name: varname,
    args: args,
    filters: filters,
    escape: escape
  };
};

exports.parse = function (data, tags, autoescape) {
  var rawtokens = data.replace(/(^\s+)|(\s+$)/g, '').split(/(\{%[^\r]*?%\}|\{\{.*?\}\}|\{#[^\r]*?#\})/),
    escape = !!autoescape,
    last_escape = escape,
    stack = [[]],
    index = 0,
    i = 0,
    j = rawtokens.length,
    filters = [],
    filter_name,
    varname,
    token,
    parts,
    part,
    names,
    matches,
    tagname,
    lines = 1,
    curline = 1,
    newlines = null,
    lastToken,
    rawStart = /^\{\% *raw *\%\}/,
    rawEnd = /\{\% *endraw *\%\}$/,
    inRaw = false,
    stripAfter = false,
    stripBefore = false,
    stripStart = false,
    stripEnd = false;

  for (i; i < j; i += 1) {
    token = rawtokens[i];
    curline = lines;
    newlines = token.match(/\n/g);
    stripAfter = false;
    stripBefore = false;
    stripStart = false;
    stripEnd = false;

    if (newlines) {
      lines += newlines.length;
    }

    if (inRaw !== false && !rawEnd.test(token)) {
      inRaw += token;
      continue;
    }

    // Ignore empty strings and comments
    if (token.length === 0 || commentRegexp.test(token)) {
      continue;
    } else if (/^(\s|\n)+$/.test(token)) {
      token = token.replace(/ +/, ' ').replace(/\n+/, '\n');
    } else if (variableRegexp.test(token)) {
      token = exports.parseVariable(token, escape);
    } else if (logicRegexp.test(token)) {
      if (rawEnd.test(token)) {
        // Don't care about the content in a raw tag, so end tag may not start correctly
        token = inRaw + token.replace(rawEnd, '');
        inRaw = false;
        stack[index].push(token);
        continue;
      }

      if (rawStart.test(token)) {
        // Have to check the whole token directly, not just parts, as the tag may not end correctly while in raw
        inRaw = token.replace(rawStart, '');
        continue;
      }

      parts = token.replace(/^\{%\s*|\s*%\}$/g, '').split(' ');
      if (parts[0] === '-') {
        stripBefore = true;
        parts.shift();
      }
      tagname = parts.shift();
      if (_.last(parts) === '-') {
        stripAfter = true;
        parts.pop();
      }

      if (index > 0 && (/^end/).test(tagname)) {
        lastToken = _.last(stack[stack.length - 2]);
        if ('end' + lastToken.name === tagname) {
          if (lastToken.name === 'autoescape') {
            escape = last_escape;
          }
          lastToken.strip.end = stripBefore;
          lastToken.strip.after = stripAfter;
          stack.pop();
          index -= 1;
          continue;
        }

        throw new Error('Expected end tag for "' + lastToken.name + '", but found "' + tagname + '" at line ' + lines + '.');
      }

      if (!tags.hasOwnProperty(tagname)) {
        throw new Error('Unknown logic tag at line ' + lines + ': "' + tagname + '".');
      }

      if (tagname === 'autoescape') {
        last_escape = escape;
        escape = (!parts.length || parts[0] === 'true') ? ((parts.length >= 2) ? parts[1] : true) : false;
      }

      token = {
        type: LOGIC_TOKEN,
        line: curline,
        name: tagname,
        compile: tags[tagname],
        parent: _.uniq(stack[stack.length - 2] || []),
        strip: {
          before: stripBefore,
          after: stripAfter,
          start: false,
          end: false
        }
      };
      token.args = getTokenArgs(token, parts);

      if (tags[tagname].ends) {
        token.strip.after = false;
        token.strip.start = stripAfter;
        stack[index].push(token);
        stack.push(token.tokens = []);
        index += 1;
        continue;
      }
    }

    // Everything else is treated as a string
    stack[index].push(token);
  }

  if (inRaw !== false) {
    throw new Error('Missing expected end tag for "raw" on line ' + curline + '.');
  }

  if (index !== 0) {
    lastToken = _.last(stack[stack.length - 2]);
    throw new Error('Missing end tag for "' + lastToken.name + '" that was opened on line ' + lastToken.line + '.');
  }

  return stack[index];
};

function precompile(indent, context) {
  var filepath,
    extendsHasVar,
    parentTokens,
    preservedTokens = [];

  // Precompile - extract blocks and create hierarchy based on 'extends' tags
  // TODO: make block and extends tags accept context variables

  // Only precompile at the template level
  if (this.type === TEMPLATE) {

    _.each(this.tokens, function (token, index) {

      if (!extendsHasVar) {
        // Load the parent template
        if (token.name === 'extends') {
          filepath = token.args[0];

          if (!helpers.isStringLiteral(filepath)) {

            if (!context) {
              extendsHasVar = true;
              return;
            }
            filepath = "\"" + getContextVar(filepath, context) + "\"";
          }

          if (!helpers.isStringLiteral(filepath) || token.args.length > 1) {
            throw new Error('Extends tag on line ' + token.line + ' accepts exactly one string literal as an argument.');
          }
          if (index > 0) {
            throw new Error('Extends tag must be the first tag in the template, but "extends" found on line ' + token.line + '.');
          }
          token.template = this.compileFile(filepath.replace(/['"]/g, ''), true);
          this.parent = token.template;

          // inherit tokens/blocks from parent.
          this.blocks = _.extend({}, this.parent.blocks, this.blocks);

        } else if (token.name === 'block') { // Make a list of blocks
          var blockname = token.args[0],
            parentBlockIndex;

          if (!helpers.isValidBlockName(blockname) || token.args.length !== 1) {
            throw new Error('Invalid block tag name "' + blockname + '" on line ' + token.line + '.');
          }

          // store blocks as flat reference list on top-level
          // template object
          this.blocks[blockname] = token;

          // child tokens may contain more blocks at this template
          // level - apply to flat this.blocks object
          findSubBlocks(token, this.blocks);

          // search parent list for a matching block, replacing the
          // parent template block tokens with the derived token.
          if (this.parent) {

            // Store parent token object on a derived block
            token.parentBlock = this.parent.blocks[blockname];

            // this will return -1 for a nested block
            parentBlockIndex = _.indexOf(this.parent.tokens,
                this.parent.blocks[blockname]);
            if (parentBlockIndex >= 0) {
              this.parent.tokens[parentBlockIndex] = token;
            }

          }
        } else if (token.type === LOGIC_TOKEN) {
          // Preserve any template logic from the extended template.
          preservedTokens.push(token);
        }
        // else, discard any tokens that are not under a LOGIC_TOKEN
        // or VAR_TOKEN (for example, static strings).

      }
    }, this);

    // If extendsHasVar == true, then we know {% extends %} is not using a string literal, thus we can't
    // compile until render is called, so we return false.
    if (extendsHasVar) {
      return false;
    }

    if (this.parent && this.parent.tokens) {
      this.tokens = preservedTokens.concat(this.parent.tokens);
    }
  }
}

exports.compile = function compile(indent, context, template) {
  var code = '',
    wrappedInMethod,
    blockname,
    parentBlock;

  indent = indent || '';

  // Template parameter is optional (not used at the top-level), initialize
  if (this.type === TEMPLATE) {
    template = this;
  }

  // Initialize blocks
  if (!this.blocks) {
    this.blocks = {};
  }

  // Precompile step - process block inheritence into true token hierarchy
  if (precompile.call(this, indent, context) === false) {
    return false;
  }

  // If this is not a template then just iterate through its tokens
  _.each(this.tokens, function (token, index) {
    var name, key, args, prev, next;
    if (typeof token === 'string') {
      prev = this.tokens[index - 1];
      next = this.tokens[index + 1];
      if (prev && prev.strip && prev.strip.after) {
        token = token.replace(/^\s+/, '');
      }
      if (next && next.strip && next.strip.before) {
        token = token.replace(/\s+$/, '');
      }
      code += '_output += "' + doubleEscape(token).replace(/\n/g, '\\n').replace(/\r/g, '\\r').replace(/"/g, '\\"') + '";\n';
      return code;
    }

    if (typeof token !== 'object') {
      return; // Tokens can be either strings or objects
    }

    if (token.type === VAR_TOKEN) {
      name = token.name.replace(/\W/g, '_');
      key = (helpers.isLiteral(name)) ? '["' + name + '"]' : '.' + name;
      args = (token.args && token.args.length) ? token.args : '';

      code += 'if (typeof _context !== "undefined" && typeof _context' + key + ' === "function") {\n';
      wrappedInMethod = helpers.wrapMethod('', { name: name, args: args }, '_context');
      code += '  _output = (typeof _output === "undefined") ? ' + wrappedInMethod + ': _output + ' + wrappedInMethod + ';\n';
      if (helpers.isValidName(name)) {
        code += '} else if (typeof ' + name + ' === "function") {\n';
        wrappedInMethod = helpers.wrapMethod('', { name: name, args: args });
        code += '  _output = (typeof _output === "undefined") ? ' + wrappedInMethod + ': _output + ' + wrappedInMethod + ';\n';
      }
      code += '} else {\n';
      code += helpers.setVar('__' + name, token);
      code += '  _output = (typeof _output === "undefined") ? __' + name + ': _output + __' + name + ';\n';
      code += '}\n';
    }

    if (token.type !== LOGIC_TOKEN) {
      return; // Tokens can be either VAR_TOKEN or LOGIC_TOKEN
    }

    if (token.name === 'block') {
      blockname = token.args[0];

      // Sanity check - the template should be in the flat block list.
      if (!template.blocks.hasOwnProperty(blockname)) {
        throw new Error('Unrecognized nested block.  Block \"' + blockname +
                '\" at line ' + token.line + ' of \"' + template.id +
                '\" is not in template block list.');
      }

      code += compile.call(template.blocks[token.args[0]], indent + '  ', context, template);
    } else if (token.name === 'parent') {
      parentBlock = getParentBlock(token);
      if (!parentBlock) {
        throw new Error('No parent block found for parent tag at line ' +
                token.line + '.');
      }

      code += compile.call(parentBlock, indent + '  ', context);
    } else if (token.hasOwnProperty("compile")) {
      if (token.strip.start && token.tokens.length && typeof token.tokens[0] === 'string') {
        token.tokens[0] = token.tokens[0].replace(/^\s+/, '');
      }
      if (token.strip.end && token.tokens.length && typeof _.last(token.tokens) === 'string') {
        token.tokens[token.tokens.length - 1] = _.last(token.tokens).replace(/\s+$/, '');
      }
      code += token.compile(indent + '  ', exports);
    } else {
      code += compile.call(token, indent + '  ', context);
    }

  }, this);

  return code;
};

<|endoftext|>


<commit_before>var utils = require('./utils');

/**
 * Main markup syndax parser
 * @param  {string}   data     markup text
 * @param  {Function} callback callback function
 * @return {undefined}         return undefined when error occurs
 */
module.exports = function(data, callback) {

  /**********************************
   * Parse scraping result,         *
   * which is in the format of JSON *
   **********************************/

  try {
    content = JSON.parse(data);
  } catch(e) {
    callback(e);
    return;
  }

  if (!content.query) { callback(new Error('Query Not Found')); return; }

  /**
   * Get JSON data
   */
  var json = content.query.pages;
  var key  = Object.keys(json);

  if (key.indexOf('-1') === 0) {
    callback(new Error('Page Index Not Found'));
    return;
  } else if(!json[key]){
    callback(new Error('Malformed Response Payload'));
    return;
  } else if (json[key].revisions[0]['*'].indexOf('REDIRECT') > -1) {
    callback(new Error(json[key].revisions[0]['*']));
    return;
  }

  /**
   * Get the JSON data that contains infobox section
   */
  var reg    = new RegExp('{{[Ii]nfobox(.|\n)*}}', 'g');
  var text   = reg.exec(json[key].revisions[0]['*']);
  if (!text) { callback(new Error('Infobox Not Found')); return; }
  text = text[0];

  /************************
   * Remove useless marks *
   ************************/

  /*
   * Remove comments
   */
  text = utils.replaceAll('', '', text);
  /*
   * Remove reference
   * TODO: support reference in advanced model
   */
  text = utils.replaceAll('<ref.*(/>|>.*</ref>)', '', text);
  /*
   * Remove all HTML tags like '
', etc.
   */
  text = utils.replaceAll('<[^>]+>', '', text);
  /*
   * Remove footnote
   * TODO; support footnote in advanced model
   */
  text = utils.replaceAll('\{\{refn[^\}\}]*?\}\}', '', text);

  /*
   * Merge order, bulleted, unbulleted, Pagelist
   * list items to one line
   */
  var lists = text.match(/\{\{(order|bulleted|unbulleted|Pagelist)(.*\n)*?\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      text = text.replace(l, l.replace('{{', '').replace('}}', '')
                              .replace(/(order|bulleted|unbulleted)\slist\n\|/g, '')
                              .split('\n|').join(', '));
    });
  }

  /*
   * Parse URL
   */
  lists = text.match(/\{\{(URL)(.*)\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      var tmp = l.replace('{{', '').replace('}}', '').split('|');
      text = (tmp && tmp.length > 0) ? text.replace(l, tmp[tmp.length - 1]) : text;
    });
  }

  /*
   * Parse Start date
   */
  lists = text.match(/\{\{(Start\sdate)(.*)\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      var tmp = l.replace('{{', '').replace('}}', '').split('|');
      /* Pop first element: 'Start date' */
      tmp.shift();
      text = (tmp) ? text.replace(l, tmp.join('/')) : text;
    });
  }

  /*****************************
   * Analyze each line of text *
   *****************************/

  var result = {};
  text.split('\n|').forEach(function(item) {
    /*
     * FIXME: Each item is not always seperated by ' = ',
     *        sometimes it is ' =', '= ', etc.
     *        It's not possible to split them by simply using '=',
     *        because the item might contains several '=',
     *        like: 'students = 300 + 600 = 900'
     */
    var itemIndex = item.indexOf('=');
    if (itemIndex != -1) {
      var item_name = item.substr(0, itemIndex).trim();
      var item_content = item.substr(itemIndex + 1).trim().split('\n')[0];

      /*
       * Extract all simple texts inside '[[ ]]'
       * such as [[France]], [[Language French|French]], etc.
       */
      var find = item_content.match(/\[\[.*?\]\]/g);
      if (find) {
        find.forEach(function(substring) {
          var barestring = substring.replace('[[', '').replace(']]', '');
          var arr = barestring.split('|');
          /**
           * TODO: support link.
           * Reference: https://en.wikipedia.org/wiki/Help:Wiki_markup#Links_and_URLs
           */
          item_content = item_content.replace(substring, arr[arr.length - 1]);
        });
      }

      /*
       * Remove font style
       * {{fake clarify}}
       * {{fake citation needed}}
       * {{fake elucidate}}
       * {{fake heading}}
       * {{fake notes and references}}
       * {{dummy ref}}
       * {{dummy backlink}}
       * {{dummy footnote}}
       * {{break}}
       * {{break|5}}
       * {{clear}}
       * {{clear|left}}
       * {{clear|right}}
       * {{plainlist}}
       * {{startflatlist}}
       * {{flatlist}}
       * {{hlist|first item|second item|third item|...}}
       * {{bulleted list |item1 |item2 |...}}
       * {{pagelist}}
       * {{nowrap}}
       * {{italics}}
       * {{smallcaps|small caps}}
       * {{pad|4.0em}}
       */
      while (item_content.indexOf('{{nowrap|') !== -1) {
        item_content = item_content.replace('{{nowrap|', '');
        item_content = item_content.replace('}}', '');
      }

      while (item_content.indexOf('{{small|') !== -1) {
        item_content = item_content.replace('{{small|', '');
        item_content = item_content.replace('}}', '');
      }

      if (item_content.indexOf('{{native') !== -1) {
        find = item_content.match(/\{\{native[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          item_content = item_content.replace(substring, substring.split('|')[2]);
        });
      }

      /* Remove simple vertical list tag */
      if (item_content.indexOf('{{vunblist') !== -1 &&
          item_content.split('{{').length < 3) {

        find = item_content.match(/\{\{vunblist[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          var tmp = substring.split('|');
          tmp.shift();
          item_content = item_content.replace(substring, tmp.join(',').replace('}}', ''));
        });
      }

      /* Remove horizon list tag */
      if (item_content.indexOf('{{hlist') !== -1) {
        find = item_content.match(/\{\{hlist[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          var tmp = substring.split('|');
          tmp.shift();
          item_content = item_content.replace(substring, tmp.join(',').replace('}}', ''));
        });
      }

      /* Remove efn tag */
      if (item_content.indexOf('{{efn') !== -1) {
        find = item_content.match(/\{\{efn[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          item_content = item_content.replace(substring, '');
        });
      }

      item_content = utils.replaceAll('&nbsp', ' ', item_content);
      item_content = utils.replaceAll('\n\}\}', '', item_content);
      result[item_name] = item_content;
    }
  });

  callback(null, JSON.stringify(result));
  return;
};
<commit_msg>Update comment.<commit_after>var utils = require('./utils');

/**
 * Main markup syndax parser
 * @param  {string}   data     markup text
 * @param  {Function} callback callback function
 * @return {undefined}         return undefined when error occurs
 */
module.exports = function(data, callback) {

  /**********************************
   * Parse scraping result,         *
   * which is in the format of JSON *
   **********************************/

  try {
    content = JSON.parse(data);
  } catch(e) {
    callback(e);
    return;
  }

  if (!content.query) { callback(new Error('Query Not Found')); return; }

  /**
   * Get JSON data
   */
  var json = content.query.pages;
  var key  = Object.keys(json);

  if (key.indexOf('-1') === 0) {
    callback(new Error('Page Index Not Found'));
    return;
  } else if(!json[key]){
    callback(new Error('Malformed Response Payload'));
    return;
  } else if (json[key].revisions[0]['*'].indexOf('REDIRECT') > -1) {
    callback(new Error(json[key].revisions[0]['*']));
    return;
  }

  /**
   * Get the JSON data that contains infobox section
   */
  var reg    = new RegExp('{{[Ii]nfobox(.|\n)*}}', 'g');
  var text   = reg.exec(json[key].revisions[0]['*']);
  if (!text) { callback(new Error('Infobox Not Found')); return; }
  text = text[0];

  /************************
   * Remove useless marks *
   ************************/

  /*
   * Remove comments
   */
  text = utils.replaceAll('', '', text);
  /*
   * Remove reference
   * TODO: support reference in advanced model
   */
  text = utils.replaceAll('<ref.*(/>|>.*</ref>)', '', text);
  /*
   * Remove all HTML tags like '
', etc.
   */
  text = utils.replaceAll('<[^>]+>', '', text);
  /*
   * Remove footnote
   * TODO; support footnote in advanced model
   */
  text = utils.replaceAll('\{\{refn[^\}\}]*?\}\}', '', text);

  /*
   * Merge order, bulleted, unbulleted, Pagelist
   * list items to one line
   */
  var lists = text.match(/\{\{(order|bulleted|unbulleted|Pagelist)(.*\n)*?\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      text = text.replace(l, l.replace('{{', '').replace('}}', '')
                              .replace(/(order|bulleted|unbulleted)\slist\n\|/g, '')
                              .split('\n|').join(', '));
    });
  }

  /*
   * Parse URL
   */
  lists = text.match(/\{\{(URL)(.*)\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      var tmp = l.replace('{{', '').replace('}}', '').split('|');
      text = (tmp && tmp.length > 0) ? text.replace(l, tmp[tmp.length - 1]) : text;
    });
  }

  /*
   * Parse Start date
   */
  lists = text.match(/\{\{(Start\sdate)(.*)\}\}/g);
  if (lists && lists.length) {
    lists.forEach(function(l) {
      var tmp = l.replace('{{', '').replace('}}', '').split('|');
      /* Pop first element: 'Start date' */
      tmp.shift();
      text = (tmp) ? text.replace(l, tmp.join('/')) : text;
    });
  }

  /*****************************
   * Analyze each line of text *
   *****************************/

  var result = {};
  text.split('\n|').forEach(function(item) {
    /**
     * Extract {item_name, item_content} from each item
     */
    var itemIndex = item.indexOf('=');
    if (itemIndex != -1) {
      var item_name = item.substr(0, itemIndex).trim();
      var item_content = item.substr(itemIndex + 1).trim().split('\n')[0];

      /*
       * Extract all simple texts inside '[[ ]]'
       * such as [[France]], [[Language French|French]], etc.
       */
      var find = item_content.match(/\[\[.*?\]\]/g);
      if (find) {
        find.forEach(function(substring) {
          var barestring = substring.replace('[[', '').replace(']]', '');
          var arr = barestring.split('|');
          /**
           * TODO: support link.
           * Reference: https://en.wikipedia.org/wiki/Help:Wiki_markup#Links_and_URLs
           */
          item_content = item_content.replace(substring, arr[arr.length - 1]);
        });
      }

      /*
       * Remove font style
       * {{fake clarify}}
       * {{fake citation needed}}
       * {{fake elucidate}}
       * {{fake heading}}
       * {{fake notes and references}}
       * {{dummy ref}}
       * {{dummy backlink}}
       * {{dummy footnote}}
       * {{break}}
       * {{break|5}}
       * {{clear}}
       * {{clear|left}}
       * {{clear|right}}
       * {{plainlist}}
       * {{startflatlist}}
       * {{flatlist}}
       * {{hlist|first item|second item|third item|...}}
       * {{bulleted list |item1 |item2 |...}}
       * {{pagelist}}
       * {{nowrap}}
       * {{italics}}
       * {{smallcaps|small caps}}
       * {{pad|4.0em}}
       */
      while (item_content.indexOf('{{nowrap|') !== -1) {
        item_content = item_content.replace('{{nowrap|', '');
        item_content = item_content.replace('}}', '');
      }

      while (item_content.indexOf('{{small|') !== -1) {
        item_content = item_content.replace('{{small|', '');
        item_content = item_content.replace('}}', '');
      }

      if (item_content.indexOf('{{native') !== -1) {
        find = item_content.match(/\{\{native[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          item_content = item_content.replace(substring, substring.split('|')[2]);
        });
      }

      /* Remove simple vertical list tag */
      if (item_content.indexOf('{{vunblist') !== -1 &&
          item_content.split('{{').length < 3) {

        find = item_content.match(/\{\{vunblist[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          var tmp = substring.split('|');
          tmp.shift();
          item_content = item_content.replace(substring, tmp.join(',').replace('}}', ''));
        });
      }

      /* Remove horizon list tag */
      if (item_content.indexOf('{{hlist') !== -1) {
        find = item_content.match(/\{\{hlist[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          var tmp = substring.split('|');
          tmp.shift();
          item_content = item_content.replace(substring, tmp.join(',').replace('}}', ''));
        });
      }

      /* Remove efn tag */
      if (item_content.indexOf('{{efn') !== -1) {
        find = item_content.match(/\{\{efn[^\}\}]*?\}\}/g);
        find && find.forEach(function(substring) {
          item_content = item_content.replace(substring, '');
        });
      }

      item_content = utils.replaceAll('&nbsp', ' ', item_content);
      item_content = utils.replaceAll('\n\}\}', '', item_content);
      result[item_name] = item_content;
    }
  });

  callback(null, JSON.stringify(result));
  return;
};
<|endoftext|>


<commit_before>var fs = require('fs')
var path = require('path')
var Stream = require('stream').Stream
var vm = require('vm')

var inherits = require('inherits')
var requireLike = require('require-like')
// var Atok = require('atok')
var Atok = require('../../node-atok')
var Helpers = require('./helpers')
var Tracker = require('./tracker')

function noOp () {}

// Copy properties from one object to another - not replacing if required
function merge (a, b, soft) {
  for (var k in b)
    if (!soft || !a.hasOwnProperty(k)) a[k] = b[k]
  return a
}

// Show special characters for display
function inspect (s) {
  return s
    .replace(/\\/g, '\\\\')
    .replace(/\n/g, '\\n')
    .replace(/\t/g, '\\t')
    .replace(/"/g, '\\"')
    .replace(/\r/g, '\\r')
}

merge(Atok.prototype, Helpers)
exports.Helpers = Helpers

/**
 * createParser(file, parserOptions, atokOptions)
 * - file (String): file to read the parser from(.js extension is optional)
 * - parserOptions (String): list of named arguments supplied to the parser
 * - atokOptions (Object): tokenizer options
 *
 * Return a parser class (Function) based on the input file.
 * The following variables are made available to the parser js:
 * - atok {Object}: atok tokenizer instanciated with provided options. Also
 *   set as this.atok *DO NOT DELETE*
 * - self (Object): reference to this
 * Predefined methods:
 * - write
 * - end
 * - pause
 * - resume
 * - track
 * Events automatically forwarded from tokenizer to parser:
 * - drain
 * - debug
**/
exports.createParser = function (file, parserOptions, atokOptions) {
  var filename = file.slice(-3) === '.js' ? file : file + '.js'
  filename = path.resolve( path.dirname(module.parent.filename), filename )

  var content = []
  content.push(
    'function Parser (' + (parserOptions || '') + ') {'
  , 'if (!(this instanceof Parser)) {'
  , 'return new Parser(' + (parserOptions || '') + ') }'
  , 'Stream.call(this)'
  , 'this.readable = true'
  , 'this.writable = true'
  , 'var self = this'
  , 'var atok = new Atok(' + (atokOptions ? JSON.stringify(atokOptions): '') + ')'
  , 'atok.on("drain", function () { self.emit("drain") })'
  , 'atok.on("debug", function (msg) { self.emit("debug", msg) })'
  , 'atok.helpersCache = {}'
  , 'this.atok = atok'
  , 'this.atokTracker = new Tracker(atok)'
  , fs.readFileSync(filename).toString()
  , '}'
  , 'inherits(Parser, Stream)'
  , 'exports = Parser'
  , ''
  )

  // Expose standard Node globals + atok specifics
  var exports = {}
  var sandbox = {
    exports: exports
  , module: { exports: exports }
  , require: requireLike(module.parent.filename)
  , __filename: filename
  , __dirname: path.dirname(filename)
  // Custom exposed globals
  , Atok: Atok
  , Tracker: Tracker
  , Stream: Stream
  , inherits: inherits
  }
  sandbox.global = sandbox
  // Make the global properties available to the parser
  merge(sandbox, global)

  // Build the parser constructor
  vm.createScript(content.join(';'), filename)
    .runInNewContext(sandbox)

  var Parser = sandbox.exports

  // Apply the parser stream methods to the tokenizer
  Parser.prototype.pause = function () {
    this.atok.pause()
    return this
  }
  Parser.prototype.resume = function () {
    this.atok.resume()
    return this
  }
  Parser.prototype.write = function (data) {
    return this.atok.write(data)
  }
  Parser.prototype.end = function (data) {
    return this.atok.end(data)
  }
  Parser.prototype.destroy = noOp
  // Track current line and column
  // type: {String | Boolean} platform type (used to identify newline characters)
  Parser.prototype.track = function (type) {
    if (type) { // Turn tracking ON
      this.atokTracker.start(type)
    } else { // Turn tracking OFF
      this.atokTracker.end()
    }

    return this
  }
  // Build error message based on the current position in the buffer
  Parser.prototype.trackError = function (err, data, delta) {
    err = err || { message: '' }
    data = data || ''
    delta = typeof delta === 'number' ? delta : 10

    var atok = this.atok
    var size = data.length
    var pos = atok.offset - size

    var tracker = this.atokTracker

    var msg = ''
    msg += 'rule: ' + atok.getRuleSet()

    if (tracker.running) {
      var margin = 4

      var line = tracker.y
      // var column = atok.bytesRead - tracker.x + 1
      var column = tracker.x + 1

      // Extract some data from the current position: some left, some right
      var left = Math.max(0, pos - delta)
      var leftExtract = atok._slice(left, pos)
      var leftInspected = inspect(leftExtract)

      var right = Math.min(atok.length, pos + delta)
      var rightExtract = atok._slice(pos, right)
      var rightInspected = inspect(rightExtract)

      // Position of the cursor below the displayed data
      var cursor = pos - left + margin + leftInspected.length - leftExtract.length

      msg += ', line: ' + line
      msg += ', column: ' + column
      msg += '\n'
      // Data sample around the current buffer position
      msg += Array(margin + 1).join(' ')
      msg += leftInspected
      msg += rightInspected
      msg += '\n'
      // Offset the cursor
      msg += Array(cursor).join(' ')
      // Add the cursor
      msg += Array((Math.min(size, right - pos) || 1) + 1).join('^')
    } else {
      msg += ', token: ' + inspect( data )
    }
    msg += '\n'

    err.message += err.message.length ? ', ' + msg : msg
    return merge(err, {
      line: line
    , column: column
    , token: data
    , rule: atok.getRuleSet()
    })
  }

  return Parser
}<commit_msg>Version 0.0.5<commit_after>var fs = require('fs')
var path = require('path')
var Stream = require('stream').Stream
var vm = require('vm')

var inherits = require('inherits')
var requireLike = require('require-like')
var Atok = require('atok')
var Helpers = require('./helpers')
var Tracker = require('./tracker')

function noOp () {}

// Copy properties from one object to another - not replacing if required
function merge (a, b, soft) {
  for (var k in b)
    if (!soft || !a.hasOwnProperty(k)) a[k] = b[k]
  return a
}

// Show special characters for display
function inspect (s) {
  return s
    .replace(/\\/g, '\\\\')
    .replace(/\n/g, '\\n')
    .replace(/\t/g, '\\t')
    .replace(/"/g, '\\"')
    .replace(/\r/g, '\\r')
}

merge(Atok.prototype, Helpers)
exports.Helpers = Helpers

/**
 * createParser(file, parserOptions, atokOptions)
 * - file (String): file to read the parser from(.js extension is optional)
 * - parserOptions (String): list of named arguments supplied to the parser
 * - atokOptions (Object): tokenizer options
 *
 * Return a parser class (Function) based on the input file.
 * The following variables are made available to the parser js:
 * - atok {Object}: atok tokenizer instanciated with provided options. Also
 *   set as this.atok *DO NOT DELETE*
 * - self (Object): reference to this
 * Predefined methods:
 * - write
 * - end
 * - pause
 * - resume
 * - track
 * Events automatically forwarded from tokenizer to parser:
 * - drain
 * - debug
**/
exports.createParser = function (file, parserOptions, atokOptions) {
  var filename = file.slice(-3) === '.js' ? file : file + '.js'
  filename = path.resolve( path.dirname(module.parent.filename), filename )

  var content = []
  content.push(
    'function Parser (' + (parserOptions || '') + ') {'
  , 'if (!(this instanceof Parser)) {'
  , 'return new Parser(' + (parserOptions || '') + ') }'
  , 'Stream.call(this)'
  , 'this.readable = true'
  , 'this.writable = true'
  , 'var self = this'
  , 'var atok = new Atok(' + (atokOptions ? JSON.stringify(atokOptions): '') + ')'
  , 'atok.on("drain", function () { self.emit("drain") })'
  , 'atok.on("debug", function (msg) { self.emit("debug", msg) })'
  , 'atok.helpersCache = {}'
  , 'this.atok = atok'
  , 'this.atokTracker = new Tracker(atok)'
  , fs.readFileSync(filename).toString()
  , '}'
  , 'inherits(Parser, Stream)'
  , 'exports = Parser'
  , ''
  )

  // Expose standard Node globals + atok specifics
  var exports = {}
  var sandbox = {
    exports: exports
  , module: { exports: exports }
  , require: requireLike(module.parent.filename)
  , __filename: filename
  , __dirname: path.dirname(filename)
  // Custom exposed globals
  , Atok: Atok
  , Tracker: Tracker
  , Stream: Stream
  , inherits: inherits
  }
  sandbox.global = sandbox
  // Make the global properties available to the parser
  merge(sandbox, global)

  // Build the parser constructor
  vm.createScript(content.join(';'), filename)
    .runInNewContext(sandbox)

  var Parser = sandbox.exports

  // Apply the parser stream methods to the tokenizer
  Parser.prototype.pause = function () {
    this.atok.pause()
    return this
  }
  Parser.prototype.resume = function () {
    this.atok.resume()
    return this
  }
  Parser.prototype.write = function (data) {
    return this.atok.write(data)
  }
  Parser.prototype.end = function (data) {
    return this.atok.end(data)
  }
  Parser.prototype.destroy = noOp
  // Track current line and column
  // type: {String | Boolean} platform type (used to identify newline characters)
  Parser.prototype.track = function (type) {
    if (type) { // Turn tracking ON
      this.atokTracker.start(type)
    } else { // Turn tracking OFF
      this.atokTracker.end()
    }

    return this
  }
  // Build error message based on the current position in the buffer
  Parser.prototype.trackError = function (err, data, delta) {
    err = err || { message: '' }
    data = data || ''
    delta = typeof delta === 'number' ? delta : 10

    var atok = this.atok
    var size = data.length
    var pos = atok.offset - size

    var tracker = this.atokTracker

    var msg = ''
    msg += 'rule: ' + atok.getRuleSet()

    if (tracker.running) {
      var margin = 4

      var line = tracker.y
      // var column = atok.bytesRead - tracker.x + 1
      var column = tracker.x + 1

      // Extract some data from the current position: some left, some right
      var left = Math.max(0, pos - delta)
      var leftExtract = atok._slice(left, pos)
      var leftInspected = inspect(leftExtract)

      var right = Math.min(atok.length, pos + delta)
      var rightExtract = atok._slice(pos, right)
      var rightInspected = inspect(rightExtract)

      // Position of the cursor below the displayed data
      var cursor = pos - left + margin + leftInspected.length - leftExtract.length

      msg += ', line: ' + line
      msg += ', column: ' + column
      msg += '\n'
      // Data sample around the current buffer position
      msg += Array(margin + 1).join(' ')
      msg += leftInspected
      msg += rightInspected
      msg += '\n'
      // Offset the cursor
      msg += Array(cursor).join(' ')
      // Add the cursor
      msg += Array((Math.min(size, right - pos) || 1) + 1).join('^')
    } else {
      msg += ', token: ' + inspect( data )
    }
    msg += '\n'

    err.message += err.message.length ? ', ' + msg : msg
    return merge(err, {
      line: line
    , column: column
    , token: data
    , rule: atok.getRuleSet()
    })
  }

  return Parser
}<|endoftext|>


<commit_before>/** @module lib/parser
    @description Parse HTML files to extract assessment information
*/

/*
* node.js requires:
* 'cheerio', ('../logger').appLogger
*
* browser requires:
* jQuery
*
* notice that for cheerio: $(this) == this -> true
* but in jQuery the above result in false, this why you see the
* following at various places:
* var $this = $(this);
*/
'use strict';

//shared requires
var when      = require('when')
  , _         = require('lodash')
  , isBrowser = require('./utils').isBrowser()

  //to be set (if needed)
  , jQuery
  , cheerio
  , logger
  , getOuterHTML ;

//private function to initialize environment
(function configureEnv_(){
  //choose appropriate configuration
  isBrowser ? configure4Browser_() : configure4Node_();

})();

function configure4Node_(){
  //avoid browserify cause cheerio is incompatible
  // FIXME: keep an eye on cheerio for compatibility
  cheerio = eval('require(\'cheerio\')');
  // TODO: Allow logger argument
  // logger = require('../logger').appLogger;
  logger = console;

  getOuterHTML = function($, $el) {
    return $.html($el)
  };
}

function configure4Browser_(){
  $ = jQuery = require('jquery');
  logger = console;

  getOuterHTML = function($, $el) {
    $($('

').html($el.clone())).html();
  };
}

var Parser = module.exports = function(){
  // default options for template names, selectors etc.
  this.defaultOptions={
    outputFormat : 'Array',
    //TODO: test this
    defaultMultiChoiceInput: 'checkbox',
    questionSelectors: {
      multiChoice : 'multi-choice',
      codeInput : 'code-input',
      textInput : 'text-input',
      highlight : 'highlight'
    },
    inputTypeSelectors: {
      'choose-1'   : 'radio' ,
      'choose-0-n' : 'checkbox',
      'choose-1-n' : 'checkbox'
    }
  };

  this.options={};
  //placeholder for the dom manipulation library
  this.$ = null;
  // Log all erros during parsing.
  this.errors = [];
};

(function(){
  /**
   * Return form input type based on first class found indicating such a type
   * @param {Object} $el - jQuery Object for which the input types needs to be determined.
   *
   * The input type is determined by the first valid class encountered.
   **/
  this.getFormButtonType = function ($el) {
    var i
    , hasOwn     = Object.prototype.hasOwnProperty
    , inputTypes = this.options.inputTypeSelectors;
    for (i in inputTypes) {
      if (hasOwn.call(inputTypes, i) && $el.hasClass(i)) {
        return inputTypes[i];
      }
    }
    return this.options.defaultMultiChoiceInput;

  }

  this.getQuestionType = function($el){
    var classes = []
      , opts = this.options;

    for (var key in opts.questionSelectors){
      if ($el.hasClass(opts.questionSelectors[key])){
        classes.push(opts.questionSelectors[key]);
      }
    }

    if(classes.length > 1) {
      this.pushError([
        'Multiple question types:', classes.toString(), 'for', $el.html()
      ].join(' '));
      return false;
    } else if(classes.length < 1) {
      this.pushError('No question type found for ' + $el.html())
      return false;
    } else {
      return classes[0];
    }
  }

  this.parseMultiChoice = function($, $el, opts) {
    //options for each question
    var questionOptions = []
    , opts              = this.options // TODO: Why do we have this!? (Fix it)
    , self              = this;

    //get options
    var choices = $el.children('ol.options').children('li.option');
    //console.log(choices);
    choices.each(function(){
      var $this = $(this);

      questionOptions.push({
        text: $this.html(),
        classList: $this.attr('class'),
        correct: $this.attr('data-correct') === 'true' ? true:false
      });
    });

    // find input types
    var inputType = this.getFormButtonType($el);
    if (inputType ===''){
      var optSels = opts.inputTypeSelectors;
      var validSelectors = optSels.sel1 + ' ' + optSels.sel0_n + ' ' + optSels.sel1_n ;
      self.pushError('No valid input type selectors for question. Valid types are: ' + validSelectors)
    }

    // find parent slide id
    var parentSlideId;
    var $slideParent = $el.parents('.step');
    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var question = {
      stem : getOuterHTML($, $el.find('.stem')),
      stemText: $el.find('.stem').html(),
      htmlId: $el.attr('id'),
      slideHtmlId : parentSlideId,
      questionType: 'multi-choice',
      formButtonType : inputType,
      questionOptions : questionOptions
    };

    return question;
  }

  this.parseTextInput = function ($, $el, opts) {
    // find parent slide id
    var parentSlideId
      , $slideParent = $el.parents('.step');

    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var $stem =$el.find('.stem')
      , answer = $stem.attr('data-correct-answer')
      , question = {
        stem : getOuterHTML($, $stem),
        stemText: $stem.html(),
        htmlId: $el.attr('id'),
        slideHtmlId : parentSlideId,
        questionType: 'text-input',
        correctAnswer : typeof answer== 'undefined' ? null : answer
      };

    return question;
  }

  this.parseCodeInput = function ($, $el, opts) {
    // find parent slide id
    var parentSlideId
      , $slideParent = $el.parents('.step');

    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var $stem =$el.find('.stem')
      , answer = $stem.attr('data-correct-answer')
      , question = {
        stem : getOuterHTML($, $stem),
        stemText: $stem.html(),
        htmlId: $el.attr('id'),
        slideHtmlId : parentSlideId,
        questionType: 'code-input',
        correctAnswer : typeof answer === 'undefined' ? null : answer
      };

    return question;
  }

  this.parseMCQRubric = function($, $rubric, rubric) {
    var criteria = [];

    //criteria are list items with the option class
    var $criteria = $rubric.find('ol li.option') // TODO: prefix option with asq (also for standard questions)
    , i, max;
    for (i = 0, max = $criteria.length; i < max; i++) {
      var $criterion = $($criteria[i]);
      var points = parseInt($criterion.attr('data-asq-rubric-points'));
      if (points !== 0 && ! points) { // Default point value from pos in list
        points = max - i;             // if unable to get from data attr.
      }
      // Label from data attr if given otherwise from points.
      var label =  $criterion.attr('data-asq-rubric-label') || '' + points;
      var desc  = $criterion.html(); //description
      criteria.push({
        points : points,
        label  : label,
        desc   : desc
      });
    }

    rubric.criteria       = criteria;
    rubric.formButtonType = this.getFormButtonType($rubric);
    rubric.statTypes      = ['rubric-mcq']; // Setting specific stats for rubrics
    return rubric;
  }

  /**
   *  Parse a rubric and return a JSON of the formated rubric.
   *  @param {Object} $rubric - jQuery node of the rubric to parse.
   **/
  this.parseRubric = function($, $rubric, questionId) {
    var self = this;

    // check for multiple question types
    var rubricType = self.getQuestionType($rubric);
    if (!rubricType) { // Something went wrong while getting the question
      return false;    // type. It's been handled, we just return false.
    }

    var rubric = {};
    rubric.questionType = rubricType;
    rubric.question     = questionId;

    //Stem
    var $stem = $rubric.find('.stem');
    rubric.stem     = getOuterHTML($, $stem);
    rubric.stemText = $stem.html();

    var questionTypes = this.options.questionSelectors;
    switch(rubricType) { //TODO: Switch !? Come on... we can do nicer than that!
      case questionTypes.codeInput :
        self.pushError('Code Input not implemented for rubrics (question ' +
          rubric.question +')');
        break;

      case questionTypes.multiChoice :
        self.parseMCQRubric($, $rubric, rubric);
        break;

      case questionTypes.textInput :
        self.pushError('Code Input not implemented for rubrics (question ' +
          rubric.question + ')');
        break;
    }

    return rubric;
  }

  // search for selector starting from root
  // module.exports.findFromRoot = function($el, selector) {
  //  console.log($el.find(selector))
  //  // return $el.filter(selector).add($el.find(selector));
  // }

  this.parse = function(html, options, callback){

    //configure dom selector lib implementation
    var $ = this.$ = isBrowser ? jQuery : cheerio.load(html);

    //argument overload
    options = options==undefined? {} : options;

    // Support callback as second arg.
    if (typeof options === 'function') {
        callback = options;
        options  = {};
    }

    if(typeof options !== 'undefined' && typeof options != 'Function') {
      _.extend(this.options, this.defaultOptions, options);
    }

    var opts= this.options;

    var deferred = when.defer()
    , start      = new Date()
    , questions  = opts.outputFormat === 'Array'? [] : {}
    , rubrics    = []
    , stats      = []
    , data       = null;

    // assessments parsing
    var $assessment = $('.assessment')
    , self          = this;

    // filter already processed elements
    $assessment.filter(function filterProcessed() {
      return $(this).find('form').length === 0;
    }).each( //Parse each assessment
      function parseAssessment() {
        var $this = $(this);

        //check for empty ids
        if($this.attr('id') === undefined || $this.attr('id').trim() === '') {
          self.pushError('found question without id');
          //TODO: try 5 times to assign a random id
          // the following code tries to assign a random id. For now disabled
          // because we have to see if we will save the html file.
          /*for (var i=0; i<5; i++){
            newId = 'question-' + Math.floor( Math.random()*999999 );
            if( $('#'+newId).length == 0){
              $this.attr('id',newId);
              console.log('Assigned id: ' + newId);
              break;
            }else if(i==4){
              errors = [{'error':  'Failed to assign newId'}]

              //break .each()
              return false;

            }
          }*/
        }

        // check for double question ids
        var ids = $('.assessment[id="'+ $this.attr('id')+'"]');
        if(ids.length > 1) {
          self.pushError('Multiple IDs #'+$this.attr('id'))
          return false;
        }

        // check for multiple question types
        var questionType = self.getQuestionType($this);
        if (! questionType) { // Something went wrong while getting the question
          return false;      // type. It's been handled, we just return false.
        }

        var question;
        switch(questionType) { //TODO: Switch !? Come on... we can do nicer than that!
          case opts.questionSelectors.codeInput :
            question = self.parseCodeInput($, $this, opts);
            break;

          case opts.questionSelectors.multiChoice :
            question = self.parseMultiChoice($, $this, opts);
            question.statTypes = ['correctness']; //Horrible hack to add correctness stats to MCQ...
            break;

          case opts.questionSelectors.textInput :
            question = self.parseTextInput($, $this, opts);
            break;
        }

        // Parsing rubric
        var $rubrics = $this.children('.asq-rubric');

        if ($rubrics.length > 0) {
          question.rubrics = [];
          var i, max;
          for (i = 0, max = $rubrics.length; i < max; i++){
            var rubric = self.parseRubric($, $($rubrics[i]), question.htmlId);
            if (!! rubric) { // Otherwise the rubric is not valid and is ignored
              rubrics.push(rubric); // and an error has been logged.
            }
          }
        }

        if (opts.outputFormat === 'Array') {
          questions.push(question);
        } else {
          questions[$this.attr('id')] = question;
        }
    });

    //stats parsing
    $('.stats').each(function(){
      var $this = $(this);
      stats.push({
        questionHtmlId  : $this.attr('data-target-assessment'),
        slideHtmlId     : $this.parents('.step').attr('id')
      });
    })

    if(self.errors.length === 0) { // No errors during parsing
      data = { questions : questions, rubrics : rubrics, stats : stats };
    }

    var errors = self.errors.length === 0 // Set errors to null if there are
      ? null                              // no errors.
      : self.errors;
    returnData(errors, data, callback, deferred, logger, start);
    return deferred.promise;
  }

  // instantiates error array if not instantiated
  // and pushes the error
  this.pushError = function(error) {
    this.errors.push(error);
  }

}).call(Parser.prototype)

// returns date either by callback or promise
// copied from https://github.com/glennjones/microformat-node/
// full credit goes to Glenn Jones
function returnData(errors, data, callback, deferred, logger, start){
  if (errors) {
   logger.error(JSON.stringify(errors));
  }
  if (start) {
    logger.info('ASQ parser took: ' + (new Date().getTime() - start.getTime()) + 'ms');
  }
  if (callback && (typeof(callback) === 'function')) {
    callback(errors, data);
  } else if (errors) {
    deferred.reject(errors);
  } else {
    deferred.resolve(data);
  }
}<commit_msg>lib/parser: added maxScore<commit_after>/** @module lib/parser
    @description Parse HTML files to extract assessment information
*/

/*
* node.js requires:
* 'cheerio', ('../logger').appLogger
*
* browser requires:
* jQuery
*
* notice that for cheerio: $(this) == this -> true
* but in jQuery the above result in false, this why you see the
* following at various places:
* var $this = $(this);
*/
'use strict';

//shared requires
var when      = require('when')
  , _         = require('lodash')
  , isBrowser = require('./utils').isBrowser()

  //to be set (if needed)
  , jQuery
  , cheerio
  , logger
  , getOuterHTML ;

//private function to initialize environment
(function configureEnv_(){
  //choose appropriate configuration
  isBrowser ? configure4Browser_() : configure4Node_();

})();

function configure4Node_(){
  //avoid browserify cause cheerio is incompatible
  // FIXME: keep an eye on cheerio for compatibility
  cheerio = eval('require(\'cheerio\')');
  // TODO: Allow logger argument
  // logger = require('../logger').appLogger;
  logger = console;

  getOuterHTML = function($, $el) {
    return $.html($el)
  };
}

function configure4Browser_(){
  $ = jQuery = require('jquery');
  logger = console;

  getOuterHTML = function($, $el) {
    $($('

').html($el.clone())).html();
  };
}

var Parser = module.exports = function(){
  // default options for template names, selectors etc.
  this.defaultOptions={
    outputFormat : 'Array',
    //TODO: test this
    defaultMultiChoiceInput: 'checkbox',
    questionSelectors: {
      multiChoice : 'multi-choice',
      codeInput : 'code-input',
      textInput : 'text-input',
      highlight : 'highlight'
    },
    inputTypeSelectors: {
      'choose-1'   : 'radio' ,
      'choose-0-n' : 'checkbox',
      'choose-1-n' : 'checkbox'
    }
  };

  this.options={};
  //placeholder for the dom manipulation library
  this.$ = null;
  // Log all erros during parsing.
  this.errors = [];
};

(function(){
  /**
   * Return form input type based on first class found indicating such a type
   * @param {Object} $el - jQuery Object for which the input types needs to be determined.
   *
   * The input type is determined by the first valid class encountered.
   **/
  this.getFormButtonType = function ($el) {
    var i
    , hasOwn     = Object.prototype.hasOwnProperty
    , inputTypes = this.options.inputTypeSelectors;
    for (i in inputTypes) {
      if (hasOwn.call(inputTypes, i) && $el.hasClass(i)) {
        return inputTypes[i];
      }
    }
    return this.options.defaultMultiChoiceInput;

  }

  this.getQuestionType = function($el){
    var classes = []
      , opts = this.options;

    for (var key in opts.questionSelectors){
      if ($el.hasClass(opts.questionSelectors[key])){
        classes.push(opts.questionSelectors[key]);
      }
    }

    if(classes.length > 1) {
      this.pushError([
        'Multiple question types:', classes.toString(), 'for', $el.html()
      ].join(' '));
      return false;
    } else if(classes.length < 1) {
      this.pushError('No question type found for ' + $el.html())
      return false;
    } else {
      return classes[0];
    }
  }

  this.parseMultiChoice = function($, $el, opts) {
    //options for each question
    var questionOptions = []
    , opts              = this.options // TODO: Why do we have this!? (Fix it)
    , self              = this;

    //get options
    var choices = $el.children('ol.options').children('li.option');
    // console.log($choices);
    choices.each(function(){
      var $this = $(this);

      questionOptions.push({
        text: $this.html(),
        classList: $this.attr('class'),
        correct: $this.attr('data-correct') === 'true' ? true:false
      });
    });

    // find input types
    var inputType = this.getFormButtonType($el);
    if (inputType ===''){
      var optSels = opts.inputTypeSelectors;
      var validSelectors = optSels.sel1 + ' ' + optSels.sel0_n + ' ' + optSels.sel1_n ;
      self.pushError('No valid input type selectors for question. Valid types are: ' + validSelectors)
    }

    // find parent slide id
    var parentSlideId;
    var $slideParent = $el.parents('.step');
    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var question = {
      stem : getOuterHTML($, $el.find('.stem')),
      stemText: $el.find('.stem').html(),
      htmlId: $el.attr('id'),
      slideHtmlId : parentSlideId,
      questionType: 'multi-choice',
      formButtonType : inputType,
      questionOptions : questionOptions
    };

    return question;
  }

  this.parseTextInput = function ($, $el, opts) {
    // find parent slide id
    var parentSlideId
      , $slideParent = $el.parents('.step');

    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var $stem =$el.find('.stem')
      , answer = $stem.attr('data-correct-answer')
      , question = {
        stem : getOuterHTML($, $stem),
        stemText: $stem.html(),
        htmlId: $el.attr('id'),
        slideHtmlId : parentSlideId,
        questionType: 'text-input',
        correctAnswer : typeof answer== 'undefined' ? null : answer
      };

    return question;
  }

  this.parseCodeInput = function ($, $el, opts) {
    // find parent slide id
    var parentSlideId
      , $slideParent = $el.parents('.step');

    if($slideParent.length>0){
      parentSlideId = $slideParent.attr('id') ? $slideParent.attr('id'):'';
    }

    var $stem =$el.find('.stem')
      , answer = $stem.attr('data-correct-answer')
      , question = {
        stem : getOuterHTML($, $stem),
        stemText: $stem.html(),
        htmlId: $el.attr('id'),
        slideHtmlId : parentSlideId,
        questionType: 'code-input',
        correctAnswer : typeof answer === 'undefined' ? null : answer
      };

    return question;
  }

  this.parseMCQRubric = function($, $rubric, rubric) {
    var criteria = [];

    //criteria are list items with the option class
    var $criteria = $rubric.find('ol li.option') // TODO: prefix option with asq (also for standard questions)
    , i, max;
    var points = [];
    for (i = 0, max = $criteria.length; i < max; i++) {
      var $criterion = $($criteria[i]);
      var point = parseInt($criterion.attr('data-asq-rubric-points'));
      if (point !== 0 && ! point) { // Default point value from pos in list
        point = max - i;             // if unable to get from data attr.
      }
      points.push(point);
      // Label from data attr if given otherwise from points.
      var label =  $criterion.attr('data-asq-rubric-label') || '' + point;
      var desc  = $criterion.html(); //description
      criteria.push({
        points : point,
        label  : label,
        desc   : desc
      });
    }
    i = points.length;
    var positive = null;
    var maxScore = 0;
    while(i--) {
      if (points[i] === 0) {
        continue;
      } else if (positive === null) {
        positive = points[i] > 0;
        maxScore = positive ? points[i] : - points[i]
      } else if (positive && points[i] > 0) {
        maxScore += points[i];
      } else if (!positive && points[i] < 0) {
        maxScore -= points[i];
      } else {
        this.pushError(new Error(
          'Mixed signs for criteria scores for rubric ' + rubric.stemText));
      }
    }

    rubric.criteria       = criteria;
    rubric.maxScore       = maxScore;
    rubric.formButtonType = this.getFormButtonType($rubric);
    rubric.statTypes      = ['rubric-mcq']; // Setting specific stats for rubrics
    return rubric;
  }

  /**
   *  Parse a rubric and return a JSON of the formated rubric.
   *  @param {Object} $rubric - jQuery node of the rubric to parse.
   **/
  this.parseRubric = function($, $rubric, questionId) {
    var self = this;

    // check for multiple question types
    var rubricType = self.getQuestionType($rubric);
    if (!rubricType) { // Something went wrong while getting the question
      return false;    // type. It's been handled, we just return false.
    }

    var rubric = {};
    rubric.questionType = rubricType;
    rubric.question     = questionId;

    //Stem
    var $stem = $rubric.find('.stem');
    rubric.stem     = getOuterHTML($, $stem);
    rubric.stemText = $stem.html();

    var questionTypes = this.options.questionSelectors;
    switch(rubricType) { //TODO: Switch !? Come on... we can do nicer than that!
      case questionTypes.codeInput :
        self.pushError('Code Input not implemented for rubrics (question ' +
          rubric.question +')');
        break;

      case questionTypes.multiChoice :
        self.parseMCQRubric($, $rubric, rubric);
        break;

      case questionTypes.textInput :
        self.pushError('Code Input not implemented for rubrics (question ' +
          rubric.question + ')');
        break;
    }

    return rubric;
  }

  // search for selector starting from root
  // module.exports.findFromRoot = function($el, selector) {
  //  console.log($el.find(selector))
  //  // return $el.filter(selector).add($el.find(selector));
  // }

  this.parse = function(html, options, callback){

    //configure dom selector lib implementation
    var $ = this.$ = isBrowser ? jQuery : cheerio.load(html);

    //argument overload
    options = options==undefined? {} : options;

    // Support callback as second arg.
    if (typeof options === 'function') {
        callback = options;
        options  = {};
    }

    if(typeof options !== 'undefined' && typeof options != 'Function') {
      _.extend(this.options, this.defaultOptions, options);
    }

    var opts= this.options;

    var deferred = when.defer()
    , start      = new Date()
    , questions  = opts.outputFormat === 'Array'? [] : {}
    , rubrics    = []
    , stats      = []
    , data       = null;

    // assessments parsing
    var $assessment = $('.assessment')
    , self          = this;

    // filter already processed elements
    $assessment.filter(function filterProcessed() {
      return $(this).find('form').length === 0;
    }).each( //Parse each assessment
      function parseAssessment() {
        var $this = $(this);

        //check for empty ids
        if($this.attr('id') === undefined || $this.attr('id').trim() === '') {
          self.pushError('found question without id');
          //TODO: try 5 times to assign a random id
          // the following code tries to assign a random id. For now disabled
          // because we have to see if we will save the html file.
          /*for (var i=0; i<5; i++){
            newId = 'question-' + Math.floor( Math.random()*999999 );
            if( $('#'+newId).length == 0){
              $this.attr('id',newId);
              console.log('Assigned id: ' + newId);
              break;
            }else if(i==4){
              errors = [{'error':  'Failed to assign newId'}]

              //break .each()
              return false;

            }
          }*/
        }

        // check for double question ids
        var ids = $('.assessment[id="'+ $this.attr('id')+'"]');
        if(ids.length > 1) {
          self.pushError('Multiple IDs #'+$this.attr('id'))
          return false;
        }

        // check for multiple question types
        var questionType = self.getQuestionType($this);
        if (! questionType) { // Something went wrong while getting the question
          return false;      // type. It's been handled, we just return false.
        }

        var question;
        switch(questionType) { //TODO: Switch !? Come on... we can do nicer than that!
          case opts.questionSelectors.codeInput :
            question = self.parseCodeInput($, $this, opts);
            break;

          case opts.questionSelectors.multiChoice :
            question = self.parseMultiChoice($, $this, opts);
            question.statTypes = ['correctness']; //Horrible hack to add correctness stats to MCQ...
            break;

          case opts.questionSelectors.textInput :
            question = self.parseTextInput($, $this, opts);
            break;
        }

        // Parsing rubric
        var $rubrics = $this.children('.asq-rubric');

        if ($rubrics.length > 0) {
          question.rubrics = [];
          var i, max;
          for (i = 0, max = $rubrics.length; i < max; i++){
            var rubric = self.parseRubric($, $($rubrics[i]), question.htmlId);
            if (!! rubric) { // Otherwise the rubric is not valid and is ignored
              rubrics.push(rubric); // and an error has been logged.
            }
          }
        }

        if (opts.outputFormat === 'Array') {
          questions.push(question);
        } else {
          questions[$this.attr('id')] = question;
        }
    });

    //stats parsing
    $('.stats').each(function(){
      var $this = $(this);
      stats.push({
        questionHtmlId  : $this.attr('data-target-assessment'),
        slideHtmlId     : $this.parents('.step').attr('id')
      });
    })

    if(self.errors.length === 0) { // No errors during parsing
      data = { questions : questions, rubrics : rubrics, stats : stats };
    }

    var errors = self.errors.length === 0 // Set errors to null if there are
      ? null                              // no errors.
      : self.errors;
    returnData(errors, data, callback, deferred, logger, start);
    return deferred.promise;
  }

  // instantiates error array if not instantiated
  // and pushes the error
  this.pushError = function(error) {
    this.errors.push(error);
  }

}).call(Parser.prototype)

// returns date either by callback or promise
// copied from https://github.com/glennjones/microformat-node/
// full credit goes to Glenn Jones
function returnData(errors, data, callback, deferred, logger, start){
  if (errors) {
   logger.error(JSON.stringify(errors));
  }
  if (start) {
    logger.info('ASQ parser took: ' + (new Date().getTime() - start.getTime()) + 'ms');
  }
  if (callback && (typeof(callback) === 'function')) {
    callback(errors, data);
  } else if (errors) {
    deferred.reject(errors);
  } else {
    deferred.resolve(data);
  }
}<|endoftext|>


<commit_before>const pug = require("pug");
const pugRuntime = require("pug-runtime");
const pugRuntimeWrap = require("pug-runtime/wrap");

const path = require("path");
const YAML = require("js-yaml");
const getCodeBlock = require("pug-code-block");
const detectIndent = require("detect-indent");
const rebaseIndent = require("rebase-indent");
const pugdocArguments = require("./arguments");

const MIXIN_NAME_REGEX = /^mixin +([-\w]+)?/;
const DOC_REGEX = /^\s*\/\/-\s+?\@pugdoc\s*$/;
const DOC_STRING = "//- @pugdoc";
const CAPTURE_ALL = "all";
const CAPTURE_SECTION = "section";
const EXAMPLE_BLOCK = "block";

/**
 * Returns all pugdoc comment and code blocks for the given code
 *
 * @param templateSrc {string}
 * @return {{lineNumber: number, comment: string, code: string}[]}
 */

function extractPugdocBlocks(templateSrc) {
  return (
    templateSrc
      .split("\n")
      // Walk through every line and look for a pugdoc comment
      .map(function (line, lineIndex) {
        // If the line does not contain a pugdoc comment skip it
        if (!line.match(DOC_REGEX)) {
          return undefined;
        }

        // If the line contains a pugdoc comment return
        // the comment block and the next code block
        const comment = getCodeBlock.byLine(templateSrc, lineIndex + 1);
        const meta = parsePugdocComment(comment);

        // add number of captured blocks
        if (meta.capture <= 0) {
          return undefined;
        }

        let capture = 2;
        if (meta.capture) {
          if (meta.capture === CAPTURE_ALL) {
            capture = Infinity;
          } else if (meta.capture === CAPTURE_SECTION) {
            capture = Infinity;
          } else {
            capture = meta.capture + 1;
          }
        }

        // get all code blocks
        let code = getCodeBlock.byLine(templateSrc, lineIndex + 1, capture);

        // make string
        if (Array.isArray(code)) {
          // remove comment
          code.shift();

          // join all code
          code = code.join("\n");
        } else {
          return undefined;
        }

        // filter out all but current pugdoc section
        if (meta.capture === CAPTURE_SECTION) {
          const nextPugDocIndex = code.indexOf(DOC_STRING);
          if (nextPugDocIndex > -1) {
            code = code.substr(0, nextPugDocIndex);
          }
        }

        // if no code and no comment, skip
        if (comment.match(DOC_REGEX) && code === "") {
          return undefined;
        }

        return {
          lineNumber: lineIndex + 1,
          comment: comment,
          code: code,
        };
      })
      // Remove skiped lines
      .filter(function (result) {
        return result !== undefined;
      })
  );
}

/**
 * Returns all pugdocDocuments for the given code
 *
 * @param templateSrc {string}
 * @param filename {string}
 */

function getPugdocDocuments(templateSrc, filename, locals) {
  return extractPugdocBlocks(templateSrc).map(function (pugdocBlock) {
    const meta = parsePugdocComment(pugdocBlock.comment);
    const fragments = [];

    // parse jsdoc style arguments list
    if (meta.arguments) {
      meta.arguments = meta.arguments.map(function (arg) {
        return pugdocArguments.parse(arg, true);
      });
    }

    // parse jsdoc style attributes list
    if (meta.attributes) {
      meta.attributes = meta.attributes.map(function (arg) {
        return pugdocArguments.parse(arg, true);
      });
    }

    let source = pugdocBlock.code;
    source = source.replace(/\u2028|\u200B/g, "");

    if (meta.example && meta.example !== false) {
      if (meta.beforeEach) {
        meta.example = `${meta.beforeEach}\n${meta.example}`;
      }
      if (meta.afterEach) {
        meta.example = `${meta.example}\n${meta.afterEach}`;
      }
    }

    // get example objects and add them to parent example
    // also return them as separate pugdoc blocks
    if (meta.examples) {
      for (let i = 0, l = meta.examples.length; i < l; ++i) {
        let x = meta.examples[i];

        // do nothing for simple examples
        if (typeof x === "string") {
          if (meta.beforeEach) {
            meta.examples[i] = `${meta.beforeEach}\n${x}`;
          }
          if (meta.afterEach) {
            meta.examples[i] = `${x}\n${meta.afterEach}`;
          }
          continue;
        }

        if (meta.beforeEach && typeof x.beforeEach === "undefined") {
          x.example = `${meta.beforeEach}\n${x.example}`;
        }

        if (meta.afterEach && typeof x.afterEach === "undefined") {
          x.example = `${x.example}\n${meta.afterEach}`;
        }

        // merge example/examples with parent examples
        meta.examples[i] = getExamples(x).reduce(
          (acc, val) => acc.concat(val),
          []
        );

        // add fragments
        fragments.push(x);
      }

      meta.examples = meta.examples.reduce((acc, val) => acc.concat(val), []);
    }

    // fix pug compilation for boolean use of example
    const exampleClone = meta.example;
    if (typeof meta.example === "boolean") {
      meta.example = "";
    }

    const obj = {
      // get meta
      meta: meta,
      // add file path
      file: path.relative(".", filename),
      // get pug code block matching the comments indent
      source: source,
      // get html output
      output: compilePug(source, meta, filename, locals),
    };

    // remove output if example = false
    if (exampleClone === false) {
      obj.output = null;
    }

    // add fragments
    if (fragments && fragments.length) {
      obj.fragments = fragments.map((subexample) => {
        return {
          // get meta
          meta: subexample,
          // get html output
          output: compilePug(source, subexample, filename, locals),
        };
      });
    }

    if (obj.output || obj.fragments) {
      return obj;
    }

    return null;
  });
}

/**
 * Extract pug attributes from comment block
 */

function parsePugdocComment(comment) {
  // remove first line (@pugdoc)
  if (comment.indexOf("\n") === -1) {
    return {};
  }

  comment = comment.substr(comment.indexOf("\n"));
  comment = pugdocArguments.escapeArgumentsYAML(comment, "arguments");
  comment = pugdocArguments.escapeArgumentsYAML(comment, "attributes");

  // parse YAML
  return YAML.safeLoad(comment) || {};
}

/**
 * get all examples from the meta object
 * either one or both of meta.example and meta.examples can be given
 */

function getExamples(meta) {
  let examples = [];
  if (meta.example) {
    examples = examples.concat(meta.example);
  }
  if (meta.examples) {
    examples = examples.concat(meta.examples);
  }
  return examples;
}

/**
 * Compile Pug
 */

function compilePug(src, meta, filename, locals) {
  let newSrc = [src];

  // add example calls
  getExamples(meta).forEach(function (example, i) {
    // append to pug if it's a mixin example
    if (MIXIN_NAME_REGEX.test(src)) {
      newSrc.push(example);

      // replace example block with src
    } else {
      if (i === 0) {
        newSrc = [];
      }

      const lines = example.split("\n");
      lines.forEach(function (line) {
        if (line.trim() === EXAMPLE_BLOCK) {
          const indent = detectIndent(line).indent.length;
          line = rebaseIndent(src.split("\n"), indent).join("\n");
        }
        newSrc.push(line);
      });
    }
  });

  newSrc = newSrc.join("\n");

  locals = Object.assign({}, locals, meta.locals);

  // compile pug
  const compiled = pug.compileClient(newSrc, {
    name: "tmp",
    externalRuntime: true,
    filename: filename,
  });

  try {
    const templateFunc = pugRuntimeWrap(compiled, "tmp");
    return templateFunc(locals || {});
  } catch (err) {
    try {
      const compiledDebug = pug.compileClient(newSrc, {
        name: "tmp",
        externalRuntime: true,
        filename: filename,
        compileDebug: true,
      });
      const templateFuncDebug = pugRuntimeWrap(compiledDebug, "tmp");
      templateFuncDebug(locals || {});
    } catch (debugErr) {
      process.stderr.write(
        `\n\nPug-doc error: ${JSON.stringify(meta, null, 2)}`
      );
      process.stderr.write(`\n\n${debugErr.toString()}`);
      return null;
    }
  }
}

// Exports
module.exports = {
  extractPugdocBlocks: extractPugdocBlocks,
  getPugdocDocuments: getPugdocDocuments,
  parsePugdocComment: parsePugdocComment,
  getExamples: getExamples,
};
<commit_msg>remove unused require<commit_after>const pug = require("pug");
const pugRuntimeWrap = require("pug-runtime/wrap");

const path = require("path");
const YAML = require("js-yaml");
const getCodeBlock = require("pug-code-block");
const detectIndent = require("detect-indent");
const rebaseIndent = require("rebase-indent");
const pugdocArguments = require("./arguments");

const MIXIN_NAME_REGEX = /^mixin +([-\w]+)?/;
const DOC_REGEX = /^\s*\/\/-\s+?\@pugdoc\s*$/;
const DOC_STRING = "//- @pugdoc";
const CAPTURE_ALL = "all";
const CAPTURE_SECTION = "section";
const EXAMPLE_BLOCK = "block";

/**
 * Returns all pugdoc comment and code blocks for the given code
 *
 * @param templateSrc {string}
 * @return {{lineNumber: number, comment: string, code: string}[]}
 */

function extractPugdocBlocks(templateSrc) {
  return (
    templateSrc
      .split("\n")
      // Walk through every line and look for a pugdoc comment
      .map(function (line, lineIndex) {
        // If the line does not contain a pugdoc comment skip it
        if (!line.match(DOC_REGEX)) {
          return undefined;
        }

        // If the line contains a pugdoc comment return
        // the comment block and the next code block
        const comment = getCodeBlock.byLine(templateSrc, lineIndex + 1);
        const meta = parsePugdocComment(comment);

        // add number of captured blocks
        if (meta.capture <= 0) {
          return undefined;
        }

        let capture = 2;
        if (meta.capture) {
          if (meta.capture === CAPTURE_ALL) {
            capture = Infinity;
          } else if (meta.capture === CAPTURE_SECTION) {
            capture = Infinity;
          } else {
            capture = meta.capture + 1;
          }
        }

        // get all code blocks
        let code = getCodeBlock.byLine(templateSrc, lineIndex + 1, capture);

        // make string
        if (Array.isArray(code)) {
          // remove comment
          code.shift();

          // join all code
          code = code.join("\n");
        } else {
          return undefined;
        }

        // filter out all but current pugdoc section
        if (meta.capture === CAPTURE_SECTION) {
          const nextPugDocIndex = code.indexOf(DOC_STRING);
          if (nextPugDocIndex > -1) {
            code = code.substr(0, nextPugDocIndex);
          }
        }

        // if no code and no comment, skip
        if (comment.match(DOC_REGEX) && code === "") {
          return undefined;
        }

        return {
          lineNumber: lineIndex + 1,
          comment: comment,
          code: code,
        };
      })
      // Remove skiped lines
      .filter(function (result) {
        return result !== undefined;
      })
  );
}

/**
 * Returns all pugdocDocuments for the given code
 *
 * @param templateSrc {string}
 * @param filename {string}
 */

function getPugdocDocuments(templateSrc, filename, locals) {
  return extractPugdocBlocks(templateSrc).map(function (pugdocBlock) {
    const meta = parsePugdocComment(pugdocBlock.comment);
    const fragments = [];

    // parse jsdoc style arguments list
    if (meta.arguments) {
      meta.arguments = meta.arguments.map(function (arg) {
        return pugdocArguments.parse(arg, true);
      });
    }

    // parse jsdoc style attributes list
    if (meta.attributes) {
      meta.attributes = meta.attributes.map(function (arg) {
        return pugdocArguments.parse(arg, true);
      });
    }

    let source = pugdocBlock.code;
    source = source.replace(/\u2028|\u200B/g, "");

    if (meta.example && meta.example !== false) {
      if (meta.beforeEach) {
        meta.example = `${meta.beforeEach}\n${meta.example}`;
      }
      if (meta.afterEach) {
        meta.example = `${meta.example}\n${meta.afterEach}`;
      }
    }

    // get example objects and add them to parent example
    // also return them as separate pugdoc blocks
    if (meta.examples) {
      for (let i = 0, l = meta.examples.length; i < l; ++i) {
        let x = meta.examples[i];

        // do nothing for simple examples
        if (typeof x === "string") {
          if (meta.beforeEach) {
            meta.examples[i] = `${meta.beforeEach}\n${x}`;
          }
          if (meta.afterEach) {
            meta.examples[i] = `${x}\n${meta.afterEach}`;
          }
          continue;
        }

        if (meta.beforeEach && typeof x.beforeEach === "undefined") {
          x.example = `${meta.beforeEach}\n${x.example}`;
        }

        if (meta.afterEach && typeof x.afterEach === "undefined") {
          x.example = `${x.example}\n${meta.afterEach}`;
        }

        // merge example/examples with parent examples
        meta.examples[i] = getExamples(x).reduce(
          (acc, val) => acc.concat(val),
          []
        );

        // add fragments
        fragments.push(x);
      }

      meta.examples = meta.examples.reduce((acc, val) => acc.concat(val), []);
    }

    // fix pug compilation for boolean use of example
    const exampleClone = meta.example;
    if (typeof meta.example === "boolean") {
      meta.example = "";
    }

    const obj = {
      // get meta
      meta: meta,
      // add file path
      file: path.relative(".", filename),
      // get pug code block matching the comments indent
      source: source,
      // get html output
      output: compilePug(source, meta, filename, locals),
    };

    // remove output if example = false
    if (exampleClone === false) {
      obj.output = null;
    }

    // add fragments
    if (fragments && fragments.length) {
      obj.fragments = fragments.map((subexample) => {
        return {
          // get meta
          meta: subexample,
          // get html output
          output: compilePug(source, subexample, filename, locals),
        };
      });
    }

    if (obj.output || obj.fragments) {
      return obj;
    }

    return null;
  });
}

/**
 * Extract pug attributes from comment block
 */

function parsePugdocComment(comment) {
  // remove first line (@pugdoc)
  if (comment.indexOf("\n") === -1) {
    return {};
  }

  comment = comment.substr(comment.indexOf("\n"));
  comment = pugdocArguments.escapeArgumentsYAML(comment, "arguments");
  comment = pugdocArguments.escapeArgumentsYAML(comment, "attributes");

  // parse YAML
  return YAML.safeLoad(comment) || {};
}

/**
 * get all examples from the meta object
 * either one or both of meta.example and meta.examples can be given
 */

function getExamples(meta) {
  let examples = [];
  if (meta.example) {
    examples = examples.concat(meta.example);
  }
  if (meta.examples) {
    examples = examples.concat(meta.examples);
  }
  return examples;
}

/**
 * Compile Pug
 */

function compilePug(src, meta, filename, locals) {
  let newSrc = [src];

  // add example calls
  getExamples(meta).forEach(function (example, i) {
    // append to pug if it's a mixin example
    if (MIXIN_NAME_REGEX.test(src)) {
      newSrc.push(example);

      // replace example block with src
    } else {
      if (i === 0) {
        newSrc = [];
      }

      const lines = example.split("\n");
      lines.forEach(function (line) {
        if (line.trim() === EXAMPLE_BLOCK) {
          const indent = detectIndent(line).indent.length;
          line = rebaseIndent(src.split("\n"), indent).join("\n");
        }
        newSrc.push(line);
      });
    }
  });

  newSrc = newSrc.join("\n");

  locals = Object.assign({}, locals, meta.locals);

  // compile pug
  const compiled = pug.compileClient(newSrc, {
    name: "tmp",
    externalRuntime: true,
    filename: filename,
  });

  try {
    const templateFunc = pugRuntimeWrap(compiled, "tmp");
    return templateFunc(locals || {});
  } catch (err) {
    try {
      const compiledDebug = pug.compileClient(newSrc, {
        name: "tmp",
        externalRuntime: true,
        filename: filename,
        compileDebug: true,
      });
      const templateFuncDebug = pugRuntimeWrap(compiledDebug, "tmp");
      templateFuncDebug(locals || {});
    } catch (debugErr) {
      process.stderr.write(
        `\n\nPug-doc error: ${JSON.stringify(meta, null, 2)}`
      );
      process.stderr.write(`\n\n${debugErr.toString()}`);
      return null;
    }
  }
}

// Exports
module.exports = {
  extractPugdocBlocks: extractPugdocBlocks,
  getPugdocDocuments: getPugdocDocuments,
  parsePugdocComment: parsePugdocComment,
  getExamples: getExamples,
};
<|endoftext|>