<commit_before>import jsx from "babel-plugin-syntax-jsx";

export function element(tagName, attributes, children) {
  const tag = document.createElement(tagName);

  for (let [key, value] of Object.entries(attributes)) {
    tag.setAttribute(key, value);
  }

  children.forEach(child => tag.appendChild(child));

  return tag;
}

export function attributes(...attrs) {
  return Object.assign({}, ...attrs);
}

export function attributeStringLiteral(key, value) {
  return { [key]: value };
}

export function attributeEmpty(key) {
  return { [key]: key };
}

export function attributeExpression(key, value) {
  return { [key]: value.toString() };
}

export function attributeSpread(obj) {
  return obj;
}

export function children(...children) {
  return [].concat(...children);
}

export function childText(text) {
  return [document.createTextNode(text)];
}

export function childElement(jSXElement) {
  return [jSXElement];
}

export function childExpression(expression) {
  return [expression];
}

export function childSpread(array) {
  lively.notify(array);
  return array;
}

function detectUnsupportedNodes(path, filename) {
  function gainPrintableFullPath(path) {
    let fullPath = [];

    while(path) {
      fullPath.unshift(path.node.type);
      path = path.parentPath;
    }

    return fullPath.map((nodeType, index) => '  '.repeat(index) + nodeType).join('\n');
  }

  path.traverse({
    /**
     * No support for JSXMemberExpression yet. #TODO: what are the semantics outside of react for this?
     *
     * <foo.bar></foo.bar>
     */
    JSXMemberExpression(path, state) {
      throw new SyntaxError(`JSXMemberExpression not yet supported.
${gainPrintableFullPath(path)}`, filename, path.node.loc.start.line);
    },
    /**
     * No support for JSXEmptyExpression yet. #TODO: where is this feature even useful?
     *
     *

     * or
     *
{}

     */
    JSXEmptyExpression(path, state) {
      throw new SyntaxError(`JSXEmptyExpression not yet supported.
${gainPrintableFullPath(path)}`, filename);
    },
    /**
     * No support for JSXSpreadAttribute yet.
     *
     * let obj = { id: "myObj", class: "foo"};
     * ;
     */
    JSXSpreadAttribute(path, state) {
      throw new SyntaxError(`JSXSpreadAttribute not yet supported.
${gainPrintableFullPath(path)}`, filename, path.node.loc.start.line);
    },
    /**
     * No support for jSXNamespacedName yet.
     *
     *
;
     */
    JSXNamespacedName(path, state) {
      throw new SyntaxError(`jSXNamespacedName not yet supported.
${gainPrintableFullPath(path)}`, filename, path.node.loc.start.line);
    }
  });
}

export default function ({ types: t, template, traverse }) {
  const GENERATED_IMPORT_IDENTIFIER = Symbol("generated import identifier");

  // #TODO: duplicate with aexpr transform -> extract it
  function addCustomTemplate(file, name) {
    let declar = file.declarations[name];
    if (declar) return declar;

    let identifier = file.declarations[name] = file.addImport("https://lively-kernel.org/lively4/lively4-core/src/external/babel-plugin-jsx-lively.js", name, name);
    identifier[GENERATED_IMPORT_IDENTIFIER] = true;
    return identifier;
  }

  return {
    inherits: jsx,
    visitor: {
      Program(path, state) {
        detectUnsupportedNodes(path, state && state.opts && state.opts.filename);

        function transformPath(path, programState) {
          function jSXAttributeToBuilder(path) {

            function getCallExpressionFor(functionName, ...additionalParameters) {
              return t.callExpression(
                addCustomTemplate(programState.file, functionName), // builder function
                [
                  t.stringLiteral(path.get("name").node.name), // key
                  ...additionalParameters
                ]
              );
            }

            let attributeValue = path.get("value");
            if(!path.node.value) {
              return getCallExpressionFor("attributeEmpty");
            } else if(attributeValue.isStringLiteral()) {
              return getCallExpressionFor("attributeStringLiteral", attributeValue.node);
            } else if(attributeValue.isJSXExpressionContainer()) {
              return getCallExpressionFor("attributeExpression", attributeValue.node.expression);
            } else if(attributeValue.isJSXElement()) {
              // #TODO: what would that even mean?
              throw new SyntaxError(`JSXElement as property value of JSXAttribute not yet supported.`);
            }

            throw new Error('unknown node type in JSXAttribute value ' + attributeValue.node.type);
          }

          function jSXChildrenToBuilder(child) {
            function getCallExpressionFor(functionName, childSpec) {
              return t.callExpression(
                addCustomTemplate(programState.file, functionName), // builder function
                [
                  childSpec
                ]
              );
            }

            if(child.isJSXText()) {
              return getCallExpressionFor("childText", t.stringLiteral(child.node.value));
            }
            if(child.isJSXElement()) {
              return getCallExpressionFor("childElement", child.node);
            }
            if(child.isJSXExpressionContainer()) {
              return getCallExpressionFor("childExpression", child.get("expression").node);
            }
            if(child.isJSXSpreadChild()) {
              return getCallExpressionFor("childSpread", child.get("expression").node);
              //throw new SyntaxError(`JSXSpreadChild as child of JSXElement not yet supported.`);
            }
            throw new Error('unknown node type in children of JSXElement ' + child.node.type);
          }

          path.traverse({
            JSXElement(path, state) {
              const jSXAttributes = path.get("openingElement").get("attributes");
              const jSXChildren = path.get("children");
              lively.notify(jSXChildren.length);

              let newNode = t.callExpression(
                addCustomTemplate(programState.file, "element"),
                [
                  t.stringLiteral(path.get("openingElement").get("name").node.name),
                  t.callExpression(
                    addCustomTemplate(programState.file, "attributes"),
                    jSXAttributes.map(jSXAttributeToBuilder)
                  ),
                  t.callExpression(
                    addCustomTemplate(programState.file, "children"),
                    jSXChildren.map(jSXChildrenToBuilder)
                  ),
                ]
              );

              path.replaceWith(newNode);
            }
          })

          return path;
        }

        transformPath(path, state);
      }
    }
  };
}
<commit_msg>added spread attribute for transformation and logic<commit_after>import jsx from "babel-plugin-syntax-jsx";

export function element(tagName, attributes, children) {
  const tag = document.createElement(tagName);

  for (let [key, value] of Object.entries(attributes)) {
    tag.setAttribute(key, value);
  }

  children.forEach(child => tag.appendChild(child));

  return tag;
}

export function attributes(...attrs) {
  return Object.assign({}, ...attrs);
}

export function attributeStringLiteral(key, value) {
  return { [key]: value };
}

export function attributeEmpty(key) {
  return { [key]: key };
}

export function attributeExpression(key, value) {
  return { [key]: value.toString() };
}

export function attributeSpread(obj) {
  return obj;
}

export function children(...children) {
  return [].concat(...children);
}

export function childText(text) {
  return [document.createTextNode(text)];
}

export function childElement(jSXElement) {
  return [jSXElement];
}

export function childExpression(expression) {
  return [expression];
}

export function childSpread(array) {
  lively.notify(array);
  return array;
}

function detectUnsupportedNodes(path, filename) {
  function gainPrintableFullPath(path) {
    let fullPath = [];

    while(path) {
      fullPath.unshift(path.node.type);
      path = path.parentPath;
    }

    return fullPath.map((nodeType, index) => '  '.repeat(index) + nodeType).join('\n');
  }

  path.traverse({
    /**
     * No support for JSXMemberExpression yet. #TODO: what are the semantics outside of react for this?
     *
     * <foo.bar></foo.bar>
     */
    JSXMemberExpression(path, state) {
      throw new SyntaxError(`JSXMemberExpression not yet supported.
${gainPrintableFullPath(path)}`, filename, path.node.loc.start.line);
    },
    /**
     * No support for JSXEmptyExpression yet. #TODO: where is this feature even useful?
     *
     *

     * or
     *
{}

     */
    JSXEmptyExpression(path, state) {
      throw new SyntaxError(`JSXEmptyExpression not yet supported.
${gainPrintableFullPath(path)}`, filename);
    },
    /**
     * No support for jSXNamespacedName yet.
     *
     *
;
     */
    JSXNamespacedName(path, state) {
      throw new SyntaxError(`jSXNamespacedName not yet supported.
${gainPrintableFullPath(path)}`, filename, path.node.loc.start.line);
    }
  });
}

export default function ({ types: t, template, traverse }) {
  const GENERATED_IMPORT_IDENTIFIER = Symbol("generated import identifier");

  // #TODO: duplicate with aexpr transform -> extract it
  function addCustomTemplate(file, name) {
    let declar = file.declarations[name];
    if (declar) return declar;

    let identifier = file.declarations[name] = file.addImport("https://lively-kernel.org/lively4/lively4-core/src/external/babel-plugin-jsx-lively.js", name, name);
    identifier[GENERATED_IMPORT_IDENTIFIER] = true;
    return identifier;
  }

  return {
    inherits: jsx,
    visitor: {
      Program(path, state) {
        detectUnsupportedNodes(path, state && state.opts && state.opts.filename);

        function transformPath(path, programState) {
          function jSXAttributeToBuilder(path) {

            function getCallExpressionFor(functionName, ...additionalParameters) {
              return t.callExpression(
                addCustomTemplate(programState.file, functionName), // builder function
                [
                  t.stringLiteral(path.get("name").node.name), // key
                  ...additionalParameters
                ]
              );
            }

            let attributeValue = path.get("value");
            if(path.isJSXSpreadAttribute()) {
              return t.callExpression(
                addCustomTemplate(programState.file, "attributeSpread"),
                [ path.get("argument").node ]
              );
            } else if(!path.node.value) {
              return getCallExpressionFor("attributeEmpty");
            } else if(attributeValue.isStringLiteral()) {
              return getCallExpressionFor("attributeStringLiteral", attributeValue.node);
            } else if(attributeValue.isJSXExpressionContainer()) {
              return getCallExpressionFor("attributeExpression", attributeValue.node.expression);
            } else if(attributeValue.isJSXElement()) {
              // #TODO: what would that even mean?
              throw new SyntaxError(`JSXElement as property value of JSXAttribute not yet supported.`);
            }

            throw new Error('unknown node type in JSXAttribute value ' + attributeValue.node.type);
          }

          function jSXChildrenToBuilder(child) {
            function getCallExpressionFor(functionName, childSpec) {
              return t.callExpression(
                addCustomTemplate(programState.file, functionName), // builder function
                [
                  childSpec
                ]
              );
            }

            if(child.isJSXText()) {
              return getCallExpressionFor("childText", t.stringLiteral(child.node.value));
            }
            if(child.isJSXElement()) {
              return getCallExpressionFor("childElement", child.node);
            }
            if(child.isJSXExpressionContainer()) {
              return getCallExpressionFor("childExpression", child.get("expression").node);
            }
            if(child.isJSXSpreadChild()) {
              return getCallExpressionFor("childSpread", child.get("expression").node);
              //throw new SyntaxError(`JSXSpreadChild as child of JSXElement not yet supported.`);
            }
            throw new Error('unknown node type in children of JSXElement ' + child.node.type);
          }

          path.traverse({
            JSXElement(path, state) {
              const jSXAttributes = path.get("openingElement").get("attributes");
              const jSXChildren = path.get("children");
              lively.notify(jSXChildren.length);

              let newNode = t.callExpression(
                addCustomTemplate(programState.file, "element"),
                [
                  t.stringLiteral(path.get("openingElement").get("name").node.name),
                  t.callExpression(
                    addCustomTemplate(programState.file, "attributes"),
                    jSXAttributes.map(jSXAttributeToBuilder)
                  ),
                  t.callExpression(
                    addCustomTemplate(programState.file, "children"),
                    jSXChildren.map(jSXChildrenToBuilder)
                  ),
                ]
              );

              path.replaceWith(newNode);
            }
          })

          return path;
        }

        transformPath(path, state);
      }
    }
  };
}
<|endoftext|>


<commit_before>const moment = require('moment')

module.exports = function (date) {
  return moment(date).format('dddd D MMMM YYYY')
}
<commit_msg>Updated date helper to not add hour when after DST as already added (#528)<commit_after>const moment = require('moment')

module.exports = function (date) {
  return moment.utc(date).format('dddd D MMMM YYYY')
}
<|endoftext|>


<commit_before>/**
 * RtK Kanji flashcard review.
 *
 * Options:
 *
 *   fcr_options   Options to setup uiFlashcardReview
 *   end_url       Url to redirect to at the end of the review
 *
 */
// @ts-check

import $$, { DomJS, asHtmlElement, domGetById, hasClass } from "@lib/dom";
import AjaxDialog from "@old/ajaxdialog";
import DictLookupDialog from "@old/components/DictLookupDialog";
import EditFlashcardDialog from "@old/components/EditFlashcardDialog";
import EditStoryDialog from "@old/components/EditStoryDialog";
import FlashcardReview, { FCRATE } from "@app/review/FlashcardReview";
import ReviewPage from "@app/review/ReviewPage";

/** @typedef {{end_url: string, editstory_url: string}} TReviewProps */

export default class KanjiReview {
  /** @type {TReviewProps} */
  options;

  /** @type {FlashcardReview} */
  oReview;

  /** @type {DomJS<Element>}*/
  $elStats;
  /** @type {HTMLElement} */
  elProgressBar;

  /** @type {DictLookupDialog?} */
  dictDialog = null;

  /** @type {EditFlashcardDialog?} */
  oEditFlashcard = null;
  /** @type {TUcsId} */
  oEditFlashcardId = 0;

  /** @type {EditStoryDialog?} */
  editStoryDialog = null;

  /** @type {number[]} */
  deletedCards = [];

  /**
   *
   * @param {TReviewOptions} fcrOptions ... options for FlashcardReview instance
   * @param {TReviewProps} props ... props for Vue component (TBD refactor)
   */
  constructor(fcrOptions, props) {
    this.options = props;

    fcrOptions.events = {
      onEndReview: this.onEndReview,
      onFlashcardCreate: this.onFlashcardCreate,
      onFlashcardDestroy: this.onFlashcardDestroy,
      onFlashcardState: this.onFlashcardState,
      onFlashcardUndo: this.onFlashcardUndo,
    };
    fcrOptions.scope = this;

    this.oReview = new FlashcardReview(fcrOptions);

    this.reviewPage = new ReviewPage(this.onAction.bind(this));

    this.reviewPage.addShortcutKey("f", "flip");
    this.reviewPage.addShortcutKey(" ", "flip");
    this.reviewPage.addShortcutKey(96, "flip"); // NUMPAD_0

    this.reviewPage.addShortcutKey("n", "no");
    this.reviewPage.addShortcutKey("g", "again");
    this.reviewPage.addShortcutKey("h", "hard");
    this.reviewPage.addShortcutKey("y", "yes");
    this.reviewPage.addShortcutKey("e", "easy");

    // added number keys to answer with just left hand
    this.reviewPage.addShortcutKey("1", "no");
    this.reviewPage.addShortcutKey("2", "hard");
    this.reviewPage.addShortcutKey("3", "yes");
    this.reviewPage.addShortcutKey("4", "easy");

    // same for numpad keys
    this.reviewPage.addShortcutKey(97, "no"); // NUMPAD_1
    this.reviewPage.addShortcutKey(98, "hard"); // NUMPAD_2
    this.reviewPage.addShortcutKey(99, "yes"); // NUMPDA_3
    this.reviewPage.addShortcutKey(100, "easy"); // NUMPAD_4

    this.reviewPage.addShortcutKey("u", "undo");
    this.reviewPage.addShortcutKey("s", "story");
    this.reviewPage.addShortcutKey("d", "dict");

    // skip flashcard (110 = comma)
    this.reviewPage.addShortcutKey("k", "skip");
    this.reviewPage.addShortcutKey(110, "skip"); // NUMPAD_DECIMAL

    // Disabled because it's next to (F)lip Card
    //this.reviewPage.addShortcutKey('d', 'delete');

    // flashcad container
    // this.elFlashcard = $$('.uiFcCard')[0];

    // stats panel
    this.$elStats = $$("#uiFcStats");
    this.elsCount = $$("#uiFcProgressBar .count");
    this.elProgressBar = asHtmlElement($$("#review-progress span")[0]);

    // answer stats
    this.elAnswerPass = this.$elStats.down(".JsPass")[0];
    this.elAnswerFail = this.$elStats.down(".JsFail")[0];
    this.countYes = 0;
    this.countNo = 0;

    this.countDeleted = 0;
    this.deletedCards = [];

    // end review div
    this.elFinish = asHtmlElement(this.$elStats.down(".JsFinish")[0]);
  }

  /**
   * Returns an option value
   *
   * @param {keyof TReviewProps} name
   */
  getOption(name) {
    return this.options[name];
  }

  // proxy which typecasts the card data, and *always* returns a valid card
  getFlashcardData() {
    return /**@type {TCardData}*/ (this.oReview.getFlashcardData());
  }

  /**
   * Update the visible stats to the latest server hit,
   * and setup form data for redirection to the Review Summary page.
   *
   */
  onEndReview() {
    console.log("KanjiReview.onEndReview()");

    this.updateStatsPanel();

    var elFrm = /**@type {HTMLFormElement}*/ (domGetById("uiFcRedirectForm"));

    // set form data and redirect to summary with POST
    elFrm.method = "post";
    elFrm.action = this.getOption("end_url");
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_pass")).value = "" + this.countYes;
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_fail")).value = "" + this.countNo;
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_deld")).value = this.deletedCards.join(",");
    elFrm.submit();
  }

  onFlashcardCreate() {
    // console.log("KanjiReview.onFlashcardCreate()");

    // Show panels when first card is loaded
    if (this.oReview.getPosition() === 0) {
      this.$elStats.display();
    }

    // Show undo action if available
    $$("#JsBtnUndo").display(this.oReview.getNumUndos() > 0);

    this.updateStatsPanel();
  }

  /**
   * Hide buttons until next card shows up.
   *
   */
  onFlashcardDestroy() {
    $$("#uiFcButtons0").display(false);
    $$("#uiFcButtons1").display(false);
  }

  /**
   *
   * @param {TCardAnswer} oAnswer
   */
  onFlashcardUndo(oAnswer) {
    //  console.log('onFlashcardUndo(%o)', oAnswer);

    // correct the Yes / No totals
    this.updateAnswerStats(oAnswer.id, oAnswer.r, true);
  }

  /** @param {number} iState */
  onFlashcardState(iState) {
    // console.log("onFlashcardState(%d)", iState);
    $$("#uiFcButtons0").display(iState === 0);
    $$("#uiFcButtons1").display(iState !== 0);
  }

  /**
   *
   * @param {string} sActionId cf. eventdispatcher
   * @param {Event} oEvent ...
   * @returns
   */
  onAction(sActionId, oEvent) {
    /** @type {TReviewRating=} */
    let cardRating;

    console.log("KanjiReview.onAction(%o)", sActionId);

    // help dialog
    if (sActionId === "help") {
      var dlg = new AjaxDialog("#JsFcHelpDlg", {
        useMarkup: true,
        context: ["JsBtnHelp", "tl", "bl", null, [0, 0]],
        skin: "rtk-skin-dlg",
        mobile: true,
        close: false,
      });
      dlg.show();

      return false;
    }

    // flashcard is loading
    if (!this.oReview.getFlashcard()) {
      return false;
    }

    if (sActionId === "story") {
      this.toggleEditStory();
      return false;
    }

    if (sActionId === "dict") {
      this.toggleDictDialog();
      return false;
    }

    switch (sActionId) {
      case "fcmenu":
        this.flashcardMenu();
        break;
      case "delete":
        this.answerCard("delete");
        break;

      case "flip":
        if (oEvent.type === "click" && hasClass(asHtmlElement(oEvent.target), "JsKeywordLink")) {
          // pass through so the link functions
          return true;
        }
        if (this.oReview.getFlashcardState() === 0) {
          this.oReview.setFlashcardState(1);
        }
        break;

      case "undo":
        if (this.oReview.getNumUndos() > 0) {
          this.oReview.backward();
        }
        break;

      case "end":
        this.elFinish.style.display = "none";
        this.oReview.endReview(); // this will notify onEndReview()
        break;

      case "skip":
        this.answerCard("skip");
        break;

      case "no":
        cardRating = "no";
        break;

      case "again":
        cardRating = "again";
        break;

      case "hard":
        cardRating = "hard";
        break;

      case "yes":
        cardRating = "yes";
        break;

      case "easy":
        cardRating = "easy";
        break;
    }

    if (cardRating) {
      // "No" answer doesn't require flipping the card first (issue #163)
      if (sActionId === "no" || this.oReview.getFlashcardState() > 0) {
        this.answerCard(cardRating);
      }
    }

    return false;
  }

  toggleEditStory() {
    if (this.editStoryDialog && this.editStoryDialog.isVisible()) {
      this.editStoryDialog.hide();
    } else {
      const oCardData = this.getFlashcardData();

      if (!this.editStoryDialog) {
        // initialize Story Window and its position
        //var left = this.elFlashcard.offsetLeft + (this.elFlashcard.offsetWidth /2) - (520/2);
        //var top = this.elFlashcard.offsetTop + 61;
        this.editStoryDialog = new EditStoryDialog(this.getOption("editstory_url"), oCardData.id);
      } else {
        this.editStoryDialog.load(oCardData.id);
        this.editStoryDialog.show();
      }
    }
  }

  toggleDictDialog() {
    if (this.dictDialog && this.dictDialog.isVisible()) {
      this.dictDialog.hide();
    } else {
      const oCardData = this.getFlashcardData();
      const ucsId = oCardData.id;

      if (!this.dictDialog) {
        this.dictDialog = new DictLookupDialog();
      }

      this.dictDialog.show();
      this.dictDialog.load(ucsId);
    }
  }

  /**
   * Rate card (or mark action like delete/skip), then forward.
   *
   * @param {TReviewRating} answer
   */
  answerCard(answer) {
    const oCardData = this.getFlashcardData();

    /** @type {TCardAnswer} */
    let oAnswer = { id: oCardData.id, r: answer };

    this.oReview.answerCard(oAnswer);
    this.updateAnswerStats(oAnswer.id, oAnswer.r, false);
    this.oReview.forward();
  }

  /**
   * The little wrench icon that opens the menu contains:
   *
   *  data-param  {"review":1}    JSON data passed on the the menu ajax post (plus ucs id)
   *  data-uri                    Flashcard Edit Dialog ajax url
   *
   */
  flashcardMenu() {
    let el = /** @type {HTMLElement} */ ($$("#uiFcMenu")[0]);

    let data = /** @type {{uri: string, param: string}} */ (el.dataset);

    let oCardData = /** @type {TCardData}*/ (this.oReview.getFlashcardData());

    const onMenuHide = () => {
      // clear icon focus state when dialog closes
      el.classList.remove("active");
    };

    /** @param {string} menuid */
    const onMenuItem = (menuid) => {
      if (menuid === "confirm-delete") {
        // set flashcard answer that tells server to delete the card
        this.answerCard("delete");
        return true;
      } else if (menuid === "skip") {
        this.answerCard("skip");
        return true;
      }

      // does not close dialog
      return false;
    };

    el.classList.add("active");

    // reload the edit flashcard menu when changed flashcard
    if (oCardData.id !== this.oEditFlashcardId) {
      this.oEditFlashcardId = oCardData.id;

      if (this.oEditFlashcard) {
        this.oEditFlashcard.destroy();
        this.oEditFlashcard = null;
      }
    }

    if (!this.oEditFlashcard) {
      var params = {
        ...JSON.parse(data.param),
        ...{ ucs: oCardData.id },
      };
      // console.log("zomg %o", params);return false;

      this.oEditFlashcard = new EditFlashcardDialog(data.uri, params, [el, "tr", "br"], {
        events: {
          onMenuHide: onMenuHide,
          onMenuItem: onMenuItem,
        },
      });
    } else {
      this.oEditFlashcard.show();
    }

    return false;
  }

  updateStatsPanel() {
    //  console.log('KanjiReview.updateStatsPanel()');
    const total = this.oReview.numCards;
    const pos = this.oReview.numRated;

    // update review count, don't show "4 of 3" after answering last card
    this.elsCount[0].innerHTML = "" + (total - pos);

    let elCount = $$("#uiFcProgressBar h3")[0];
    elCount.innerHTML = `Rated: <em>${pos}/${this.oReview.numCards}</em>  Again: <em>${this.oReview.numAgain}</em>`;

    // update progress bar
    let pct = pos > 0 ? Math.ceil((pos * 100) / total) : 0;
    pct = Math.min(pct, 100);
    this.elProgressBar.style.width = (pct > 0 ? pct : 0) + "%";
  }

  /**
   *
   * @param  {TUcsId} id ... the card's id (the kanji UCS code)
   * @param {TReviewRating} rating
   * @param  {boolean} isUndo
   */
  updateAnswerStats(id, rating, isUndo) {
    // cf. uiFlashcardReview.php const
    let yes = rating === FCRATE.YES || rating === FCRATE.EASY ? 1 : 0;
    let no = rating === FCRATE.NO || rating === FCRATE.HARD ? 1 : 0;
    let deld = rating === FCRATE.DELETE ? 1 : 0;

    if (isUndo) {
      yes = -yes;
      no = -no;
      deld = -deld;
    }

    this.countYes += yes;
    this.countNo += no;
    this.elAnswerPass.innerHTML = "" + this.countYes;
    this.elAnswerFail.innerHTML = "" + this.countNo;

    if (deld !== 0) {
      this.updateDeletedCards(id, deld);
    }
  }

  /**
   *
   * @param {TUcsId} ucsId
   * @param {number} count
   */
  updateDeletedCards(ucsId, count) {
    this.countDeleted += count;

    if (count > 0) {
      this.deletedCards.push(ucsId);
    } else if (count < 0) {
      this.deletedCards.pop();
    }

    $$("#uiFcStDeld").display(this.countDeleted > 0);
    $$("#uiFcStDeld em")[0].innerHTML = "" + this.countDeleted;
    $$("#uiFcStDeldK span")[0].innerHTML = this.getDeletedCards();
  }

  getDeletedCards() {
    return "&#" + this.deletedCards.join(";&#") + ";";
  }

  /**
   * Sets buttons (children of element) to default state, or disabled state
   *
   * @param {HTMLElement} elParent
   * @param {boolean} bEnabled
   */
  setButtonState(elParent, bEnabled) {
    $$(".uiIBtn", elParent).each((el) => {
      el.classList.toggle("uiFcBtnDisabled", bEnabled);
    });
  }
}
<commit_msg>Use "a" key shortcut for Again rating (instead of "g")<commit_after>/**
 * RtK Kanji flashcard review.
 *
 * Options:
 *
 *   fcr_options   Options to setup uiFlashcardReview
 *   end_url       Url to redirect to at the end of the review
 *
 */
// @ts-check

import $$, { DomJS, asHtmlElement, domGetById, hasClass } from "@lib/dom";
import AjaxDialog from "@old/ajaxdialog";
import DictLookupDialog from "@old/components/DictLookupDialog";
import EditFlashcardDialog from "@old/components/EditFlashcardDialog";
import EditStoryDialog from "@old/components/EditStoryDialog";
import FlashcardReview, { FCRATE } from "@app/review/FlashcardReview";
import ReviewPage from "@app/review/ReviewPage";

/** @typedef {{end_url: string, editstory_url: string}} TReviewProps */

export default class KanjiReview {
  /** @type {TReviewProps} */
  options;

  /** @type {FlashcardReview} */
  oReview;

  /** @type {DomJS<Element>}*/
  $elStats;
  /** @type {HTMLElement} */
  elProgressBar;

  /** @type {DictLookupDialog?} */
  dictDialog = null;

  /** @type {EditFlashcardDialog?} */
  oEditFlashcard = null;
  /** @type {TUcsId} */
  oEditFlashcardId = 0;

  /** @type {EditStoryDialog?} */
  editStoryDialog = null;

  /** @type {number[]} */
  deletedCards = [];

  /**
   *
   * @param {TReviewOptions} fcrOptions ... options for FlashcardReview instance
   * @param {TReviewProps} props ... props for Vue component (TBD refactor)
   */
  constructor(fcrOptions, props) {
    this.options = props;

    fcrOptions.events = {
      onEndReview: this.onEndReview,
      onFlashcardCreate: this.onFlashcardCreate,
      onFlashcardDestroy: this.onFlashcardDestroy,
      onFlashcardState: this.onFlashcardState,
      onFlashcardUndo: this.onFlashcardUndo,
    };
    fcrOptions.scope = this;

    this.oReview = new FlashcardReview(fcrOptions);

    this.reviewPage = new ReviewPage(this.onAction.bind(this));

    this.reviewPage.addShortcutKey("f", "flip");
    this.reviewPage.addShortcutKey(" ", "flip");
    this.reviewPage.addShortcutKey(96, "flip"); // NUMPAD_0

    this.reviewPage.addShortcutKey("n", "no");
    this.reviewPage.addShortcutKey("a", "again");
    this.reviewPage.addShortcutKey("h", "hard");
    this.reviewPage.addShortcutKey("y", "yes");
    this.reviewPage.addShortcutKey("e", "easy");

    // added number keys to answer with just left hand
    this.reviewPage.addShortcutKey("1", "no");
    this.reviewPage.addShortcutKey("2", "hard");
    this.reviewPage.addShortcutKey("3", "yes");
    this.reviewPage.addShortcutKey("4", "easy");

    // same for numpad keys
    this.reviewPage.addShortcutKey(97, "no"); // NUMPAD_1
    this.reviewPage.addShortcutKey(98, "hard"); // NUMPAD_2
    this.reviewPage.addShortcutKey(99, "yes"); // NUMPDA_3
    this.reviewPage.addShortcutKey(100, "easy"); // NUMPAD_4

    this.reviewPage.addShortcutKey("u", "undo");
    this.reviewPage.addShortcutKey("s", "story");
    this.reviewPage.addShortcutKey("d", "dict");

    // skip flashcard (110 = comma)
    this.reviewPage.addShortcutKey("k", "skip");
    this.reviewPage.addShortcutKey(110, "skip"); // NUMPAD_DECIMAL

    // Disabled because it's next to (F)lip Card
    //this.reviewPage.addShortcutKey('d', 'delete');

    // flashcad container
    // this.elFlashcard = $$('.uiFcCard')[0];

    // stats panel
    this.$elStats = $$("#uiFcStats");
    this.elsCount = $$("#uiFcProgressBar .count");
    this.elProgressBar = asHtmlElement($$("#review-progress span")[0]);

    // answer stats
    this.elAnswerPass = this.$elStats.down(".JsPass")[0];
    this.elAnswerFail = this.$elStats.down(".JsFail")[0];
    this.countYes = 0;
    this.countNo = 0;

    this.countDeleted = 0;
    this.deletedCards = [];

    // end review div
    this.elFinish = asHtmlElement(this.$elStats.down(".JsFinish")[0]);
  }

  /**
   * Returns an option value
   *
   * @param {keyof TReviewProps} name
   */
  getOption(name) {
    return this.options[name];
  }

  // proxy which typecasts the card data, and *always* returns a valid card
  getFlashcardData() {
    return /**@type {TCardData}*/ (this.oReview.getFlashcardData());
  }

  /**
   * Update the visible stats to the latest server hit,
   * and setup form data for redirection to the Review Summary page.
   *
   */
  onEndReview() {
    console.log("KanjiReview.onEndReview()");

    this.updateStatsPanel();

    var elFrm = /**@type {HTMLFormElement}*/ (domGetById("uiFcRedirectForm"));

    // set form data and redirect to summary with POST
    elFrm.method = "post";
    elFrm.action = this.getOption("end_url");
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_pass")).value = "" + this.countYes;
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_fail")).value = "" + this.countNo;
    /**@type{HTMLInputElement}*/ (elFrm.elements.namedItem("fc_deld")).value = this.deletedCards.join(",");
    elFrm.submit();
  }

  onFlashcardCreate() {
    // console.log("KanjiReview.onFlashcardCreate()");

    // Show panels when first card is loaded
    if (this.oReview.getPosition() === 0) {
      this.$elStats.display();
    }

    // Show undo action if available
    $$("#JsBtnUndo").display(this.oReview.getNumUndos() > 0);

    this.updateStatsPanel();
  }

  /**
   * Hide buttons until next card shows up.
   *
   */
  onFlashcardDestroy() {
    $$("#uiFcButtons0").display(false);
    $$("#uiFcButtons1").display(false);
  }

  /**
   *
   * @param {TCardAnswer} oAnswer
   */
  onFlashcardUndo(oAnswer) {
    //  console.log('onFlashcardUndo(%o)', oAnswer);

    // correct the Yes / No totals
    this.updateAnswerStats(oAnswer.id, oAnswer.r, true);
  }

  /** @param {number} iState */
  onFlashcardState(iState) {
    // console.log("onFlashcardState(%d)", iState);
    $$("#uiFcButtons0").display(iState === 0);
    $$("#uiFcButtons1").display(iState !== 0);
  }

  /**
   *
   * @param {string} sActionId cf. eventdispatcher
   * @param {Event} oEvent ...
   * @returns
   */
  onAction(sActionId, oEvent) {
    /** @type {TReviewRating=} */
    let cardRating;

    console.log("KanjiReview.onAction(%o)", sActionId);

    // help dialog
    if (sActionId === "help") {
      var dlg = new AjaxDialog("#JsFcHelpDlg", {
        useMarkup: true,
        context: ["JsBtnHelp", "tl", "bl", null, [0, 0]],
        skin: "rtk-skin-dlg",
        mobile: true,
        close: false,
      });
      dlg.show();

      return false;
    }

    // flashcard is loading
    if (!this.oReview.getFlashcard()) {
      return false;
    }

    if (sActionId === "story") {
      this.toggleEditStory();
      return false;
    }

    if (sActionId === "dict") {
      this.toggleDictDialog();
      return false;
    }

    switch (sActionId) {
      case "fcmenu":
        this.flashcardMenu();
        break;
      case "delete":
        this.answerCard("delete");
        break;

      case "flip":
        if (oEvent.type === "click" && hasClass(asHtmlElement(oEvent.target), "JsKeywordLink")) {
          // pass through so the link functions
          return true;
        }
        if (this.oReview.getFlashcardState() === 0) {
          this.oReview.setFlashcardState(1);
        }
        break;

      case "undo":
        if (this.oReview.getNumUndos() > 0) {
          this.oReview.backward();
        }
        break;

      case "end":
        this.elFinish.style.display = "none";
        this.oReview.endReview(); // this will notify onEndReview()
        break;

      case "skip":
        this.answerCard("skip");
        break;

      case "no":
        cardRating = "no";
        break;

      case "again":
        cardRating = "again";
        break;

      case "hard":
        cardRating = "hard";
        break;

      case "yes":
        cardRating = "yes";
        break;

      case "easy":
        cardRating = "easy";
        break;
    }

    if (cardRating) {
      // "No" answer doesn't require flipping the card first (issue #163)
      if (sActionId === "no" || this.oReview.getFlashcardState() > 0) {
        this.answerCard(cardRating);
      }
    }

    return false;
  }

  toggleEditStory() {
    if (this.editStoryDialog && this.editStoryDialog.isVisible()) {
      this.editStoryDialog.hide();
    } else {
      const oCardData = this.getFlashcardData();

      if (!this.editStoryDialog) {
        // initialize Story Window and its position
        //var left = this.elFlashcard.offsetLeft + (this.elFlashcard.offsetWidth /2) - (520/2);
        //var top = this.elFlashcard.offsetTop + 61;
        this.editStoryDialog = new EditStoryDialog(this.getOption("editstory_url"), oCardData.id);
      } else {
        this.editStoryDialog.load(oCardData.id);
        this.editStoryDialog.show();
      }
    }
  }

  toggleDictDialog() {
    if (this.dictDialog && this.dictDialog.isVisible()) {
      this.dictDialog.hide();
    } else {
      const oCardData = this.getFlashcardData();
      const ucsId = oCardData.id;

      if (!this.dictDialog) {
        this.dictDialog = new DictLookupDialog();
      }

      this.dictDialog.show();
      this.dictDialog.load(ucsId);
    }
  }

  /**
   * Rate card (or mark action like delete/skip), then forward.
   *
   * @param {TReviewRating} answer
   */
  answerCard(answer) {
    const oCardData = this.getFlashcardData();

    /** @type {TCardAnswer} */
    let oAnswer = { id: oCardData.id, r: answer };

    this.oReview.answerCard(oAnswer);
    this.updateAnswerStats(oAnswer.id, oAnswer.r, false);
    this.oReview.forward();
  }

  /**
   * The little wrench icon that opens the menu contains:
   *
   *  data-param  {"review":1}    JSON data passed on the the menu ajax post (plus ucs id)
   *  data-uri                    Flashcard Edit Dialog ajax url
   *
   */
  flashcardMenu() {
    let el = /** @type {HTMLElement} */ ($$("#uiFcMenu")[0]);

    let data = /** @type {{uri: string, param: string}} */ (el.dataset);

    let oCardData = /** @type {TCardData}*/ (this.oReview.getFlashcardData());

    const onMenuHide = () => {
      // clear icon focus state when dialog closes
      el.classList.remove("active");
    };

    /** @param {string} menuid */
    const onMenuItem = (menuid) => {
      if (menuid === "confirm-delete") {
        // set flashcard answer that tells server to delete the card
        this.answerCard("delete");
        return true;
      } else if (menuid === "skip") {
        this.answerCard("skip");
        return true;
      }

      // does not close dialog
      return false;
    };

    el.classList.add("active");

    // reload the edit flashcard menu when changed flashcard
    if (oCardData.id !== this.oEditFlashcardId) {
      this.oEditFlashcardId = oCardData.id;

      if (this.oEditFlashcard) {
        this.oEditFlashcard.destroy();
        this.oEditFlashcard = null;
      }
    }

    if (!this.oEditFlashcard) {
      var params = {
        ...JSON.parse(data.param),
        ...{ ucs: oCardData.id },
      };
      // console.log("zomg %o", params);return false;

      this.oEditFlashcard = new EditFlashcardDialog(data.uri, params, [el, "tr", "br"], {
        events: {
          onMenuHide: onMenuHide,
          onMenuItem: onMenuItem,
        },
      });
    } else {
      this.oEditFlashcard.show();
    }

    return false;
  }

  updateStatsPanel() {
    //  console.log('KanjiReview.updateStatsPanel()');
    const total = this.oReview.numCards;
    const pos = this.oReview.numRated;

    // update review count, don't show "4 of 3" after answering last card
    this.elsCount[0].innerHTML = "" + (total - pos);

    let elCount = $$("#uiFcProgressBar h3")[0];
    elCount.innerHTML = `Rated: <em>${pos}/${this.oReview.numCards}</em>  Again: <em>${this.oReview.numAgain}</em>`;

    // update progress bar
    let pct = pos > 0 ? Math.ceil((pos * 100) / total) : 0;
    pct = Math.min(pct, 100);
    this.elProgressBar.style.width = (pct > 0 ? pct : 0) + "%";
  }

  /**
   *
   * @param  {TUcsId} id ... the card's id (the kanji UCS code)
   * @param {TReviewRating} rating
   * @param  {boolean} isUndo
   */
  updateAnswerStats(id, rating, isUndo) {
    // cf. uiFlashcardReview.php const
    let yes = rating === FCRATE.YES || rating === FCRATE.EASY ? 1 : 0;
    let no = rating === FCRATE.NO || rating === FCRATE.HARD ? 1 : 0;
    let deld = rating === FCRATE.DELETE ? 1 : 0;

    if (isUndo) {
      yes = -yes;
      no = -no;
      deld = -deld;
    }

    this.countYes += yes;
    this.countNo += no;
    this.elAnswerPass.innerHTML = "" + this.countYes;
    this.elAnswerFail.innerHTML = "" + this.countNo;

    if (deld !== 0) {
      this.updateDeletedCards(id, deld);
    }
  }

  /**
   *
   * @param {TUcsId} ucsId
   * @param {number} count
   */
  updateDeletedCards(ucsId, count) {
    this.countDeleted += count;

    if (count > 0) {
      this.deletedCards.push(ucsId);
    } else if (count < 0) {
      this.deletedCards.pop();
    }

    $$("#uiFcStDeld").display(this.countDeleted > 0);
    $$("#uiFcStDeld em")[0].innerHTML = "" + this.countDeleted;
    $$("#uiFcStDeldK span")[0].innerHTML = this.getDeletedCards();
  }

  getDeletedCards() {
    return "&#" + this.deletedCards.join(";&#") + ";";
  }

  /**
   * Sets buttons (children of element) to default state, or disabled state
   *
   * @param {HTMLElement} elParent
   * @param {boolean} bEnabled
   */
  setButtonState(elParent, bEnabled) {
    $$(".uiIBtn", elParent).each((el) => {
      el.classList.toggle("uiFcBtnDisabled", bEnabled);
    });
  }
}
<|endoftext|>


<commit_before>export { MobileNetV1 };

import * as ValueDesc from "../../../Unpacker/ValueDesc.js";
import { Params } from "../Block_Params.js";
import { Base } from "./Base.js";

/**
 * Provide parameters for MobileNetV1 (i.e. no-add-inut-to-output, pointwise1 is same size of pointwise21).
 *
 * Although this the simplest pointwise1-depthwise-pointwise2 architecture, it may be the most efficient neural network if
 * using CLIP_BY_VALUE_N3_P3 (instead of RELU) as activation function.
 *
 * The reasons are:
 *
 *   - Inference speed faster than MobileNetV2: According to experience of ShuffleNetV2_ByMobileNetV1, the CLIP_BY_VALUE_N3_P3
 *       activation function could achieve skipping connection (i.e. residual connection) without add-input-to-output (i.e MobileNetV2).
 *
 *   - Inference speed could be faster than ShuffleNetV2_ByMobileNetV1: MobileNetV1_Xxx's all steps could have no pointwise1.
 *       ShuffleNetV2_ByMobileNetV1's step0 always have pointwise1 (even if ( blockParams.bPointwise1 == false ), it still exists
 *       internally). (But ShuffleNetV2_ByMobileNetV1's learning speed is faster than MobileNetv1 because less filter weights
 *       need to be learned.)
 *
 *
 *
 */
class MobileNetV1 extends Base {

  /** @override */
  configTo_beforeStep0() {
    super.configTo_beforeStep0(); // step0's inputHeight0, inputWidth0, bias, activation.

    let blockParams = this.blockParams;

    this.channelCount0_pointwise1Before = blockParams.sourceChannelCount; // Step0 uses the original input channel count (as input0).

    // In MobileNetV1, all steps (include step0) do not use input1.
    this.channelCount1_pointwise1Before = ValueDesc.channelCount1_pointwise1Before.Singleton.Ids.ONE_INPUT;

    if ( blockParams.bPointwise1 == false ) {
      this.pointwise1ChannelCount = 0;                                  // NoPointwise1.
      this.depthwise_AvgMax_Or_ChannelMultiplier = 2;                   // Double of input0. (Same as pointwise21.)

    } else {
      this.pointwise1ChannelCount = blockParams.sourceChannelCount * 2; // Double of input0. (Same as pointwise21.)
      this.depthwise_AvgMax_Or_ChannelMultiplier = 1;
    }

    // All steps' output0 is double depth of source input0.
    //
    // Note: In original MobileNet(V2) design, it is not always "twice". We choose "twice" just for comparing with ShuffleNetV2.
    //
    this.pointwise21ChannelCount = blockParams.sourceChannelCount * 2;

    this.bOutput1Requested = false; // In MobileNet, all steps do not have output1.

    this.outChannels0 = this.pointwise21ChannelCount;
    this.outChannels1 = 0;
  }

  /** @override */
  configTo_afterStep0() {
    super.configTo_afterStep0(); // step1, 2, 3, ...'s inputHeight0, inputWidth0.

    // The input0 of all steps (except step0) have the same depth as previous (also step0's) step's output0.
    this.channelCount0_pointwise1Before = this.outChannels0;

    // In MobileNetV1, all steps (include step0) do not use input1.
    this.channelCount1_pointwise1Before = ValueDesc.channelCount1_pointwise1Before.Singleton.Ids.ONE_INPUT;
  }

  /** @override */
  configTo_beforeStepLast() {
    super.configTo_beforeStepLast(); // stepLast's pointwise21 bias.
  }

}

<commit_msg>Update MobileNetV1.js<commit_after>export { MobileNetV1 };

import * as ValueDesc from "../../../Unpacker/ValueDesc.js";
import { Params } from "../Block_Params.js";
import { Base } from "./Base.js";

/**
 * Provide parameters for MobileNetV1 (i.e. no-add-inut-to-output, pointwise1 is same size of pointwise21).
 *
 * Although this the simplest pointwise1-depthwise-pointwise2 architecture, it may be the most efficient neural network if
 * using CLIP_BY_VALUE_N3_P3 (instead of RELU) as activation function.
 *
 * The reasons are:
 *
 *   - Inference speed faster than MobileNetV2: According to experience of ShuffleNetV2_ByMobileNetV1, the CLIP_BY_VALUE_N3_P3
 *       activation function could achieve skipping connection (i.e. residual connection) without add-input-to-output (i.e MobileNetV2).
 *
 *   - Inference speed could be faster than ShuffleNetV2_ByMobileNetV1: MobileNetV1_Xxx's all steps could have no pointwise1.
 *       ShuffleNetV2_ByMobileNetV1's step0 always have pointwise1 (even if ( blockParams.bPointwise1 == false ), it still exists
 *       internally). (But ShuffleNetV2_ByMobileNetV1's learning speed is faster than MobileNetv1 because less filter weights
 *       need to be learned.)
 *
 *
 *
 */
class MobileNetV1 extends Base {

  constructor( blockParams ) {
    super( blockParams );
  }

  /** @override */
  configTo_beforeStep0() {
    super.configTo_beforeStep0(); // step0's inputHeight0, inputWidth0, bias, activation.

    let blockParams = this.blockParams;

    this.channelCount0_pointwise1Before = blockParams.sourceChannelCount; // Step0 uses the original input channel count (as input0).

    // In MobileNetV1, all steps (include step0) do not use input1.
    this.channelCount1_pointwise1Before = ValueDesc.channelCount1_pointwise1Before.Singleton.Ids.ONE_INPUT;

    if ( blockParams.bPointwise1 == false ) {
      this.pointwise1ChannelCount = 0;                                  // NoPointwise1.
      this.depthwise_AvgMax_Or_ChannelMultiplier = 2;                   // Double of input0. (Same as pointwise21.)

    } else {
      this.pointwise1ChannelCount = blockParams.sourceChannelCount * 2; // Double of input0. (Same as pointwise21.)
      this.depthwise_AvgMax_Or_ChannelMultiplier = 1;
    }

    // All steps' output0 is double depth of source input0.
    //
    // Note: In original MobileNet(V2) design, it is not always "twice". We choose "twice" just for comparing with ShuffleNetV2.
    //
    this.pointwise21ChannelCount = blockParams.sourceChannelCount * 2;

    this.bOutput1Requested = false; // In MobileNet, all steps do not have output1.

    this.outChannels0 = this.pointwise21ChannelCount;
    this.outChannels1 = 0;
  }

  /** @override */
  configTo_afterStep0() {
    super.configTo_afterStep0(); // step1, 2, 3, ...'s inputHeight0, inputWidth0.

    // The input0 of all steps (except step0) have the same depth as previous (also step0's) step's output0.
    this.channelCount0_pointwise1Before = this.outChannels0;

    // In MobileNetV1, all steps (include step0) do not use input1.
    this.channelCount1_pointwise1Before = ValueDesc.channelCount1_pointwise1Before.Singleton.Ids.ONE_INPUT;
  }

  /** @override */
  configTo_beforeStepLast() {
    super.configTo_beforeStepLast(); // stepLast's pointwise21 bias.
  }

}

<|endoftext|>


<commit_before>define( ['dojo/_base/array',
         'JBrowse/Util',
         './Util'
        ],
        function( array, Util, BAMUtil ) {

var SEQRET_DECODER = ['=', 'A', 'C', 'x', 'G', 'x', 'x', 'x', 'T', 'x', 'x', 'x', 'x', 'x', 'x', 'N'];
var CIGAR_DECODER  = ['M', 'I', 'D', 'N', 'S', 'H', 'P', '=', 'X', '?', '?', '?', '?', '?', '?', '?'];

var readInt   = BAMUtil.readInt;
var readShort = BAMUtil.readShort;

var Feature = Util.fastDeclare(

/**
 * @lends JBrowse.Store.BAM.Feature
 */
{

    /**
     * Feature object used for the JBrowse BAM backend.
     * @param args.store the BAM store this feature comes from
     * @param args.record optional BAM record (a plain object) containing the data for this feature
     * @constructs
     */
    constructor: function( args ) {
        this.file = args.file;
        this.store = args.store;

        var data = args.data
            || args.bytes && this._fromBytes( args.bytes.byteArray, args.bytes.start, args.bytes.end )
            || args.record && dojo.clone(args.record);

        // figure out start and end
        data.start = data.start || data.pos;
        data.end = data.end || ( data.lref ? data.pos + data.lref : data.seq ? data.pos + data.seq.length : undefined );

        // trying to determine orientation from 'XS' optional field,
        // or from the seq_reverse_complemented flag
        data.strand = ('XS' in data )                ? ( data.XS == '-' ? -1 : 1 ) :
                       data.seq_reverse_complemented ? -1                          :
                                                       1;

        data.score = data.mapping_quality || data.MQ || data.mq;
        data.type = data.type || 'match';
        data.source = args.store.source;

        if( data.qual && data.qual.join )
            data.qual = data.qual.join(' ');

        if( data.readName ) {
            data.name = data.readName;
            delete data.readName;
        }
        delete data.pos;

        this.data = data;
        this._subCounter = 0;
        this._uniqueID = args.parent ? args.parent._uniqueID + '-' + ++args.parent._subCounter
                                     : this.data.name+' at '+ data.segment + ':' + data.start + '..' + data.end;

        var cigar = data.CIGAR || data.cigar;
        // this.data.subfeatures = [];
        // if( cigar ) {
        //     this.data.Gap = this._cigarToGap( cigar );
        //     this.data.subfeatures.push.apply( this.data.subfeatures, this._cigarToSubfeats( cigar, this ) );
        // }
    },

    _fromBytes: function( byteArray, blockStart, blockEnd ) {
        var record = {};

        var refID = readInt(byteArray, blockStart + 4);
        var pos = readInt(byteArray, blockStart + 8);

        var bmn = readInt(byteArray, blockStart + 12);
        //var bin = (bmn & 0xffff0000) >> 16;
        var mq = (bmn & 0xff00) >> 8;
        var nl = bmn & 0xff;

        var flag_nc = readInt(byteArray, blockStart + 16);
        this._decodeFlags( record, (flag_nc & 0xffff0000) >> 16 );

        record.template_length = readInt(byteArray, blockStart + 32);

        var lseq = readInt(byteArray, blockStart + 20);
        record.seq_length = lseq;

        // If the read is unmapped, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        var numCigarOps = flag_nc & 0xffff;

        // var nextRef  = readInt(byteArray, blockStart + 24);
        // var nextPos = readInt(byteArray, blockStart + 28);

        if( ! record.unmapped ) {
            var readName = '';
            for (var j = 0; j < nl-1; ++j) {
                readName += String.fromCharCode(byteArray[blockStart + 36 + j]);
            }
        }

        var p = blockStart + 36 + nl;
        var cigar = '';
        var lref = 0;
        for (var c = 0; c < numCigarOps; ++c) {
            var cigop = readInt(byteArray, p);
            var lop = cigop >> 4;
            var op = CIGAR_DECODER[cigop & 0xf];
            cigar = cigar + lop + op;
            switch (op) {
            case 'M':
            case 'D':
            case 'N':
            case '=':
            case 'X':
                lref += lop;
                break;
            }
            p += 4;
        }
        if( ! record.unmapped ) {
            record.cigar = cigar;
            record.length_on_ref = lref;
        }

        var seq = '';
        var seqBytes = (lseq + 1) >> 1;
        for (var j = 0; j < seqBytes; ++j) {
            var sb = byteArray[p + j];
            seq += SEQRET_DECODER[(sb & 0xf0) >> 4];
            seq += SEQRET_DECODER[(sb & 0x0f)];
        }
        p += seqBytes;
        record.seq = seq;

        if( ! record.unmapped ) {
            var qseq = [];
            for (var j = 0; j < lseq; ++j) {
                qseq.push( byteArray[p + j] );
            }
        }

        p += lseq;

        if( ! record.unmapped ) {
            record.qual = qseq;

            record.pos = pos;
            if( mq != 255 ) // value of 255 means MQ is not available
                record.mapping_quality = mq;
            record.readName = readName;
            record.segment = this.file.indexToChr[refID];
            record._refID = refID;
        }

        while (p <= blockEnd) {
            var tag = String.fromCharCode(byteArray[p]) + String.fromCharCode(byteArray[p + 1]);
            var type = String.fromCharCode(byteArray[p + 2]);
            var value;

            if (type == 'A') {
                value = String.fromCharCode(byteArray[p + 3]);
                p += 4;
            } else if (type == 'i' || type == 'I') {
                value = readInt(byteArray, p + 3);
                p += 7;
            } else if (type == 'c' || type == 'C') {
                value = byteArray[p + 3];
                p += 4;
            } else if (type == 's' || type == 'S') {
                value = readShort(byteArray, p + 3);
                p += 5;
            } else if (type == 'f') {
                throw 'FIXME need floats';
            } else if (type == 'Z') {
                p += 3;
                value = '';
                for (;;) {
                    var cc = byteArray[p++];
                    if (cc == 0) {
                        break;
                    } else {
                        value += String.fromCharCode(cc);
                    }
                }
            } else {
                throw 'Unknown type '+ type;
            }
            record[tag] = value;
        }
        return record;
    },

    /**
     * Decode the BAM flags field and set them in the record.
     */
    _decodeFlags: function( record, flags ) {
        // the following explanations are taken verbatim from the SAM/BAM spec

        // 0x1 template having multiple segments in sequencing
        // If 0x1 is unset, no assumptions can be made about 0x2, 0x8, 0x20,
        // 0x40 and 0x80.
        if( flags & 0x1 ) {
            record.multi_segment_template = true;
            // 0x2 each segment properly aligned according to the aligner
            record.multi_segment_all_aligned = !!( flags & 0x2 );
            // 0x8 next segment in the template unmapped
            record.multi_segment_next_segment_unmapped = !!( flags & 0x8 );
            // 0x20 SEQ of the next segment in the template being reversed
            record.multi_segment_next_segment_reversed = !!( flags & 0x20 );

            // 0x40 the first segment in the template
            var first = !!( flags & 0x40 );
            // 0x80 the last segment in the template
            var last =  !!( flags & 0x80 );
            // * If 0x40 and 0x80 are both set, the segment is part of a linear
            // template, but it is neither the first nor the last segment. If both
            // 0x40 and 0x80 are unset, the index of the segment in the template is
            // unknown. This may happen for a non-linear template or the index is
            // lost in data processing.
            if( first && last ) {
                record.multi_segment_inner = true;
            }
            else if( first && !last ) {
                record.multi_segment_first = true;
            }
            else if( !first && last ) {
                record.multi_segment_last = true;
            }
            else {
                record.multi_segment_index_unknown = true;
            }
        }

        // 0x4 segment unmapped
        // * Bit 0x4 is the only reliable place to tell whether the segment is
        // unmapped. If 0x4 is set, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        // only set unmapped if true
        if( flags & 0x4 ) {
            record.unmapped = true;
        } else {
            // 0x10 SEQ being reverse complemented
            record.seq_reverse_complemented = !!(flags & 0x10);

            // 0x100 secondary alignment
            // * Bit 0x100 marks the alignment not to be used in certain analyses
            // when the tools in use are aware of this bit.
            if( flags & 0x100 )
                record.secondary_alignment = true;
        }

        // 0x200 not passing quality controls
        // only set qc_failed if it is true
        if( flags & 0x200 )
            record.qc_failed = true;

        // 0x400 PCR or optical duplicate
        // only set duplicate if true
        if ( flags & 0x400 )
            record.duplicate = true;
    },

    _parseCigar: function( cigar ) {
        return array.map( cigar.match(/\d+\D/g), function( op ) {
           return [ op.match(/\D/)[0].toUpperCase(), parseInt( op ) ];
        });
    },

    /**
     *  take a cigar string, and initial position, return an array of subfeatures
     */
    _cigarToSubfeats: function(cigar, parent)    {
        var subfeats = [];
        var min = parent.get('start');
        var max;
        var ops = this._parseCigar( cigar );
        for (var i = 0; i < ops.length; i++)  {
            var lop = ops[i][1];
            var op = ops[i][0];  // operation type
            // converting "=" to "E" to avoid possible problems later with non-alphanumeric type name
            if (op === "=")  { op = "E"; }

            switch (op) {
            case 'M':
            case 'D':
            case 'N':
            case 'E':
            case 'X':
                max = min + lop;
                break;
            case 'I':
                max = min;
                break;
            case 'P':  // not showing padding deletions (possibly change this later -- could treat same as 'I' ?? )
            case 'H':  // not showing hard clipping (since it's unaligned, and offset arg meant to be beginning of aligned part)
            case 'S':  // not showing soft clipping (since it's unaligned, and offset arg meant to be beginning of aligned part)
                break;
                // other possible cases
            }
            var subfeat = new Feature({
                store: this.store,
                file: this.file,
                data: {
                    type: 'match_part',
                    start: min,
                    end: max,
                    strand: parent.get('strand'),
                    cigar_op: lop+op
                },
                parent: this
            });
            if (op !== 'N')  {
                subfeats.push(subfeat);
            }
            min = max;
        }
        return subfeats;
    },

    /**
     * Takes a CIGAR string, translates to a GFF3 Gap attribute
     */
    _cigarToGap: function(cigar)    {
        return array.map( this._parseCigar( cigar ), function( op ) {
            return ( {
                         // map the CIGAR operations to the less-descriptive
                         // GFF3 gap operations
                         M: 'M',
                         I: 'I',
                         D: 'D',
                         N: 'D',
                         S: 'M',
                         H: 'M',
                         P: 'I'
                     }[op[0]] || op[0]
                   )+op[1];
        }).join(' ');
    },

    get: function(name) {
        return this.data[ name ];
    },

    tags: function() {
        var t = [];
        var d = this.data;
        for( var k in d ) {
            if( d.hasOwnProperty( k ) )
                t.push( k );
        }
        return t;
    }

});

return Feature;
});
<commit_msg>optimize BAM record parsing a bit<commit_after>define( ['dojo/_base/array',
         'JBrowse/Util',
         './Util'
        ],
        function( array, Util, BAMUtil ) {

var SEQRET_DECODER = ['=', 'A', 'C', 'x', 'G', 'x', 'x', 'x', 'T', 'x', 'x', 'x', 'x', 'x', 'x', 'N'];
var CIGAR_DECODER  = ['M', 'I', 'D', 'N', 'S', 'H', 'P', '=', 'X', '?', '?', '?', '?', '?', '?', '?'];

var readInt   = BAMUtil.readInt;
var readShort = BAMUtil.readShort;

var Feature = Util.fastDeclare(

/**
 * @lends JBrowse.Store.BAM.Feature
 */
{

    /**
     * Feature object used for the JBrowse BAM backend.
     * @param args.store the BAM store this feature comes from
     * @param args.record optional BAM record (a plain object) containing the data for this feature
     * @constructs
     */
    constructor: function( args ) {
        this.file = args.file;
        this.store = args.store;

        var data = args.data
            || args.bytes && this._fromBytes( args.bytes.byteArray, args.bytes.start, args.bytes.end )
            || args.record && dojo.clone(args.record);

        // figure out start and end
        data.start = data.start || data.pos;
        data.end = data.end || ( data.lref ? data.pos + data.lref : data.seq ? data.pos + data.seq.length : undefined );

        // trying to determine orientation from 'XS' optional field,
        // or from the seq_reverse_complemented flag
        data.strand = ('XS' in data )                ? ( data.XS == '-' ? -1 : 1 ) :
                       data.seq_reverse_complemented ? -1                          :
                                                       1;

        data.score = data.mapping_quality || data.MQ || data.mq;
        data.type = data.type || 'match';
        data.source = args.store.source;

        if( data.qual && data.qual.join )
            data.qual = data.qual.join(' ');

        if( data.readName ) {
            data.name = data.readName;
            delete data.readName;
        }
        delete data.pos;

        this.data = data;
        this._subCounter = 0;
        this._uniqueID = args.parent ? args.parent._uniqueID + '-' + ++args.parent._subCounter
                                     : this.data.name+' at '+ data.segment + ':' + data.start + '..' + data.end;

        var cigar = data.CIGAR || data.cigar;
        // this.data.subfeatures = [];
        // if( cigar ) {
        //     this.data.Gap = this._cigarToGap( cigar );
        //     this.data.subfeatures.push.apply( this.data.subfeatures, this._cigarToSubfeats( cigar, this ) );
        // }
    },

    _fromBytes: function( byteArray, blockStart, blockEnd ) {
        var record = {};

        var intArray = new Int32Array( byteArray.buffer.slice(blockStart+4,blockStart+36) );
        var refID = intArray[0];
        var pos = intArray[1];

        var bmn = intArray[2];
        //var bin = (bmn & 0xffff0000) >> 16;
        var mq = (bmn & 0xff00) >> 8;
        var nl = bmn & 0xff;

        var flag_nc = intArray[3];
        this._decodeFlags( record, (flag_nc & 0xffff0000) >> 16 );

        var lseq = intArray[4];
        record.seq_length = lseq;

        record.template_length = intArray[7];

        // If the read is unmapped, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        var numCigarOps = flag_nc & 0xffff;

        // var nextRef  = readInt(byteArray, blockStart + 24);
        // var nextPos = readInt(byteArray, blockStart + 28);

        if( ! record.unmapped ) {
            var readName = '';
            for (var j = 0; j < nl-1; ++j) {
                readName += String.fromCharCode(byteArray[blockStart + 36 + j]);
            }
        }

        var p = blockStart + 36 + nl;
        var cigar = '';
        var lref = 0;
        for (var c = 0; c < numCigarOps; ++c) {
            var cigop = readInt(byteArray, p);
            var lop = cigop >> 4;
            var op = CIGAR_DECODER[cigop & 0xf];
            cigar = cigar + lop + op;
            switch (op) {
            case 'M':
            case 'D':
            case 'N':
            case '=':
            case 'X':
                lref += lop;
                break;
            }
            p += 4;
        }
        if( ! record.unmapped ) {
            record.cigar = cigar;
            record.length_on_ref = lref;
        }

        var seq = '';
        var seqBytes = (lseq + 1) >> 1;
        for (var j = 0; j < seqBytes; ++j) {
            var sb = byteArray[p + j];
            seq += SEQRET_DECODER[(sb & 0xf0) >> 4];
            seq += SEQRET_DECODER[(sb & 0x0f)];
        }
        p += seqBytes;
        record.seq = seq;

        if( ! record.unmapped ) {
            var qseq = [];
            for (var j = 0; j < lseq; ++j) {
                qseq.push( byteArray[p + j] );
            }
        }

        p += lseq;

        if( ! record.unmapped ) {
            record.qual = qseq;

            record.pos = pos;
            if( mq != 255 ) // value of 255 means MQ is not available
                record.mapping_quality = mq;
            record.readName = readName;
            record.segment = this.file.indexToChr[refID];
            record._refID = refID;
        }

        while (p <= blockEnd) {
            var tag = String.fromCharCode(byteArray[p]) + String.fromCharCode(byteArray[p + 1]);
            var type = String.fromCharCode(byteArray[p + 2]);
            var value;

            if (type == 'A') {
                value = String.fromCharCode(byteArray[p + 3]);
                p += 4;
            } else if (type == 'i' || type == 'I') {
                value = readInt(byteArray, p + 3);
                p += 7;
            } else if (type == 'c' || type == 'C') {
                value = byteArray[p + 3];
                p += 4;
            } else if (type == 's' || type == 'S') {
                value = readShort(byteArray, p + 3);
                p += 5;
            } else if (type == 'f') {
                throw 'FIXME need floats';
            } else if (type == 'Z') {
                p += 3;
                value = '';
                for (;;) {
                    var cc = byteArray[p++];
                    if (cc == 0) {
                        break;
                    } else {
                        value += String.fromCharCode(cc);
                    }
                }
            } else {
                throw 'Unknown type '+ type;
            }
            record[tag] = value;
        }
        return record;
    },

    /**
     * Decode the BAM flags field and set them in the record.
     */
    _decodeFlags: function( record, flags ) {
        // the following explanations are taken verbatim from the SAM/BAM spec

        // 0x1 template having multiple segments in sequencing
        // If 0x1 is unset, no assumptions can be made about 0x2, 0x8, 0x20,
        // 0x40 and 0x80.
        if( flags & 0x1 ) {
            record.multi_segment_template = true;
            // 0x2 each segment properly aligned according to the aligner
            record.multi_segment_all_aligned = !!( flags & 0x2 );
            // 0x8 next segment in the template unmapped
            record.multi_segment_next_segment_unmapped = !!( flags & 0x8 );
            // 0x20 SEQ of the next segment in the template being reversed
            record.multi_segment_next_segment_reversed = !!( flags & 0x20 );

            // 0x40 the first segment in the template
            var first = !!( flags & 0x40 );
            // 0x80 the last segment in the template
            var last =  !!( flags & 0x80 );
            // * If 0x40 and 0x80 are both set, the segment is part of a linear
            // template, but it is neither the first nor the last segment. If both
            // 0x40 and 0x80 are unset, the index of the segment in the template is
            // unknown. This may happen for a non-linear template or the index is
            // lost in data processing.
            if( first && last ) {
                record.multi_segment_inner = true;
            }
            else if( first && !last ) {
                record.multi_segment_first = true;
            }
            else if( !first && last ) {
                record.multi_segment_last = true;
            }
            else {
                record.multi_segment_index_unknown = true;
            }
        }

        // 0x4 segment unmapped
        // * Bit 0x4 is the only reliable place to tell whether the segment is
        // unmapped. If 0x4 is set, no assumptions can be made about RNAME, POS,
        // CIGAR, MAPQ, bits 0x2, 0x10 and 0x100 and the bit 0x20 of the next
        // segment in the template.
        // only set unmapped if true
        if( flags & 0x4 ) {
            record.unmapped = true;
        } else {
            // 0x10 SEQ being reverse complemented
            record.seq_reverse_complemented = !!(flags & 0x10);

            // 0x100 secondary alignment
            // * Bit 0x100 marks the alignment not to be used in certain analyses
            // when the tools in use are aware of this bit.
            if( flags & 0x100 )
                record.secondary_alignment = true;
        }

        // 0x200 not passing quality controls
        // only set qc_failed if it is true
        if( flags & 0x200 )
            record.qc_failed = true;

        // 0x400 PCR or optical duplicate
        // only set duplicate if true
        if ( flags & 0x400 )
            record.duplicate = true;
    },

    _parseCigar: function( cigar ) {
        return array.map( cigar.match(/\d+\D/g), function( op ) {
           return [ op.match(/\D/)[0].toUpperCase(), parseInt( op ) ];
        });
    },

    /**
     *  take a cigar string, and initial position, return an array of subfeatures
     */
    _cigarToSubfeats: function(cigar, parent)    {
        var subfeats = [];
        var min = parent.get('start');
        var max;
        var ops = this._parseCigar( cigar );
        for (var i = 0; i < ops.length; i++)  {
            var lop = ops[i][1];
            var op = ops[i][0];  // operation type
            // converting "=" to "E" to avoid possible problems later with non-alphanumeric type name
            if (op === "=")  { op = "E"; }

            switch (op) {
            case 'M':
            case 'D':
            case 'N':
            case 'E':
            case 'X':
                max = min + lop;
                break;
            case 'I':
                max = min;
                break;
            case 'P':  // not showing padding deletions (possibly change this later -- could treat same as 'I' ?? )
            case 'H':  // not showing hard clipping (since it's unaligned, and offset arg meant to be beginning of aligned part)
            case 'S':  // not showing soft clipping (since it's unaligned, and offset arg meant to be beginning of aligned part)
                break;
                // other possible cases
            }
            var subfeat = new Feature({
                store: this.store,
                file: this.file,
                data: {
                    type: 'match_part',
                    start: min,
                    end: max,
                    strand: parent.get('strand'),
                    cigar_op: lop+op
                },
                parent: this
            });
            if (op !== 'N')  {
                subfeats.push(subfeat);
            }
            min = max;
        }
        return subfeats;
    },

    /**
     * Takes a CIGAR string, translates to a GFF3 Gap attribute
     */
    _cigarToGap: function(cigar)    {
        return array.map( this._parseCigar( cigar ), function( op ) {
            return ( {
                         // map the CIGAR operations to the less-descriptive
                         // GFF3 gap operations
                         M: 'M',
                         I: 'I',
                         D: 'D',
                         N: 'D',
                         S: 'M',
                         H: 'M',
                         P: 'I'
                     }[op[0]] || op[0]
                   )+op[1];
        }).join(' ');
    },

    get: function(name) {
        return this.data[ name ];
    },

    tags: function() {
        var t = [];
        var d = this.data;
        for( var k in d ) {
            if( d.hasOwnProperty( k ) )
                t.push( k );
        }
        return t;
    }

});

return Feature;
});
<|endoftext|>


<commit_before>const async = require('async');
const url = require('url');

const Logger = require('werelogs').Logger;

const config = require('../../conf/Config');
const zookeeper = require('../clients/zookeeper');
const IngestionReader = require('./IngestionReader');
const MetricsConsumer = require('../MetricsConsumer');
const MetricsProducer = require('../MetricsProducer');
const { metricsExtension } = require('../../extensions/ingestion/constants');

class IngestionPopulator {
    /**
     * Create an ingestion populator object to populate various kafka
     * queues from the metadata log
     *
     * @constructor
     * @param {Object} zkConfig - zookeeper configuration object
     * @param {String} zkConfig.connectionString - zookeeper
     *   connection string as "host:port[/chroot]"
     * @param {Object} kafkaConfig - kafka configuration object
     * @param {String} kafkaConfig.hosts - kafka hosts list
     * as "host:port[,host:port...]"
     * @param {Object} qpConfig - queue populator configuration
     * @param {String} qpConfig.zookeeperPath - sub-path to use for
     *   storing populator state in zookeeper
     * @param {String} qpConfig.logSource - type of source
     *   log: "bucketd" (raft log) or "dmd" (bucketfile)
     * @param {Object} [qpConfig.bucketd] - bucketd source
     *   configuration (mandatory if logSource is "bucket")
     * @param {Object} [qpConfig.dmd] - dmd source
     *   configuration (mandatory if logSource is "dmd")
     * @param {Object} [qpConfig.mongo] - mongo source
     *   configuration (mandatory if logSource is "mongo")
     * @param {Object} mConfig - metrics configuration object
     * @param {String} mConfig.topic - metrics topic
     * @param {Object} rConfig - redis configuration object
     * @param {Object} ingestionConfig - ingestion extension configurations
     * @param {Object} s3Config - configuration to connect to S3 Client
     */
    constructor(zkConfig, kafkaConfig, qpConfig, mConfig, rConfig,
                ingestionConfig, s3Config) {
        this.zkConfig = zkConfig;
        this.kafkaConfig = kafkaConfig;
        this.qpConfig = qpConfig;
        this.mConfig = mConfig;
        this.rConfig = rConfig;
        this.ingestionConfig = ingestionConfig;
        this.s3Config = s3Config;

        this.log = new Logger('Backbeat:IngestionPopulator');

        // list of active log readers
        this.logReaders = [];

        // list of updated log readers, if any
        this.logReadersUpdate = [];
        // metrics clients
        this._mProducer = null;
        this._mConsumer = null;

        // active ingestion readers
        // i.e.: { zenko-bucket-name: new IngestionReader() }
        this._activeIngestionSources = {};
    }

    /**
     * Open the ingestion populator
     *
     * @param {function} cb - callback function
     * @return {undefined}
     */
    open(cb) {
        this._loadExtension();
        async.series([
            next => this._setupMetricsClients(err => {
                if (err) {
                    this.log.error('error setting up metrics client', {
                        method: 'IngestionPopulator.open',
                        error: err,
                    });
                }
                return next(err);
            }),
            next => this._setupIngestionExtension(err => {
                if (err) {
                    this.log.error(
                        'error setting up ingestion extension', {
                            method: 'IngestionPopulator.open',
                            error: err,
                        });
                }
                return next(err);
            }),
            next => this._setupZookeeper(next),
        ], err => {
            if (err) {
                this.log.error('error starting up ingestion populator',
                    { method: 'IngestionPopulator.open',
                        error: err });
                return cb(err);
            }
            return cb();
        });
    }

    _setupMetricsClients(cb) {
        // Metrics Consumer
        this._mConsumer = new MetricsConsumer(this.rConfig, this.mConfig,
            this.kafkaConfig, metricsExtension);
        this._mConsumer.start();

        // Metrics Producer
        this._mProducer = new MetricsProducer(this.kafkaConfig, this.mConfig);
        this._mProducer.setupProducer(cb);
    }

    /**
     * Apply updates when ingestion buckets are updated in Config
     * @return {undefined}
     */
    applyUpdates() {
        // Example buckets param object:
        // [
        //      {
        //          accessKey: ...,
        //          secretKey: ...,
        //          endpoint: 'http://10.100.1.128:8000',
        //          locationType: 'scality_s3',
        //          bucketName: 'source-bucket-1',
        //          zenkoBucket: 'my-zenko-bucket',
        //          ingestion: { status: 'enabled' },
        //          locationConstraint: 'my-ring',
        //      }
        // ]
        const buckets = config.getIngestionBuckets();

        const currentActiveSources = Object.keys(this._activeIngestionSources);
        buckets.forEach(bucket => {
            const { zenkoBucket } = bucket;
            // if the zenko bucket has already been setup and is already
            // active, stop tracking bucket from `currentActiveSources`
            const isAlreadyActive =
                currentActiveSources.indexOf(zenkoBucket) >= 0;
            if (isAlreadyActive) {
                currentActiveSources.splice(
                    currentActiveSources.indexOf(zenkoBucket), 1);
            } else if (!currentActiveSources.includes(zenkoBucket)) {
                // Add new ingestion source
                const newSourceInfo =
                    this._getNewSourceInformation(bucket);
                this.addNewLogSource(newSourceInfo);
            }
        });

        // Any leftover `currentActiveSources` are no longer active and
        // must be removed.
        currentActiveSources.forEach(source => {
            this.closeLogState(source);
        });
    }

    /**
     * Get necessary information for setting up a new IngestionReader and form
     * as single object
     * @param {Object} ingestionInfo - ingestion bucket/location info
     * @return {Object} new source config details
     */
    _getNewSourceInformation(ingestionInfo) {
        const {
            accessKey, secretKey, endpoint, locationType,
            bucketName, zenkoBucket, locationConstraint,
        } = ingestionInfo;

        const urlObject = url.parse(endpoint);
        // TODO: will change for non-ring sources
        const auth = { accessKey, secretKey };

        return {
            // target zenko bucket name
            name: zenkoBucket,
            // source bucket name to ingest from
            bucket: bucketName,
            // source (s3c) endpoint
            host: urlObject.hostname,
            port: parseInt(urlObject.port, 10) || 80,
            https: urlObject.protocol.startsWith('https'),
            type: locationType,
            locationConstraint,
            auth,
        };
    }

    /**
     * add an Ingestion Reader
     * @param {Object} source - new source object
     * @return {undefined}
     */
    addNewLogSource(source) {
        const zenkoBucket = source.name;

        const newRaftReader = new IngestionReader({
            zkClient: this.zkClient,
            kafkaConfig: this.kafkaConfig,
            bucketdConfig: source,
            logger: this.log,
            extensions: [this._extension],
            metricsProducer: this._mProducer,
            qpConfig: this.qpConfig,
            s3Config: this.s3Config,
        });

        // add to active ingestion sources list
        this._activeIngestionSources[zenkoBucket] = newRaftReader;
        // add to log readers update list
        this.logReadersUpdate.push(newRaftReader);
    }

    _setupZookeeper(done) {
        const populatorZkPath = this.ingestionConfig.zookeeperPath;
        const zookeeperUrl =
            `${this.zkConfig.connectionString}${populatorZkPath}`;
        this.log.info('opening zookeeper connection for persisting ' +
                      'populator state',
                      { zookeeperUrl });
        this.zkClient = zookeeper.createClient(zookeeperUrl, {
            autoCreateNamespace: this.zkConfig.autoCreateNamespace,
        });
        this.zkClient.connect();
        this.zkClient.once('error', done);
        this.zkClient.once('ready', () => {
            // just in case there would be more 'error' events emitted
            this.zkClient.removeAllListeners('error');
            done();
        });
    }

    _loadExtension() {
        const index = require('../../extensions/ingestion/index.js');
        if (!index.queuePopulatorExtension) {
            this.log.fatal('Missing ingestion populator extension file');
            process.exit(1);
        }
        // eslint-disable-next-line new-cap
        const ext = new index.queuePopulatorExtension({
            config: this.ingestionConfig,
            logger: this.log,
        });
        ext.setZkConfig(this.zkConfig);
        this._extension = ext;

        this.log.info('ingestion extension is active');
    }

    _setupIngestionExtension(cb) {
        return this._extension.setupZookeeper(cb);
    }

    _setupUpdatedReaders(done) {
        const newReaders = this.logReadersUpdate;
        this.logReadersUpdate = [];
        async.each(newReaders, (logReader, cb) => logReader.setup(cb),
                   err => {
                       if (err) {
                           return done(err);
                       }
                       this.logReaders.push(...newReaders);
                       return done();
                   });
    }

    /**
     * Remove all ingestion sources
     * @return {undefined}
     */
    close() {
        // TODO: remove/cleanup zk paths
        Object.keys(this._activeIngestionSources).forEach(source => {
            this.closeLogState(source);
        });
    }

    /**
     * Remove an inactive IngestionReader
     * @param {String} key - source key (zenko bucket name)
     * @return {undefined}
     */
    closeLogState(key) {
        // TODO: remove/cleanup zk paths
        delete this._activeIngestionSources[key];
        this._checkAndRemoveLogReader(key, this.logReaders);
        this._checkAndRemoveLogReader(key, this.logReadersUpdate);
    }

    _checkAndRemoveLogReader(name, list) {
        const index = list.findIndex(lr =>
            lr.getTargetZenkoBucketName() === name);
        if (index !== -1) {
            list.splice(index, 1);
        }
    }

    _processLogEntries(params, done) {
        return async.map(
            this.logReaders,
            (logReader, cb) => logReader.processLogEntries(params, cb),
            done);
    }

    processLogEntries(params, done) {
        if (this.logReadersUpdate.length >= 1) {
            return this._setupUpdatedReaders(err => {
                if (err) {
                    return done(err);
                }
                return this._processLogEntries(params, done);
            });
        }
        return this._processLogEntries(params, done);
    }

    zkStatus() {
        return this.zkClient.getState();
    }
}

module.exports = IngestionPopulator;
<commit_msg>rf: ZENKO-1608 error handling ingestion populator<commit_after>const async = require('async');
const url = require('url');

const Logger = require('werelogs').Logger;

const config = require('../../conf/Config');
const zookeeper = require('../clients/zookeeper');
const IngestionReader = require('./IngestionReader');
const MetricsConsumer = require('../MetricsConsumer');
const MetricsProducer = require('../MetricsProducer');
const { metricsExtension } = require('../../extensions/ingestion/constants');

class IngestionPopulator {
    /**
     * Create an ingestion populator object to populate various kafka
     * queues from the metadata log
     *
     * @constructor
     * @param {Object} zkConfig - zookeeper configuration object
     * @param {String} zkConfig.connectionString - zookeeper
     *   connection string as "host:port[/chroot]"
     * @param {Object} kafkaConfig - kafka configuration object
     * @param {String} kafkaConfig.hosts - kafka hosts list
     * as "host:port[,host:port...]"
     * @param {Object} qpConfig - queue populator configuration
     * @param {String} qpConfig.zookeeperPath - sub-path to use for
     *   storing populator state in zookeeper
     * @param {String} qpConfig.logSource - type of source
     *   log: "bucketd" (raft log) or "dmd" (bucketfile)
     * @param {Object} [qpConfig.bucketd] - bucketd source
     *   configuration (mandatory if logSource is "bucket")
     * @param {Object} [qpConfig.dmd] - dmd source
     *   configuration (mandatory if logSource is "dmd")
     * @param {Object} [qpConfig.mongo] - mongo source
     *   configuration (mandatory if logSource is "mongo")
     * @param {Object} mConfig - metrics configuration object
     * @param {String} mConfig.topic - metrics topic
     * @param {Object} rConfig - redis configuration object
     * @param {Object} ingestionConfig - ingestion extension configurations
     * @param {Object} s3Config - configuration to connect to S3 Client
     */
    constructor(zkConfig, kafkaConfig, qpConfig, mConfig, rConfig,
                ingestionConfig, s3Config) {
        this.zkConfig = zkConfig;
        this.kafkaConfig = kafkaConfig;
        this.qpConfig = qpConfig;
        this.mConfig = mConfig;
        this.rConfig = rConfig;
        this.ingestionConfig = ingestionConfig;
        this.s3Config = s3Config;

        this.log = new Logger('Backbeat:IngestionPopulator');

        // list of active log readers
        this.logReaders = [];

        // list of updated log readers, if any
        this.logReadersUpdate = [];
        // metrics clients
        this._mProducer = null;
        this._mConsumer = null;

        // active ingestion readers
        // i.e.: { zenko-bucket-name: new IngestionReader() }
        this._activeIngestionSources = {};
    }

    /**
     * Open the ingestion populator
     *
     * @param {function} cb - callback function
     * @return {undefined}
     */
    open(cb) {
        this._loadExtension();
        async.series([
            next => this._setupMetricsClients(err => {
                if (err) {
                    this.log.error('error setting up metrics client', {
                        method: 'IngestionPopulator.open',
                        error: err,
                    });
                }
                return next(err);
            }),
            next => this._setupIngestionExtension(err => {
                if (err) {
                    this.log.error(
                        'error setting up ingestion extension', {
                            method: 'IngestionPopulator.open',
                            error: err,
                        });
                }
                return next(err);
            }),
            next => this._setupZookeeper(next),
        ], err => {
            if (err) {
                this.log.error('error starting up ingestion populator',
                    { method: 'IngestionPopulator.open',
                        error: err });
                return cb(err);
            }
            return cb();
        });
    }

    _setupMetricsClients(cb) {
        // Metrics Consumer
        this._mConsumer = new MetricsConsumer(this.rConfig, this.mConfig,
            this.kafkaConfig, metricsExtension);
        this._mConsumer.start();

        // Metrics Producer
        this._mProducer = new MetricsProducer(this.kafkaConfig, this.mConfig);
        this._mProducer.setupProducer(cb);
    }

    /**
     * Apply updates when ingestion buckets are updated in Config
     * @return {undefined}
     */
    applyUpdates() {
        // Example buckets param object:
        // [
        //      {
        //          accessKey: ...,
        //          secretKey: ...,
        //          endpoint: 'http://10.100.1.128:8000',
        //          locationType: 'scality_s3',
        //          bucketName: 'source-bucket-1',
        //          zenkoBucket: 'my-zenko-bucket',
        //          ingestion: { status: 'enabled' },
        //          locationConstraint: 'my-ring',
        //      }
        // ]
        const buckets = config.getIngestionBuckets();

        const currentActiveSources = Object.keys(this._activeIngestionSources);
        buckets.forEach(bucket => {
            const { zenkoBucket } = bucket;
            // if the zenko bucket has already been setup and is already
            // active, stop tracking bucket from `currentActiveSources`
            const isAlreadyActive =
                currentActiveSources.indexOf(zenkoBucket) >= 0;
            if (isAlreadyActive) {
                currentActiveSources.splice(
                    currentActiveSources.indexOf(zenkoBucket), 1);
            } else if (!currentActiveSources.includes(zenkoBucket)) {
                // Add new ingestion source
                const newSourceInfo =
                    this._getNewSourceInformation(bucket);
                this.addNewLogSource(newSourceInfo);
            }
        });

        // Any leftover `currentActiveSources` are no longer active and
        // must be removed.
        currentActiveSources.forEach(source => {
            this.closeLogState(source);
        });
    }

    /**
     * Get necessary information for setting up a new IngestionReader and form
     * as single object
     * @param {Object} ingestionInfo - ingestion bucket/location info
     * @return {Object} new source config details
     */
    _getNewSourceInformation(ingestionInfo) {
        const {
            accessKey, secretKey, endpoint, locationType,
            bucketName, zenkoBucket, locationConstraint,
        } = ingestionInfo;

        const urlObject = url.parse(endpoint);
        // TODO: will change for non-ring sources
        const auth = { accessKey, secretKey };

        return {
            // target zenko bucket name
            name: zenkoBucket,
            // source bucket name to ingest from
            bucket: bucketName,
            // source (s3c) endpoint
            host: urlObject.hostname,
            port: parseInt(urlObject.port, 10) || 80,
            https: urlObject.protocol.startsWith('https'),
            type: locationType,
            locationConstraint,
            auth,
        };
    }

    /**
     * add an Ingestion Reader
     * @param {Object} source - new source object
     * @return {undefined}
     */
    addNewLogSource(source) {
        const zenkoBucket = source.name;

        const newRaftReader = new IngestionReader({
            zkClient: this.zkClient,
            kafkaConfig: this.kafkaConfig,
            bucketdConfig: source,
            logger: this.log,
            extensions: [this._extension],
            metricsProducer: this._mProducer,
            qpConfig: this.qpConfig,
            s3Config: this.s3Config,
        });

        // add to active ingestion sources list
        this._activeIngestionSources[zenkoBucket] = newRaftReader;
        // add to log readers update list
        this.logReadersUpdate.push(newRaftReader);
    }

    _setupZookeeper(done) {
        const populatorZkPath = this.ingestionConfig.zookeeperPath;
        const zookeeperUrl =
            `${this.zkConfig.connectionString}${populatorZkPath}`;
        this.log.info('opening zookeeper connection for persisting ' +
                      'populator state',
                      { zookeeperUrl });
        this.zkClient = zookeeper.createClient(zookeeperUrl, {
            autoCreateNamespace: this.zkConfig.autoCreateNamespace,
        });
        this.zkClient.connect();
        this.zkClient.once('error', done);
        this.zkClient.once('ready', () => {
            // just in case there would be more 'error' events emitted
            this.zkClient.removeAllListeners('error');
            done();
        });
    }

    _loadExtension() {
        const index = require('../../extensions/ingestion/index.js');
        if (!index.queuePopulatorExtension) {
            this.log.fatal('Missing ingestion populator extension file');
            process.exit(1);
        }
        // eslint-disable-next-line new-cap
        const ext = new index.queuePopulatorExtension({
            config: this.ingestionConfig,
            logger: this.log,
        });
        ext.setZkConfig(this.zkConfig);
        this._extension = ext;

        this.log.info('ingestion extension is active');
    }

    _setupIngestionExtension(cb) {
        return this._extension.setupZookeeper(cb);
    }

    _setupUpdatedReaders(done) {
        const newReaders = this.logReadersUpdate;
        this.logReadersUpdate = [];
        async.each(newReaders, (logReader, cb) => logReader.setup(cb),
                   err => {
                       if (err) {
                           return done(err);
                       }
                       this.logReaders.push(...newReaders);
                       return done();
                   });
    }

    /**
     * Remove all ingestion sources
     * @return {undefined}
     */
    close() {
        // TODO: remove/cleanup zk paths
        Object.keys(this._activeIngestionSources).forEach(source => {
            this.closeLogState(source);
        });
    }

    /**
     * Remove an inactive IngestionReader
     * @param {String} key - source key (zenko bucket name)
     * @return {undefined}
     */
    closeLogState(key) {
        // TODO: remove/cleanup zk paths
        delete this._activeIngestionSources[key];
        this._checkAndRemoveLogReader(key, this.logReaders);
        this._checkAndRemoveLogReader(key, this.logReadersUpdate);
    }

    _checkAndRemoveLogReader(name, list) {
        const index = list.findIndex(lr =>
            lr.getTargetZenkoBucketName() === name);
        if (index !== -1) {
            list.splice(index, 1);
        }
    }

    _processLogEntries(params, done) {
        return async.eachLimit(this.logReaders, 10,
            (logReader, cb) => {
                logReader.processLogEntries(params, error => {
                    if (error) {
                        this.log.warn('error processing log entries', {
                            error,
                            zenkoBucket: logReader.getTargetZenkoBucketName(),
                            location: logReader.getLocationConstraint(),
                        });
                    }
                    // do not escalate error to avoid crashing pod
                    // instead retry on next cron
                    return cb();
                });
            }, done);
    }

    processLogEntries(params, done) {
        if (this.logReadersUpdate.length >= 1) {
            return this._setupUpdatedReaders(err => {
                if (err) {
                    return done(err);
                }
                return this._processLogEntries(params, done);
            });
        }
        return this._processLogEntries(params, done);
    }

    zkStatus() {
        return this.zkClient.getState();
    }
}

module.exports = IngestionPopulator;
<|endoftext|>