Source: lib/text/text_engine.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.text.TextEngine');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.Deprecate');
  11. goog.require('shaka.media.ClosedCaptionParser');
  12. goog.require('shaka.text.Cue');
  13. goog.require('shaka.util.BufferUtils');
  14. goog.require('shaka.util.IDestroyable');
  15. goog.require('shaka.util.MimeUtils');
  18. // TODO: revisit this when Closure Compiler supports partially-exported classes.
  19. /**
  20.  * @summary Manages text parsers and cues.
  21.  * @implements {shaka.util.IDestroyable}
  22.  * @export
  23.  */
  24. shaka.text.TextEngine = class {
  25.   /** @param {shaka.extern.TextDisplayer} displayer */
  26.   constructor(displayer) {
  27.     /** @private {?shaka.extern.TextParser} */
  28.     this.parser_ = null;
  30.     /** @private {shaka.extern.TextDisplayer} */
  31.     this.displayer_ = displayer;
  33.     /** @private {boolean} */
  34.     this.segmentRelativeVttTiming_ = false;
  36.     /** @private {number} */
  37.     this.timestampOffset_ = 0;
  39.     /** @private {number} */
  40.     this.appendWindowStart_ = 0;
  42.     /** @private {number} */
  43.     this.appendWindowEnd_ = Infinity;
  45.     /** @private {?number} */
  46.     this.bufferStart_ = null;
  48.     /** @private {?number} */
  49.     this.bufferEnd_ = null;
  51.     /** @private {string} */
  52.     this.selectedClosedCaptionId_ = '';
  54.     /** @private {shaka.extern.TextParser.ModifyCueCallback} */
  55.     this.modifyCueCallback_ = (cue, uri) => {};
  57.     /**
  58.      * The closed captions map stores the CEA closed captions by closed captions
  59.      * id and start and end time.
  60.      * It's used as the buffer of closed caption text streams, to show captions
  61.      * when we start displaying captions or switch caption tracks, we need to be
  62.      * able to get the cues for the other language and display them without
  63.      * re-fetching the video segments they were embedded in.
  64.      * Structure of closed caption map:
  65.      * closed caption id -> {start and end time -> cues}
  66.      * @private {!Map.<string, !Map.<string, !Array.<shaka.text.Cue>>>} */
  67.     this.closedCaptionsMap_ = new Map();
  68.   }
  70.   /**
  71.    * @param {string} mimeType
  72.    * @param {!shaka.extern.TextParserPlugin} plugin
  73.    * @export
  74.    */
  75.   static registerParser(mimeType, plugin) {
  76.     shaka.text.TextEngine.parserMap_[mimeType] = plugin;
  77.   }
  79.   /**
  80.    * @param {string} mimeType
  81.    * @export
  82.    */
  83.   static unregisterParser(mimeType) {
  84.     delete shaka.text.TextEngine.parserMap_[mimeType];
  85.   }
  87.   /**
  88.    * @return {?shaka.extern.TextParserPlugin}
  89.    * @export
  90.    */
  91.   static findParser(mimeType) {
  92.     return shaka.text.TextEngine.parserMap_[mimeType];
  93.   }
  95.   /**
  96.    * @param {string} mimeType
  97.    * @return {boolean}
  98.    */
  99.   static isTypeSupported(mimeType) {
  100.     if (shaka.text.TextEngine.parserMap_[mimeType]) {
  101.       // An actual parser is available.
  102.       return true;
  103.     }
  104.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  105.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE ) {
  106.       return !!shaka.media.ClosedCaptionParser.findDecoder();
  107.     }
  108.     return false;
  109.   }
  111.   // TODO: revisit this when the compiler supports partially-exported classes.
  112.   /**
  113.    * @override
  114.    * @export
  115.    */
  116.   destroy() {
  117.     this.parser_ = null;
  118.     this.displayer_ = null;
  119.     this.closedCaptionsMap_.clear();
  121.     return Promise.resolve();
  122.   }
  124.   /**
  125.    * @param {!shaka.extern.TextDisplayer} displayer
  126.    */
  127.   setDisplayer(displayer) {
  128.     this.displayer_ = displayer;
  129.   }
  131.   /**
  132.    * Initialize the parser.  This can be called multiple times, but must be
  133.    * called at least once before appendBuffer.
  134.    *
  135.    * @param {string} mimeType
  136.    * @param {boolean} sequenceMode
  137.    * @param {boolean} segmentRelativeVttTiming
  138.    * @param {string} manifestType
  139.    */
  140.   initParser(mimeType, sequenceMode, segmentRelativeVttTiming, manifestType) {
  141.     // No parser for CEA, which is extracted from video and side-loaded
  142.     // into TextEngine and TextDisplayer.
  143.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  144.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE) {
  145.       this.parser_ = null;
  146.       return;
  147.     }
  149.     const factory = shaka.text.TextEngine.parserMap_[mimeType];
  150.     goog.asserts.assert(
  151.         factory, 'Text type negotiation should have happened already');
  152.     this.parser_ = factory();
  153.     if (this.parser_.setSequenceMode) {
  154.       this.parser_.setSequenceMode(sequenceMode);
  155.     } else {
  156.       shaka.Deprecate.deprecateFeature(5,
  157.           'Text parsers w/ setSequenceMode',
  158.           'Text parsers should have a "setSequenceMode" method!');
  159.     }
  160.     if (this.parser_.setManifestType) {
  161.       this.parser_.setManifestType(manifestType);
  162.     } else {
  163.       shaka.Deprecate.deprecateFeature(5,
  164.           'Text parsers w/ setManifestType',
  165.           'Text parsers should have a "setManifestType" method!');
  166.     }
  167.     this.segmentRelativeVttTiming_ = segmentRelativeVttTiming;
  168.   }
  170.   /** @param {shaka.extern.TextParser.ModifyCueCallback} modifyCueCallback */
  171.   setModifyCueCallback(modifyCueCallback) {
  172.     this.modifyCueCallback_ = modifyCueCallback;
  173.   }
  175.   /**
  176.    * @param {BufferSource} buffer
  177.    * @param {?number} startTime relative to the start of the presentation
  178.    * @param {?number} endTime relative to the start of the presentation
  179.    * @param {?string=} uri
  180.    * @return {!Promise}
  181.    */
  182.   async appendBuffer(buffer, startTime, endTime, uri) {
  183.     goog.asserts.assert(
  184.         this.parser_, 'The parser should already be initialized');
  186.     // Start the operation asynchronously to avoid blocking the caller.
  187.     await Promise.resolve();
  189.     // Check that TextEngine hasn't been destroyed.
  190.     if (!this.parser_ || !this.displayer_) {
  191.       return;
  192.     }
  194.     if (startTime == null || endTime == null) {
  195.       this.parser_.parseInit(shaka.util.BufferUtils.toUint8(buffer));
  196.       return;
  197.     }
  199.     const vttOffset = this.segmentRelativeVttTiming_ ?
  200.         startTime : this.timestampOffset_;
  202.     /** @type {shaka.extern.TextParser.TimeContext} **/
  203.     const time = {
  204.       periodStart: this.timestampOffset_,
  205.       segmentStart: startTime,
  206.       segmentEnd: endTime,
  207.       vttOffset: vttOffset,
  208.     };
  210.     // Parse the buffer and add the new cues.
  211.     const allCues = this.parser_.parseMedia(
  212.         shaka.util.BufferUtils.toUint8(buffer), time, uri);
  213.     for (const cue of allCues) {
  214.       this.modifyCueCallback_(cue, uri || null, time);
  215.     }
  216.     const cuesToAppend = allCues.filter((cue) => {
  217.       return cue.startTime >= this.appendWindowStart_ &&
  218.           cue.startTime < this.appendWindowEnd_;
  219.     });
  221.     this.displayer_.append(cuesToAppend);
  223.     // NOTE: We update the buffered range from the start and end times
  224.     // passed down from the segment reference, not with the start and end
  225.     // times of the parsed cues.  This is important because some segments
  226.     // may contain no cues, but we must still consider those ranges
  227.     // buffered.
  228.     if (this.bufferStart_ == null) {
  229.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  230.     } else {
  231.       // We already had something in buffer, and we assume we are extending
  232.       // the range from the end.
  233.       goog.asserts.assert(
  234.           this.bufferEnd_ != null,
  235.           'There should already be a buffered range end.');
  236.       goog.asserts.assert(
  237.           (startTime - this.bufferEnd_) <= 1,
  238.           'There should not be a gap in text references >1s');
  239.     }
  240.     this.bufferEnd_ = Math.min(endTime, this.appendWindowEnd_);
  241.   }
  243.   /**
  244.    * @param {number} startTime relative to the start of the presentation
  245.    * @param {number} endTime relative to the start of the presentation
  246.    * @return {!Promise}
  247.    */
  248.   async remove(startTime, endTime) {
  249.     // Start the operation asynchronously to avoid blocking the caller.
  250.     await Promise.resolve();
  252.     if (this.displayer_ && this.displayer_.remove(startTime, endTime)) {
  253.       if (this.bufferStart_ == null) {
  254.         goog.asserts.assert(
  255.             this.bufferEnd_ == null, 'end must be null if startTime is null');
  256.       } else {
  257.         goog.asserts.assert(
  258.             this.bufferEnd_ != null,
  259.             'end must be non-null if startTime is non-null');
  261.         // Update buffered range.
  262.         if (endTime <= this.bufferStart_ || startTime >= this.bufferEnd_) {
  263.           // No intersection.  Nothing was removed.
  264.         } else if (startTime <= this.bufferStart_ &&
  265.                    endTime >= this.bufferEnd_) {
  266.           // We wiped out everything.
  267.           this.bufferStart_ = this.bufferEnd_ = null;
  268.         } else if (startTime <= this.bufferStart_ &&
  269.                    endTime < this.bufferEnd_) {
  270.           // We removed from the beginning of the range.
  271.           this.bufferStart_ = endTime;
  272.         } else if (startTime > this.bufferStart_ &&
  273.                    endTime >= this.bufferEnd_) {
  274.           // We removed from the end of the range.
  275.           this.bufferEnd_ = startTime;
  276.         } else {
  277.           // We removed from the middle?  StreamingEngine isn't supposed to.
  278.           goog.asserts.assert(
  279.               false, 'removal from the middle is not supported by TextEngine');
  280.         }
  281.       }
  282.     }
  283.   }
  285.   /** @param {number} timestampOffset */
  286.   setTimestampOffset(timestampOffset) {
  287.     this.timestampOffset_ = timestampOffset;
  288.   }
  290.   /**
  291.    * @param {number} appendWindowStart
  292.    * @param {number} appendWindowEnd
  293.    */
  294.   setAppendWindow(appendWindowStart, appendWindowEnd) {
  295.     this.appendWindowStart_ = appendWindowStart;
  296.     this.appendWindowEnd_ = appendWindowEnd;
  297.   }
  299.   /**
  300.    * @return {?number} Time in seconds of the beginning of the buffered range,
  301.    *   or null if nothing is buffered.
  302.    */
  303.   bufferStart() {
  304.     return this.bufferStart_;
  305.   }
  307.   /**
  308.    * @return {?number} Time in seconds of the end of the buffered range,
  309.    *   or null if nothing is buffered.
  310.    */
  311.   bufferEnd() {
  312.     return this.bufferEnd_;
  313.   }
  315.   /**
  316.    * @param {number} t A timestamp
  317.    * @return {boolean}
  318.    */
  319.   isBuffered(t) {
  320.     if (this.bufferStart_ == null || this.bufferEnd_ == null) {
  321.       return false;
  322.     }
  323.     return t >= this.bufferStart_ && t < this.bufferEnd_;
  324.   }
  326.   /**
  327.    * @param {number} t A timestamp
  328.    * @return {number} Number of seconds ahead of 't' we have buffered
  329.    */
  330.   bufferedAheadOf(t) {
  331.     if (this.bufferEnd_ == null || this.bufferEnd_ < t) {
  332.       return 0;
  333.     }
  335.     goog.asserts.assert(
  336.         this.bufferStart_ != null,
  337.         'start should not be null if end is not null');
  339.     return this.bufferEnd_ - Math.max(t, this.bufferStart_);
  340.   }
  342.   /**
  343.    * Set the selected closed captions id.
  344.    * Append the cues stored in the closed captions map until buffer end time.
  345.    * This is to fill the gap between buffered and unbuffered captions, and to
  346.    * avoid duplicates that would be caused by any future video segments parsed
  347.    * for captions.
  348.    *
  349.    * @param {string} id
  350.    * @param {number} bufferEndTime Load any stored cues up to this time.
  351.    */
  352.   setSelectedClosedCaptionId(id, bufferEndTime) {
  353.     this.selectedClosedCaptionId_ = id;
  355.     const captionsMap = this.closedCaptionsMap_.get(id);
  356.     if (captionsMap) {
  357.       for (const startAndEndTime of captionsMap.keys()) {
  358.         /** @type {Array.<!shaka.text.Cue>} */
  359.         const cues = captionsMap.get(startAndEndTime)
  360.             .filter((c) => c.endTime <= bufferEndTime);
  361.         if (cues) {
  362.           this.displayer_.append(cues);
  363.         }
  364.       }
  365.     }
  366.   }
  368.   /**
  369.    * @param {!shaka.text.Cue} cue the cue to apply the timestamp to recursively
  370.    * @param {number} videoTimestampOffset the timestamp offset of the video
  371.    * @private
  372.    */
  373.   applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset) {
  374.     cue.startTime += videoTimestampOffset;
  375.     cue.endTime += videoTimestampOffset;
  376.     for (const nested of cue.nestedCues) {
  377.       this.applyVideoTimestampOffsetRecursive_(nested, videoTimestampOffset);
  378.     }
  379.   }
  381.   /**
  382.    * Store the closed captions in the text engine, and append the cues to the
  383.    * text displayer.  This is a side-channel used for embedded text only.
  384.    *
  385.    * @param {!Array<!shaka.extern.ICaptionDecoder.ClosedCaption>} closedCaptions
  386.    * @param {?number} startTime relative to the start of the presentation
  387.    * @param {?number} endTime relative to the start of the presentation
  388.    * @param {number} videoTimestampOffset the timestamp offset of the video
  389.    *   stream in which these captions were embedded
  390.    */
  391.   storeAndAppendClosedCaptions(
  392.       closedCaptions, startTime, endTime, videoTimestampOffset) {
  393.     const startAndEndTime = startTime + ' ' + endTime;
  394.     /** @type {!Map.<string, !Map.<string, !Array.<!shaka.text.Cue>>>} */
  395.     const captionsMap = new Map();
  397.     for (const caption of closedCaptions) {
  398.       const id = caption.stream;
  399.       const cue = caption.cue;
  400.       if (!captionsMap.has(id)) {
  401.         captionsMap.set(id, new Map());
  402.       }
  403.       if (!captionsMap.get(id).has(startAndEndTime)) {
  404.         captionsMap.get(id).set(startAndEndTime, []);
  405.       }
  407.       // Adjust CEA captions with respect to the timestamp offset of the video
  408.       // stream in which they were embedded.
  409.       this.applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset);
  411.       const keepThisCue =
  412.           cue.startTime >= this.appendWindowStart_ &&
  413.           cue.startTime < this.appendWindowEnd_;
  414.       if (!keepThisCue) {
  415.         continue;
  416.       }
  418.       captionsMap.get(id).get(startAndEndTime).push(cue);
  419.       if (id == this.selectedClosedCaptionId_) {
  420.         this.displayer_.append([cue]);
  421.       }
  422.     }
  424.     for (const id of captionsMap.keys()) {
  425.       if (!this.closedCaptionsMap_.has(id)) {
  426.         this.closedCaptionsMap_.set(id, new Map());
  427.       }
  428.       for (const startAndEndTime of captionsMap.get(id).keys()) {
  429.         const cues = captionsMap.get(id).get(startAndEndTime);
  430.         this.closedCaptionsMap_.get(id).set(startAndEndTime, cues);
  431.       }
  432.     }
  434.     if (this.bufferStart_ == null) {
  435.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  436.     } else {
  437.       this.bufferStart_ = Math.min(
  438.           this.bufferStart_, Math.max(startTime, this.appendWindowStart_));
  439.     }
  441.     this.bufferEnd_ = Math.max(
  442.         this.bufferEnd_, Math.min(endTime, this.appendWindowEnd_));
  443.   }
  445.   /**
  446.    * Get the number of closed caption channels.
  447.    *
  448.    * This function is for TESTING ONLY. DO NOT USE in the library.
  449.    *
  450.    * @return {number}
  451.    */
  452.   getNumberOfClosedCaptionChannels() {
  453.     return this.closedCaptionsMap_.size;
  454.   }
  456.   /**
  457.    * Get the number of closed caption cues for a given channel. If there is
  458.    * no channel for the given channel id, this will return 0.
  459.    *
  460.    * This function is for TESTING ONLY. DO NOT USE in the library.
  461.    *
  462.    * @param {string} channelId
  463.    * @return {number}
  464.    */
  465.   getNumberOfClosedCaptionsInChannel(channelId) {
  466.     const channel = this.closedCaptionsMap_.get(channelId);
  467.     return channel ? channel.size : 0;
  468.   }
  469. };
  471. /** @private {!Object.<string, !shaka.extern.TextParserPlugin>} */
  472. shaka.text.TextEngine.parserMap_ = {};