From 0cb042129b536ee31370c692e3d8b09fa0acb508 Mon Sep 17 00:00:00 2001
From: Dan Olson <olson_dan@yahoo.com>
Date: Tue, 18 Jun 2024 20:36:47 -0400
Subject: [PATCH 1/2] Use <source> and <track> elements

---
 src/components/canvas-instance.ts | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/components/canvas-instance.ts b/src/components/canvas-instance.ts
index 96dfa06..a327d90 100644
--- a/src/components/canvas-instance.ts
+++ b/src/components/canvas-instance.ts
@@ -5,7 +5,7 @@ import { convertToPercentage } from '../helpers/convert-to-percentage';
 
 const $ = require('jquery');
 import { Annotation, AnnotationBody, Canvas, Duration, Range, Utils } from 'manifesto.js';
-import { Behavior, MediaType } from '@iiif/vocabulary/dist-commonjs';
+import { Behavior, MediaType, RenderingFormat } from '@iiif/vocabulary/dist-commonjs';
 import { BaseComponent, IBaseComponentOptions } from '@iiif/base-component';
 import { IAVCanvasInstanceData } from '../interfaces/canvas-instance-data';
 import { MediaElement } from '../elements/media-element';
@@ -35,6 +35,13 @@ import { getHls } from '../helpers/get-hls';
 import 'waveform-panel';
 import { WaveformPanel } from 'waveform-panel';
 
+type AugmentedRenderingFormat = RenderingFormat | "text/vtt";
+type TextTrackDescriptor = {
+  language?: string;
+  label?: string;
+  id: string;
+};
+
 export class CanvasInstance extends BaseComponent {
   private _$canvasContainer: JQuery;
   private _$canvasDuration: JQuery;
@@ -589,6 +596,16 @@ export class CanvasInstance extends BaseComponent {
       const offsetStart = ot[0] ? parseInt(ot[0]) : ot[0],
         offsetEnd = ot[1] ? parseInt(ot[1]) : ot[1];
 
+      const captions: Array<TextTrackDescriptor> = item
+        .getRenderings()
+        .filter(r => r.getFormat() as AugmentedRenderingFormat === "text/vtt")
+        .map(vttRendering => ({
+          label:
+            vttRendering.getLabel().getValue() ??
+            vttRendering.getFormat().toString(),
+          id: vttRendering.id,
+        }));
+
       // todo: type this
       const itemData: any = {
         active: false,
@@ -603,6 +620,7 @@ export class CanvasInstance extends BaseComponent {
         top: percentageTop,
         type: type,
         width: percentageWidth,
+        captions: captions
       };
 
       this._renderMediaElement(itemData);
@@ -1334,7 +1352,16 @@ export class CanvasInstance extends BaseComponent {
         //});
       }
     } else {
-      $mediaElement.attr('src', data.source);
+      $mediaElement.append(
+        $(`<source src="${data.source}" type="${data.format}">`)
+      );
+      if (data.captions && data.captions.length) {
+        data.captions.forEach(subtitle => {
+          $mediaElement.append(
+            $(`<track label="${subtitle.label}" kind="subtitles" srclang="${subtitle.language}" src="${subtitle.id}" ${data.captions.indexOf(subtitle) === 0 ? "default" : ""}>`)
+          );
+        })
+      }
     }
 
     $mediaElement

From 7647f08bcc7389a61a24e64d5993bf46b0962d0a Mon Sep 17 00:00:00 2001
From: Dan Olson <olson_dan@yahoo.com>
Date: Wed, 3 Jul 2024 20:27:47 -0400
Subject: [PATCH 2/2] Add caption support

This provides support for rendering captions when the manifest's
annotation items have a text type rendering of format 'text/vtt'.

The implementation is modeled after that of MediaElement.js
---
 src/components/canvas-instance.ts | 153 ++++++++++++++++++++++++++++--
 src/css/iiif-av-component.less    | 108 ++++++++++++++++++++-
 src/css/img/cc.png                | Bin 0 -> 915 bytes
 3 files changed, 249 insertions(+), 12 deletions(-)
 create mode 100644 src/css/img/cc.png

diff --git a/src/components/canvas-instance.ts b/src/components/canvas-instance.ts
index a327d90..2a45095 100644
--- a/src/components/canvas-instance.ts
+++ b/src/components/canvas-instance.ts
@@ -50,6 +50,8 @@ export class CanvasInstance extends BaseComponent {
   private _$canvasTime: JQuery;
   private _$canvasTimelineContainer: JQuery;
   private _$controlsContainer: JQuery;
+  private _$captionsButton: JQuery;
+  private _$captionsContainer: JQuery;
   private _$durationHighlight: JQuery;
   private _$hoverPreviewTemplate: JQuery;
   private _$nextButton: JQuery;
@@ -69,6 +71,7 @@ export class CanvasInstance extends BaseComponent {
   private _canvasClockTime: TimelineTime = 0 as TimelineTime;
   private _canvasHeight = 0;
   private _canvasWidth = 0;
+  private _captionsSelectedIndex = 0; // hide captions by default
   private _waveformPanel?: WaveformPanel;
   private _contentAnnotations: any[]; // todo: type as HTMLMediaElement?
   private _data: IAVCanvasInstanceData = this.data();
@@ -102,7 +105,6 @@ export class CanvasInstance extends BaseComponent {
     this._$element = $(this.options.target);
     this._data = this.options.data;
     this.$playerElement = $('<div class="player player--loading"></div>');
-    this.$playerElement.hide();
   }
 
   public loaded(): void {
@@ -274,12 +276,19 @@ export class CanvasInstance extends BaseComponent {
                                     ${this._data.content.fastRewind || ''}
                                     </span>
                                 </button>`);
+    this._$captionsButton = $(`
+                                <button class="btn captions-button hide" title="Captions">
+                                    <i class="av-icon av-icon-captions" aria-hidden="true"></i>
+                                    <span class="sr-only>
+                                    </span>
+                                </button>`);
 
     this._$timeDisplay = $(
       '<div class="time-display"><span class="canvas-time"></span> / <span class="canvas-duration"></span></div>'
     );
     this._$canvasTime = this._$timeDisplay.find('.canvas-time');
     this._$canvasDuration = this._$timeDisplay.find('.canvas-duration');
+    this._$captionsContainer = $('<div class="captions-container"><span class="captions-text"></span></div>');
 
     if (this.isVirtual()) {
       this.$playerElement.addClass('virtual');
@@ -307,7 +316,8 @@ export class CanvasInstance extends BaseComponent {
       this._data.enableFastForward ? this._$fastForward : null,
       this._$nextButton,
       this._$timeDisplay,
-      $volume
+      $volume,
+      this._$captionsButton
     );
     this._$canvasTimelineContainer.append(
       this._$canvasHoverPreview,
@@ -321,7 +331,7 @@ export class CanvasInstance extends BaseComponent {
       this._$controlsContainer
     );
     this.$playerElement.append(this._$canvasContainer, this._$optionsContainer);
-
+    this._$canvasContainer.append(this._$captionsContainer);
     this._$canvasHoverPreview.hide();
     this._$rangeHoverPreview.hide();
 
@@ -452,6 +462,17 @@ export class CanvasInstance extends BaseComponent {
       return this._setCurrentTime(start);
     });
 
+    this._$captionsButton.on("mouseenter focusin", () => {
+      this._$captionsButton.find(".caption-choices").removeClass("offscreen");
+    });
+
+    this._$captionsButton.on("mouseleave focusout", () => {
+      const choices = this._$captionsButton.find(".caption-choices");
+      if (!choices.is(":hover")) {
+        choices.addClass("offscreen");
+      }
+    });
+
     if (newRanges) {
       this._$canvasTimelineContainer.slider({
         value: 0,
@@ -604,7 +625,19 @@ export class CanvasInstance extends BaseComponent {
             vttRendering.getLabel().getValue() ??
             vttRendering.getFormat().toString(),
           id: vttRendering.id,
-        }));
+        }))
+        .sort((a, b) => {
+          if (a.label > b.label) {
+            return 1;
+          } else if (b.label > a.label) {
+            return -1;
+          } else {
+            return 0;
+          }
+        });
+      if (captions.length) {
+        captions.unshift({ id: "none", label: "None" });
+      }
 
       // todo: type this
       const itemData: any = {
@@ -641,6 +674,34 @@ export class CanvasInstance extends BaseComponent {
         });
       }
     }
+
+    const contentAnnotation = this._contentAnnotations[0];
+    this._showApplicableCaptionChoices(contentAnnotation);
+  }
+
+  // Applies the caption choices UI (if any) for the given contentAnnotation
+  // to the captions button in the controls container. Since there are multiple
+  // contentAnnotations but just one controls container, we need to swap out
+  // the caption choices whenever a new contentAnnotation becomes active.
+  private _showApplicableCaptionChoices(contentAnnotation: any) {
+    // detach any caption choices pertaining to previously active media
+    this._$captionsButton.children('.caption-choices').detach();
+
+    if (contentAnnotation.captionChoices) {
+      // add caption choices for active media
+      this._$captionsButton.append(contentAnnotation.captionChoices);
+      this._$captionsButton.removeClass("hide");
+    } else {
+      this._$captionsButton.addClass("hide");
+    }
+  }
+
+  private _getSelectedCaptionsIndex() {
+    return this._captionsSelectedIndex;
+  }
+
+  private _setSelectedCaptionsIndex(index: number) {
+    this._captionsSelectedIndex = index;
   }
 
   private _getBody(bodies: AnnotationBody[]): AnnotationBody | null {
@@ -1356,11 +1417,32 @@ export class CanvasInstance extends BaseComponent {
         $(`<source src="${data.source}" type="${data.format}">`)
       );
       if (data.captions && data.captions.length) {
-        data.captions.forEach(subtitle => {
+        const $captionChoices: JQuery = $('<div class="caption-choices offscreen"></div>');
+        const $captionChoicesList: JQuery = $('<ul class="caption-choices-list"></ul>');
+        data.captions.forEach((subtitle, idx) => {
+          // add the track to the media element
           $mediaElement.append(
-            $(`<track label="${subtitle.label}" kind="subtitles" srclang="${subtitle.language}" src="${subtitle.id}" ${data.captions.indexOf(subtitle) === 0 ? "default" : ""}>`)
+            $(`<track label="${subtitle.label}" kind="captions" srclang="${subtitle.language}" src="${subtitle.id}" ${idx === 0 ? "default" : ""}>`)
           );
-        })
+
+          const isSelected = idx === this._getSelectedCaptionsIndex();
+
+          // generate an entry in the caption choices list
+          const $captionButton: JQuery = $(
+            `<li class="caption-choices-list-item">
+              <input type="radio" class="caption-choices-input" name="caption-choice" id="caption-choice-${idx}" />
+              <label for="caption-choice-${idx}" class="caption-choices-label ${isSelected ? "caption-choices-selected" : ""}">${subtitle.label}</label>
+            </li>`
+          );
+          $captionButton.find("input").on("click", () => {
+            this._setSelectedCaptionsIndex(idx);
+            this._synchronizeCaptions();
+          });
+          $captionChoicesList.append($captionButton);
+        });
+        $captionChoices.append($captionChoicesList);
+
+        data.captionChoices = $captionChoices;
       }
     }
 
@@ -1965,7 +2047,7 @@ export class CanvasInstance extends BaseComponent {
     for (let i = 0; i < this._contentAnnotations.length; i++) {
       contentAnnotation = this._contentAnnotations[i];
 
-      if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
+      if (this._isActiveContentAnnotation(contentAnnotation)) {
         this._checkMediaSynchronization();
 
         if (!contentAnnotation.active) {
@@ -2002,6 +2084,55 @@ export class CanvasInstance extends BaseComponent {
     }
   }
 
+  private _isActiveContentAnnotation(contentAnnotation: { start: number; end: number; }): boolean {
+    return contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime;
+  }
+
+  // If the currently active contentAnnotation has captions, set up an event
+  // handler to render the caption text over the canvas.
+  private _synchronizeCaptions(activeAnnotation: any = undefined): void {
+    activeAnnotation = activeAnnotation || this._contentAnnotations.find(annotation => {
+      return this._isActiveContentAnnotation(annotation);
+    });
+    if (!activeAnnotation) {
+      return;
+    }
+
+    const mediaElement = activeAnnotation.element[0];
+    const selectedCaptionsIdx = this._getSelectedCaptionsIndex();
+    const tracksList = mediaElement.textTracks;
+
+    this._setCaptionsText("");
+
+    $(".caption-choices-label")
+      .removeClass("caption-choices-selected");
+    $(`label[for="caption-choice-${selectedCaptionsIdx}"]`)
+      .addClass("caption-choices-selected");
+
+    let textTrack: TextTrack;
+    for (let i = 0; i < tracksList.length; i++) {
+      textTrack = mediaElement.textTracks[i];
+      textTrack.mode = "hidden";
+      if (i === selectedCaptionsIdx) {
+        $(textTrack).on("cuechange", (event: Event) => {
+          // @ts-ignore
+          const activeCues = event.target.activeCues;
+          const text = activeCues.length ? activeCues[0].text : "";
+          this._setCaptionsText(text);
+        });
+      } else {
+        $(textTrack).off("cuechange");
+      }
+    }
+  }
+
+  private _setCaptionsText(text: string) {
+    const textHolder = this._$captionsContainer.find(".captions-text");
+    textHolder.text(text);
+    // hide the textHolder if there is no text
+    textHolder.toggle(!!text);
+  }
+
   private _synchronizeMedia(): void {
     if (AVComponent.newRanges && this.isVirtual()) {
       return;
@@ -2017,8 +2148,10 @@ export class CanvasInstance extends BaseComponent {
         this._canvasClockTime - contentAnnotation.start + contentAnnotation.startOffset
       );
 
-      if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
+      if (this._isActiveContentAnnotation(contentAnnotation)) {
+        this._showApplicableCaptionChoices(contentAnnotation);
         if (this._isPlaying) {
+          this._synchronizeCaptions(contentAnnotation);
           if (contentAnnotation.element[0].paused) {
             const promise = contentAnnotation.element[0].play();
             if (promise) {
@@ -2064,7 +2197,7 @@ export class CanvasInstance extends BaseComponent {
     for (let i = 0, l = this._contentAnnotations.length; i < l; i++) {
       contentAnnotation = this._contentAnnotations[i];
 
-      if (contentAnnotation.start <= this._canvasClockTime && contentAnnotation.end >= this._canvasClockTime) {
+      if (this._isActiveContentAnnotation(contentAnnotation)) {
         if (this._isPlaying) {
           if (contentAnnotation.element[0].readyState < 3) {
             this._buffering = true;
diff --git a/src/css/iiif-av-component.less b/src/css/iiif-av-component.less
index e7f5825..1df5eba 100644
--- a/src/css/iiif-av-component.less
+++ b/src/css/iiif-av-component.less
@@ -105,7 +105,7 @@
                 border-right: 6px solid transparent;
             }
         }
-        
+
         &:focus .av-icon,
         &:focus .uv-icon {
             opacity: 1;
@@ -172,6 +172,25 @@
         .circular-slider-handle();
     }
 
+    .captions-container {
+        position: absolute;
+        bottom: 0;
+        height: 70px;
+        width: 100%;
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        z-index: 1;
+    }
+
+    .captions-text {
+        background-color: rgba(20, 20, 20, 0.5);
+        color: white;
+        font-size: 16px;
+        padding: 0;
+        display: none;
+    }
+
     // .ui-slider-handle:after {
     //     position: absolute;
     //     top: 0px;
@@ -183,7 +202,6 @@
     // }
 
     .controls-container {
-        overflow: hidden;
         margin-top: 5px;
         height: 36px;
     }
@@ -284,6 +302,92 @@
 
     }
 
+    i.av-icon-captions {
+        background-image: data-uri('@{img-path}cc.png');
+        width: 19px;
+        height: 15px
+    }
+
+    .captions-button {
+        display: flex;
+        justify-content: center;
+        align-items: center;
+        position: relative;
+        width: 30px;
+        height: 30px;
+
+        &.hide {
+            display: none;
+        }
+    }
+
+    .caption-choices {
+        background: rgba(50, 50, 50, 0.9);
+        border: solid 1px transparent;
+        border-radius: 0;
+        bottom: 90%;
+        padding: 0;
+        position: absolute;
+        visibility: visible;
+        width: 86px;
+        z-index: 10;
+    }
+
+    .caption-choices-list {
+        padding: 0;
+        margin: 0;
+        text-align: left;
+    }
+
+    .caption-choices-list-item {
+        color: #fff;
+        cursor: pointer;
+        display: block;
+        list-style-type: none !important;
+        margin: 0;
+        padding-bottom: 5px;
+        overflow: hidden;
+    }
+
+    .caption-choices-list-item:hover {
+        background-color: rgb(200, 200, 200) !important;
+        background-color: rgba(255, 255, 255, 0.4) !important;
+    }
+
+    .caption-choices-input {
+        clear: both;
+        float: left;
+        left: -1000px;
+        margin: 3px 3px 0 5px;
+        position: absolute;
+    }
+
+    .caption-choices-label {
+        cursor: pointer;
+        float: left;
+        font-size: 10px;
+        line-height: 15px;
+        padding: 4px 10px 0;
+        width: 100%;
+    }
+
+    .caption-choices-selected {
+        color: rgba(33, 248, 248, 1);
+    }
+
+    .offscreen {
+        border: 0;
+        clip: rect(1px, 1px, 1px, 1px);
+        clip-path: inset(50%);
+        height: 1px;
+        margin: -1px;
+        overflow: hidden;
+        padding: 0;
+        position: absolute;
+        width: 1px;
+        word-wrap: normal;
+    }
+
     .av-icon-fast-forward {
         background-image: data-uri('@{img-path}fast-forward.svg');
     }
diff --git a/src/css/img/cc.png b/src/css/img/cc.png
new file mode 100644
index 0000000000000000000000000000000000000000..625e2b6a8ea002b61a130d9c7f353185833c8277
GIT binary patch
literal 915
zcmeAS@N?(olHy`uVBq!ia0vp^!a&T=!3HGRT)yZ4DaPU;cPEB*=VV?2IRU8=o@u_m
z3|c@o2LltM5Cbzv2@o>0XELyW*$fP7K$?Mpc>yCrb^#MyR(Jt3oXrnXIDHX|HPDdA
z%#etZ2wxwo<osN{#FYG`RK1Ga0+0d*n+hu+GdHy)QK2F?C$HG5!d3~a!V1U+3F|8<
zfR&VF+bTu)1}He^7o{qg=^5x5DA{o-C@9zzrKDK}xwt{K19`Se86_nJR{Hwo<>h+i
z#(Mch>H3D2mX`VkM*2oZx<D1W#g%y_i50qe#mW#vVCJ}_7AF^F7L;V>=P7{9O-#x!
zEwNQn0$BtH5O<bjrXg&~D~5Ut=s|sujJ~0sfj-=9pjI1@sFi<FW-8ETkok6oHei(~
zVo-Z*^g)&*IR@fWuqe>0c3d|4@L;p!l8k%E4h-ZTPZ!4!3&FL4z8*}00`rR2&%0Wu
z!oT*)RXxv1sVymTuKljNE-mf5dUp4Wn=9`$b*=c46npxrN5535*1>=a>q<phzM8&o
zJe~I-wJ_mc&F6hH^V9D!@m%Im-2CGGi>kNc(&aaOINp6-wc=OGk!OJ+b0iKe3rU|N
z=WN`2BYZF4MGuy+>9z$Mw)3BL3;rpvjxjw`uqIBwg^BURy;O&3sjcY@-C8Bz^R|22
zbmtZ6bv@Y{rMfa@QVw^Zm*MF&qm-k2UA**FKgH-wTHEUzq&M}qNiW}l=FX?`^E(9i
zR(tY(TmRF;ags^-f1&nGX{)a?&D)-RJS;W$)NS_9TS5dkN39K5UAyYm)`Cf&<eMv_
zayyPDF&#71Qkyr&Y5C=as}DQtu?N}lbL)u*n@aUg=s2EuD($fR<%Y#a6la`K+Yq7C
zb6olC*NKgfAN=~p))%a);lVoNStUnXQ?1mszjY5bJbU$M($UK0ldo#9a<~<Aak;hs
z3R0i9IRDzUn``o3Cw}t1pU(D|cW?W;<8>P{+qM2PGIK6GoTIz!JSd5Ky85}Sb4q9e
E0AJojhX4Qo

literal 0
HcmV?d00001