diff --git a/.gitignore b/.gitignore index c2337f8..19d779f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ node_modules/ .prettierrc -dist/ \ No newline at end of file +dist/ +**/.DS_Store +.vscode \ No newline at end of file diff --git a/engine/playhead_state.js b/engine/playhead_state.js index b07e3c9..766c3c1 100644 --- a/engine/playhead_state.js +++ b/engine/playhead_state.js @@ -57,7 +57,8 @@ class PlayheadStateStore extends SharedStateStore { tickInterval: 3, mediaSeq: 0, vodMediaSeqVideo: 0, - vodMediaSeqAudio: 0, + vodMediaSeqAudio: 0, + vodMediaSeqSubtitle: 0, }); } diff --git a/engine/server.ts b/engine/server.ts index 1424ed7..3ab41b1 100644 --- a/engine/server.ts +++ b/engine/server.ts @@ -7,6 +7,7 @@ const Session = require('./session.js'); const SessionLive = require('./session_live.js'); const StreamSwitcher = require('./stream_switcher.js'); const EventStream = require('./event_stream.js'); +const SubtitleSlicer = require('./subtitle_slicer.js'); const { SessionStateStore } = require('./session_state.js'); const { SessionLiveStateStore } = require('./session_live_state.js'); @@ -22,6 +23,8 @@ const sessionsLive = {}; // Should be a persistent store... const sessionSwitchers = {}; // Should be a persistent store... const switcherStatus = {}; // Should be a persistent store... const eventStreams = {}; +const DefaultDummySubtitleEndpointPath = "dummyUrl" +const DefaultSubtitleSpliceEndpointPath = "sliceUrl" export interface ChannelEngineOpts { defaultSlateUri?: string; @@ -38,6 +41,9 @@ export interface ChannelEngineOpts { maxTickInterval?: number; cloudWatchMetrics?: boolean; useDemuxedAudio?: boolean; + dummySubtitleEndpoint?: string; + subtitleSliceEndpoint?: string; + useVTTSubtitles?: boolean; alwaysNewSegments?: boolean; diffCompensationRate?: number; staticDirectory?: string; @@ -110,6 +116,7 @@ export interface Channel { id: string; profile: ChannelProfile[]; audioTracks?: AudioTracks[]; + subtitleTracks?: SubtitleTracks[]; closedCaptions?: ClosedCaptions[]; } @@ -127,6 +134,11 @@ export interface AudioTracks { default?: boolean; enforceAudioGroupId?: string; } +export interface SubtitleTracks { + language: string; + name: string; + default?: boolean; +} export interface IChannelManager { getChannels: () => Channel[]; @@ -155,6 +167,9 @@ export interface IStreamSwitchManager { export class ChannelEngine { private options?: ChannelEngineOpts; private useDemuxedAudio: boolean; + private dummySubtitleEndpoint: string; + private subtitleSliceEndpoint: string; + private useVTTSubtitles: boolean; private alwaysNewSegments: boolean; private defaultSlateUri?: string; private slateDuration?: number; @@ -172,7 +187,7 @@ export class ChannelEngine { private logCloudWatchMetrics: boolean; private adCopyMgrUri?: string; private adXchangeUri?: string; - + constructor(assetMgr: IAssetManager, options?: ChannelEngineOpts) { this.options = options; if (options && options.adCopyMgrUri) { @@ -185,6 +200,11 @@ export class ChannelEngine { if (options && options.useDemuxedAudio) { this.useDemuxedAudio = true; } + + this.useVTTSubtitles = (options && options.useVTTSubtitles) ? options.useVTTSubtitles : false ; + this.dummySubtitleEndpoint = (options && options.dummySubtitleEndpoint) ? options.dummySubtitleEndpoint : DefaultDummySubtitleEndpointPath; + this.subtitleSliceEndpoint = (options && options.subtitleSliceEndpoint) ? options.dummySubtitleEndpoint : DefaultSubtitleSpliceEndpointPath; + this.alwaysNewSegments = false; if (options && options.alwaysNewSegments) { this.alwaysNewSegments = true; @@ -278,6 +298,11 @@ export class ChannelEngine { req.params[1] = m[2]; req.params[2] = m[3]; await this._handleAudioManifest(req, res, next); + } else if (m = req.params.file.match(/subtitles-(\S+)_(\S+).m3u8;session=(.*)$/)) { + req.params[0] = m[1]; + req.params[1] = m[2]; + req.params[2] = m[3]; + await this._handleSubtitleManifest(req, res, next); } }; this.server.get('/live/:file', async (req, res, next) => { @@ -310,6 +335,8 @@ export class ChannelEngine { this.server.get('/health', this._handleAggregatedSessionHealth.bind(this)); this.server.get('/health/:sessionId', this._handleSessionHealth.bind(this)); this.server.get('/reset', this._handleSessionReset.bind(this)); + this.server.get('/channels/:channelId/' + this.dummySubtitleEndpoint, this._handleDummySubtitleEndpoint.bind(this)); + this.server.get('/channels/:channelId/' + this.subtitleSliceEndpoint, this._handleSubtitleSliceEndpoint.bind(this)); this.server.on('NotFound', (req, res, err, next) => { res.header("X-Instance-Id", this.instanceId + `<${version}>`); @@ -407,6 +434,9 @@ export class ChannelEngine { sessionId: channel.id, averageSegmentDuration: channel.options && channel.options.averageSegmentDuration ? channel.options.averageSegmentDuration : this.streamerOpts.defaultAverageSegmentDuration, useDemuxedAudio: options.useDemuxedAudio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + useVTTSubtitles: this.useVTTSubtitles, alwaysNewSegments: options.alwaysNewSegments, noSessionDataTags: options.noSessionDataTags, playheadDiffThreshold: channel.options && channel.options.playheadDiffThreshold ? channel.options.playheadDiffThreshold : this.streamerOpts.defaultPlayheadDiffThreshold, @@ -416,6 +446,7 @@ export class ChannelEngine { diffCompensationRate: channel.options && channel.options.diffCompensationRate ? channel.options.diffCompensationRate : this.streamerOpts.diffCompensationRate, profile: channel.profile, audioTracks: channel.audioTracks, + subtitleTracks: channel.subtitleTracks, closedCaptions: channel.closedCaptions, slateUri: channel.slate && channel.slate.uri ? channel.slate.uri : this.defaultSlateUri, slateRepetitions: channel.slate && channel.slate.repetitions ? channel.slate.repetitions : this.slateRepetitions, @@ -426,6 +457,9 @@ export class ChannelEngine { sessionsLive[channel.id] = new SessionLive({ sessionId: channel.id, useDemuxedAudio: options.useDemuxedAudio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + useVTTSubtitles: this.useVTTSubtitles, cloudWatchMetrics: this.logCloudWatchMetrics, profile: channel.profile, }, this.sessionLiveStore); @@ -475,7 +509,7 @@ export class ChannelEngine { const session = sessions[channelId]; const sessionLive = sessionsLive[channelId]; if (!this.monitorTimer[channelId]) { - this.monitorTimer[channelId] = setInterval(async () => { await this._monitorAsync(session, sessionLive) }, 5000); + this.monitorTimer[channelId] = setInterval(async () => { await this._monitorAsync(session, sessionLive) }, 5000); } session.startPlayheadAsync(); await sessionLive.startPlayheadAsync(); @@ -598,6 +632,34 @@ export class ChannelEngine { } } + async getSubtitleManifests(channelId) { + if (sessions[channelId]) { + const allSubtitleM3U8 = {}; + let promises = []; + const session = sessions[channelId]; + const addM3U8 = async (groupId, lang) => { + let subtitleM3U8 = await session.getCurrentSubtitleManifestAsync(groupId, lang); + if (!allSubtitleM3U8[groupId]) { + allSubtitleM3U8[groupId] = {}; + } + allSubtitleM3U8[groupId][lang] = subtitleM3U8; + } + // Get m3u8s for all langauges for all groups + const subtitleGroupsAndLangs = await session.getSubtitleGroupsAndLangs(); + for (const [subtitleGroup, languages] of Object.entries(subtitleGroupsAndLangs)) { + (>languages).forEach((lang) => { + promises.push(addM3U8(subtitleGroup, lang)); + }); + } + await Promise.all(promises); + + return allSubtitleM3U8; + } else { + const err = new errs.NotFoundError('Invalid session'); + return Promise.reject(err) + } + } + _handleHeartbeat(req, res, next) { debug('req.url=' + req.url); res.send(200); @@ -626,6 +688,9 @@ export class ChannelEngine { options.adXchangeUri = this.adXchangeUri; options.averageSegmentDuration = this.streamerOpts.defaultAverageSegmentDuration; options.useDemuxedAudio = this.useDemuxedAudio; + options.dummySubtitleEndpoint = this.dummySubtitleEndpoint; + options.subtitleSliceEndpoint = this.subtitleSliceEndpoint; + options.useVTTSubtitles = this.useVTTSubtitles; options.alwaysNewSegments = this.alwaysNewSegments; options.playheadDiffThreshold = this.streamerOpts.defaultPlayheadDiffThreshold; options.maxTickInterval = this.streamerOpts.defaultMaxTickInterval; @@ -709,6 +774,71 @@ export class ChannelEngine { } } + async _handleSubtitleManifest(req, res, next) { + debug(`req.url=${req.url}`); + const session = sessions[req.params[2]]; + if (session) { + try { + const body = await session.getCurrentSubtitleManifestAsync( + req.params[0], + req.params[1], + req.headers["x-playback-session-id"] + ); + res.sendRaw(200, Buffer.from(body, 'utf8'), { + "Content-Type": "application/vnd.apple.mpegurl", + "Access-Control-Allow-Origin": "*", + "Cache-Control": `max-age=${this.streamerOpts.cacheTTL || '4'}`, + "X-Instance-Id": this.instanceId + `<${version}>`, + }); + next(); + } catch (err) { + next(this._gracefulErrorHandler(err)); + } + } else { + const err = new errs.NotFoundError('Invalid session'); + next(err); + } + } + + async _handleDummySubtitleEndpoint(req,res,next) { + debug(`req.url=${req.url}`); + const session = sessions[req.params.channelId]; + if (session) { + try { + const body = `WEBVTT`; + res.sendRaw(200, Buffer.from(body, 'utf8'), { + "Content-Type": "application/vnd.apple.mpegurl", + "Access-Control-Allow-Origin": "*", + "Cache-Control": `max-age=${this.streamerOpts.cacheTTL || '4'}`, + "X-Instance-Id": this.instanceId + `<${version}>`, + }); + next(); + } catch (err) { + next(this._gracefulErrorHandler(err)); + } + } else { + const err = new errs.NotFoundError('Invalid session'); + next(err); + } + } + + async _handleSubtitleSliceEndpoint(req,res,next) { + debug(`req.url=${req.url}`); + try { + const slicer = new SubtitleSlicer(); + const body = await slicer.generateVtt(req.query); + res.sendRaw(200, Buffer.from(body, 'utf8'), { + "Content-Type": "application/vnd.apple.mpegurl", + "Access-Control-Allow-Origin": "*", + "Cache-Control": `max-age=${this.streamerOpts.cacheTTL || '4'}`, + "X-Instance-Id": this.instanceId + `<${version}>`, + }); + next(); + } catch (err) { + next(this._gracefulErrorHandler(err)); + } + } + async _handleMediaManifest(req, res, next) { debug(`x-playback-session-id=${req.headers["x-playback-session-id"]} req.url=${req.url}`); debug(req.params); diff --git a/engine/session.js b/engine/session.js index 53d04b9..aec12f6 100644 --- a/engine/session.js +++ b/engine/session.js @@ -40,6 +40,9 @@ class Session { this._events = []; this.averageSegmentDuration = AVERAGE_SEGMENT_DURATION; this.use_demuxed_audio = false; + this.use_vtt_subtitles = false; + this.dummySubtitleEndpoint = ""; + this.subtitleSliceEndpoint = ""; this.cloudWatchLogging = false; this.playheadDiffThreshold = DEFAULT_PLAYHEAD_DIFF_THRESHOLD; this.maxTickInterval = DEFAULT_MAX_TICK_INTERVAL; @@ -49,11 +52,13 @@ class Session { this.timePositionOffset = 0; this.prevVodMediaSeq = { video: null, - audio: null + audio: null, + subtitle: null } this.prevMediaSeqOffset = { video: null, - audio: null + audio: null, + subtitle: null } this.waitingForNextVod = false; this.leaderIsSettingNextVod = false; @@ -85,6 +90,15 @@ class Session { if (config.useDemuxedAudio) { this.use_demuxed_audio = true; } + if (config.dummySubtitleEndpoint) { + this.dummySubtitleEndpoint = config.dummySubtitleEndpoint; + } + if (config.subtitleSliceEndpoint) { + this.subtitleSliceEndpoint = config.subtitleSliceEndpoint; + } + if (config.useVTTSubtitles) { + this.use_vtt_subtitles = config.useVTTSubtitles; + } if (config.startWithId) { this.startWithId = config.startWithId; } @@ -94,6 +108,9 @@ class Session { if (config.audioTracks) { this._audioTracks = config.audioTracks; } + if (config.subtitleTracks) { + this._subtitleTracks = config.subtitleTracks; + } if (config.closedCaptions) { this._closedCaptions = config.closedCaptions; } @@ -683,18 +700,94 @@ class Session { throw new Error("Engine not ready"); } } + + async getCurrentSubtitleManifestAsync(subtitleGroupId, subtitleLanguage) { + if (!this._sessionState) { + throw new Error('Session not ready'); + } + let currentVod = null; + const sessionState = await this._sessionState.getValues(["discSeqSubtitle", "vodMediaSeqSubtitle"]); + let playheadState = await this._playheadState.getValues(["mediaSeqSubtitle", "vodMediaSeqSubtitle"]); + + if (playheadState.vodMediaSeqSubtitle > sessionState.vodMediaSeqSubtitle || (playheadState.vodMediaSeqSubtitle < sessionState.vodMediaSeqSubtitle && playheadState.mediaSeqSubtitle === this.prevMediaSeqOffset.subtitle)) { + const state = await this._sessionState.get("state"); + const DELAY_TIME_MS = 1000; + const ACTION = [SessionState.VOD_RELOAD_INIT, SessionState.VOD_RELOAD_INITIATING].includes(state) ? "Reloaded" : "Loaded Next"; + debug(`[${this._sessionId}]: Recently ${ACTION} Vod. PlayheadState not up-to-date (${playheadState.vodMediaSeqSubtitle}_${sessionState.vodMediaSeqSubtitle}). Waiting ${DELAY_TIME_MS}ms before reading from store again`); + await timer(DELAY_TIME_MS); + playheadState = await this._playheadState.getValues(["mediaSeqSubtitle", "vodMediaSeqSubtitle"]); + } + // local store the prev values + if (this.prevVodMediaSeq.subtitle === null) { + this.prevVodMediaSeq.subtitle = playheadState.vodMediaSeqSubtitle; + } + if (this.prevMediaSeqOffset.subtitle === null) { + this.prevMediaSeqOffset.subtitle = playheadState.mediaSeqSubtitle; + } + currentVod = await this._sessionState.getCurrentVod(); + if (currentVod) { + // condition suggesting that a new vod should exist + if (playheadState.vodMediaSeqSubtitle < 2 || playheadState.mediaSeqSubtitle !== this.prevMediaSeqOffset.subtitle) { + const AGE_THRESH = this.averageSegmentDuration * 2; + let cacheAge = null; + if (this._sessionState.cache && this._sessionState.cache.currentVod.ts) { + cacheAge = Date.now() - this._sessionState.cache.currentVod.ts; + } + if (cacheAge !== null && cacheAge > AGE_THRESH) { + await timer(500); + debug(`[${this._sessionId}]: While requesting subtitle manifest for ${subtitleGroupId}-${subtitleLanguage}, (mseq=${playheadState.vodMediaSeqSubtitle})(vod cache age=${cacheAge})`); + await this._sessionState.clearCurrentVodCache(); // force reading up from shared store + currentVod = await this._sessionState.getCurrentVod(); + } + } + try { + let manifestMseq = playheadState.mediaSeqSubtitle + playheadState.vodMediaSeqSubtitle; + let manifestDseq = sessionState.discSeqSubtitle + currentVod.discontinuitiesSubtitle[playheadState.vodMediaSeqSubtitle]; + if (currentVod.sequenceAlwaysContainNewSegments) { + const mediaSequenceValue = currentVod.mediaSequenceValuesSubtitle[playheadState.vodMediaSeqSubtitle]; + debug(`[${this._sessionId}]: {${mediaSequenceValue}}_{${currentVod.getLastSequenceMediaSequenceValueSubtitle()}} SUBTITLES`); + manifestMseq = playheadState.mediaSeqSubtitle + mediaSequenceValue; + } + debug(`[${this._sessionId}]: [${playheadState.vodMediaSeqSubtitle}]_[${currentVod.getLiveMediaSequencesCount("subtitle")}] SUBTITLES (${subtitleGroupId})`); + const m3u8 = currentVod.getLiveMediaSubtitleSequences( + playheadState.mediaSeqSubtitle, + subtitleGroupId, + subtitleLanguage, + playheadState.vodMediaSeqSubtitle, + sessionState.discSeqSubtitle, + this.targetDurationPadding, + this.forceTargetDuration + ); + // # Case: current VOD does not have the selected track. + if (!m3u8) { + debug(`[${this._sessionId}]: [${playheadState.mediaSeqSubtitle + playheadState.vodMediaSeqSubtitle}] Request Failed for current subtitle manifest for ${subtitleGroupId}-${subtitleLanguage}`); + } + debug(`[${this._sessionId}]: [${manifestMseq}][${manifestDseq}] Current subtitle manifest for ${subtitleGroupId}-${subtitleLanguage} requested`); + this.prevVodMediaSeq.subtitle = playheadState.vodMediaSeqSubtitle; + this.prevMediaSeqOffset.subtitle = playheadState.mediaSeqSubtitle; + return m3u8; + } catch (err) { + logerror(this._sessionId, err); + await this._sessionState.clearCurrentVodCache(); // force reading up from shared store + throw new Error("Failed to generate subtitle manifest: " + JSON.stringify(playheadState)); + } + } else { + throw new Error("Engine not ready"); + } + } async incrementAsync() { await this._tickAsync(); const isLeader = await this._sessionStateStore.isLeader(this._instanceId); let sessionState = await this._sessionState.getValues( - ["state", "mediaSeq", "mediaSeqAudio", "discSeq", "discSeqAudio", "vodMediaSeqVideo", "vodMediaSeqAudio"]); - let playheadState = await this._playheadState.getValues(["mediaSeq", "mediaSeqAudio", "vodMediaSeqVideo", "vodMediaSeqAudio"]); + ["state", "mediaSeq", "mediaSeqAudio", "mediaSeqSubtitle", "discSeq", "discSeqAudio", "discSeqSubtitle", "vodMediaSeqVideo", "vodMediaSeqAudio", "vodMediaSeqSubtitle"]); + let playheadState = await this._playheadState.getValues(["mediaSeq", "mediaSeqAudio", "mediaSeqSubtitle", "vodMediaSeqVideo", "vodMediaSeqAudio", "vodMediaSubtitle"]); let currentVod = await this._sessionState.getCurrentVod(); if (!currentVod || sessionState.vodMediaSeqVideo === null || sessionState.vodMediaSeqAudio === null || + sessionState.vodMediaSeqSubtitle === null || sessionState.state === null || sessionState.mediaSeq === null || sessionState.discSeq === null) { @@ -723,7 +816,7 @@ class Session { this.isAllowedToClearVodCache = true; } } else { - sessionState.vodMediaSeqVideo = await this._sessionState.increment("vodMediaSeqVideo"); + sessionState.vodMediaSeqVideo = await this._sessionState.increment("vodMediaSeqVideo", 1); let audioIncrement; if (this.use_demuxed_audio) { let positionV = 0; @@ -761,6 +854,11 @@ class Session { } debug(`[${this._sessionId}]: Will increment audio with ${audioIncrement}`); sessionState.vodMediaSeqAudio = await this._sessionState.increment("vodMediaSeqAudio", audioIncrement); + + if (this.use_vtt_subtitles) { + debug(`[${this._sessionId}]: Will increment subtitle with 1`); + sessionState.vodMediaSeqSubtitle = await this._sessionState.increment("vodMediaSeqSubtitle", 1); + } } if (sessionState.vodMediaSeqVideo >= currentVod.getLiveMediaSequencesCount() - 1) { @@ -772,6 +870,10 @@ class Session { sessionState.vodMediaSeqAudio = await this._sessionState.set("vodMediaSeqAudio", currentVod.getLiveMediaSequencesCount("audio") - 1); } + if (sessionState.vodMediaSeqSubtitle >= currentVod.getLiveMediaSequencesCount("subtitle") - 1) { + sessionState.vodMediaSeqSubtitle = await this._sessionState.set("vodMediaSeqSubtitle", currentVod.getLiveMediaSequencesCount("subtitle") - 1); + } + if (this.isSwitchingBackToV2L) { sessionState.state = await this._sessionState.set("state", SessionState.VOD_RELOAD_INIT); this.isSwitchingBackToV2L = false; @@ -782,8 +884,10 @@ class Session { } playheadState.mediaSeq = await this._playheadState.set("mediaSeq", sessionState.mediaSeq, isLeader); playheadState.mediaSeqAudio = await this._playheadState.set("mediaSeqAudio", sessionState.mediaSeqAudio, isLeader); + playheadState.mediaSeqSubtitle = await this._playheadState.set("mediaSeqSubtitle", sessionState.mediaSeqSubtitle, isLeader); playheadState.vodMediaSeqVideo = await this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo, isLeader); playheadState.vodMediaSeqAudio = await this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio, isLeader); + playheadState.vodMediaSeqSubtitle = await this._playheadState.set("vodMediaSeqSubtitle", sessionState.vodMediaSeqSubtitle, isLeader); if (currentVod.sequenceAlwaysContainNewSegments) { const mediaSequenceValue = currentVod.mediaSequenceValues[playheadState.vodMediaSeqVideo]; @@ -833,12 +937,17 @@ class Session { this.prevMediaSeqOffset.audio = playheadState.mediaSeqAudio; } + if (this.use_vtt_subtitles) { + this.prevVodMediaSeq.subtitle = playheadState.vodMediaSeqSubtitle; + this.prevMediaSeqOffset.subtitle = playheadState.mediaSeqSubtitle; + } + let m3u8 = currentVod.getLiveMediaSequences(playheadState.mediaSeq, 180000, playheadState.vodMediaSeqVideo, sessionState.discSeq); await this._playheadState.setLastM3u8(m3u8); return m3u8; } - async getMediaManifestAsync(bw, opts) { + async getMediaManifestAsync(bw, opts) { // this function is no longer used and should be removed comment added 5/5-2023 await this._tickAsync(); const tsLastRequestVideo = await this._sessionState.get("tsLastRequestVideo"); let timeSinceLastRequest = (tsLastRequestVideo === null) ? 0 : Date.now() - tsLastRequestVideo; @@ -910,7 +1019,7 @@ class Session { } } - async getAudioManifestAsync(audioGroupId, audioLanguage, opts) { + async getAudioManifestAsync(audioGroupId, audioLanguage, opts) { // this function is no longer used and should be removed comment added 5/5-2023 const tsLastRequestAudio = await this._sessionState.get("tsLastRequestAudio"); let timeSinceLastRequest = (tsLastRequestAudio === null) ? 0 : Date.now() - tsLastRequestAudio; @@ -963,6 +1072,7 @@ class Session { let audioGroupIds = currentVod.getAudioGroups(); debug(`[${this._sessionId}]: currentVod.getAudioGroups()=${audioGroupIds.join(",")}`); let defaultAudioGroupId; + let defaultSubtitleGroupId; let hasClosedCaptions = this._closedCaptions && this._closedCaptions.length > 0; if (hasClosedCaptions) { this._closedCaptions.forEach(cc => { @@ -996,6 +1106,28 @@ class Session { defaultAudioGroupId = audioGroupIds[0]; } } + if (this.use_vtt_subtitles) { + let subtitleGroupIds = currentVod.getSubtitleGroups(); + if (subtitleGroupIds.length > 0) { + m3u8 += "# Subtitle groups\n"; + for (let i = 0; i < subtitleGroupIds.length; i++) { + let subtitleGroupId = subtitleGroupIds[i]; + for (let j = 0; j < this._subtitleTracks.length; j++) { + let subtitleTrack = this._subtitleTracks[j]; + // Make default track if set property is true. TODO add enforce + m3u8 += `#EXT-X-MEDIA:TYPE=SUBTITLES` + + `,GROUP-ID="${subtitleGroupId}"` + + `,LANGUAGE="${subtitleTrack.language}"` + + `,NAME="${subtitleTrack.name}"` + + `,AUTOSELECT=YES,DEFAULT=${subtitleTrack.default ? 'YES' : 'NO'}` + + `,URI="subtitles-${subtitleGroupId}_${subtitleTrack.language}.m3u8%3Bsession=${this._sessionId}"` + + "\n"; + } + } + // As of now, by default set StreamItem's SUBTITLES attribute to + defaultSubtitleGroupId = subtitleGroupIds[0]; + } + } if (this._sessionProfile) { const sessionProfile = filter ? applyFilter(this._sessionProfile, filter) : this._sessionProfile; sessionProfile.forEach(profile => { @@ -1019,6 +1151,7 @@ class Session { ',RESOLUTION=' + profile.resolution[0] + 'x' + profile.resolution[1] + ',CODECS="' + profile.codecs + '"' + `,AUDIO="${audioGroupIdToUse}"` + + (defaultSubtitleGroupId ? `,SUBTITLES="${defaultSubtitleGroupId}"` : '') + (hasClosedCaptions ? ',CLOSED-CAPTIONS="cc"' : '') + '\n'; m3u8 += "master" + profile.bw + ".m3u8%3Bsession=" + this._sessionId + "\n"; } @@ -1027,6 +1160,7 @@ class Session { ',RESOLUTION=' + profile.resolution[0] + 'x' + profile.resolution[1] + ',CODECS="' + profile.codecs + '"' + (defaultAudioGroupId ? `,AUDIO="${defaultAudioGroupId}"` : '') + + (defaultSubtitleGroupId ? `,SUBTITLES="${defaultSubtitleGroupId}"` : '') + (hasClosedCaptions ? ',CLOSED-CAPTIONS="cc"' : '') + '\n'; m3u8 += "master" + profile.bw + ".m3u8%3Bsession=" + this._sessionId + "\n"; } @@ -1037,6 +1171,7 @@ class Session { ',RESOLUTION=' + profile.resolution + ',CODECS="' + profile.codecs + '"' + (defaultAudioGroupId ? `,AUDIO="${defaultAudioGroupId}"` : '') + + (defaultSubtitleGroupId ? `,SUBTITLES="${defaultSubtitleGroupId}"` : '') + (hasClosedCaptions ? ',CLOSED-CAPTIONS="cc"' : '') + '\n'; m3u8 += "master" + profile.bw + ".m3u8%3Bsession=" + this._sessionId + "\n"; }); @@ -1068,6 +1203,21 @@ class Session { return allAudioGroupsAndTheirLanguages; } + async getSubtitleGroupsAndLangs() { + const currentVod = await this._sessionState.getCurrentVod(); + if (!currentVod) { + throw new Error('Session not ready'); + } + const subtitleGroupIds = currentVod.getSubtitleGroups(); + let allSubtitleGroupsAndTheirLanguages = {}; + subtitleGroupIds.forEach((groupId) => { + allSubtitleGroupsAndTheirLanguages[groupId] = + currentVod.getSubtitleLangsForSubtitleGroup(groupId); + }); + + return allSubtitleGroupsAndTheirLanguages; + } + consumeEvent() { return this._events.shift(); } @@ -1080,7 +1230,7 @@ class Session { return !this.disabledPlayhead; } - async _insertSlate(currentVod) { + async _insertSlate(currentVod) { // no support for subs if (this.slateUri) { console.error(`[${this._sessionId}]: Will insert slate`); const slateVod = await this._loadSlate(currentVod); @@ -1125,7 +1275,7 @@ class Session { let newVod; let sessionState = await this._sessionState.getValues( - ["state", "assetId", "vodMediaSeqVideo", "vodMediaSeqAudio", "mediaSeq", "mediaSeqAudio", "discSeq", "discSeqAudio", "nextVod"]); + ["state", "assetId", "vodMediaSeqVideo", "vodMediaSeqAudio", "vodMediaSeqSubtitle", "mediaSeq", "mediaSeqAudio", "mediaSeqSubtitle", "discSeq", "discSeqAudio", "discSeqSubtitle", "nextVod"]); let isLeader = await this._sessionStateStore.isLeader(this._instanceId); @@ -1161,7 +1311,13 @@ class Session { let loadPromise; if (!vodResponse.type) { debug(`[${this._sessionId}]: got first VOD uri=${vodResponse.uri}:${vodResponse.offset || 0}`); - const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, forcedDemuxMode: this.use_demuxed_audio }; + const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, + forcedDemuxMode: this.use_demuxed_audio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + shouldContainSubtitles: this.use_vtt_subtitles, + expectedSubtitleTracks: this._subtitleTracks + }; newVod = new HLSVod(vodResponse.uri, [], vodResponse.unixTs, vodResponse.offset * 1000, m3u8Header(this._instanceId), hlsOpts); if (vodResponse.timedMetadata) { Object.keys(vodResponse.timedMetadata).map(k => { @@ -1192,10 +1348,13 @@ class Session { //debug(newVod); sessionState.mediaSeq = await this._sessionState.set("mediaSeq", 0); sessionState.mediaSeqAudio = await this._sessionState.set("mediaSeqAudio", 0); + sessionState.mediaSeqSubtitle = await this._sessionState.set("mediaSeqSubtitle", 0); sessionState.discSeq = await this._sessionState.set("discSeq", 0); sessionState.discSeqAudio = await this._sessionState.set("discSeqAudio", 0); + sessionState.discSeqSubtitle = await this._sessionState.set("discSeqSubtitle", 0); sessionState.vodMediaSeqVideo = await this._sessionState.set("vodMediaSeqVideo", 0); sessionState.vodMediaSeqAudio = await this._sessionState.set("vodMediaSeqAudio", 0); + sessionState.vodMediaSeqSubtitle = await this._sessionState.set("vodMediaSeqSubtitle", 0); await this._playheadState.set("playheadRef", Date.now(), isLeader); this.produceEvent({ type: 'NOW_PLAYING', @@ -1246,6 +1405,14 @@ class Session { this.prevMediaSeqOffset.audio = sessionState.mediaSeqAudio; } } + if (this.use_vtt_subtitles) { + if (this.prevVodMediaSeq.subtitle === null) { + this.prevVodMediaSeq.subtitle = sessionState.vodMediaSeqSubtitle; + } + if (this.prevMediaSeqOffset.audio === null) { + this.prevMediaSeqOffset.subtitle = sessionState.mediaSeqSubtitle; + } + } // Clear Cache if prev count is HIGHER than current... if (sessionState.vodMediaSeqVideo < this.prevVodMediaSeq.video) { debug(`[${this._sessionId}]: state=VOD_PLAYING, current[${sessionState.vodMediaSeqVideo}], prev[${this.prevVodMediaSeq.video}], total[${currentVod.getLiveMediaSequencesCount()}]`); @@ -1253,6 +1420,7 @@ class Session { currentVod = await this._sessionState.getCurrentVod(); this.prevVodMediaSeq.video = sessionState.vodMediaSeqVideo; this.prevVodMediaSeq.audio = sessionState.vodMediaSeqAudio; + this.prevVodMediaSeq.subtitle = sessionState.vodMediaSeqSubtitle; } } else { // Handle edge case where Leader loaded next vod but Follower remained in state=VOD_PLAYING @@ -1263,10 +1431,10 @@ class Session { this.isAllowedToClearVodCache = true; } } - debug(`[${this._sessionId}]: state=VOD_PLAYING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")})`); + debug(`[${this._sessionId}]: state=VOD_PLAYING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}_${sessionState.vodMediaSeqSubtitle}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")}_${currentVod.getLiveMediaSequencesCount("subtitle")})`); return; case SessionState.VOD_NEXT_INITIATING: - debug(`[${this._sessionId}]: state=VOD_NEXT_INITIATING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")})`); + debug(`[${this._sessionId}]: state=VOD_NEXT_INITIATING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}_${sessionState.vodMediaSeqSubtitle}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")}_${currentVod.getLiveMediaSequencesCount("audio")}_${currentVod.getLiveMediaSequencesCount("subtitle")})`); if (!isLeader) { debug(`[${this._sessionId}]: not the leader so just waiting for the VOD to be initiated`); } @@ -1285,15 +1453,19 @@ class Session { const length = currentVod.getLiveMediaSequencesCount(); let endMseqValue; let endMseqValueAudio; + let endMseqValueSubtitle; if (currentVod.sequenceAlwaysContainNewSegments) { endMseqValue = currentVod.getLastSequenceMediaSequenceValue(); endMseqValueAudio = currentVod.getLastSequenceMediaSequenceValueAudio(); + endMseqValueSubtitle = currentVod.getLastSequenceMediaSequenceValueSubtitle(); } else { endMseqValue = currentVod.getLiveMediaSequencesCount(); endMseqValueAudio = currentVod.getLiveMediaSequencesCount("audio"); + endMseqValueSubtitle = currentVod.getLiveMediaSequencesCount("subtitle"); } const lastDiscontinuity = currentVod.getLastDiscontinuity(); const lastDiscontinuityAudio = currentVod.getLastDiscontinuityAudio(); + const lastDiscontinuitySubtitle = currentVod.getLastDiscontinuitySubtitle(); sessionState.state = await this._sessionState.set("state", SessionState.VOD_NEXT_INITIATING); let vodPromise = this._getNextVod(); if (length === 1) { @@ -1310,7 +1482,13 @@ class Session { let loadPromise; if (!vodResponse.type) { debug(`[${this._sessionId}]: got next VOD uri=${vodResponse.uri}:${vodResponse.offset}`); - const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, forcedDemuxMode: this.use_demuxed_audio }; + const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, + forcedDemuxMode: this.use_demuxed_audio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + shouldContainSubtitles: this.use_vtt_subtitles, + expectedSubtitleTracks: this._subtitleTracks + }; newVod = new HLSVod(vodResponse.uri, null, vodResponse.unixTs, vodResponse.offset * 1000, m3u8Header(this._instanceId), hlsOpts); if (vodResponse.timedMetadata) { Object.keys(vodResponse.timedMetadata).map(k => { @@ -1354,14 +1532,17 @@ class Session { debug(`[${this._sessionId}]: playhead positions [V]=${newVod.getPlayheadPositions("video")}`); debug(`[${this._sessionId}]: playhead positions [A]=${newVod.getPlayheadPositions("audio")}`); currentVod = newVod; - debug(`[${this._sessionId}]: msequences=${currentVod.getLiveMediaSequencesCount()}; audio msequences=${currentVod.getLiveMediaSequencesCount("audio")}`); + debug(`[${this._sessionId}]: msequences=${currentVod.getLiveMediaSequencesCount()}; audio msequences=${currentVod.getLiveMediaSequencesCount("audio")}; subtitle msequences=${currentVod.getLiveMediaSequencesCount("subtitle")}`); sessionState.vodMediaSeqVideo = await this._sessionState.set("vodMediaSeqVideo", 0); sessionState.vodMediaSeqAudio = await this._sessionState.set("vodMediaSeqAudio", 0); + sessionState.vodMediaSeqSubtitle = await this._sessionState.set("vodMediaSeqSubtitle", 0); sessionState.mediaSeq = await this._sessionState.set("mediaSeq", sessionState.mediaSeq + endMseqValue); sessionState.mediaSeqAudio = await this._sessionState.set("mediaSeqAudio", sessionState.mediaSeqAudio + endMseqValueAudio); + sessionState.mediaSeqSubtitle = await this._sessionState.set("mediaSeqSubtitle", sessionState.mediaSeqSubtitle + endMseqValueSubtitle); sessionState.discSeq = await this._sessionState.set("discSeq", sessionState.discSeq + lastDiscontinuity); sessionState.discSeqAudio = await this._sessionState.set("discSeqAudio", sessionState.discSeqAudio + lastDiscontinuityAudio); - debug(`[${this._sessionId}]: new sequence data set in store [${sessionState.mediaSeq}][${sessionState.discSeq}]_[${sessionState.mediaSeqAudio}][${sessionState.discSeqAudio}]`); + sessionState.discSeqSubtitle = await this._sessionState.set("discSeqSubtitle", sessionState.discSeqSubtitle + lastDiscontinuitySubtitle); + debug(`[${this._sessionId}]: new sequence data set in store [${sessionState.mediaSeq}][${sessionState.discSeq}]_[${sessionState.mediaSeqSubtitle}][${sessionState.discSeqSubtitle}]_[${sessionState.mediaSeqSubtitle}][${sessionState.discSeqSubtitle}]`); await this._sessionState.remove("nextVod"); sessionState.currentVod = await this._sessionState.setCurrentVod(currentVod, { ttl: currentVod.getDuration() * 1000 }); this.leaderIsSettingNextVod = false; @@ -1456,7 +1637,7 @@ class Session { } // ---------------------------------------------------. - // TODO: Support reloading with audioSegments as well | + // TODO: Support reloading with audioSegments and SubtitleSegments as well | // ---------------------------------------------------' await currentVod.reload(nextMseq, segments, null, reloadBehind); @@ -1464,8 +1645,10 @@ class Session { await this._sessionState.set("vodReloaded", 1); await this._sessionState.set("vodMediaSeqVideo", 0); await this._sessionState.set("vodMediaSeqAudio", 0); + await this._sessionState.set("vodMediaSeqSubtitle", 0); await this._playheadState.set("vodMediaSeqVideo", 0, isLeader); await this._playheadState.set("vodMediaSeqAudio", 0, isLeader); + await this._playheadState.set("vodMediaSeqSubtitle", 0, isLeader); await this._playheadState.set("playheadRef", Date.now(), isLeader); // 4) Log to debug and cloudwatch debug(`[${this._sessionId}]: LEADER: Set new Reloaded VOD and vodMediaSeq counts in store.`); @@ -1484,7 +1667,7 @@ class Session { } break; case SessionState.VOD_RELOAD_INITIATING: - debug(`[${this._sessionId}]: state=VOD_RELOAD_INITIATING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")})`); + debug(`[${this._sessionId}]: state=VOD_RELOAD_INITIATING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}_${sessionState.vodMediaSeqSubtitle}, ${currentVod.getLiveMediaSequencesCount()}_${currentVod.getLiveMediaSequencesCount("audio")}_${currentVod.getLiveMediaSequencesCount("subtitle")})`); if (!isLeader) { debug(`[${this._sessionId}]: not the leader so just waiting for the VOD to be reloaded`); if (sessionState.vodMediaSeqVideo === 0 || this.waitingForNextVod) { @@ -1541,7 +1724,13 @@ class Session { slateVod.load() .then(() => { - const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, forcedDemuxMode: this.use_demuxed_audio }; + const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, + forcedDemuxMode: this.use_demuxed_audio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + shouldContainSubtitles: this.use_vtt_subtitles, + expectedSubtitleTracks: this._subtitleTracks + }; const timestamp = Date.now(); hlsVod = new HLSVod(this.slateUri, null, timestamp, null, m3u8Header(this._instanceId), hlsOpts); hlsVod.addMetadata('id', `slate-${timestamp}`); @@ -1601,7 +1790,14 @@ class Session { slateVod.load() .then(() => { - const hlsOpts = { sequenceAlwaysContainNewSegments: this.alwaysNewSegments, forcedDemuxMode: this.use_demuxed_audio }; + const hlsOpts = { + sequenceAlwaysContainNewSegments: this.alwaysNewSegments, + forcedDemuxMode: this.use_demuxed_audio, + dummySubtitleEndpoint: this.dummySubtitleEndpoint, + subtitleSliceEndpoint: this.subtitleSliceEndpoint, + shouldContainSubtitles: this.use_vtt_subtitles, + expectedSubtitleTracks: this._subtitleTracks + }; const timestamp = Date.now(); hlsVod = new HLSVod(nexVodUri, null, timestamp, null, m3u8Header(this._instanceId), hlsOpts); hlsVod.addMetadata('id', `slate-${timestamp}`); @@ -1737,6 +1933,16 @@ class Session { return playheadPositions[seqIdx]; } + async _getSubtitlePlayheadPosition(seqIdx) { + const currentVod = await this._sessionState.getCurrentVod(); + const playheadPositions = currentVod.getPlayheadPositions("subtitle"); + if (seqIdx >= playheadPositions.length - 1) { + seqIdx = playheadPositions.length - 1 + } + debug(`[${this._sessionId}]: Current subtitle playhead position (${seqIdx}): ${playheadPositions[seqIdx]}`); + return playheadPositions[seqIdx]; + } + _getLastDuration(manifest) { return new Promise((resolve, reject) => { try { diff --git a/engine/session_live.js b/engine/session_live.js index 19823f1..dd783a7 100644 --- a/engine/session_live.js +++ b/engine/session_live.js @@ -446,6 +446,11 @@ class SessionLive { return "Not Implemented"; } + async getCurrentSubtitleManifestAsync(subtitleGroupId, subtitleLanguage) { + debug(`[${this.sessionId}]: getCurrentSubtitleManifestAsync is NOT Implemented`); + return "Not Implemented"; + } + /** * * @param {string} masterManifestURI The master manifest URI. diff --git a/engine/session_state.js b/engine/session_state.js index 3c8901f..1fb386d 100644 --- a/engine/session_state.js +++ b/engine/session_state.js @@ -139,13 +139,17 @@ class SessionStateStore extends SharedStateStore { discSeq: 0, mediaSeqAudio: 0, discSeqAudio: 0, + mediaSeqSubtitle: 0, + discSeqSubtitle: 0, vodMediaSeqVideo: 0, vodMediaSeqAudio: 0, // assume only one audio group now + vodMediaSeqSubtitle: 0, // assume only one subtitle group now state: SessionState.VOD_INIT, lastM3u8: {}, tsLastRequestVideo: null, tsLastRequestMaster: null, tsLastRequestAudio: null, + tsLastRequestSubtitle: null, currentVod: null, slateCount: 0, assetId: "", diff --git a/engine/subtitle_slicer.js b/engine/subtitle_slicer.js new file mode 100644 index 0000000..5674700 --- /dev/null +++ b/engine/subtitle_slicer.js @@ -0,0 +1,175 @@ +const fetch = require("node-fetch"); +const fs = require("fs"); +class SubtitleSlicer { + constructor() { + this.vttFiles = {}; + } + + async getVttFile(url) { + let resp = await fetch(url) + if (resp.status === 200) { // TODO add error handeling + let text = await resp.text(); + return text; + } + else { + return ""; + } + + } + + checkTimeStamp(line, startTime, endTime, elapsedtime) { + const times = line.split("-->"); + let startTimeTimestamp = times[0].split(":"); + let endTimeTimestamp = times[1].split(":"); + let startTimeTimestampInSec = parseInt(startTimeTimestamp[0]) * 3600; + startTimeTimestampInSec += parseInt(startTimeTimestamp[1]) * 60; + const startTimeSecondsAndFractions = startTimeTimestamp[2].split("."); + startTimeTimestampInSec += parseInt(startTimeSecondsAndFractions[0]); + + let endTimeTimestampInSec = parseInt(endTimeTimestamp[0]) * 3600; + endTimeTimestampInSec += parseInt(endTimeTimestamp[1]) * 60; + const endTimeSecondsAndFractions = endTimeTimestamp[2].split("."); + endTimeTimestampInSec += parseInt(endTimeSecondsAndFractions[0]); + startTime = parseInt(startTime); + endTime = parseInt(endTime); + elapsedtime = parseInt(elapsedtime); + startTime += elapsedtime; + endTime += elapsedtime; + + if (startTime <= startTimeTimestampInSec && startTimeTimestampInSec < endTime) { + return true; + } + if (startTime < endTimeTimestampInSec && endTimeTimestampInSec <= endTime) { + return true; + } + if (startTimeTimestampInSec < startTime && endTime < endTimeTimestampInSec) { + return true; + } + + return false + + } + + streamToString(stream) { + const chunks = []; + return new Promise((resolve, reject) => { + stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + stream.on('error', (err) => reject(err)); + stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + }) + } + + async generateVtt(params, _injectedVttFile, _injectedPreviousVttFile) { + const paramEncode = new URLSearchParams(params) + const uri = paramEncode.get("vtturi"); + const startTime = paramEncode.get("starttime"); + const endTime = paramEncode.get("endtime"); + const elapsedTime = paramEncode.get("elapsedtime"); + const previousParams = new URLSearchParams(paramEncode.get("previousvtturi")) + const previousUri = previousParams.get("vtturi"); + const previousStartTime = previousParams.get("starttime"); + const previousEndTime = previousParams.get("endtime"); + const previousElapsedTime = previousParams.get("elapsedtime"); + + let file = ""; + let previousFile = ""; + let newFile = ""; + if (uri) { + if (this.vttFiles.length) { + file = this.vttFile[uri]; + } + if (!file) { + file = await this.getVttFile(uri) + } + } else if (_injectedVttFile) { + file = await this.streamToString(_injectedVttFile) + } else { + console.error("no vtt file provided"); + } + + if (previousUri) { + if (this.vttFiles.length) { + previousFile = this.vttFile[previousUri]; + } + if (!previousFile) { + previousFile = await this.getVttFile(previousUri) + } + } else if (_injectedPreviousVttFile) { + previousFile = await this.streamToString(_injectedPreviousVttFile) + } + + const previousFileLines = previousFile.split("\n"); + let previousFileContentToAdd = ""; + for (let i = 0; i < previousFileLines.length; i++) { + const ss = previousFileLines[i]; + if (ss.match(/(\d+):(\d+):(\d+).(\d+) --> (\d+):(\d+):(\d+).(\d+)/)?.input) { + let shouldAdd = this.checkTimeStamp(ss, previousStartTime, previousEndTime, previousElapsedTime) + if (shouldAdd) { + if (previousFileLines[i - 1]) { + if (previousFileLines[i - 1].slice(0, 4) !== "NOTE") + previousFileContentToAdd += previousFileLines[i - 1] + "\n"; + } + + previousFileContentToAdd += ss + "\n"; + + let j = 1; + while (previousFileLines.length > i + j) { + if (previousFileLines[i + j]) { + previousFileContentToAdd += previousFileLines[i + j] + "\n"; + } else { + break; + } + j++; + } + previousFileContentToAdd += "\n" + } + } + } + + const lines = file.split("\n") + let addedOnce = false; + + for (let i = 0; i < lines.length; i++) { + let ss = lines[i] + switch (ss) { + case ss.match("WEBVTT")?.input: + newFile += ss + "\n" + break; + case ss.match(/X-TIMESTAMP-MAP/)?.input: + if (!ss.match(/LOCAL:00:00:00.000/) || !ss.match(/MPEGTS:0/)) { + console.warn("MPEGTS and/or LOCAL is not zero") + } + newFile += ss + "\n\n" + break; + case ss.match(/(\d+):(\d+):(\d+).(\d+) --> (\d+):(\d+):(\d+).(\d+)/)?.input: + let shouldAdd = this.checkTimeStamp(ss, startTime, endTime, elapsedTime) + if (shouldAdd) { + if (!addedOnce) { + addedOnce = true; + newFile += previousFileContentToAdd; + } + + if (lines[i - 1]) { + if (lines[i - 1].slice(0, 4) !== "NOTE") + newFile += lines[i - 1] + "\n"; + } + newFile += ss + "\n"; + let j = 1; + while (lines.length > i + j) { + if (lines[i + j]) { + newFile += lines[i + j] + "\n"; + } else { + break; + } + j++; + } + newFile += "\n" + } + break; + } + } + return newFile; + } +} + +module.exports = SubtitleSlicer; \ No newline at end of file diff --git a/index.ts b/index.ts index fbeb727..0b5fa52 100644 --- a/index.ts +++ b/index.ts @@ -8,5 +8,6 @@ export { ChannelEngine, Channel, ChannelProfile, AudioTracks, - Schedule + SubtitleTracks, + Schedule, } from "./engine/server"; diff --git a/package-lock.json b/package-lock.json index aed033e..2af79d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@eyevinn/hls-repeat": "^0.2.0", "@eyevinn/hls-truncate": "^0.2.0", - "@eyevinn/hls-vodtolive": "^2.3.12", + "@eyevinn/hls-vodtolive": "^3.0.0-rc.4", "@eyevinn/m3u8": "^0.5.3", "abort-controller": "^3.0.0", "debug": "^3.2.7", @@ -96,9 +96,9 @@ } }, "node_modules/@eyevinn/hls-vodtolive": { - "version": "2.3.12", - "resolved": "https://registry.npmjs.org/@eyevinn/hls-vodtolive/-/hls-vodtolive-2.3.12.tgz", - "integrity": "sha512-SsbZBoeX6JE2htoSP13UKg/dJ1BahrKc9N6+tjwDTrq0n6qh7V6GRm+6Mnm+P1QmqmanaRudX8qlSYKWuBtMYA==", + "version": "3.0.0-rc.4", + "resolved": "https://registry.npmjs.org/@eyevinn/hls-vodtolive/-/hls-vodtolive-3.0.0-rc.4.tgz", + "integrity": "sha512-WXd0fmCp5LT6A81LiOMfEkaKJfiETspZgK/7luw8bBEZJqwWMErxELqvO75JB0d86LNNmlOhj6Lt0RXt5vCjSA==", "dependencies": { "@eyevinn/m3u8": "^0.5.6", "abort-controller": "^3.0.0", @@ -2621,9 +2621,9 @@ } }, "@eyevinn/hls-vodtolive": { - "version": "2.3.12", - "resolved": "https://registry.npmjs.org/@eyevinn/hls-vodtolive/-/hls-vodtolive-2.3.12.tgz", - "integrity": "sha512-SsbZBoeX6JE2htoSP13UKg/dJ1BahrKc9N6+tjwDTrq0n6qh7V6GRm+6Mnm+P1QmqmanaRudX8qlSYKWuBtMYA==", + "version": "3.0.0-rc.4", + "resolved": "https://registry.npmjs.org/@eyevinn/hls-vodtolive/-/hls-vodtolive-3.0.0-rc.4.tgz", + "integrity": "sha512-WXd0fmCp5LT6A81LiOMfEkaKJfiETspZgK/7luw8bBEZJqwWMErxELqvO75JB0d86LNNmlOhj6Lt0RXt5vCjSA==", "requires": { "@eyevinn/m3u8": "^0.5.6", "abort-controller": "^3.0.0", diff --git a/package.json b/package.json index 13ff2b2..8dd810b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "dependencies": { "@eyevinn/hls-repeat": "^0.2.0", "@eyevinn/hls-truncate": "^0.2.0", - "@eyevinn/hls-vodtolive": "^2.3.12", + "@eyevinn/hls-vodtolive": "^3.0.0-rc.4", "@eyevinn/m3u8": "^0.5.3", "abort-controller": "^3.0.0", "debug": "^3.2.7", diff --git a/server-demux.ts b/server-demux.ts index 9c9e66d..aeae58d 100644 --- a/server-demux.ts +++ b/server-demux.ts @@ -12,6 +12,7 @@ import { Channel, ChannelProfile, AudioTracks, + SubtitleTracks, } from "./index"; class RefAssetManager implements IAssetManager { @@ -22,8 +23,8 @@ class RefAssetManager implements IAssetManager { 1: [ { id: 1, - title: "Sintel", - uri: "https://cdn.bitmovin.com/content/assets/sintel/hls/playlist.m3u8", + title: "Elephants dream", + uri: "https://playertest.longtailvideo.com/adaptive/elephants_dream_v4/index.m3u8", }, { id: 2, @@ -54,10 +55,6 @@ class RefAssetManager implements IAssetManager { if (this.pos[channelId] > this.assets[channelId].length - 1) { this.pos[channelId] = 0; } - vod.timedMetadata = { - 'start-date': new Date().toISOString(), - 'class': 'se.eyevinn.demo' - }; resolve(vod); } else { reject("Invalid channelId provided"); @@ -68,7 +65,7 @@ class RefAssetManager implements IAssetManager { class RefChannelManager implements IChannelManager { getChannels(): Channel[] { - return [{ id: "1", profile: this._getProfile(), audioTracks: this._getAudioTracks() }]; + return [{ id: "1", profile: this._getProfile(), audioTracks: this._getAudioTracks(), subtitleTracks: this._getSubtitleTracks() }]; } _getProfile(): ChannelProfile[] { @@ -92,7 +89,13 @@ class RefChannelManager implements IChannelManager { _getAudioTracks(): AudioTracks[] { return [ { language: "en", name: "English", default: true }, - { language: "sp", name: "Spanish", default: false } + { language: "es", name: "Spanish", default: false }, + ]; + } + _getSubtitleTracks(): SubtitleTracks[] { + return [ + { language: "zh", name: "chinese", default: true }, + { language: "fr", name: "french", default: false } ]; } } @@ -104,11 +107,12 @@ const engineOptions: ChannelEngineOpts = { heartbeat: "/", averageSegmentDuration: 2000, channelManager: refChannelManager, - defaultSlateUri: "https://maitv-vod.lab.eyevinn.technology/slate-consuo.mp4/master.m3u8", + defaultSlateUri: "https://mtoczko.github.io/hls-test-streams/test-audio-pdt/playlist.m3u8", slateRepetitions: 10, redisUrl: process.env.REDIS_URL, useDemuxedAudio: true, - alwaysNewSegments: true, + alwaysNewSegments: false, + useVTTSubtitles: true }; const engine = new ChannelEngine(refAssetManager, engineOptions); diff --git a/spec/engine/subtitle_spec.js b/spec/engine/subtitle_spec.js new file mode 100644 index 0000000..d07205a --- /dev/null +++ b/spec/engine/subtitle_spec.js @@ -0,0 +1,119 @@ +const SubtitleSlicer = require('../../engine/subtitle_slicer.js'); +const fs = require("fs"); + + + +describe("Subtitle slicer", () => { + let mockWebVtt; + let mockWebVtt2; + beforeEach(() => { + mockWebVtt = fs.createReadStream("spec/testvectors/subtitle_file.webvtt"); + mockWebVtt2 = fs.createReadStream("spec/testvectors/subtitle_file_2.webvtt"); + }); + + it("generate sliced vtt with correct time stamps", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 15); + params.append("endtime", 20); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[0]).toEqual("WEBVTT"); + expect(subStrings[1]).toEqual("X-TIMESTAMP-MAP=MPEGTS:0,LOCAL:00:00:00.000"); + expect(subStrings[3]).toEqual("00:00:15.000 --> 00:00:18.000"); + expect(subStrings[4]).toEqual("À votre gauche vous pouvez voir..."); + expect(subStrings[6]).toEqual("00:00:18.000 --> 00:00:20.000"); + expect(subStrings[7]).toEqual("À votre droite vous pouvez voir les..."); + done(); + }); + + it("generate sliced with correct line without comment", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 26); + params.append("endtime", 27); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[2]).toEqual(""); + expect(subStrings[3]).toEqual("00:00:26.000 --> 00:00:27.000"); + expect(subStrings[4]).toEqual("Emo ?"); + done(); + }); + + it("with explicit cue positions", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 51); + params.append("endtime", 53); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[3]).toEqual("00:00:51.000 --> 00:00:53.000 align:left size:50%"); + expect(subStrings[4]).toEqual("Je crois pas...|et vous ?"); + done(); + }); + + it("generate sliced with all lines relevant to timestamp", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 55); + params.append("endtime", 57); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[3]).toEqual("00:00:55.000 --> 00:00:57.000"); + expect(subStrings[4]).toEqual("Ça va."); + expect(subStrings[5]).toEqual("Ça va."); + expect(subStrings[6]).toEqual("Ça va."); + done(); + }); + + it("with chapters", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 62); + params.append("endtime", 63); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[3]).toEqual("Slide 3"); + expect(subStrings[4]).toEqual("00:01:02.000 --> 00:01:03.000"); + expect(subStrings[5]).toEqual("Allons-y."); + done(); + }); + + it("with text that is longer than one segment", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + params.append("starttime", 64); + params.append("endtime", 68); + params.append("elapsedtime", 0); + vttFile = await subslice.generateVtt(params, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[3]).toEqual("00:01:03.000 --> 00:01:09.000"); + expect(subStrings[4]).toEqual("Et après ?"); + done(); + }); + + it("that handles slice over multiple files", async (done) => { + let subslice = new SubtitleSlicer(); + let params = new URLSearchParams(); + let perviousParams = new URLSearchParams(); + perviousParams.append("starttime", 63) + perviousParams.append("endtime", 69) + perviousParams.append("elapsedtime", 0) + params.append("previousvtturi", perviousParams) + params.append("starttime", 0) + params.append("endtime", 3) + params.append("elapsedtime", 69) + vttFile = await subslice.generateVtt(params, mockWebVtt2, mockWebVtt) + const subStrings = vttFile.split("\n") + expect(subStrings[3]).toEqual("00:01:03.000 --> 00:01:09.000"); + expect(subStrings[4]).toEqual("Et après ?"); + expect(subStrings[6]).toEqual("00:01:09.000 --> 00:01:12.000"); + expect(subStrings[7]).toEqual("À votre gauche vous pouvez voir..."); + done(); + }); +}); diff --git a/spec/testvectors/subtitle_file.webvtt b/spec/testvectors/subtitle_file.webvtt new file mode 100644 index 0000000..c597a7f --- /dev/null +++ b/spec/testvectors/subtitle_file.webvtt @@ -0,0 +1,44 @@ +WEBVTT +X-TIMESTAMP-MAP=MPEGTS:0,LOCAL:00:00:00.000 + +00:00:15.000 --> 00:00:18.000 +À votre gauche vous pouvez voir... + +00:00:18.000 --> 00:00:20.000 +À votre droite vous pouvez voir les... + +00:00:20.000 --> 00:00:21.000 +... les étêteurs. + +00:00:22.000 --> 00:00:24.000 +Tout est sans danger.|Parfaitement sans danger. + +NOTE a test comment +00:00:26.000 --> 00:00:27.000 +Emo ? + +00:00:28.000 --> 00:00:30.000 +Attention ! + +00:00:46.000 --> 00:00:48.000 +Tu n’as rien ? + +00:00:51.000 --> 00:00:53.000 align:left size:50% +Je crois pas...|et vous ? + +00:00:55.000 --> 00:00:57.000 +Ça va. +Ça va. +Ça va. + +00:00:57.000 --> 00:01:01.000 +Lève-toi.|Emo, ce n’est pas sûr ici. + +Slide 3 +00:01:02.000 --> 00:01:03.000 +Allons-y. + +00:01:03.000 --> 00:01:09.000 +Et après ? + + diff --git a/spec/testvectors/subtitle_file_2.webvtt b/spec/testvectors/subtitle_file_2.webvtt new file mode 100644 index 0000000..72f2651 --- /dev/null +++ b/spec/testvectors/subtitle_file_2.webvtt @@ -0,0 +1,5 @@ +WEBVTT +X-TIMESTAMP-MAP=MPEGTS:0,LOCAL:00:00:00.000 + +00:01:09.000 --> 00:01:12.000 +À votre gauche vous pouvez voir...