Skip to content

Commit

Permalink
Feat: New Option for Partial Object Serialization (toJSON) (#114)
Browse files Browse the repository at this point in the history 
* Feat: add option to skip serializing mediaSequences
and also add new member function for generating media sequences only

* feat: add option for only patial serialization and
external media sequence generation

* test: for new partial serialization option
  • Loading branch information
@Nfrederiksen
Nfrederiksen authored Aug 11, 2023
1 parent d3102a3 commit b582faf
Show file tree
Hide file tree
Showing 15 changed files with 3,832 additions and 22 deletions.
68 changes: 46 additions & 22 deletions index.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
const m3u8 = require("@eyevinn/m3u8");
const { deserialize } = require("v8");
const debug = require("debug")("hls-vodtolive");
const verbose = require("debug")("hls-vodtolive-verbose");
const { findIndexReversed, fetchWithRetry, urlResolve, segToM3u8, findBottomSegItem } = require("./utils.js");
Expand Down Expand Up @@ -49,6 +48,7 @@ class HLSVod {
this.header = header;
this.lastUsedDiscSeq = null;
this.sequenceAlwaysContainNewSegments = false;
this.skipSerializeMediaSequences = false;
if (opts && opts.sequenceAlwaysContainNewSegments) {
this.sequenceAlwaysContainNewSegments = opts.sequenceAlwaysContainNewSegments;
}
Expand All @@ -70,6 +70,9 @@ class HLSVod {
if (opts && opts.alwaysMapBandwidthByNearest) {
this.alwaysMapBandwidthByNearest = opts.alwaysMapBandwidthByNearest;
}
if (opts && opts.skipSerializeMediaSequences) {
this.skipSerializeMediaSequences = opts.skipSerializeMediaSequences;
}
this.videoSequencesCount = 0;
this.audioSequencesCount = 0;
this.defaultAudioGroupAndLang = null;
Expand All @@ -86,7 +89,7 @@ class HLSVod {
subtitleSegments: this.subtitleSegments,
shouldContainSubtitles: this.shouldContainSubtitles,
expectedSubtitleTracks: this.expectedSubtitleTracks,
mediaSequences: this.mediaSequences,
mediaSequences: this.skipSerializeMediaSequences ? null : this.mediaSequences,
SEQUENCE_DURATION: this.SEQUENCE_DURATION,
targetDuration: this.targetDuration,
targetAudioDuration: this.targetAudioDuration,
Expand Down Expand Up @@ -114,12 +117,13 @@ class HLSVod {
forcedDemuxMode: this.forcedDemuxMode,
dummySubtitleEndpoint: this.dummySubtitleEndpoint,
subtitleSliceEndpoint: this.subtitleSliceEndpoint,
videoSequencesCount: this.videoSequencesCount,
audioSequencesCount: this.audioSequencesCount,
subtitleSequencesCount: this.subtitleSequencesCount,
videoSequencesCount: this.skipSerializeMediaSequences ? 0 : this.videoSequencesCount,
audioSequencesCount: this.skipSerializeMediaSequences ? 0 : this.audioSequencesCount,
subtitleSequencesCount: this.skipSerializeMediaSequences ? 0 : this.subtitleSequencesCount,
mediaStartExcessTime: this.mediaStartExcessTime,
audioCodecsMap: this.audioCodecsMap,
alwaysMapBandwidthByNearest: this.alwaysMapBandwidthByNearest
alwaysMapBandwidthByNearest: this.alwaysMapBandwidthByNearest,
skipSerializeMediaSequences: this.skipSerializeMediaSequences
};
return JSON.stringify(serialized);
}
Expand Down Expand Up @@ -172,6 +176,7 @@ class HLSVod {
this.mediaStartExcessTime = de.mediaStartExcessTime;
this.audioCodecsMap = de.audioCodecsMap;
this.alwaysMapBandwidthByNearest = de.alwaysMapBandwidthByNearest;
this.skipSerializeMediaSequences = de.skipSerializeMediaSequences;
}

/**
Expand Down Expand Up @@ -500,12 +505,12 @@ class HLSVod {
try {
this._loadPrevious();
this.load(_injectMasterManifest, _injectMediaManifest, _injectAudioManifest, _injectSubtitleManifest)
.then(() => {// WARNING we can never remove this.previousVod because it is used later in the code
previousVod.releasePreviousVod();
.then(() => {
this.releasePreviousVod();
resolve();
})
.catch((err) => {
previousVod.releasePreviousVod();
this.releasePreviousVod();
reject(err);
});
} catch (exc) {
Expand Down Expand Up @@ -1398,8 +1403,12 @@ class HLSVod {
}

videoSequences.push(_sequence);
this.mediaSequenceValues[seqIndex] = totalRemovedSegments;
this.discontinuities[seqIndex] = totalRemovedDiscTags;
if (!this.mediaSequenceValues[seqIndex]) {
this.mediaSequenceValues[seqIndex] = totalRemovedSegments;
}
if (!this.discontinuities[seqIndex]) {
this.discontinuities[seqIndex] = totalRemovedDiscTags;
}
sequence = _sequence;
seqIndex++;
} catch (err) {
Expand Down Expand Up @@ -1529,7 +1538,7 @@ class HLSVod {
let shiftOnce = true;
let shiftedSegmentsCount = 0;
// 2 - Shift excess segments and keep count of what has been removed (per variant)
while (totalSeqDur >= this.SEQUENCE_DURATION || (shiftOnce && segIdx !== 0)) { // TODO continue here
while (totalSeqDur >= this.SEQUENCE_DURATION || (shiftOnce && segIdx !== 0)) {
shiftOnce = false;
let timeToRemove = 0;
let incrementDiscSeqCount = false;
Expand Down Expand Up @@ -1602,11 +1611,19 @@ class HLSVod {
sequences.push(_sequence);

if (type === "audio") {
this.discontinuitiesAudio[seqIndex] = totalRemovedDiscTags;
this.mediaSequenceValuesAudio[seqIndex] = totalRemovedSegments;
if (!this.discontinuitiesAudio[seqIndex]) {
this.discontinuitiesAudio[seqIndex] = totalRemovedDiscTags;
}
if (!this.mediaSequenceValuesAudio[seqIndex]) {
this.mediaSequenceValuesAudio[seqIndex] = totalRemovedSegments;
}
} else if (type === "subtitle") {
this.discontinuitiesSubtitle[seqIndex] = totalRemovedDiscTags;
this.mediaSequenceValuesSubtitle[seqIndex] = totalRemovedSegments;
if (!this.discontinuitiesSubtitle[seqIndex]) {
this.discontinuitiesSubtitle[seqIndex] = totalRemovedDiscTags;
}
if (!this.mediaSequenceValuesSubtitle[seqIndex]) {
this.mediaSequenceValuesSubtitle[seqIndex] = totalRemovedSegments;
}
}
sequence = _sequence;
seqIndex++;
Expand All @@ -1617,6 +1634,13 @@ class HLSVod {
return sequences;
}

generateMediaSequences() {
this.mediaSequences = [];
return new Promise((resolve, reject) => {
this._createMediaSequences(1).then(resolve).catch(reject);
});
}

calculateDeltaAndPositionExtraMedia(type) {
let prevLastSegment = null;
let discSeqNo = 0;
Expand Down Expand Up @@ -1959,7 +1983,7 @@ class HLSVod {
}
}

_createMediaSequences() {
_createMediaSequences(mode = false) {
return new Promise((resolve, reject) => {
const bw = this._getFirstBwWithSegments();
const audioGroupId = this._getFirstAudioGroupWithSegments();
Expand Down Expand Up @@ -1993,7 +2017,7 @@ class HLSVod {
if (subtitleGroupId) {
this._removeDoubleDiscontinuitiesFromExtraMedia(this.subtitleSegments)
}
if (this.shouldContainSubtitles) {
if (this.shouldContainSubtitles && !mode) {
// we are doing all this to figure out the entire duration of the new vod so we can create a long subtitle segment that we can later chunk to smaller segments
let duration = this.getDuration();
let offset = 0;
Expand Down Expand Up @@ -2092,9 +2116,7 @@ class HLSVod {
let subtitleSequences = [];
if (this.shouldContainSubtitles) {
subtitleSequences = this.generateSequencesTypeBExtraMedia(this.subtitleSegments, subtitleGroupId, firstSubtitleLanguage, "subtitle");

}

// Append newly generated video/audio/subtitle sequences
videoSequences.map((_, index) => {
this.mediaSequences.push({
Expand Down Expand Up @@ -2125,15 +2147,18 @@ class HLSVod {
});
}
}

this.videoSequencesCount = videoSequences.length;
this.audioSequencesCount = audioSequences.length;
this.subtitleSequencesCount = subtitleSequences.length
}

if (!this.mediaSequences) {
reject("Failed to init media sequences");
} else if (this.mediaSequences.length > 0 && this.skipSerializeMediaSequences && this.deltaTimes.length > 0 && this.mediaSequences.length === this.deltaTimes.length) {
debug(`Discontinuities, MediaSequencesValues, Delta times & positions have already been calculated. Skipping this step!`);
resolve()
} else {
// Calculate Delta Times and Position for Video
let prevLastSegment = null;
let discSeqNo = 0;
this.deltaTimes.push({
Expand Down Expand Up @@ -2238,7 +2263,6 @@ class HLSVod {
if (this.mediaSequences[0].subtitleSegments) {
this.calculateDeltaAndPositionExtraMedia("subtitle")
}

resolve();
}
});
Expand Down
184 changes: 184 additions & 0 deletions spec/hlsvod_spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2099,6 +2099,7 @@ describe("HLSVod with mixed target durations", () => {
describe("HLSVod serializing", () => {
let mockMasterManifest;
let mockMediaManifest;
let mockM3u8Item1;

beforeEach(() => {
mockMasterManifest = function () {
Expand All @@ -2108,6 +2109,21 @@ describe("HLSVod serializing", () => {
mockMediaManifest = function (bandwidth) {
return fs.createReadStream("testvectors/hls1/" + bandwidth + ".m3u8");
};

mockM3u8Item1 = {
master: () => {
return fs.createReadStream("testvectors/hls_abr7/master.m3u8");
},
media: (bw) => {
return fs.createReadStream(`testvectors/hls_abr7/master${bw}.m3u8`);
},
audio: (g, l) => {
return fs.createReadStream(`testvectors/hls_abr7/master-${g}_${l}.m3u8`);
},
subtitle: (g, l) => {
return fs.createReadStream(`testvectors/hls_abr7/master-${g}_${l}.m3u8`);
},
};
});

it("can be serialized", (done) => {
Expand Down Expand Up @@ -2161,6 +2177,174 @@ describe("HLSVod serializing", () => {
done();
});
});

it("can handle a sequence of VODs, with skipSerializeMediaSequences set -> true (TYPE-A)", (done) => {
mockVod = new HLSVod("http://mock.com/mock.m3u8", null, 0, 0, null, {
sequenceAlwaysContainNewSegments: 0,
forcedDemuxMode: 1,
shouldContainSubtitles: 1,
subtitleSliceEndpoint: "https://test.test.com",
expectedSubtitleTracks: [
{ language: "sv", name: "sv" },
{ language: "en", name: "en" },
{ language: "no", name: "no" },
],
dummySubtitleEndpoint: "fake/subs/here",
alwaysMapBandwidthByNearest: 1,
skipSerializeMediaSequences: false,
});
mockVod2 = new HLSVod("http://mock.com/mock2.m3u8", null, 0, 0, null, {
sequenceAlwaysContainNewSegments: 0,
forcedDemuxMode: 1,
shouldContainSubtitles: 1,
subtitleSliceEndpoint: "https://test.test.com",
expectedSubtitleTracks: [
{ language: "sv", name: "sv" },
{ language: "en", name: "en" },
{ language: "no", name: "no" },
],
dummySubtitleEndpoint: "fake/subs/here",
alwaysMapBandwidthByNearest: 1,
skipSerializeMediaSequences: false,
});
mockVod
.load(mockM3u8Item1.master, mockM3u8Item1.media, mockM3u8Item1.audio, mockM3u8Item1.subtitle)
.then(() => {
return mockVod2.loadAfter(mockVod, mockM3u8Item1.master, mockM3u8Item1.media, mockM3u8Item1.audio, mockM3u8Item1.subtitle);
})
.then(() => {
const bytesToMB = (bytes) => {
const megabytes = bytes / (1024 * 1024);
return megabytes.toFixed(2);
}
const expectedJSONSizeInMBFull = "5.26";
const expectedJSONSizeInMBPartial = "0.38";
const expectedJSONSizeInMBFullAgain = expectedJSONSizeInMBFull;
const expectedMseqCountFull = 145;
const expectedMseqCountPartial = 0;
const expectedMseqCountFullAgain = expectedMseqCountFull;

const mseqCountFull = mockVod2.getLiveMediaSequencesCount();
const fullySerialized = mockVod2.toJSON();
const sizeFull = Buffer.byteLength(JSON.stringify(fullySerialized));
const sizeInMBFull = bytesToMB(sizeFull);
const fullyDeserializedVod = new HLSVod();
fullyDeserializedVod.fromJSON(fullySerialized);

fullyDeserializedVod.skipSerializeMediaSequences = true;
const partiallySerialized = fullyDeserializedVod.toJSON();
const partiallyDeserializedVod = new HLSVod();
partiallyDeserializedVod.fromJSON(partiallySerialized);

const mseqCountPart = partiallyDeserializedVod.getLiveMediaSequencesCount();
const sizePart = Buffer.byteLength(JSON.stringify(partiallySerialized));
const sizeInMBPart = bytesToMB(sizePart);

partiallyDeserializedVod.generateMediaSequences().then(() => {
const mseqCountFullAgain = partiallyDeserializedVod.getLiveMediaSequencesCount();
partiallyDeserializedVod.skipSerializeMediaSequences = false;
const fullySerializedAgain = partiallyDeserializedVod.toJSON();
const sizeFullAgain = Buffer.byteLength(JSON.stringify(fullySerializedAgain));
const sizeInMBFullAgain = bytesToMB(sizeFullAgain);

expect(sizeInMBFull).toBe(expectedJSONSizeInMBFull);
expect(sizeInMBPart).toBe(expectedJSONSizeInMBPartial);
expect(sizeInMBFullAgain).toBe(expectedJSONSizeInMBFullAgain);
expect(mseqCountFull).toBe(expectedMseqCountFull);
expect(mseqCountPart).toBe(expectedMseqCountPartial);
expect(mseqCountFullAgain).toBe(expectedMseqCountFullAgain);

expect(sizeInMBFull).toEqual(sizeInMBFullAgain);
expect(fullySerialized).toEqual(fullySerializedAgain);

done();
});
});
});

it("can handle a sequence of VODs, with skipSerializeMediaSequences set -> true (TYPE-B)", (done) => {
mockVod = new HLSVod("http://mock.com/mock.m3u8", null, 0, 0, null, {
sequenceAlwaysContainNewSegments: 1,
forcedDemuxMode: 1,
shouldContainSubtitles: 1,
subtitleSliceEndpoint: "https://test.test.com",
expectedSubtitleTracks: [
{ language: "sv", name: "sv" },
{ language: "en", name: "en" },
{ language: "no", name: "no" },
],
dummySubtitleEndpoint: "fake/subs/here",
alwaysMapBandwidthByNearest: 1,
skipSerializeMediaSequences: false,
});
mockVod2 = new HLSVod("http://mock.com/mock2.m3u8", null, 0, 0, null, {
sequenceAlwaysContainNewSegments: 1,
forcedDemuxMode: 1,
shouldContainSubtitles: 1,
subtitleSliceEndpoint: "https://test.test.com",
expectedSubtitleTracks: [
{ language: "sv", name: "sv" },
{ language: "en", name: "en" },
{ language: "no", name: "no" },
],
dummySubtitleEndpoint: "fake/subs/here",
alwaysMapBandwidthByNearest: 1,
skipSerializeMediaSequences: false,
});
mockVod
.load(mockM3u8Item1.master, mockM3u8Item1.media, mockM3u8Item1.audio, mockM3u8Item1.subtitle)
.then(() => {
return mockVod2.loadAfter(mockVod, mockM3u8Item1.master, mockM3u8Item1.media, mockM3u8Item1.audio, mockM3u8Item1.subtitle);
})
.then(() => {
const bytesToMB = (bytes) => {
const megabytes = bytes / (1024 * 1024);
return megabytes.toFixed(2);
}
const expectedJSONSizeInMBFull = "5.29";
const expectedJSONSizeInMBPartial = "0.39";
const expectedJSONSizeInMBFullAgain = expectedJSONSizeInMBFull;
const expectedMseqCountFull = 146;
const expectedMseqCountPartial = 0;
const expectedMseqCountFullAgain = expectedMseqCountFull;

const mseqCountFull = mockVod2.getLiveMediaSequencesCount();
const fullySerialized = mockVod2.toJSON();
const sizeFull = Buffer.byteLength(JSON.stringify(fullySerialized));
const sizeInMBFull = bytesToMB(sizeFull);
const fullyDeserializedVod = new HLSVod();
fullyDeserializedVod.fromJSON(fullySerialized);

fullyDeserializedVod.skipSerializeMediaSequences = true;
const partiallySerialized = fullyDeserializedVod.toJSON();
const partiallyDeserializedVod = new HLSVod();
partiallyDeserializedVod.fromJSON(partiallySerialized);

const mseqCountPart = partiallyDeserializedVod.getLiveMediaSequencesCount();
const sizePart = Buffer.byteLength(JSON.stringify(partiallySerialized));
const sizeInMBPart = bytesToMB(sizePart);

partiallyDeserializedVod.generateMediaSequences().then(() => {
const mseqCountFullAgain = partiallyDeserializedVod.getLiveMediaSequencesCount();
partiallyDeserializedVod.skipSerializeMediaSequences = false;
const fullySerializedAgain = partiallyDeserializedVod.toJSON();
const sizeFullAgain = Buffer.byteLength(JSON.stringify(fullySerializedAgain));
const sizeInMBFullAgain = bytesToMB(sizeFullAgain);

expect(sizeInMBFull).toBe(expectedJSONSizeInMBFull);
expect(sizeInMBPart).toBe(expectedJSONSizeInMBPartial);
expect(sizeInMBFullAgain).toBe(expectedJSONSizeInMBFullAgain);
expect(mseqCountFull).toBe(expectedMseqCountFull);
expect(mseqCountPart).toBe(expectedMseqCountPartial);
expect(mseqCountFullAgain).toBe(expectedMseqCountFullAgain);

expect(sizeInMBFull).toEqual(sizeInMBFullAgain);
expect(fullySerialized).toEqual(fullySerializedAgain);

done();
});
});
});
});

describe("HLSVod time metadata", () => {
Expand Down
Loading

0 comments on commit b582faf

Please sign in to comment.