From aebb3c108a7f96720d463a878f51dbcf578d0904 Mon Sep 17 00:00:00 2001 From: Christian Bernier Date: Mon, 6 Apr 2020 09:57:45 -0400 Subject: [PATCH 1/3] Adds a way to manipulate the raw audio data for a specific DataSource before that it gets encoded. - DataSources now have an optional handler: "PostProcessor" that will be called for each input and output buffers, just like the resample ore remix classes, but the goal in this case is to do a custom manipulation of the final raw data. It can also be use to prevent the data from being sent to the encoder (by not not filling the outputBuffer in the postProcess method) . For example, it can be use to merge two audio DataSource. - Adds two PostProcessors that can be used to mix the audio of two DataSources: MixerSourceAudioPostProcessor is used accumulate and not write the raw audio data of the first DataSouce. MixerTargetAudioPostProcessor is used to do the mixing and write the mixed audio data to the output buffer when processing the second DataSource. - Adds an AudioPostProcessor to change the volume of a DataSource --- .../transcoder/TranscoderOptions.java | 4 +- .../transcoder/engine/Engine.java | 38 +++++++++++++------ .../postprocessor/AudioPostProcessor.java | 16 ++++++++ .../DefaultAudioPostProcessor.java | 17 +++++++++ .../postprocessor/PostProcessor.java | 10 +++++ .../VolumeAudioPostProcessor.java | 36 ++++++++++++++++++ .../source/BlankAudioDataSource.java | 14 +++++++ .../transcoder/source/DataSource.java | 17 +++++++++ .../transcoder/source/DataSourceWrapper.java | 14 +++++++ .../transcoder/source/DefaultDataSource.java | 14 +++++++ .../transcoder/time/PresentationTime.java | 11 ++++++ .../transcode/AudioTrackTranscoder.java | 14 ++++++- .../transcode/internal/AudioEngine.java | 23 +++++++++-- 13 files changed, 209 insertions(+), 19 deletions(-) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/AudioPostProcessor.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/DefaultAudioPostProcessor.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/PostProcessor.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/VolumeAudioPostProcessor.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/time/PresentationTime.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java index 81c12a4d..c5898348 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/TranscoderOptions.java @@ -349,7 +349,9 @@ private List buildAudioDataSources() if (dataSource.getTrackFormat(TrackType.AUDIO) != null) { result.add(dataSource); } else { - result.add(new BlankAudioDataSource(dataSource.getDurationUs())); + DataSource blankDataSource = new BlankAudioDataSource(dataSource.getDurationUs()); + blankDataSource.setPostProcessor(dataSource.getPostProcessor()); + result.add(blankDataSource); } } return result; diff --git a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java index 742fd1cf..f7441fae 100644 --- a/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java +++ b/lib/src/main/java/com/otaliastudios/transcoder/engine/Engine.java @@ -20,10 +20,12 @@ import com.otaliastudios.transcoder.TranscoderOptions; import com.otaliastudios.transcoder.internal.TrackTypeMap; import com.otaliastudios.transcoder.internal.ValidatorException; +import com.otaliastudios.transcoder.postprocessor.AudioPostProcessor; import com.otaliastudios.transcoder.sink.DataSink; import com.otaliastudios.transcoder.sink.InvalidOutputFormatException; import com.otaliastudios.transcoder.source.DataSource; import com.otaliastudios.transcoder.strategy.TrackStrategy; +import com.otaliastudios.transcoder.time.PresentationTime; import com.otaliastudios.transcoder.time.TimeInterpolator; import com.otaliastudios.transcoder.transcode.AudioTrackTranscoder; import com.otaliastudios.transcoder.transcode.NoOpTrackTranscoder; @@ -70,6 +72,7 @@ public interface ProgressCallback { private final TrackTypeMap mOutputFormats = new TrackTypeMap<>(); private volatile double mProgress; private final ProgressCallback mProgressCallback; + private final PresentationTime mAudioPresentationTime = new PresentationTime(); public Engine(@Nullable ProgressCallback progressCallback) { mProgressCallback = progressCallback; @@ -177,7 +180,9 @@ private void openCurrentStep(@NonNull TrackType type, @NonNull TranscoderOptions transcoder = new AudioTrackTranscoder(dataSource, mDataSink, interpolator, options.getAudioStretcher(), - options.getAudioResampler()); + options.getAudioResampler(), + (AudioPostProcessor)dataSource.getPostProcessor(), + mAudioPresentationTime); break; default: throw new RuntimeException("Unknown type: " + type); @@ -253,17 +258,22 @@ public long interpolate(@NonNull TrackType type, long time) { }; } - private long getTrackDurationUs(@NonNull TrackType type) { + private long getTrackDurationUs(@NonNull TrackType type, boolean processedDuration) { if (!mStatuses.require(type).isTranscoding()) return 0L; int current = mCurrentStep.require(type); long totalDurationUs = 0; for (int i = 0; i < mDataSources.require(type).size(); i++) { DataSource source = mDataSources.require(type).get(i); + long dataSourceDurationUs; if (i < current) { // getReadUs() is a better approximation for sure. - totalDurationUs += source.getReadUs(); + dataSourceDurationUs = source.getReadUs(); } else { - totalDurationUs += source.getDurationUs(); + dataSourceDurationUs = source.getDurationUs(); } + if (processedDuration && source.getPostProcessor() != null) { + dataSourceDurationUs = source.getPostProcessor().calculateNewDurationUs(dataSourceDurationUs); + } + totalDurationUs += dataSourceDurationUs; } return totalDurationUs; } @@ -271,19 +281,23 @@ private long getTrackDurationUs(@NonNull TrackType type) { private long getTotalDurationUs() { boolean hasVideo = hasVideoSources() && mStatuses.requireVideo().isTranscoding(); boolean hasAudio = hasAudioSources() && mStatuses.requireAudio().isTranscoding(); - long video = hasVideo ? getTrackDurationUs(TrackType.VIDEO) : Long.MAX_VALUE; - long audio = hasAudio ? getTrackDurationUs(TrackType.AUDIO) : Long.MAX_VALUE; + long video = hasVideo ? getTrackDurationUs(TrackType.VIDEO, true) : Long.MAX_VALUE; + long audio = hasAudio ? getTrackDurationUs(TrackType.AUDIO, true) : Long.MAX_VALUE; return Math.min(video, audio); } - private long getTrackReadUs(@NonNull TrackType type) { + private long getTrackProgressUs(@NonNull TrackType type, boolean processedDuration) { if (!mStatuses.require(type).isTranscoding()) return 0L; int current = mCurrentStep.require(type); long completedDurationUs = 0; for (int i = 0; i < mDataSources.require(type).size(); i++) { DataSource source = mDataSources.require(type).get(i); if (i <= current) { - completedDurationUs += source.getReadUs(); + long dataSourceReadUs = source.getReadUs(); + if (processedDuration && source.getPostProcessor() != null) { + dataSourceReadUs = source.getPostProcessor().calculateNewDurationUs(dataSourceReadUs); + } + completedDurationUs += dataSourceReadUs; } } return completedDurationUs; @@ -291,8 +305,8 @@ private long getTrackReadUs(@NonNull TrackType type) { private double getTrackProgress(@NonNull TrackType type) { if (!mStatuses.require(type).isTranscoding()) return 0.0D; - long readUs = getTrackReadUs(type); - long totalUs = getTotalDurationUs(); + long readUs = getTrackProgressUs(type, false); + long totalUs = getTrackDurationUs(type, false); LOG.v("getTrackProgress - readUs:" + readUs + ", totalUs:" + totalUs); if (totalUs == 0) totalUs = 1; // Avoid NaN return (double) readUs / (double) totalUs; @@ -361,8 +375,8 @@ public void transcode(@NonNull TranscoderOptions options) throws InterruptedExce // This can happen, for example, if user adds 1 minute (video only) with 20 seconds // of audio. The video track must be stopped once the audio stops. long totalUs = getTotalDurationUs() + 100 /* tolerance */; - forceAudioEos = getTrackReadUs(TrackType.AUDIO) > totalUs; - forceVideoEos = getTrackReadUs(TrackType.VIDEO) > totalUs; + forceAudioEos = getTrackProgressUs(TrackType.AUDIO, true) > totalUs; + forceVideoEos = getTrackProgressUs(TrackType.VIDEO, true) > totalUs; // Now step for transcoders that are not completed. audioCompleted = isCompleted(TrackType.AUDIO); diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/AudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/AudioPostProcessor.java new file mode 100644 index 00000000..5b07c39e --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/AudioPostProcessor.java @@ -0,0 +1,16 @@ +package com.otaliastudios.transcoder.postprocessor; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +public interface AudioPostProcessor extends PostProcessor { + /** + * Manipulates the raw audio data inside inputBuffer and put the result in outputBuffer + * @param inputBuffer the input data (as raw audio data) + * @param outputBuffer the data after the manipulation + * @param bufferDurationUs the duration of the input data + * @return the duration of the output data + */ + long postProcess(@NonNull final ShortBuffer inputBuffer, @NonNull final ShortBuffer outputBuffer, long bufferDurationUs); +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/DefaultAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/DefaultAudioPostProcessor.java new file mode 100644 index 00000000..13f11c1d --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/DefaultAudioPostProcessor.java @@ -0,0 +1,17 @@ +package com.otaliastudios.transcoder.postprocessor; + +import androidx.annotation.NonNull; +import java.nio.ShortBuffer; + +public class DefaultAudioPostProcessor implements AudioPostProcessor { + @Override + public long calculateNewDurationUs(long durationUs) { + return durationUs; + } + + @Override + public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { + outputBuffer.put(inputBuffer); + return bufferDurationUs; + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/PostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/PostProcessor.java new file mode 100644 index 00000000..c2289790 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/PostProcessor.java @@ -0,0 +1,10 @@ +package com.otaliastudios.transcoder.postprocessor; + +public interface PostProcessor { + /** + * Returns the duration of the data source on it has been processed (after calling the postProcess() method) + * @param durationUs the original duratin in Us + * @return the new duration in Us + */ + long calculateNewDurationUs(long durationUs); +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/VolumeAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/VolumeAudioPostProcessor.java new file mode 100644 index 00000000..d976fcc4 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/VolumeAudioPostProcessor.java @@ -0,0 +1,36 @@ +package com.otaliastudios.transcoder.postprocessor; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +public class VolumeAudioPostProcessor implements AudioPostProcessor { + private float mVolume; + + public VolumeAudioPostProcessor(float volume) { + mVolume = volume; + } + + @Override + public long calculateNewDurationUs(long durationUs) { + return durationUs; + } + + private short applyVolume(short sample) { + float sampleAtVolume = sample * mVolume; + if (sampleAtVolume < Short.MIN_VALUE) + sampleAtVolume = Short.MIN_VALUE; + else if (sampleAtVolume > Short.MAX_VALUE) + sampleAtVolume = Short.MAX_VALUE; + return (short)sampleAtVolume; + } + + @Override + public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { + int inputRemaining = inputBuffer.remaining(); + for (int i=0; i Date: Mon, 13 Apr 2020 12:40:57 -0400 Subject: [PATCH 2/3] Add an example of AudioPostProcessor to mix the audio of 2 data sources. --- .../MixerSourceAudioPostProcessor.java | 28 ++++++++++ .../MixerTargetAudioPostProcessor.java | 54 +++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java create mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java new file mode 100644 index 00000000..7555fb85 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java @@ -0,0 +1,28 @@ +package com.otaliastudios.transcoder.postprocessor; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; +import java.util.ArrayDeque; +import java.util.Queue; + +public class MixerSourceAudioPostProcessor implements AudioPostProcessor { + + Queue mBuffers = new ArrayDeque<>(); + + @Override + public long calculateNewDurationUs(long durationUs) { + return 0; + } + + @Override + public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { + if (inputBuffer.remaining() > 0) { + ShortBuffer sourceBuffer = ShortBuffer.allocate(inputBuffer.limit()); + sourceBuffer.put(inputBuffer); + sourceBuffer.rewind(); + mBuffers.add(sourceBuffer); + } + return 0; + } +} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java new file mode 100644 index 00000000..e3d1b504 --- /dev/null +++ b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java @@ -0,0 +1,54 @@ +package com.otaliastudios.transcoder.postprocessor; + +import androidx.annotation.NonNull; + +import java.nio.ShortBuffer; + +import static java.lang.Math.min; + +public class MixerTargetAudioPostProcessor implements AudioPostProcessor { + private MixerSourceAudioPostProcessor mSource; + private float mSourceVolume; + private float mTargetVolume; + + @Override + public long calculateNewDurationUs(long durationUs) { + return durationUs; + } + + public MixerTargetAudioPostProcessor(MixerSourceAudioPostProcessor source, float sourceVolume, float targetVolume) + { + mSource = source; + mSourceVolume = sourceVolume; + mTargetVolume = targetVolume; + } + + private short mixSamples(short sourceSample, short targetSample) { + float mixedSample = (sourceSample * mSourceVolume) + (targetSample * mTargetVolume); + if (mixedSample < Short.MIN_VALUE) + mixedSample = Short.MIN_VALUE; + else if (mixedSample > Short.MAX_VALUE) + mixedSample = Short.MAX_VALUE; + return (short)mixedSample; + } + + @Override + public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { + ShortBuffer sourceBuffer = mSource.mBuffers.peek(); + int inputRemaining = inputBuffer.remaining(); + while (sourceBuffer != null && inputRemaining > 0) { + int sourceRemaining = sourceBuffer.remaining(); + int remaining = min(inputRemaining, sourceRemaining); + for (int i=0; i= 0) { + mSource.mBuffers.remove(); + sourceBuffer = mSource.mBuffers.peek(); + } + } + outputBuffer.put(inputBuffer); + return bufferDurationUs; + } +} From c02976470ba534cbabf53b9910fdf866dd6938b7 Mon Sep 17 00:00:00 2001 From: Christian Bernier Date: Mon, 13 Apr 2020 12:42:21 -0400 Subject: [PATCH 3/3] Revert "Add an example of AudioPostProcessor to mix the audio of 2 data sources." This reverts commit 68198ce38c0549b4238ceca976859d0dd83f7002. --- .../MixerSourceAudioPostProcessor.java | 28 ---------- .../MixerTargetAudioPostProcessor.java | 54 ------------------- 2 files changed, 82 deletions(-) delete mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java delete mode 100644 lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java deleted file mode 100644 index 7555fb85..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerSourceAudioPostProcessor.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.otaliastudios.transcoder.postprocessor; - -import androidx.annotation.NonNull; - -import java.nio.ShortBuffer; -import java.util.ArrayDeque; -import java.util.Queue; - -public class MixerSourceAudioPostProcessor implements AudioPostProcessor { - - Queue mBuffers = new ArrayDeque<>(); - - @Override - public long calculateNewDurationUs(long durationUs) { - return 0; - } - - @Override - public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { - if (inputBuffer.remaining() > 0) { - ShortBuffer sourceBuffer = ShortBuffer.allocate(inputBuffer.limit()); - sourceBuffer.put(inputBuffer); - sourceBuffer.rewind(); - mBuffers.add(sourceBuffer); - } - return 0; - } -} diff --git a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java b/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java deleted file mode 100644 index e3d1b504..00000000 --- a/lib/src/main/java/com/otaliastudios/transcoder/postprocessor/MixerTargetAudioPostProcessor.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.otaliastudios.transcoder.postprocessor; - -import androidx.annotation.NonNull; - -import java.nio.ShortBuffer; - -import static java.lang.Math.min; - -public class MixerTargetAudioPostProcessor implements AudioPostProcessor { - private MixerSourceAudioPostProcessor mSource; - private float mSourceVolume; - private float mTargetVolume; - - @Override - public long calculateNewDurationUs(long durationUs) { - return durationUs; - } - - public MixerTargetAudioPostProcessor(MixerSourceAudioPostProcessor source, float sourceVolume, float targetVolume) - { - mSource = source; - mSourceVolume = sourceVolume; - mTargetVolume = targetVolume; - } - - private short mixSamples(short sourceSample, short targetSample) { - float mixedSample = (sourceSample * mSourceVolume) + (targetSample * mTargetVolume); - if (mixedSample < Short.MIN_VALUE) - mixedSample = Short.MIN_VALUE; - else if (mixedSample > Short.MAX_VALUE) - mixedSample = Short.MAX_VALUE; - return (short)mixedSample; - } - - @Override - public long postProcess(@NonNull ShortBuffer inputBuffer, @NonNull ShortBuffer outputBuffer, long bufferDurationUs) { - ShortBuffer sourceBuffer = mSource.mBuffers.peek(); - int inputRemaining = inputBuffer.remaining(); - while (sourceBuffer != null && inputRemaining > 0) { - int sourceRemaining = sourceBuffer.remaining(); - int remaining = min(inputRemaining, sourceRemaining); - for (int i=0; i= 0) { - mSource.mBuffers.remove(); - sourceBuffer = mSource.mBuffers.peek(); - } - } - outputBuffer.put(inputBuffer); - return bufferDurationUs; - } -}