Javacpp ffmpeg upgrade (#247)

* Upgraded javacpp-ffmpeg version and updated imports.

* Fixed and updated dependencies.

* Updated to newest ffmpeg snapshot and tested on macOS ARM.

* Fixed incorrectly merged import.

* Upgraded ffmpeg to release version.

* Cleaned up ffmpeg related classes.

* Bump minor version.

Former-commit-id: 6271f74

build.gradle

@@ -11,7 +11,7 @@ allprojects {
     group = 'org.vitrivr'
 
     /* Our current version, on dev branch this should always be release+1-SNAPSHOT */
-    version = '3.7.3'
+    version = '3.8.0'
 
     apply plugin: 'java-library'
     apply plugin: 'maven-publish'
@@ -36,6 +36,10 @@ subprojects {
 
     apply plugin: 'com.github.johnrengelman.shadow'
 
+    shadowJar {
+        zip64 true // Required to allow more than 65535 entries in final archive
+    }
+
     sourceCompatibility = 1.11
     targetCompatibility = 1.11
 

cineast-core/build.gradle

@@ -141,7 +141,7 @@ dependencies {
 
     /** JavaCPP. */
     api group: "org.bytedeco", name: "javacpp", version: version_javacpp
-    api group: "org.bytedeco.javacpp-presets", name: "ffmpeg", version: version_ffmpeg
+    api group: "org.bytedeco", name: "ffmpeg-platform", version: version_ffmpeg
 
     /** OpenCV. */
     api group: 'org.openpnp', name: 'opencv', version: version_opencv

cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/audio/FFMpegAudioDecoder.java

@@ -5,27 +5,25 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayDeque;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.bytedeco.ffmpeg.avcodec.AVCodec;
+import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
+import org.bytedeco.ffmpeg.avcodec.AVPacket;
+import org.bytedeco.ffmpeg.avformat.AVFormatContext;
+import org.bytedeco.ffmpeg.avutil.AVDictionary;
+import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.avutil.AVRational;
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.ffmpeg.global.avformat;
+import org.bytedeco.ffmpeg.global.avutil;
+import org.bytedeco.ffmpeg.global.swresample;
+import org.bytedeco.ffmpeg.swresample.SwrContext;
 import org.bytedeco.javacpp.IntPointer;
 import org.bytedeco.javacpp.Pointer;
 import org.bytedeco.javacpp.PointerPointer;
-import org.bytedeco.javacpp.avcodec;
-import org.bytedeco.javacpp.avcodec.AVCodec;
-import org.bytedeco.javacpp.avcodec.AVCodecContext;
-import org.bytedeco.javacpp.avcodec.AVPacket;
-import org.bytedeco.javacpp.avformat;
-import org.bytedeco.javacpp.avformat.AVFormatContext;
-import org.bytedeco.javacpp.avutil;
-import org.bytedeco.javacpp.avutil.AVDictionary;
-import org.bytedeco.javacpp.avutil.AVFrame;
-import org.bytedeco.javacpp.avutil.AVRational;
-import org.bytedeco.javacpp.swresample;
-import org.bytedeco.javacpp.swresample.SwrContext;
 import org.vitrivr.cineast.core.config.CacheConfig;
 import org.vitrivr.cineast.core.config.DecoderConfig;
 import org.vitrivr.cineast.core.data.frames.AudioDescriptor;
@@ -45,19 +43,7 @@ public class FFMpegAudioDecoder implements AudioDecoder {
    * <p>
    * TODO: List may not be complete yet.
    */
-  private static final Set<String> supportedFiles;
-
-  static {
-    HashSet<String> tmp = new HashSet<>();
-    tmp.add("multimedia/mp4"); /* They share the same suffix with audio (.mp4). */
-    tmp.add("audio/mp4");
-    tmp.add("audio/aac");
-    tmp.add("audio/mpeg");
-    tmp.add("audio/ogg");
-    tmp.add("audio/wav");
-    tmp.add("audio/flac");
-    supportedFiles = Collections.unmodifiableSet(tmp);
-  }
+  private static final Set<String> supportedFiles = Set.of("multimedia/mp4", "audio/mp4", "audio/aac", "audio/mpeg", "audio/ogg", "audio/wav", "audio/flac");
 
   /**
    * Property name and default value for channel settings.
@@ -80,7 +66,7 @@ public class FFMpegAudioDecoder implements AudioDecoder {
   /**
    * Internal data structure used to hold decoded AudioFrames.
    */
-  private ArrayDeque<AudioFrame> frameQueue = new ArrayDeque<>();
+  private final ArrayDeque<AudioFrame> frameQueue = new ArrayDeque<>();
 
   private AVFormatContext pFormatCtx = null;
   private AVCodecContext pCodecCtx = null;
@@ -91,11 +77,11 @@ public class FFMpegAudioDecoder implements AudioDecoder {
   private AVFrame decodedFrame = null;
   private AVFrame resampledFrame = null;
 
-  private IntPointer out_linesize = new IntPointer();
+  private final IntPointer out_linesize = new IntPointer();
 
   private SwrContext swr_ctx = null;
 
-  private AtomicBoolean complete = new AtomicBoolean(false);
+  private final AtomicBoolean complete = new AtomicBoolean(false);
 
   private AudioDescriptor descriptor = null;
 
@@ -183,7 +169,7 @@ private void readOriginal(int samples) {
   private void readResampled(int samples) {
     /* Convert decoded frame. Break if resampling fails.*/
     if (swresample.swr_convert(this.swr_ctx, null, 0, this.decodedFrame.data(), samples) < 0) {
-      LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName());
+      LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).");
       return;
     }
 

cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AbstractAVStreamContainer.java

@@ -2,20 +2,26 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.bytedeco.javacpp.avcodec;
-import org.bytedeco.javacpp.avformat;
-import org.bytedeco.javacpp.avutil;
+import org.bytedeco.ffmpeg.avcodec.AVCodec;
+import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
+import org.bytedeco.ffmpeg.avcodec.AVPacket;
+import org.bytedeco.ffmpeg.avformat.AVFormatContext;
+import org.bytedeco.ffmpeg.avformat.AVStream;
+import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.ffmpeg.global.avformat;
+import org.bytedeco.ffmpeg.global.avutil;
 
 public abstract class AbstractAVStreamContainer {
 
   private static final Logger LOGGER = LogManager.getLogger();
-  avcodec.AVPacket pkt;
-  protected avformat.AVStream st;
-  protected avcodec.AVCodec codec;
-  avcodec.AVCodecContext c;
-  private avformat.AVFormatContext oc;
+  AVPacket pkt;
+  protected AVStream st;
+  protected AVCodec codec;
+  AVCodecContext c;
+  private AVFormatContext oc;
 
-  AbstractAVStreamContainer(avformat.AVFormatContext oc, int codec_id) {
+  AbstractAVStreamContainer(AVFormatContext oc, int codec_id) {
     this.oc = oc;
     st = avformat.avformat_new_stream(oc, null);
     if (st == null) {
@@ -42,7 +48,7 @@ public abstract class AbstractAVStreamContainer {
   }
 
 
-  void encode(avcodec.AVCodecContext enc_ctx, avutil.AVFrame frame, avcodec.AVPacket pkt) {
+  void encode(AVCodecContext enc_ctx, AVFrame frame, AVPacket pkt) {
 
     int ret = avcodec.avcodec_send_frame(enc_ctx, frame);
     if (ret < 0) {

cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/AudioOutputStreamContainer.java

@@ -1,32 +1,33 @@
 package org.vitrivr.cineast.core.extraction.decode.video;
 
-import static org.bytedeco.javacpp.avcodec.AV_CODEC_CAP_VARIABLE_FRAME_SIZE;
-import static org.bytedeco.javacpp.avcodec.avcodec_free_context;
-import static org.bytedeco.javacpp.avcodec.avcodec_parameters_from_context;
-import static org.bytedeco.javacpp.avutil.AVDictionary;
-import static org.bytedeco.javacpp.avutil.AVFrame;
-import static org.bytedeco.javacpp.avutil.AVRational;
-import static org.bytedeco.javacpp.avutil.AV_ROUND_UP;
-import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_FLTP;
-import static org.bytedeco.javacpp.avutil.AV_SAMPLE_FMT_S16;
-import static org.bytedeco.javacpp.avutil.av_frame_free;
-import static org.bytedeco.javacpp.avutil.av_frame_make_writable;
-import static org.bytedeco.javacpp.avutil.av_opt_set_int;
-import static org.bytedeco.javacpp.avutil.av_opt_set_sample_fmt;
-import static org.bytedeco.javacpp.avutil.av_rescale_q;
-import static org.bytedeco.javacpp.avutil.av_rescale_rnd;
-import static org.bytedeco.javacpp.swresample.swr_alloc;
-import static org.bytedeco.javacpp.swresample.swr_convert;
-import static org.bytedeco.javacpp.swresample.swr_free;
-import static org.bytedeco.javacpp.swresample.swr_get_delay;
-import static org.bytedeco.javacpp.swresample.swr_init;
+import static org.bytedeco.ffmpeg.global.avcodec.AV_CODEC_CAP_VARIABLE_FRAME_SIZE;
+import static org.bytedeco.ffmpeg.global.avcodec.avcodec_free_context;
+import static org.bytedeco.ffmpeg.global.avcodec.avcodec_parameters_from_context;
+import static org.bytedeco.ffmpeg.global.avutil.AV_ROUND_UP;
+import static org.bytedeco.ffmpeg.global.avutil.AV_SAMPLE_FMT_FLTP;
+import static org.bytedeco.ffmpeg.global.avutil.AV_SAMPLE_FMT_S16;
+import static org.bytedeco.ffmpeg.global.avutil.av_frame_free;
+import static org.bytedeco.ffmpeg.global.avutil.av_frame_make_writable;
+import static org.bytedeco.ffmpeg.global.avutil.av_opt_set_int;
+import static org.bytedeco.ffmpeg.global.avutil.av_opt_set_sample_fmt;
+import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
+import static org.bytedeco.ffmpeg.global.avutil.av_rescale_rnd;
+import static org.bytedeco.ffmpeg.global.swresample.swr_alloc;
+import static org.bytedeco.ffmpeg.global.swresample.swr_convert;
+import static org.bytedeco.ffmpeg.global.swresample.swr_free;
+import static org.bytedeco.ffmpeg.global.swresample.swr_get_delay;
+import static org.bytedeco.ffmpeg.global.swresample.swr_init;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.bytedeco.javacpp.avcodec;
-import org.bytedeco.javacpp.avformat;
-import org.bytedeco.javacpp.avutil;
-import org.bytedeco.javacpp.swresample;
+import org.bytedeco.ffmpeg.avformat.AVFormatContext;
+import org.bytedeco.ffmpeg.avutil.AVDictionary;
+import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.avutil.AVRational;
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.ffmpeg.global.avformat;
+import org.bytedeco.ffmpeg.global.avutil;
+import org.bytedeco.ffmpeg.swresample.SwrContext;
 import org.vitrivr.cineast.core.data.frames.AudioFrame;
 
 class AudioOutputStreamContainer extends AbstractAVStreamContainer {
@@ -36,12 +37,12 @@ class AudioOutputStreamContainer extends AbstractAVStreamContainer {
   AVFrame tmp_frame;
   private int samples_count;
   private AVFrame frame;
-  private swresample.SwrContext swr_ctx;
+  private SwrContext swr_ctx;
   private AVRational rat = new AVRational();
 
   private final int channels = 1;
 
-  AudioOutputStreamContainer(avformat.AVFormatContext oc, int codec_id, int sampleRate, int bitRate, AVDictionary opt) {
+  AudioOutputStreamContainer(AVFormatContext oc, int codec_id, int sampleRate, int bitRate, AVDictionary opt) {
     super(oc, codec_id);
 
     long channellayout = avutil.av_get_default_channel_layout(channels);

cineast-core/src/main/java/org/vitrivr/cineast/core/extraction/decode/video/FFMpegVideoDecoder.java

@@ -5,27 +5,30 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayDeque;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.bytedeco.ffmpeg.avcodec.AVCodec;
+import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
+import org.bytedeco.ffmpeg.avcodec.AVPacket;
+import org.bytedeco.ffmpeg.avformat.AVFormatContext;
+import org.bytedeco.ffmpeg.avutil.AVDictionary;
+import org.bytedeco.ffmpeg.avutil.AVFrame;
+import org.bytedeco.ffmpeg.avutil.AVRational;
+import org.bytedeco.ffmpeg.global.avcodec;
+import org.bytedeco.ffmpeg.global.avformat;
+import org.bytedeco.ffmpeg.global.avutil;
+import org.bytedeco.ffmpeg.global.swresample;
+import org.bytedeco.ffmpeg.global.swscale;
+import org.bytedeco.ffmpeg.swresample.SwrContext;
+import org.bytedeco.ffmpeg.swscale.SwsContext;
 import org.bytedeco.javacpp.BytePointer;
 import org.bytedeco.javacpp.DoublePointer;
 import org.bytedeco.javacpp.IntPointer;
 import org.bytedeco.javacpp.Pointer;
 import org.bytedeco.javacpp.PointerPointer;
-import org.bytedeco.javacpp.avcodec;
-import org.bytedeco.javacpp.avcodec.AVCodec;
-import org.bytedeco.javacpp.avformat;
-import org.bytedeco.javacpp.avformat.AVFormatContext;
-import org.bytedeco.javacpp.avutil;
-import org.bytedeco.javacpp.avutil.AVDictionary;
-import org.bytedeco.javacpp.avutil.AVRational;
-import org.bytedeco.javacpp.swresample;
-import org.bytedeco.javacpp.swscale;
 import org.vitrivr.cineast.core.config.CacheConfig;
 import org.vitrivr.cineast.core.config.DecoderConfig;
 import org.vitrivr.cineast.core.data.frames.AudioDescriptor;
@@ -100,61 +103,50 @@ public class FFMpegVideoDecoder implements Decoder<VideoFrame> {
    * <p>
    * TODO: List may not be complete yet.
    */
-  public static final Set<String> supportedFiles;
-
-  static {
-    HashSet<String> tmp = new HashSet<>();
-    tmp.add("multimedia/mp4"); /* They share the same suffix with video (.mp4). */
-    tmp.add("video/mp4");
-    tmp.add("video/avi");
-    tmp.add("video/mpeg");
-    tmp.add("video/quicktime");
-    tmp.add("video/webm");
-    supportedFiles = Collections.unmodifiableSet(tmp);
-  }
+  public static final Set<String> supportedFiles = Set.of("multimedia/mp4", "video/mp4", "video/avi", "video/mpeg", "video/quicktime", "video/webm");
 
   private byte[] bytes;
   private int[] pixels;
 
   /**
    * Internal data structure used to hold decoded VideoFrames and the associated timestamp.
    */
-  private ArrayDeque<VideoFrame> videoFrameQueue = new ArrayDeque<>();
+  private final ArrayDeque<VideoFrame> videoFrameQueue = new ArrayDeque<>();
 
   /**
    * Internal data structure used to hold decoded AudioFrames and the associated timestamp.
    */
-  private ArrayDeque<AudioFrame> audioFrameQueue = new ArrayDeque<>();
+  private final ArrayDeque<AudioFrame> audioFrameQueue = new ArrayDeque<>();
 
   private AVFormatContext pFormatCtx;
 
   private int videoStream = -1;
   private int audioStream = -1;
-  private avcodec.AVCodecContext pCodecCtxVideo = null;
-  private avcodec.AVCodecContext pCodecCtxAudio = null;
+  private AVCodecContext pCodecCtxVideo = null;
+  private AVCodecContext pCodecCtxAudio = null;
 
   /**
    * Field for raw frame as returned by decoder (regardless of being audio or video).
    */
-  private avutil.AVFrame pFrame = null;
+  private AVFrame pFrame = null;
 
   /**
    * Field for RGB frame (decoded video frame).
    */
-  private avutil.AVFrame pFrameRGB = null;
+  private AVFrame pFrameRGB = null;
 
   /**
    * Field for re-sampled audio-sample.
    */
-  private avutil.AVFrame resampledFrame = null;
+  private AVFrame resampledFrame = null;
 
-  private avcodec.AVPacket packet;
+  private AVPacket packet;
   private BytePointer buffer = null;
 
-  private IntPointer out_linesize = new IntPointer();
+  private final IntPointer out_linesize = new IntPointer();
 
-  private swscale.SwsContext sws_ctx = null;
-  private swresample.SwrContext swr_ctx = null;
+  private SwsContext sws_ctx = null;
+  private SwrContext swr_ctx = null;
 
   private VideoDescriptor videoDescriptor = null;
   private AudioDescriptor audioDescriptor = null;
@@ -273,7 +265,7 @@ private void enqueueOriginalAudio() {
   private void enqueueResampledAudio() {
     /* Convert decoded frame. Break if resampling fails.*/
     if (swresample.swr_convert(this.swr_ctx, null, 0, this.pFrame.data(), this.pFrame.nb_samples()) < 0) {
-      LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).", this.getClass().getName());
+      LOGGER.error("Could not convert sample (FFMPEG swr_convert() failed).");
       return;
     }