Module FPS decrease when restarting sources

[denisvmedyantsev]

I used 4 videos (30+30+25+25=110 FPS) and a dump module

FPS logs on start

video-source-camera-1-1  | Processed 1000 frames, 30.05 FPS. Source location: /data/cam1.mp4.
video-source-camera-2-1  | Processed 1000 frames, 29.89 FPS. Source location: /data/cam2.mp4.
video-source-camera-4-1  | Processed 1000 frames, 25.05 FPS. Source location: /data/cam4.mp4.
video-source-camera-3-1  | Processed 1000 frames, 25.03 FPS. Source location: /data/cam3.mp4.
...
module-1                 |  INFO  savant_core::pipeline::stats                       > Frame-based FPS counter triggered: FPS = 109.72, OPS = 0.00, frame_delta = 10000, time_delta = 91.137 sec, period=[1729682379758, 1729682470895] ms

after ~1 hour

module-1                 |  INFO  savant_core::pipeline::stats                       > Frame-based FPS counter triggered: FPS = 68.90, OPS = 0.00, frame_delta = 10000, time_delta = 145.139 sec, period=[1729685309770, 1729685454909] ms

docker-compose.yml

services:

  video-source-camera-1:
    image: ghcr.io/insight-platform/savant-adapters-gstreamer:0.4.7
    restart: unless-stopped
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
      - ./downloads_cache:/tmp/video-loop-source-downloads
      - ./data:/data
    environment:
      - LOCATION=/data/cam1.mp4
      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
      - ZMQ_ENDPOINT=dealer+connect:ipc:///tmp/zmq-sockets/input-video.ipc
      - SOURCE_ID=camera-1
      - FILE_TYPE=video
      - READ_METADATA=FALSE
      - SYNC_OUTPUT=True
    entrypoint: /opt/savant/adapters/gst/sources/media_files.sh
    depends_on:
      module:
        condition: service_healthy

  video-source-camera-2:
    image: ghcr.io/insight-platform/savant-adapters-gstreamer:0.4.7
    restart: unless-stopped
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
      - ./downloads_cache:/tmp/video-loop-source-downloads
      - ./data:/data
    environment:
      - LOCATION=/data/cam2.mp4
      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
      - ZMQ_ENDPOINT=dealer+connect:ipc:///tmp/zmq-sockets/input-video.ipc
      - SOURCE_ID=camera-2
      - FILE_TYPE=video
      - READ_METADATA=FALSE
      - SYNC_OUTPUT=True
    entrypoint: /opt/savant/adapters/gst/sources/media_files.sh
    depends_on:
      module:
        condition: service_healthy

  video-source-camera-3:
    image: ghcr.io/insight-platform/savant-adapters-gstreamer:0.4.7
    restart: unless-stopped
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
      - ./downloads_cache:/tmp/video-loop-source-downloads
      - ./data:/data
    environment:
      - LOCATION=/data/cam3.mp4
      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
      - ZMQ_ENDPOINT=dealer+connect:ipc:///tmp/zmq-sockets/input-video.ipc
      - SOURCE_ID=camera-3
      - FILE_TYPE=video
      - READ_METADATA=FALSE
      - SYNC_OUTPUT=True
    entrypoint: /opt/savant/adapters/gst/sources/media_files.sh
    depends_on:
      module:
        condition: service_healthy

  video-source-camera-4:
    image: ghcr.io/insight-platform/savant-adapters-gstreamer:0.4.7
    restart: unless-stopped
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
      - ./downloads_cache:/tmp/video-loop-source-downloads
      - ./data:/data
    environment:
      - LOCATION=/data/cam4.mp4
      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
      - ZMQ_ENDPOINT=dealer+connect:ipc:///tmp/zmq-sockets/input-video.ipc
      - SOURCE_ID=camera-4
      - FILE_TYPE=video
      - READ_METADATA=FALSE
      - SYNC_OUTPUT=True
    entrypoint: /opt/savant/adapters/gst/sources/media_files.sh
    depends_on:
      module:
        condition: service_healthy

  module:
    image: ghcr.io/insight-platform/savant-deepstream:0.4.7-6.4
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
    entrypoint: >
      bash -c "echo '
        name: test-module
        parameters:
          batch_size: 4' | python -m savant.entrypoint
      "
    environment:
      - LOGLEVEL=info
      - ZMQ_SRC_ENDPOINT=router+bind:ipc:///tmp/zmq-sockets/input-video.ipc
      - ZMQ_SINK_ENDPOINT=pub+bind:ipc:///tmp/zmq-sockets/output-video.ipc
    runtime: nvidia

volumes:
  zmq_sockets:

[bwsw]

Backport to 0.4

[tomskikh]

Changed compose file to faster testing:

• changed media_file.sh to video_loop.sh (to avoid restarting containers);
• set SYNC_OUTPUT=False and EOS_ON_LOOP_END=True.

services:

  video-source-camera-1:
#    image: ghcr.io/insight-platform/savant-adapters-gstreamer:0.4.7
    image: ghcr.io/insight-platform/savant-adapters-gstreamer:latest
#    restart: unless-stopped
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
      - ./downloads_cache:/tmp/video-loop-source-downloads
      - ./data:/data
    environment:
      - LOCATION=/data/cam1.mp4
      - SOURCE_ID=camera-1
      - DOWNLOAD_PATH=/tmp/video-loop-source-downloads
      - ZMQ_ENDPOINT=dealer+connect:ipc:///tmp/zmq-sockets/input-video.ipc
      - FILE_TYPE=video
      - READ_METADATA=FALSE
      - SYNC_OUTPUT=False
      - EOS_ON_LOOP_END=True
    entrypoint: /opt/savant/adapters/gst/sources/video_loop.sh
    depends_on:
      module:
        condition: service_healthy

  module:
#    image: ghcr.io/insight-platform/savant-deepstream:0.4.7-6.4
    image: ghcr.io/insight-platform/savant-deepstream:latest
    volumes:
      - zmq_sockets:/tmp/zmq-sockets
    entrypoint: >
      bash -c "echo '
        name: test-module
        parameters:
          batch_size: 1' | python -m savant.entrypoint
      "
    environment:
      - LOGLEVEL=info
      - ZMQ_SRC_ENDPOINT=router+bind:ipc:///tmp/zmq-sockets/input-video.ipc
      - ZMQ_SINK_ENDPOINT=pub+bind:ipc:///tmp/zmq-sockets/output-video.ipc
      - METRICS_TIME_PERIOD=30
    runtime: nvidia

volumes:
  zmq_sockets:

On branch bugfixes/04x looks like the time between processing EOS and reconnection of the source is increasing.
On develop there's another issue: the module stops processing frames after some time (1-2 minutes).