From 0c75714d1ad309c575972663727a252c47019886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jeremy=20Lain=C3=A9?= Date: Wed, 1 Nov 2023 23:27:32 +0100 Subject: [PATCH 1/3] Simplify linter tests --- .github/workflows/tests.yml | 29 +++++------------------------ Makefile | 5 +++-- scripts/test | 14 -------------- tests/requirements.txt | 5 +++-- 4 files changed, 11 insertions(+), 42 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 996a60bf4..edca189f9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,41 +12,22 @@ on: - '**.txt' jobs: style: - name: "${{ matrix.config.suite }}" runs-on: ubuntu-latest - strategy: - matrix: - config: - - {suite: black} - - {suite: flake8} - - {suite: isort} - - env: - PYAV_PYTHON: python3 - PYAV_LIBRARY: ffmpeg-4.3 # doesn't matter steps: - - uses: actions/checkout@v4 - name: Checkout + - name: Checkout + uses: actions/checkout@v4 - name: Python uses: actions/setup-python@v4 with: python-version: 3.8 - - name: Environment - run: env | sort - - name: Packages - run: | - . scripts/activate.sh - # A bit of a hack that we can get away with this. - python -m pip install ${{ matrix.config.suite }} + run: pip install -r tests/requirements.txt - - name: "${{ matrix.config.suite }}" - run: | - . scripts/activate.sh - ./scripts/test ${{ matrix.config.suite }} + - name: Linters + run: make lint nix: name: "py-${{ matrix.config.python }} lib-${{ matrix.config.ffmpeg }} ${{matrix.config.os}}" diff --git a/Makefile b/Makefile index 6cb954dea..fd4d4ed51 100644 --- a/Makefile +++ b/Makefile @@ -34,8 +34,9 @@ fate-suite: rsync -vrltLW rsync://fate-suite.ffmpeg.org/fate-suite/ tests/assets/fate-suite/ lint: - TESTSUITE=flake8 scripts/test - TESTSUITE=isort scripts/test + black --check av examples tests + flake8 av examples tests + isort --check-only --diff av examples tests test: $(PYTHON) setup.py test diff --git a/scripts/test b/scripts/test index 806011859..3eca2feb7 100755 --- a/scripts/test +++ b/scripts/test @@ -18,20 +18,6 @@ istest() { return $? } -if istest black; then - $PYAV_PYTHON -m black --check av examples tests -fi - -if istest flake8; then - # Settings are in setup.cfg - $PYAV_PYTHON -m flake8 av examples tests -fi - -if istest isort; then - # More settings in setup.cfg - $PYAV_PYTHON -m isort --check-only --diff av examples tests -fi - if istest main; then $PYAV_PYTHON setup.py test fi diff --git a/tests/requirements.txt b/tests/requirements.txt index 2a321a28d..36d21ecf9 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,8 +1,9 @@ autopep8 -Cython +black +cython editorconfig flake8 isort numpy -Pillow +pillow sphinx < 4.4 From 0f978bbc6888aaf49c94c742d06a9033ba54a7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jeremy=20Lain=C3=A9?= Date: Thu, 2 Nov 2023 00:03:54 +0100 Subject: [PATCH 2/3] Prune Makefile targets, drop unused scripts Remove some unused Makefile targets, as well as scripts: - build-debug-python - clean-branches - vagrant-test We can also remove the `sdist` step as this is exercised by the `package-source` job. --- .github/workflows/tests.yml | 6 -- .gitignore | 1 - Makefile | 56 +++-------------- scripts/build-debug-python | 33 ---------- scripts/clean-branches | 120 ------------------------------------ scripts/test | 5 -- scripts/vagrant-test | 7 --- 7 files changed, 8 insertions(+), 220 deletions(-) delete mode 100755 scripts/build-debug-python delete mode 100755 scripts/clean-branches delete mode 100755 scripts/vagrant-test diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index edca189f9..70af7ea14 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,12 +109,6 @@ jobs: . scripts/activate.sh ffmpeg-${{ matrix.config.ffmpeg }} scripts/test examples - - name: Source Distribution - if: matrix.config.extras - run: | - . scripts/activate.sh ffmpeg-${{ matrix.config.ffmpeg }} - scripts/test sdist - windows: name: "py-${{ matrix.config.python }} lib-${{ matrix.config.ffmpeg }} ${{matrix.config.os}}" runs-on: ${{ matrix.config.os }} diff --git a/.gitignore b/.gitignore index bc6c09077..6e5a2b48e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ # Testing. *.spyderproject .idea -/.vagrant /sandbox /tests/assets /tests/samples diff --git a/Makefile b/Makefile index fd4d4ed51..e8ad38277 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PYAV_PYTHON ?= python PYTHON := $(PYAV_PYTHON) -.PHONY: default build cythonize clean clean-all info lint test fate-suite test-assets docs +.PHONY: default build clean docs fate-suite lint test default: build @@ -13,21 +13,13 @@ default: build build: CFLAGS=$(CFLAGS) LDFLAGS=$(LDFLAGS) $(PYTHON) setup.py build_ext --inplace --debug -cythonize: - $(PYTHON) setup.py cythonize - - - -wheel: build-mingw32 - $(PYTHON) setup.py bdist_wheel - -build-mingw32: - # before running, set PKG_CONFIG_PATH to the pkgconfig dir of the ffmpeg build. - # set PKG_CONFIG_PATH=D:\dev\3rd\media-autobuild_suite\local32\bin-video\ffmpegSHARED\lib\pkgconfig - CFLAGS=$(CFLAGS) LDFLAGS=$(LDFLAGS) $(PYTHON) setup.py build_ext --inplace -c mingw32 - mv *.pyd av - - +clean: + - find av -name '*.so' -delete + - rm -rf build + - rm -rf sandbox + - rm -rf src + - rm -rf tmp + - make -C docs clean fate-suite: # Grab ALL of the samples from the ffmpeg site. @@ -41,16 +33,6 @@ lint: test: $(PYTHON) setup.py test - - -vagrant: - vagrant box list | grep -q precise32 || vagrant box add precise32 http://files.vagrantup.com/precise32.box - -vtest: - vagrant ssh -c /vagrant/scripts/vagrant-test - - - tmp/ffmpeg-git: @ mkdir -p tmp/ffmpeg-git git clone --depth=1 git://source.ffmpeg.org/ffmpeg.git tmp/ffmpeg-git @@ -67,25 +49,3 @@ docs: tmp/tagfile.xml deploy-docs: docs ./docs/upload docs - - - - -clean-build: - - rm -rf build - - find av -name '*.so' -delete - -clean-sandbox: - - rm -rf sandbox/201* - - rm sandbox/last - -clean-src: - - rm -rf src - -clean-docs: - - rm tmp/Doxyfile - - rm tmp/tagfile.xml - - make -C docs clean - -clean: clean-build clean-sandbox clean-src -clean-all: clean-build clean-sandbox clean-src clean-docs diff --git a/scripts/build-debug-python b/scripts/build-debug-python deleted file mode 100755 index 757946f14..000000000 --- a/scripts/build-debug-python +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -export PYAV_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.."; pwd)" - -export PYAV_PYTHON_VERSION="2.7.13" -export PYAV_PLATFORM_SLUG="$(uname -s).$(uname -r)" -export PYAV_VENV_NAME="$PYAV_PLATFORM_SLUG.cpython-$PYAV_PYTHON_VERSION-debug" -export PYAV_VENV="$PYAV_ROOT/venvs/$PYAV_VENV_NAME" - - -export PYAV_PYTHON_SRC="$PYAV_ROOT/vendor/Python-$PYAV_PYTHON_VERSION" - -if [[ ! -d "$PYAV_PYTHON_SRC" ]]; then - url="https://www.python.org/ftp/python/$PYAV_PYTHON_VERSION/Python-$PYAV_PYTHON_VERSION.tgz" - echo "Downloading $url" - wget -O "$PYAV_PYTHON_SRC.tgz" "$url" || exit 2 - tar -C "$PYAV_ROOT/vendor" -xvzf "$PYAV_PYTHON_SRC.tgz" || exit 3 -fi - -cd "$PYAV_PYTHON_SRC" || exit 4 - -# TODO: Make generic. -export CPPFLAGS="-I$(brew --prefix openssl)/include" -export LDFLAGS="-L$(brew --prefix openssl)/lib" - -# --with-pymalloc \ -./configure \ - --with-pydebug \ - --prefix "$PYAV_VENV" \ - || exit 5 - -make -j 12 || exit 6 - diff --git a/scripts/clean-branches b/scripts/clean-branches deleted file mode 100755 index 83f82f6fb..000000000 --- a/scripts/clean-branches +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -import subprocess - - -# These are remote branches that got rebased or something, but I don't have -# control over. -ignored_hashes = set(''' - 078daa1148a84849ea0890b388353acd7333fcea - 0832d2bdaa048fca1393f3d3810b8b8810535f9f - 08e86996736d77df7d694d0a67a126fc7eac7e94 - 097631535fa4cdb87204980541d3f9bb9c7a9ffb - 16ce34ba1d1f1ed4327326a10565f1a9f07107b0 - 183cf1447571aa48baf0665d17d41be1e48f2cc6 - 245a4dca69cf0fff674c6ad5bcfb7929c7347bf6 - 28b4b3988981471ff173679db3643418ff3f5aaa - 3977b5d5be22922f2eb4288e2682f1de8fad8e12 - 5b9f192165855942918f9bd957c30e918b97cbeb - 6091e89de0ae4aff2ba76d21b1110409ef174b78 - 636afe3f0b5b07233edae8e333db35c044c36b30 - 74f79ef74ec281f5e0da51bcfd0b1051aa53edbf - 7737ef6e9e7307c40f326e61cc9291047540bc49 - 8618940d333f44ff960d561dda34167d4dbb81d4 - a2fb55e97788809b5f33b1b0c9241fc77312f606 - aa044b3f62a6d7bf4dde18cba91b1d0dd8a0816a - aa7d01ba458025ede1757e56b638002375bb864a - aafe064e209b667f565c4f57a94b098474d0b184 - afac2d8f89673c012d1f4b845b006911f55d1d86 - b115786b950c87ef9c422752e014297903bca393 - b737c6ceb6750d00f62dfdaa40fee3e757c680a3 - b7bf427a485736e6e1c71605bdce101214bae09f - ba02afa7ea160328b5a3be111c7e276fb9d3c961 - bc5ffe456345286a64ce33ffe5ce6a2ee8b63f40 - c45a337fe49875b1cc28a0501a704890be444765 - c6b1a5ac03e775ea46bffac7bbfea9d73cd03b87 - c9c0d63b09c450d494fba1c4073fbe18851dfaff - cc270d6790c02e6c5e93313d1e6499ce534350b9 - cdd8e4c085a55e258bd551f7bcf4fee60474aa05 - eac71881c24d42f801e9c18e448855a402333960 - efd12926b1f446c32f5a239c0b2d351fa2d78101 - f04dce0e80b4f290482eba4fb3c3ec68f353bf01 - f0d1e82dee788085cf4afad7656a90966e40f7a0 - f518f6e7bf47e00fe0c73a5098ae40813920400f - f779c4371fdace76ee572053b4acb3999ffd4107 -'''.strip().split()) - - -def get_branches(*args): - cmd = ['git', 'branch', '-v', '--abbrev=40'] - cmd.extend(args) - res = {} - for line in subprocess.check_output(cmd).decode().splitlines(): - parts = line[2:].strip().split() - name = parts[0] - hash_ = parts[1] - res[name] = hash_ - return res - -def rm(*args): - subprocess.check_call(('git', 'branch', '-D') + args) - - -# Clean up everything that was merged -for line in subprocess.check_output(['git', 'branch', '--merged']).decode().splitlines(): - line = line.strip() - if not line: - continue - parts = line.split() - if parts[0] == '*': - continue - if parts[-1] in ('develop', 'master'): - continue - rm(parts[-1]) - -for line in subprocess.check_output(['git', 'branch', '-r', '--merged']).decode().splitlines(): - name = line.strip().split()[-1] - if not name: - continue - if name.split('/', 1)[-1] in ('develop', 'master'): - continue - rm('-r', name) - - -our_branches = get_branches() -for name, hash_ in get_branches('-r').items(): - - if hash_ in ignored_hashes: - print("Removing ignored", name) - rm('-r', name) - continue - - if name.startswith('origin/'): - our_branches[name] = hash_ - - -for name in get_branches('-r', '--merged'): - if name.startswith('origin/'): - continue - print("Removing merged", name) - rm('-r', name) - -for name, hash_ in get_branches('-r', '--no-merged').items(): - - remote, branch = name.split('/', 1) - if remote == 'origin': - continue - - for prefix in '', 'origin/': - our_name = prefix + branch - if our_branches.get(our_name) == hash_: - print("Removing identical", name) - rm('-r', name) - break - -# Anything that doesn't root at the same place as us. -for name in get_branches('-r', '--no-contains', 'e105c0b4e64a0471f3f5375a86342c33cb942e23'): - rm('-r', name) - - - diff --git a/scripts/test b/scripts/test index 3eca2feb7..c34734456 100755 --- a/scripts/test +++ b/scripts/test @@ -22,11 +22,6 @@ if istest main; then $PYAV_PYTHON setup.py test fi -if istest sdist; then - $PYAV_PYTHON setup.py build_ext - $PYAV_PYTHON setup.py sdist -fi - if istest doctest; then make -C docs test fi diff --git a/scripts/vagrant-test b/scripts/vagrant-test deleted file mode 100755 index dbdab607a..000000000 --- a/scripts/vagrant-test +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd /vagrant - -./scripts/build-deps -./scripts/build -./scripts/test From 1fa1e494c7fe0378cef9dca323ac2358564997df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jeremy=20Lain=C3=A9?= Date: Thu, 2 Nov 2023 09:18:28 +0100 Subject: [PATCH 3/3] Simplify documentation generation Do not depend on how FFmpeg was built, just fetch a fresh clone of the FFmpeg repository to generate doxygen tags. --- .github/workflows/tests.yml | 2 +- Makefile | 20 +----------------- docs/Makefile | 6 ++++-- docs/generate-tagfile | 42 +++++++++---------------------------- scripts/test | 4 ---- 5 files changed, 16 insertions(+), 58 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 70af7ea14..8037ba891 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,7 +101,7 @@ jobs: if: matrix.config.extras run: | . scripts/activate.sh ffmpeg-${{ matrix.config.ffmpeg }} - scripts/test doctest + make -C docs test - name: Examples if: matrix.config.extras diff --git a/Makefile b/Makefile index e8ad38277..9a0b0a95f 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PYAV_PYTHON ?= python PYTHON := $(PYAV_PYTHON) -.PHONY: default build clean docs fate-suite lint test +.PHONY: default build clean fate-suite lint test default: build @@ -18,7 +18,6 @@ clean: - rm -rf build - rm -rf sandbox - rm -rf src - - rm -rf tmp - make -C docs clean fate-suite: @@ -32,20 +31,3 @@ lint: test: $(PYTHON) setup.py test - -tmp/ffmpeg-git: - @ mkdir -p tmp/ffmpeg-git - git clone --depth=1 git://source.ffmpeg.org/ffmpeg.git tmp/ffmpeg-git - -tmp/Doxyfile: tmp/ffmpeg-git - cp tmp/ffmpeg-git/doc/Doxyfile $@ - echo "GENERATE_TAGFILE = ../tagfile.xml" >> $@ - -tmp/tagfile.xml: tmp/Doxyfile - cd tmp/ffmpeg-git; doxygen ../Doxyfile - -docs: tmp/tagfile.xml - PYTHONPATH=.. make -C docs html - -deploy-docs: docs - ./docs/upload docs diff --git a/docs/Makefile b/docs/Makefile index 65a1e540e..f26ef7dc3 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,6 +2,7 @@ SPHINXOPTS = SPHINXBUILD = sphinx-build BUILDDIR = _build +FFMPEGDIR = _ffmpeg ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) . @@ -12,7 +13,8 @@ default: html TAGFILE := _build/doxygen/tagfile.xml $(TAGFILE) : - ./generate-tagfile -o $(TAGFILE) + git clone --depth=1 git://source.ffmpeg.org/ffmpeg.git $(FFMPEGDIR) + ./generate-tagfile --library $(FFMPEGDIR) -o $(TAGFILE) TEMPLATES := $(wildcard api/*.py development/*.py) @@ -24,7 +26,7 @@ _build/rst/%.rst: %.py $(TAGFILE) $(shell find ../include ../av -name '*.pyx' -o clean: - - rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR) $(FFMPEGDIR) html: $(RENDERED) $(TAGFILE) $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/generate-tagfile b/docs/generate-tagfile index f00ed7ad4..1f729de5c 100755 --- a/docs/generate-tagfile +++ b/docs/generate-tagfile @@ -1,46 +1,23 @@ #!/usr/bin/env python +import argparse import os import subprocess -import argparse parser = argparse.ArgumentParser() -parser.add_argument('-l', '--library', default=os.environ.get('PYAV_LIBRARY')) -parser.add_argument('-o', '--output', default=os.path.abspath(os.path.join( - __file__, - '..', - '_build', - 'doxygen', - 'tagfile.xml', -))) +parser.add_argument("-l", "--library", required=True) +parser.add_argument("-o", "--output", required=True) args = parser.parse_args() - -if not args.library: - print("Please provide --library or set $PYAV_LIBRARY") - exit(1) - -library = os.path.abspath(os.path.join( - __file__, - '..', '..', - 'vendor', - args.library, -)) - -if not os.path.exists(library): - print("Library does not exist:", library) - exit(2) - - output = os.path.abspath(args.output) outdir = os.path.dirname(output) if not os.path.exists(outdir): os.makedirs(outdir) - -proc = subprocess.Popen(['doxygen', '-'], stdin=subprocess.PIPE, cwd=library) -proc.communicate(''' +proc = subprocess.Popen(["doxygen", "-"], stdin=subprocess.PIPE, cwd=args.library) +proc.communicate( + """ #@INCLUDE = doc/Doxyfile GENERATE_TAGFILE = {} @@ -49,6 +26,7 @@ GENERATE_LATEX = no CASE_SENSE_NAMES = yes INPUT = libavcodec libavdevice libavfilter libavformat libavresample libavutil libswresample libswscale -'''.format(output).encode()) - - +""".format( + output + ).encode() +) diff --git a/scripts/test b/scripts/test index c34734456..01991f5a6 100755 --- a/scripts/test +++ b/scripts/test @@ -22,10 +22,6 @@ if istest main; then $PYAV_PYTHON setup.py test fi -if istest doctest; then - make -C docs test -fi - if istest examples; then for name in $(find examples -name '*.py'); do echo