[devscripts] run_tests: Create Python script (#8720)

Authored by: Grub4K
This commit is contained in:
Simon Sawicki 2023-12-26 18:30:04 +01:00 committed by GitHub
parent 65de7d204c
commit 2d1d683a54
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 95 additions and 52 deletions

View file

@ -38,18 +38,14 @@ jobs:
os: [ubuntu-latest] os: [ubuntu-latest]
# CPython 3.11 is in quick-test # CPython 3.11 is in quick-test
python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10] python-version: ['3.8', '3.9', '3.10', '3.12', pypy-3.8, pypy-3.10]
run-tests-ext: [sh]
include: include:
# atleast one of each CPython/PyPy tests must be in windows # atleast one of each CPython/PyPy tests must be in windows
- os: windows-latest - os: windows-latest
python-version: '3.8' python-version: '3.8'
run-tests-ext: bat
- os: windows-latest - os: windows-latest
python-version: '3.12' python-version: '3.12'
run-tests-ext: bat
- os: windows-latest - os: windows-latest
python-version: pypy-3.9 python-version: pypy-3.9
run-tests-ext: bat
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
@ -62,4 +58,4 @@ jobs:
continue-on-error: False continue-on-error: False
run: | run: |
python3 -m yt_dlp -v || true # Print debug head python3 -m yt_dlp -v || true # Print debug head
./devscripts/run_tests.${{ matrix.run-tests-ext }} core python3 ./devscripts/run_tests.py core

View file

@ -18,7 +18,7 @@ jobs:
run: pip install pytest -r requirements.txt run: pip install pytest -r requirements.txt
- name: Run tests - name: Run tests
continue-on-error: true continue-on-error: true
run: ./devscripts/run_tests.sh download run: python3 ./devscripts/run_tests.py download
full: full:
name: Full Download Tests name: Full Download Tests
@ -29,15 +29,12 @@ jobs:
matrix: matrix:
os: [ubuntu-latest] os: [ubuntu-latest]
python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10] python-version: ['3.10', '3.11', '3.12', pypy-3.8, pypy-3.10]
run-tests-ext: [sh]
include: include:
# atleast one of each CPython/PyPy tests must be in windows # atleast one of each CPython/PyPy tests must be in windows
- os: windows-latest - os: windows-latest
python-version: '3.8' python-version: '3.8'
run-tests-ext: bat
- os: windows-latest - os: windows-latest
python-version: pypy-3.9 python-version: pypy-3.9
run-tests-ext: bat
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }} - name: Set up Python ${{ matrix.python-version }}
@ -48,4 +45,4 @@ jobs:
run: pip install pytest -r requirements.txt run: pip install pytest -r requirements.txt
- name: Run tests - name: Run tests
continue-on-error: true continue-on-error: true
run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} download run: python3 ./devscripts/run_tests.py download

View file

@ -19,7 +19,7 @@ jobs:
- name: Run tests - name: Run tests
run: | run: |
python3 -m yt_dlp -v || true python3 -m yt_dlp -v || true
./devscripts/run_tests.sh core python3 ./devscripts/run_tests.py core
flake8: flake8:
name: Linter name: Linter
if: "!contains(github.event.head_commit.message, 'ci skip all')" if: "!contains(github.event.head_commit.message, 'ci skip all')"

View file

@ -140,12 +140,9 @@ To run yt-dlp as a developer, you don't need to build anything either. Simply ex
python -m yt_dlp python -m yt_dlp
To run the test, simply invoke your favorite test runner, or execute a test file directly; any of the following work: To run all the available core tests, use:
python -m unittest discover python devscripts/run_tests.py
python test/test_download.py
nosetests
pytest
See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
@ -187,15 +184,21 @@ After you have ensured this site is distributing its content legally, you can fo
'url': 'https://yourextractor.com/watch/42', 'url': 'https://yourextractor.com/watch/42',
'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)', 'md5': 'TODO: md5 sum of the first 10241 bytes of the video file (use --test)',
'info_dict': { 'info_dict': {
# For videos, only the 'id' and 'ext' fields are required to RUN the test:
'id': '42', 'id': '42',
'ext': 'mp4', 'ext': 'mp4',
'title': 'Video title goes here', # Then if the test run fails, it will output the missing/incorrect fields.
'thumbnail': r're:^https?://.*\.jpg$', # Properties can be added as:
# TODO more properties, either as: # * A value, e.g.
# * A value # 'title': 'Video title goes here',
# * MD5 checksum; start the string with md5: # * MD5 checksum; start the string with 'md5:', e.g.
# * A regular expression; start the string with re: # 'description': 'md5:098f6bcd4621d373cade4e832627b4f6',
# * Any Python type, e.g. int or float # * A regular expression; start the string with 're:', e.g.
# 'thumbnail': r're:^https?://.*\.jpg$',
# * A count of elements in a list; start the string with 'count:', e.g.
# 'tags': 'count:10',
# * Any Python type, e.g.
# 'view_count': int,
} }
}] }]
@ -215,8 +218,8 @@ After you have ensured this site is distributing its content legally, you can fo
} }
``` ```
1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`. 1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
1. Run `python test/test_download.py TestDownload.test_YourExtractor` (note that `YourExtractor` doesn't end with `IE`). This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, the tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. You can also run all the tests in one go with `TestDownload.test_YourExtractor_all` 1. Run `python devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
1. Make sure you have atleast one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running. 1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want. 1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): 1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):

View file

@ -1,17 +1,4 @@
@setlocal
@echo off @echo off
cd /d %~dp0..
if ["%~1"]==[""] ( >&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead
set "test_set="test"" python %~dp0run_tests.py %~1
) else if ["%~1"]==["core"] (
set "test_set="-m not download""
) else if ["%~1"]==["download"] (
set "test_set="-m "download""
) else (
echo.Invalid test type "%~1". Use "core" ^| "download"
exit /b 1
)
set PYTHONWARNINGS=error
pytest %test_set%

70
devscripts/run_tests.py Executable file
View file

@ -0,0 +1,70 @@
#!/usr/bin/env python3
import argparse
import functools
import os
import re
import subprocess
import sys
from pathlib import Path
fix_test_name = functools.partial(re.compile(r'IE(_all|_\d+)?$').sub, r'\1')
def parse_args():
parser = argparse.ArgumentParser(description='Run selected yt-dlp tests')
parser.add_argument(
'test', help='a extractor tests, or one of "core" or "download"', nargs='*')
parser.add_argument(
'-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
return parser.parse_args()
def run_tests(*tests, pattern=None):
run_core = 'core' in tests or (not pattern and not tests)
run_download = 'download' in tests
tests = list(map(fix_test_name, tests))
arguments = ['pytest', '-Werror', '--tb', 'short']
if run_core:
arguments.extend(['-m', 'not download'])
elif run_download:
arguments.extend(['-m', 'download'])
elif pattern:
arguments.extend(['-k', pattern])
else:
arguments.extend(
f'test/test_download.py::TestDownload::test_{test}' for test in tests)
print(f'Running {arguments}')
try:
subprocess.run(arguments)
return
except FileNotFoundError:
pass
arguments = [sys.executable, '-Werror', '-m', 'unittest']
if run_core:
print('"pytest" needs to be installed to run core tests', file=sys.stderr)
return
elif run_download:
arguments.append('test.test_download')
elif pattern:
arguments.extend(['-k', pattern])
else:
arguments.extend(
f'test.test_download.TestDownload.test_{test}' for test in tests)
print(f'Running {arguments}')
subprocess.run(arguments)
if __name__ == '__main__':
try:
args = parse_args()
os.chdir(Path(__file__).parent.parent)
run_tests(*args.test, pattern=args.k)
except KeyboardInterrupt:
pass

View file

@ -1,14 +1,4 @@
#!/usr/bin/env sh #!/usr/bin/env sh
if [ -z "$1" ]; then >&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead'
test_set='test' python3 devscripts/run_tests.py "$1"
elif [ "$1" = 'core' ]; then
test_set="-m not download"
elif [ "$1" = 'download' ]; then
test_set="-m download"
else
echo 'Invalid test type "'"$1"'". Use "core" | "download"'
exit 1
fi
python3 -bb -Werror -m pytest "$test_set"