1. Forum
  2. Usenet
  3. LINUX.GENTOO.DEV

  • [gentoo-dev] [PATCH 0/6] python-utils-r1/distutils-r1: EPYTEST_XDIST su

    From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:10:01 2023
    Hi,

    Here's a patchset that introduces EPYTEST_XDIST=1 variable to
    automatically use dev-python/pytest-xdist to run test suites in parallel
    via epytest. It also takes care of adding the dep if set prior to distutils_enable_tests (as expected, pkgcheck will also check for that).

    I'm also adding an EPYTEST_JOBS variable to allow overriding the job
    count independently of MAKEOPTS.

    The patches also include some prior refactoring and two example ebuild conversions.

    PR: https://github.com/gentoo/gentoo/pull/33667

    --
    Best regards,
    Michał Górny

    Michał Górny (6):
    distutils-r1.eclass: Refactor `d_e_t unittest` (NFC)
    distutils-r1.eclass: Refactor d_e_t to permit multiple test packages
    python-utils-r1.eclass: Add EPYTEST_XDIST for epytest
    distutils-r1.eclass: Add pytest-xdist dep if EPYTEST_XDIST is set
    dev-python/pyrate-limiter: Use EPYTEST_XDIST
    dev-python/numpy: Switch to EPYTEST_XDIST

    dev-python/numpy/numpy-1.26.1.ebuild | 4 +--
    .../pyrate-limiter-3.1.0.ebuild | 6 +---
    eclass/distutils-r1.eclass | 32 ++++++++++---------
    eclass/python-utils-r1.eclass | 31 ++++++++++++++++++
    4 files changed, 51 insertions(+), 22 deletions(-)

    --
    2.42.0

    --- SoupGate-Win32 v1.05
    * Origin: fsxNet Usenet Gateway (21:1/5)
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:10:01 2023
    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    eclass/distutils-r1.eclass | 17 ++++++++---------
    1 file changed, 8 insertions(+), 9 deletions(-)

    diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
    index 1cc91110dccf..9dd091921391 100644
    --- a/eclass/distutils-r1.eclass
    +++ b/eclass/distutils-r1.eclass
    @@ -608,6 +608,8 @@ distutils_enable_tests() {
    esac

    [[ ${#} -eq 1 ]] || die "${FUNCNAME} takes exactly one argument: test-runner"
    +
    + local test_deps=${RDEPEND}
    local test_pkg
    case ${1} in
    nose)
    @@ -619,7 +621,12 @@ distutils_enable_tests() {
    setup.py)
    ;;
    unittest)
    - # dep handled below
    + # unittest-or-fail is needed in py<3.12
    + test_deps+="
    + $(python_gen_cond_dep '
    + dev-python/unittest-or-fail[${PYTHON_USEDEP}]
    + ' 3.10 3.11)
    + "
    ;;
    *)
    die "${FUNCNAME}: unsupported argument: ${1}"
    @@ -628,7 +635,6 @@ distutils_enable_tests() {
    _DISTUTILS_TEST_RUNNER=${1}
    python_test() { distutils-r1_python_test; }

    - local test_deps=${RDEPEND}
    if [[ -n ${t
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:20:01 2023
    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    eclass/distutils-r1.eclass | 12 ++++++------
    1 file changed, 6 insertions(+), 6 deletions(-)

    diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
    index 9dd091921391..3d69911d9209 100644
    --- a/eclass/distutils-r1.eclass
    +++ b/eclass/distutils-r1.eclass
    @@ -610,13 +610,13 @@ distutils_enable_tests() {
    [[ ${#} -eq 1 ]] || die "${FUNCNAME} takes exactly one argument: test-runner"

    local test_deps=${RDEPEND}
    - local test_pkg
    + local test_pkgs
    case ${1} in
    nose)
    - test_pkg=">=dev-python/nose-1.3.7_p20221026"
    + test_pkgs='>=dev-python/nose-1.3.7_p20221026[${PYTHON_USEDEP}]'
    ;;
    pytest)
    - test_pkg=">=dev-python/pytest-7.3.1"
    + test_pkgs='>=dev-python/pytest-7.3.1[${PYTHON_USEDEP}]'
    ;;
    setup.py)
    ;;
    @@ -635,12 +635,12 @@ distutils_enable_tests() {
    _DISTUTILS_TEST_RUNNER=${1}
    python_test() { distutils-r1_python_test; }

    - if [[ -n ${test_pkg} ]]; then
    + if [[ -n ${test_pkgs} ]]; then
    if [[ ! ${DISTUTILS_SINGLE_IMPL} ]]; then
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:20:01 2023
    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    dev-python/numpy/numpy-1.26.1.ebuild | 4 ++--
    1 file changed, 2 insertions(+), 2 deletions(-)

    diff --git a/dev-python/numpy/numpy-1.26.1.ebuild b/dev-python/numpy/numpy-1.26.1.ebuild
    index 3255d05e2c00..92564017966e 100644
    --- a/dev-python/numpy/numpy-1.26.1.ebuild
    +++ b/dev-python/numpy/numpy-1.26.1.ebuild
    @@ -46,11 +46,11 @@ BDEPEND="
    ' 'python*')
    dev-python/charset-normalizer[${PYTHON_USEDEP}]
    >=dev-python/hypothesis-5.8.0[${PYTHON_USEDEP}]
    - dev-python/pytest-xdist[${PYTHON_USEDEP}]
    >=dev-python/pytz-2019.3[${PYTHON_USEDEP}]
    )
    "

    +EPYTEST_XDIST=1
    distutils_enable_tests pytest

    python_prepare_all() {
    @@ -143,7 +143,7 @@ python_test() {
    esac

    rm -rf numpy || die
    - epytest -n "$(makeopts_jobs)" --pyargs numpy
    + epytest --pyargs numpy
    }

    python_install_all() {
    --
    2.42.0

    --- SoupGate-Win32 v1.05
    * Origin: fsxNet Usenet Gateway (21:1/5)
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:20:01 2023
    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild | 6 +-----
    1 file changed, 1 insertion(+), 5 deletions(-)

    diff --git a/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild b/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
    index 47947453dc89..a00cb916428e 100644
    --- a/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
    +++ b/dev-python/pyrate-limiter/pyrate-limiter-3.1.0.ebuild
    @@ -26,7 +26,6 @@ RDEPEND="
    BDEPEND="
    test? (
    dev-python/pytest-asyncio[${PYTHON_USEDEP}]
    - dev-python/pytest-xdist[${PYTHON_USEDEP}]
    dev-python/pyyaml[${PYTHON_USEDEP}]
    )
    "
    @@ -37,6 +36,7 @@ EPYTEST_DESELECT=(
    # Python 3.11 is slightly faster, leading to a non-critical failure here
    "tests/test_concurrency.py::test_concurrency[ProcessPoolExecutor-SQLiteBucket]"
    )
    +EPYTEST_XDIST=1

    # TODO: package sphinx-copybutton
    # distutils_enable_sphinx docs \
    @@ -67,7 +67,3 @@ src_test() {
    # Clean up afterwards
    kill "$(<"${redis_pid}")" || die
    }
    -
    -python_test() {
    - epytest
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:20:01 2023
    Add an `EPYTEST_XDIST` variable that can be used to enable running
    the test suite in parallel via the dev-python/pytest-xdist plugin.
    This also includes user-facing `EPYTEST_JOBS` to control the job count independently of `MAKEOPTS`.

    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    eclass/python-utils-r1.eclass | 29 +++++++++++++++++++++++++++++
    1 file changed, 29 insertions(+)

    diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass index 4a538f9942f6..f2a8d4d0f65e 100644
    --- a/eclass/python-utils-r1.eclass
    +++ b/eclass/python-utils-r1.eclass
    @@ -1307,6 +1307,19 @@ _python_check_occluded_packages() {
    # parameter, when calling epytest. The listed files will be entirely
    # skipped from test collection.

    +# @VARIABLE: EPYTEST_XDIST
    +# @DEFAULT_UNSET
    +# @DESCRIPTION:
    +# If set to a non-empty value, enables running tests in parallel
    +# via pytest-xdist plugin.
    +
    +# @VARIABLE: EPYTEST_JOBS
    +# @USER_VARIABLE
    +# @DEFAULT_UNSET
    +# @DESCRIPTION:
    +# Specifies the number of jobs for parallel (pytest-xdist) test runs.
    +# When unset, defaults to -j from MAKEOPTS, or the current nproc.
    +
    # @FUNCTION: epytest
    # @USAGE: [<args>...]
    # @DESCRIPTION:
    @@ -1371,6 +1384,22 @@ epytest() {
    -p no:plus
    -p no:tavern
    )
    +
    + if [[ ${EPYTEST_XDIST
  • From =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?@21:1/5 to All on Sat Nov 4 07:20:01 2023
    Make `distutils_enable_tests pytest` automatically add
    the `dev-python/pytest-xdist` dependency if `EPYTEST_XDIST` is set.

    Signed-off-by: Michał Górny <mgorny@gentoo.org>
    ---
    eclass/distutils-r1.eclass | 3 +++
    eclass/python-utils-r1.eclass | 4 +++-
    2 files changed, 6 insertions(+), 1 deletion(-)

    diff --git a/eclass/distutils-r1.eclass b/eclass/distutils-r1.eclass
    index 3d69911d9209..3d756eaad556 100644
    --- a/eclass/distutils-r1.eclass
    +++ b/eclass/distutils-r1.eclass
    @@ -617,6 +617,9 @@ distutils_enable_tests() {
    ;;
    pytest)
    test_pkgs='>=dev-python/pytest-7.3.1[${PYTHON_USEDEP}]' + if [[ ${EPYTEST_XDIST} ]]; then
    + test_pkgs+=' dev-python/pytest-xdist[${PYTHON_USEDEP}]'
    + fi
    ;;
    setup.py)
    ;;
    diff --git a/eclass/python-utils-r1.eclass b/eclass/python-utils-r1.eclass index f2a8d4d0f65e..c61f9e2ecb94 100644
    --- a/eclass/python-utils-r1.eclass
    +++ b/eclass/python-utils-r1.eclass
    @@ -1311,7 +1311,9 @@ _python_check_occluded_packages() {
    # @DEFAULT_UNSET
    # @DESCRIPTION:
    # If set to a non-empty value, enables running tests in parallel
    -# via pytest-xdist plugin.
    +# via pytest-xdist plugin. If this variable is set prior