From fc699f7058ee279dd58fcd93c82e8333329e3b9e Mon Sep 17 00:00:00 2001 From: ori-kron-wis Date: Sun, 8 Dec 2024 12:15:37 +0200 Subject: [PATCH 1/4] Updated the scvi-tools tests schemes --- .github/workflows/test_linux_cuda.yml | 2 +- .github/workflows/test_linux_internet.yml | 74 +++++++++++++++++++++++ .github/workflows/test_linux_optional.yml | 74 +++++++++++++++++++++++ tests/conftest.py | 11 +++- 4 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test_linux_internet.yml create mode 100644 .github/workflows/test_linux_optional.yml diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 0eca4d65c8..9de73d8044 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -34,7 +34,7 @@ jobs: container: image: ghcr.io/scverse/scvi-tools:py3.12-cu12-base #image: ghcr.io/scverse/scvi-tools:py3.12-cu12-${{ env.BRANCH_NAME }}-base - options: --user root --gpus all + options: --user root --gpus all --no-cache name: integration diff --git a/.github/workflows/test_linux_internet.yml b/.github/workflows/test_linux_internet.yml new file mode 100644 index 0000000000..a2cafd3d27 --- /dev/null +++ b/.github/workflows/test_linux_internet.yml @@ -0,0 +1,74 @@ +name: test (internet) + +on: + pull_request: + branches: [main, "[0-9]+.[0-9]+.x"] + types: [labeled, synchronize, opened] + schedule: + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + # if PR has label "internet tests" or "all tests" or if scheduled or manually triggered or on push + if: >- + ( + contains(github.event.pull_request.labels.*.name, 'internet tests') || + contains(github.event.pull_request.labels.*.name, 'all tests') || + contains(github.event_name, 'schedule') || + contains(github.event_name, 'workflow_dispatch') || + contains(github.event_name, 'push') + ) + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -e {0} # -e to fail on error + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.12"] + + permissions: + id-token: write + + name: unit + + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel uv + python -m uv pip install --system "scvi-tools[tests] @ ." + + - name: Run pytest + env: + MPLBACKEND: agg + PLATFORM: ${{ matrix.os }} + DISPLAY: :42 + COLUMNS: 120 + run: | + coverage run -m pytest -v --color=yes --internet-tests + coverage report + + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test_linux_optional.yml b/.github/workflows/test_linux_optional.yml new file mode 100644 index 0000000000..4eac38adeb --- /dev/null +++ b/.github/workflows/test_linux_optional.yml @@ -0,0 +1,74 @@ +name: test (optional) + +on: + pull_request: + branches: [main, "[0-9]+.[0-9]+.x"] + types: [labeled, synchronize, opened] + schedule: + - cron: "0 10 * * *" # runs at 10:00 UTC (03:00 PST) every day + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + # if PR has label "optional tests" or "all tests" or if scheduled or manually triggered or on push + if: >- + ( + contains(github.event.pull_request.labels.*.name, 'optional tests') || + contains(github.event.pull_request.labels.*.name, 'all tests') || + contains(github.event_name, 'schedule') || + contains(github.event_name, 'workflow_dispatch') || + contains(github.event_name, 'push') + ) + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -e {0} # -e to fail on error + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python: ["3.12"] + + permissions: + id-token: write + + name: unit + + env: + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel uv + python -m uv pip install --system "scvi-tools[tests] @ ." + + - name: Run pytest + env: + MPLBACKEND: agg + PLATFORM: ${{ matrix.os }} + DISPLAY: :42 + COLUMNS: 120 + run: | + coverage run -m pytest -v --color=yes --optional + coverage report + + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/tests/conftest.py b/tests/conftest.py index f3d0ab5526..aac511cefc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,22 +56,31 @@ def pytest_collection_modifyitems(config, items): """Docstring for pytest_collection_modifyitems.""" run_internet = config.getoption("--internet-tests") skip_internet = pytest.mark.skip(reason="need --internet-tests option to run") + skip_non_internet = pytest.mark.skip(reason="test not having a pytest.mark.internet decorator") for item in items: # All tests marked with `pytest.mark.internet` get skipped unless # `--internet-tests` passed if not run_internet and ("internet" in item.keywords): item.add_marker(skip_internet) + # Skip all tests not marked with `pytest.mark.internet` if `--internet` passed + elif run_internet and ("internet" not in item.keywords): + item.add_marker(skip_non_internet) run_optional = config.getoption("--optional") skip_optional = pytest.mark.skip(reason="need --optional option to run") + skip_non_optional = pytest.mark.skip(reason="test not having a pytest.mark.optional decorator") for item in items: # All tests marked with `pytest.mark.optional` get skipped unless # `--optional` passed if not run_optional and ("optional" in item.keywords): item.add_marker(skip_optional) + # Skip all tests not marked with `pytest.mark.optional` if `--optional` passed + elif run_optional and ("optional" not in item.keywords): + item.add_marker(skip_non_optional) run_private = config.getoption("--private") skip_private = pytest.mark.skip(reason="need --private option to run") + skip_non_private = pytest.mark.skip(reason="test not having a pytest.mark.private decorator") for item in items: # All tests marked with `pytest.mark.private` get skipped unless # `--private` passed @@ -79,7 +88,7 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip_private) # Skip all tests not marked with `pytest.mark.private` if `--private` passed elif run_private and ("private" not in item.keywords): - item.add_marker(skip_private) + item.add_marker(skip_non_private) @pytest.fixture(scope="session") From 61047e06648b793daef415be61c18257b3f111a3 Mon Sep 17 00:00:00 2001 From: ori-kron-wis Date: Sun, 8 Dec 2024 14:00:10 +0200 Subject: [PATCH 2/4] fixes --- .github/workflows/test_linux_cuda.yml | 2 +- tests/data/test_built_in_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_linux_cuda.yml b/.github/workflows/test_linux_cuda.yml index 9de73d8044..22622ab3ce 100644 --- a/.github/workflows/test_linux_cuda.yml +++ b/.github/workflows/test_linux_cuda.yml @@ -34,7 +34,7 @@ jobs: container: image: ghcr.io/scverse/scvi-tools:py3.12-cu12-base #image: ghcr.io/scverse/scvi-tools:py3.12-cu12-${{ env.BRANCH_NAME }}-base - options: --user root --gpus all --no-cache + options: --user root --gpus all --pull always name: integration diff --git a/tests/data/test_built_in_data.py b/tests/data/test_built_in_data.py index 959eb117d9..44f8b7669c 100644 --- a/tests/data/test_built_in_data.py +++ b/tests/data/test_built_in_data.py @@ -91,7 +91,7 @@ def test_download_seurat_v4_pbmc(save_path: str): @pytest.mark.internet def test_download_cellxgene(save_path: str): - url = "https://cellxgene.cziscience.com/e/8d73847b-33e7-48f0-837f-e3e6579bf12d.cxg/" + url = "https://cellxgene.cziscience.com/e/de985818-285f-4f59-9dbd-d74968fddba3.cxg/" filename = "cellxgene.h5ad" scvi.data.cellxgene(url, save_path=save_path, filename=filename) anndata.read_h5ad(os.path.join(save_path, filename)) From 8e11174514cfce21d995925c4f740df5f6ba0730 Mon Sep 17 00:00:00 2001 From: ori-kron-wis Date: Sun, 8 Dec 2024 14:10:20 +0200 Subject: [PATCH 3/4] missing census in pyrpj --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f46c27aec6..744f4deac7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ loompy = ["loompy>=3.0.6"] scanpy = ["scanpy>=1.6"] optional = [ - "scvi-tools[autotune,aws,hub,loompy,pymde,regseq,scanpy]" + "scvi-tools[autotune,aws,census,hub,loompy,pymde,regseq,scanpy]" ] tutorials = [ "cell2location", From 76b1beef034f151d172539b40bf8f690492699c7 Mon Sep 17 00:00:00 2001 From: ori-kron-wis Date: Sun, 8 Dec 2024 14:22:18 +0200 Subject: [PATCH 4/4] tiledb needed for cellxgene_census --- .github/workflows/test_linux_internet.yml | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_linux_internet.yml b/.github/workflows/test_linux_internet.yml index a2cafd3d27..30ae866711 100644 --- a/.github/workflows/test_linux_internet.yml +++ b/.github/workflows/test_linux_internet.yml @@ -58,6 +58,9 @@ jobs: run: | python -m pip install --upgrade pip wheel uv python -m uv pip install --system "scvi-tools[tests] @ ." + python -m pip install tiledb + python -m pip install tiledbsoma + python -m pip install cellxgene-census - name: Run pytest env: diff --git a/pyproject.toml b/pyproject.toml index 744f4deac7..f46c27aec6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,7 +97,7 @@ loompy = ["loompy>=3.0.6"] scanpy = ["scanpy>=1.6"] optional = [ - "scvi-tools[autotune,aws,census,hub,loompy,pymde,regseq,scanpy]" + "scvi-tools[autotune,aws,hub,loompy,pymde,regseq,scanpy]" ] tutorials = [ "cell2location",