From 4ab2ae9bf26eadeea1760eab84e6e170f889c7ed Mon Sep 17 00:00:00 2001 From: Titusz Pan Date: Sat, 19 Oct 2024 20:24:07 +0200 Subject: [PATCH] Refactor pure vs cython builds --- .github/workflows/tests.yml | 26 +++++++----- build.py | 80 +++++++++++-------------------------- iscc_core/__init__.py | 13 ++++++ iscc_core/check.py | 13 +----- tests/test_check.py | 7 +--- 5 files changed, 56 insertions(+), 83 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 932570d..b6cc4ef 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,7 @@ jobs: matrix: python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] os: [ubuntu-20.04, macos-12, windows-2019] + cython: [false, true] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -20,27 +21,29 @@ jobs: with: poetry-version: 1.8.4 - - name: Install Dependencies + - name: Install Dependencies (Pure Python) + if: ${{ !matrix.cython }} run: poetry install --without dev,docs - - name: Run Tests (Pure Python with coverage) - run: poetry run pytest --cov=iscc_core --cov-report=xml -q tests + - name: Install Dependencies (with Cython) + if: ${{ matrix.cython }} + run: poetry install --without dev,docs --extras cython - - name: Build Extension modules - run: poetry install --without dev,docs --extras turbo - - - name: Run Tests (With Extension Modules) - run: poetry run pytest -q tests --turbo + - name: Run Tests + run: | + poetry run python -c "import iscc_core; print(f'Using Cython: {iscc_core.USING_CYTHON}')" + poetry run pytest -v tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.7' + if: matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.9' && !matrix.cython - name: Build Wheel + if: ${{ matrix.cython }} run: poetry build -f wheel - name: Test Wheel Installation - if: runner.os != 'Windows' + if: ${{ matrix.cython && runner.os != 'Windows' }} run: | python -m venv venv source venv/bin/activate @@ -49,7 +52,7 @@ jobs: python -c "import iscc_core; print(iscc_core.turbo())" - name: Test Wheel Installation on Windows - if: runner.os == 'Windows' + if: ${{ matrix.cython && runner.os == 'Windows' }} run: | python -m venv venv venv\Scripts\Activate.ps1 @@ -59,6 +62,7 @@ jobs: python -c "import iscc_core; print(iscc_core.turbo())" - name: Collect Wheel + if: ${{ matrix.cython }} uses: actions/upload-artifact@v3 with: path: dist/*.whl diff --git a/build.py b/build.py index 797fc1f..5840639 100644 --- a/build.py +++ b/build.py @@ -1,64 +1,32 @@ # -*- coding: utf-8 -*- -""" -Build cython extension modules. +import os +from setuptools import Extension -The shared library can also be built manually using the command: -$ cythonize -X language_level=3 -a -i ./iscc_core/cdc.py -$ cythonize -X language_level=3 -a -i ./iscc_core/minhash.py -$ cythonize -X language_level=3 -a -i ./iscc_core/simhash.py -$ cythonize -X language_level=3 -a -i ./iscc_core/dct.py -$ cythonize -X language_level=3 -a -i ./iscc_core/wtahash.py -""" -try: - from Cython.Build import cythonize, build_ext -except ImportError: - # dummy build function for poetry - def build(setup_kwargs): - pass -else: +def build(setup_kwargs): + try: + from Cython.Build import cythonize - class build_ext_gracefull(build_ext): - def run(self): - try: - print("Trying to compile C accelerator modules") - super().run() - print("Successfully comiled C accelerator modules") - except Exception as e: - print(e) - print("********************************************************************") - print("Failed to compile C accelerator module, falling back to pure python.") - print("********************************************************************") + use_cython = True + except ImportError: + use_cython = False - def build_extensions(self): - try: - print("Trying to compile C accelerator modules") - super().build_extensions() - print("Successfully comiled C accelerator modules") - except Exception as e: - print(e) - print("********************************************************************") - print("Failed to compile C accelerator module, falling back to pure python.") - print("********************************************************************") - - def build(setup_kwargs): + if use_cython: try: - setup_kwargs.update( - dict( - ext_modules=cythonize( - [ - "iscc_core/cdc.py", - "iscc_core/minhash.py", - "iscc_core/simhash.py", - "iscc_core/dct.py", - "iscc_core/wtahash.py", - ] - ), - cmdclass=dict(build_ext=build_ext_gracefull), - ) + ext_modules = cythonize( + [ + Extension("iscc_core.cdc", ["iscc_core/cdc.py"]), + Extension("iscc_core.minhash", ["iscc_core/minhash.py"]), + Extension("iscc_core.simhash", ["iscc_core/simhash.py"]), + Extension("iscc_core.dct", ["iscc_core/dct.py"]), + Extension("iscc_core.wtahash", ["iscc_core/wtahash.py"]), + ], + compiler_directives={"language_level": "3"}, ) + setup_kwargs.update({"ext_modules": ext_modules}) + print("Cython modules prepared for compilation") except Exception as e: - print(e) - print("********************************************************************") - print("Failed to compile C accelerator module, falling back to pure python.") - print("********************************************************************") + print(f"Failed to prepare Cython modules: {e}") + print("Falling back to pure Python") + else: + print("Cython not available, using pure Python") diff --git a/iscc_core/__init__.py b/iscc_core/__init__.py index 081cfe3..14f3e04 100644 --- a/iscc_core/__init__.py +++ b/iscc_core/__init__.py @@ -1,3 +1,16 @@ +import sys + + +def _using_cython_modules(): + modules = ["cdc", "minhash", "simhash", "dct", "wtahash"] + return any( + getattr(sys.modules.get(f"iscc_core.{module}"), "__file__", "").endswith((".so", ".pyd")) + for module in modules + ) + + +USING_CYTHON = _using_cython_modules() + __version__ = "1.1.0" from iscc_core.options import core_opts, conformant_options diff --git a/iscc_core/check.py b/iscc_core/check.py index 7290028..bc2535e 100644 --- a/iscc_core/check.py +++ b/iscc_core/check.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Inspect lib environment/installation""" -import inspect -from loguru import logger as log +from iscc_core import USING_CYTHON __all__ = ["turbo"] @@ -10,12 +9,4 @@ def turbo(): # pragma: no cover # type: () -> bool """Check whether all optional cython extensions have been compiled to native modules.""" - from iscc_core import cdc, minhash, simhash, dct, wtahash - - modules = (cdc, minhash, simhash, dct, wtahash) - for module in modules: - module_file = inspect.getfile(module) - log.debug(f"Module {module.__name__} file: {module_file}") - if module_file.endswith(".py") or module_file.endswith(".pyc"): - return False - return True + return USING_CYTHON diff --git a/tests/test_check.py b/tests/test_check.py index 902c158..4e20da0 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -2,8 +2,5 @@ import iscc_core as ic -def test_check_turbo(turbo): - if turbo is False: - assert ic.turbo() is False - else: - assert ic.turbo() is True +def test_check_turbo(): + assert ic.USING_CYTHON == ic.turbo()