Merge pull request #432 from Der-Henning/dev

bump v1.18.0_rc3

.devcontainer/Dockerfile

@@ -1,8 +1,8 @@
 # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.6/containers/python-3/.devcontainer/base.Dockerfile
 
 # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
-ARG VARIANT="3.9"
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+ARG VARIANT="3.11"
+FROM mcr.microsoft.com/devcontainers/python:1-${VARIANT}
 
 # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
 ARG NODE_VERSION="none"
@@ -11,7 +11,7 @@ RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/
 # Poetry
 ARG POETRY_VERSION="none"
 ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-RUN if [ "${POETRY_VERSION}" != "none" ]; then su vscode -c "umask 0002 && pip3 install poetry==${POETRY_VERSION}"; fi
+RUN if [ "${POETRY_VERSION}" != "none" ]; then su vscode -c "umask 0002 && pip3 install poetry==${POETRY_VERSION} tox"; fi
 
 # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
 # COPY requirements.txt /tmp/pip-tmp/
@@ -20,8 +20,8 @@ RUN if [ "${POETRY_VERSION}" != "none" ]; then su vscode -c "umask 0002 && pip3
 #     && rm -rf /tmp/pip-tmp
 
 # [Optional] Uncomment this section to install additional OS packages.
-# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+RUN apt-get update && apt-get upgrade -y \
+    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y gcc python3-dev
 
 # [Optional] Uncomment this line to install global node packages.
 # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -9,10 +9,10 @@
 			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
 			// Append -bullseye or -buster to pin to an OS version.
 			// Use -bullseye variants on local on arm64/Apple Silicon.
-			"VARIANT": "3.10",
+			"VARIANT": "3.11",
 			// Options
 			// "NODE_VERSION": "lts/*",
-			"POETRY_VERSION": "1.5.1"
+			"POETRY_VERSION": "1.7.1"
 		}
 	},
 	// Set *default* container specific settings.json values on container create.
@@ -22,7 +22,7 @@
 				"python.defaultInterpreterPath": "./.venv/bin/python",
 				"python.terminal.activateEnvironment": true,
 				"python.terminal.activateEnvInCurrentTerminal": true,
-				"editor.rulers": [79],
+				"editor.rulers": [130],
 				"python.testing.pytestArgs": [
 					"tests"
 				],
@@ -32,6 +32,7 @@
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				"ms-python.python",
+				"ms-toolsai.jupyter",
 				"GitHub.vscode-pull-request-github"
 			]
 		}
@@ -41,7 +42,7 @@
 		8000
 	],
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "poetry install && poetry run pre-commit install --install-hooks",
+	"postCreateCommand": "make clean && make install",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
 	"features": {

.github/actions/setup-poetry-action/action.yml

@@ -0,0 +1,55 @@
+name: Poetry Setup Action
+
+inputs:
+  poetry-version:
+    required: false
+    type: string
+    default: latest
+  python-version:
+    required: false
+    type: string
+    default: 3.11
+  install-dependencies:
+    required: false
+    type: boolean
+    default: true
+  without:
+    required: false
+    type: string
+
+runs:
+  using: composite
+  steps:
+  - uses: actions/setup-python@v4
+    with:
+      python-version: ${{ inputs.python-version }}
+  - if: ${{ inputs.poetry-version != 'latest' }}
+    shell: bash
+    run: pip install --upgrade pip setuptools wheel poetry==${{ inputs.poetry-version }}
+  - if: ${{ inputs.poetry-version == 'latest' }}
+    shell: bash
+    run: pip install --upgrade pip setuptools wheel poetry
+  - name: Configure Poetry
+    shell: bash
+    run: |
+      poetry config virtualenvs.create true
+      poetry config virtualenvs.in-project true
+      poetry config installer.max-workers 1
+  - uses: actions/cache@v3
+    if: inputs.without != '' && inputs.install-dependencies
+    with:
+      path: ./.venv
+      key: venv-without-${{ inputs.without }}-poetry-${{ inputs.poetry-version }}-python-${{ inputs.python-version }}-${{ hashFiles('poetry.lock') }}-${{ runner.os }}
+  - uses: actions/cache@v3
+    if: inputs.without == '' && inputs.install-dependencies
+    with:
+      path: ./.venv
+      key: venv-poetry-${{ inputs.poetry-version }}-python-${{ inputs.python-version }}-${{ hashFiles('poetry.lock') }}-${{ runner.os }}
+  - name: Install dependencies
+    if: inputs.without != '' && inputs.install-dependencies
+    shell: bash
+    run: poetry install --no-interaction --without ${{ inputs.without }}
+  - name: Install dependencies
+    if: inputs.without == '' && inputs.install-dependencies
+    shell: bash
+    run: poetry install --no-interaction

.github/workflows/release.yml

@@ -9,8 +9,8 @@ on:
   pull_request:
 
 env:
-  PYTHON_VERSION: '3.10'
-  POETRY_VERSION: 1.5.1
+  PYTHON_VERSION: '3.11'
+  POETRY_VERSION: 1.7.1
 
 jobs:
   docker-images:
@@ -33,16 +33,6 @@ jobs:
           context: ./
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-    - name: Setup Poetry
-      run: |
-        pip install --upgrade pip setuptools wheel poetry==${{ env.POETRY_VERSION }}
-        poetry config virtualenvs.create true --local
-        poetry config virtualenvs.in-project true --local
-    - name: Export requirements.txt
-      run: poetry export -f requirements.txt --output requirements.txt
     - uses: docker/metadata-action@v4
       id: meta
       with:
@@ -93,20 +83,11 @@ jobs:
           tag: macos
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: ./.github/actions/setup-poetry-action
       with:
+        poetry-version: ${{ env.POETRY_VERSION }}
         python-version: ${{ env.PYTHON_VERSION }}
-    - name: Setup Poetry
-      run: |
-        pip install --upgrade pip setuptools wheel poetry==${{ env.POETRY_VERSION }}
-        poetry config virtualenvs.create true --local
-        poetry config virtualenvs.in-project true --local
-    - uses: actions/cache@v3
-      with:
-        path: ./.venv
-        key: venv-build-poetry-${{ env.POETRY_VERSION }}-python-${{ env.PYTHON_VERSION }}-${{ hashFiles('poetry.lock') }}-${{ runner.os }}
-    - name: Install dependencies
-      run: poetry install --without test
+        without: test
     - name: Run PyInstaller
       run: |
         poetry run pyinstaller scanner.spec
@@ -127,7 +108,7 @@ jobs:
     - name: Upload archive
       uses: actions/upload-artifact@v3
       with:
-        name: releases
+        name: releases-${{github.sha}}
         path: ./${{ steps.filename.outputs.FILENAME }}
     - name: Add archive to release
       uses: softprops/action-gh-release@v1

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
 
 env:
-  POETRY_VERSION: 1.5.1
+  POETRY_VERSION: 1.7.1
 
 jobs:
   tests:
@@ -14,30 +14,20 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ['3.9', '3.10', '3.11']
+        python: ['3.9', '3.10', '3.11', '3.12']
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: ./.github/actions/setup-poetry-action
       with:
+        poetry-version: ${{ env.POETRY_VERSION }}
         python-version: ${{ matrix.python }}
-    - name: Setup Poetry
-      run: |
-        pip install --upgrade pip setuptools wheel poetry==${{ env.POETRY_VERSION }}
-        poetry config virtualenvs.create true --local
-        poetry config virtualenvs.in-project true --local
-        poetry config installer.max-workers 1 --local
-    - uses: actions/cache@v3
-      with:
-        path: ./.venv
-        key: venv-test-poetry-${{ env.POETRY_VERSION }}-python-${{ matrix.python }}-${{ hashFiles('poetry.lock') }}-${{ runner.os }}
+        without: build
     - uses: actions/cache@v3
       with:
         path: ~/.cache/pre-commit
         key: pre-commit-python-${{ matrix.python }}-${{ hashFiles('poetry.lock', '.pre-commit-config.yaml') }}-${{ runner.os }}
-    - name: Install dependencies
-      run: poetry install --no-interaction --without build
     - name: Run linting
       run: poetry run pre-commit run -a
     - name: Run tests
-      run: poetry run pytest -v -m "not tgtg_api" --cov --cov-report=xml
+      run: poetry run pytest -v -m "not tgtg_api" --cov=tgtg_scanner --cov-report=xml
     - uses: codecov/codecov-action@v3

.github/workflows/tgtg.yml

@@ -5,29 +5,20 @@ on:
   - cron: 0 1 * * *
 
 env:
-  PYTHON_VERSION: '3.10'
-  POETRY_VERSION: 1.5.1
+  PYTHON_VERSION: '3.11'
+  POETRY_VERSION: 1.7.1
 
 jobs:
   test:
     name: Run Test
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: ./.github/actions/setup-poetry-action
       with:
+        poetry-version: ${{ env.POETRY_VERSION }}
         python-version: ${{ env.PYTHON_VERSION }}
-    - name: Setup Poetry
-      run: |
-        pip install --upgrade pip setuptools wheel poetry==${{ env.POETRY_VERSION }}
-        poetry config virtualenvs.create true --local
-        poetry config virtualenvs.in-project true --local
-    - uses: actions/cache@v3
-      with:
-        path: ./.venv
-        key: venv-test-poetry-${{ env.POETRY_VERSION }}-python-${{ env.PYTHON_VERSION }}-${{ hashFiles('poetry.lock') }}-${{ runner.os }}
-    - name: Install dependencies
-      run: poetry install --without build
+        without: build
     - name: Run tests
       uses: nick-fields/retry@v2
       with:

.gitignore

@@ -9,7 +9,8 @@ venv
 .coverage
 coverage.xml
 .pytest_cache
-requirements.txt
+.mypy_cache
+.tox
 
 .env
 config.ini

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 exclude: (build|dist)
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: trailing-whitespace
   - id: check-added-large-files
@@ -12,43 +12,52 @@ repos:
   - id: check-toml
   - id: check-added-large-files
 
-- repo: https://github.com/pre-commit/mirrors-autopep8
-  rev: v2.0.2
+- repo: https://github.com/ambv/black
+  rev: 23.12.1
   hooks:
-  - id: autopep8
-    args:
-    - --in-place
-    - --max-line-length=79
+  - id: black
+    args: [--line-length, '130', --target-version, py38]
 
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
+    args: [--profile, black]
 
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.0.0
+  rev: 6.1.0
   hooks:
   - id: flake8
-    args:
-    - --max-line-length=79
+    args: [--max-line-length, '130']
 
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.9.0
+  rev: v2.12.0
   hooks:
   - id: pretty-format-toml
     args: [--autofix]
     exclude: poetry.lock
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2']
+  - id: pretty-format-ini
+    args: [--autofix]
 
 - repo: https://github.com/igorshubovych/markdownlint-cli
-  rev: v0.35.0
+  rev: v0.38.0
   hooks:
   - id: markdownlint
     args: [--fix]
 
 - repo: https://github.com/python-poetry/poetry
-  rev: 1.5.1    # add version here
+  rev: 1.7.0
   hooks:
   - id: poetry-check
   - id: poetry-lock
+    args: [--no-update]
+  - id: poetry-export
+    args: [-f, requirements.txt, -o, requirements.txt]
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.8.0
+  hooks:
+  - id: mypy
+    additional_dependencies: [types-requests]

Makefile

@@ -1,13 +1,27 @@
-images:
-	poetry export -f requirements.txt --output requirements.txt
-	docker build -f ./docker/Dockerfile -t tgtg-scanner:latest .
-	docker build -f ./docker/Dockerfile.alpine -t tgtg-scanner:latest-alpine .
+clean:
+	rm -rf .venv
+	rm -rf .pytest_cache
+	rm -rf .tox
+	rm -rf .mypy_cache
 
 install:
 	poetry install
+	poetry run pre-commit install --install-hooks
+
+server:
+	poetry run tgtg_server
 
 start:
-	poetry run scanner -d
+	poetry run scanner -d --base_url http://localhost:8080
+
+test:
+	poetry run pytest -v -m "not tgtg_api" --cov=tgtg_scanner
+
+lint:
+	poetry run pre-commit run -a
+
+tox:
+	tox
 
 executable:
 	rm -r ./build ||:
@@ -16,8 +30,6 @@ executable:
 	cp ./config.sample.ini ./dist/config.ini
 	zip -j ./dist/scanner.zip ./dist/*
 
-test:
-	poetry run pytest -v -m "not tgtg_api" --cov
-
-lint:
-	poetry run pre-commit run -a
+images:
+	docker build -f ./docker/Dockerfile -t tgtg-scanner:latest .
+	docker build -f ./docker/Dockerfile.alpine -t tgtg-scanner:latest-alpine .