CI: test more cln versions

.ci/test.py

@@ -256,17 +256,19 @@ def push_gather_data(data: dict, workflow: str, python_version: str):
             ]
         ).decode("utf-8")
         print(f"output from git commit: {output}")
-        for _ in range(5):
-            subprocess.run(["git", "fetch"])
-            subprocess.run(["git", "checkout", "badges"])
-            output = subprocess.run(["git", "push", "origin", "badges"])
+        for _ in range(10):
+            subprocess.run(["git", "pull", "--rebase"])
+            output = subprocess.run(["git", "push", "origin", "badges"],
+                capture_output=True,
+                text=True)
             if output.returncode == 0:
                 print("Push successful")
                 break
             else:
                 print(
                     f"Push failed with return code {output.returncode}, retrying in 2 seconds..."
                 )
+                print(f"Push failure message: {output.stderr}")
                 time.sleep(2)
     print("Done.")
 

.ci/update_badges.py

@@ -1,15 +1,18 @@
 import json
 import os
 import subprocess
+import time
 from pathlib import Path
 
 from utils import configure_git, enumerate_plugins
 
 
-def update_and_commit_badge(plugin_name: str, passed: bool, workflow: str) -> bool:
-    json_data = {"schemaVersion": 1, "label": "", "message": " ✔ ", "color": "green"}
+def update_and_commit_badge(plugin_name: str, passed: bool, workflow: str, has_tests: bool) -> bool:
+    json_data = {"schemaVersion": 1, "label": "", "message": "✔", "color": "green"}
     if not passed:
         json_data.update({"message": "✗", "color": "red"})
+    if not has_tests:
+        json_data.update({"message": "?", "color": "orange"})
 
     filename = os.path.join(".badges", f"{plugin_name}_{workflow}.json")
     with open(filename, "w") as file:
@@ -57,12 +60,28 @@ def push_badges_data(workflow: str, num_of_python_versions: int):
             if (
                 len(set(results)) == 1
                 and results[0] == "passed"
-                # and len(results) == num_of_python_versions  # TODO: Disabled as gather data for python versions is missing sporadingly.
+                and len(results) == num_of_python_versions
             ):
                 passed = True
-            any_changes |= update_and_commit_badge(plugin.name, passed, workflow)
+            any_changes |= update_and_commit_badge(plugin.name, passed, workflow, True)
+        else:
+            any_changes |= update_and_commit_badge(plugin.name, False, workflow, False)
 
     if any_changes:
+        for _ in range(10):
+            subprocess.run(["git", "pull", "--rebase"])
+            output = subprocess.run(["git", "push", "origin", "badges"],
+                capture_output=True,
+                text=True)
+            if output.returncode == 0:
+                print("Push successful")
+                break
+            else:
+                print(
+                    f"Push failed with return code {output.returncode}, retrying in 2 seconds..."
+                )
+                print(f"Push failure message: {output.stderr}")
+                time.sleep(2)
         subprocess.run(["git", "push", "origin", "badges"])
     print("Done.")
 

.ci/utils.py

@@ -32,12 +32,13 @@ def configure_git():
 
 
 def get_testfiles(p: Plugin) -> List[PosixPath]:
-    return [
-        x
-        for x in p.path.iterdir()
-        if (x.is_dir() and x.name == "tests")
-        or (x.name.startswith("test_") and x.name.endswith(".py"))
-    ]
+    test_files = []
+    for x in p.path.iterdir():
+        if x.is_dir() and x.name == "tests":
+            test_files.extend([y for y in x.iterdir() if y.is_file() and y.name.startswith("test_") and y.name.endswith(".py")])
+        elif x.is_file() and x.name.startswith("test_") and x.name.endswith(".py"):
+            test_files.append(x)
+    return test_files
 
 
 def has_testfiles(p: Plugin) -> bool:

.github/workflows/main.yml

@@ -12,15 +12,15 @@ on:
 
 jobs:
   build-and-test:
-    name: Test PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
+    name: Test CLN=${{ matrix.cln-version }}, PY=${{ matrix.python-version }}, BCD=${{ matrix.bitcoind-version }}, EXP=${{ matrix.experimental }}, DEP=${{ matrix.deprecated }}
     runs-on: ubuntu-latest
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.8", "3.12"]
         bitcoind-version: ["26.0"]
-        cln-version: ["24.05"]
+        cln-version: ["24.05", "24.02.2", "23.11.2"]
         experimental: [1]
         deprecated: [0]
 
@@ -210,9 +210,9 @@ jobs:
 
         if [[ -z "$plugin_dirs" ]]; then
             # Test all plugins if no specific plugins were changed
-            python3 .ci/test.py main ${{ matrix.python-version }} $update_badges
+            python3 .ci/test.py ${{ matrix.cln-version }} ${{ matrix.python-version }} $update_badges
         else
-            python3 .ci/test.py main ${{ matrix.python-version }} $update_badges $(echo "$plugin_dirs")
+            python3 .ci/test.py ${{ matrix.cln-version }} ${{ matrix.python-version }} $update_badges $(echo "$plugin_dirs")
         fi
 
   gather:
@@ -222,6 +222,10 @@ jobs:
     if: ${{ always() && github.event_name == 'push' }}
     needs:
       - build-and-test
+    strategy:
+      fail-fast: false
+      matrix:
+        cln-version: ["24.05", "24.02.2", "23.11.2"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -232,6 +236,6 @@ jobs:
           python-version: "3.12"
       - name: Complete
         run: |
-          echo "Updating badges data for main workflow..."
-          python3 .ci/update_badges.py main 5  # We test for 5 distinct Python versions
+          echo "Updating badges data for ${{ matrix.cln-version }} workflow..."
+          python3 .ci/update_badges.py ${{ matrix.cln-version }} 2  # Set the distinct number of Python versions we test for
           echo "CI completed."

.gitmodules

@@ -49,3 +49,21 @@
 [submodule "clnrod"]
 	path = clnrod
 	url = https://github.com/daywalker90/clnrod.git
+[submodule "c-lightning-events"]
+	path = c-lightning-events
+	url = https://github.com/rbndg/c-lightning-events
+[submodule "Lightning-Invoice-Queue"]
+	path = Lightning-Invoice-Queue
+	url = https://github.com/rbndg/Lightning-Invoice-Queue
+[submodule "listmempoolfunds"]
+	path = listmempoolfunds
+	url = https://github.com/andrewtoth/listmempoolfunds
+[submodule "NLoop"]
+	path = NLoop
+	url = https://github.com/bitbankinc/NLoop
+[submodule "paythrough"]
+	path = paythrough
+	url = https://github.com/andrewtoth/paythrough
+[submodule "c-lightning-pruning-plugin"]
+	path = c-lightning-pruning-plugin
+	url = https://github.com/Start9Labs/c-lightning-pruning-plugin