ci: Introduce llvm-lit for Linux (#1243)

This PR introduces llvm-lit, which at some point will replace our current correctness tests. For now only the functionality was introduced as well as GitHub workflows executing example lit tests.

.github/workflows/lit.yml

@@ -0,0 +1,35 @@
+name: Build
+
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+  pull_request:
+    branches: [ master ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  lit-linux-debug:
+    name: lit tests (Linux, debug build)
+    runs-on: ubuntu-latest
+    container: ghcr.io/plc-lang/rust-llvm:latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run `build.sh --lit`
+        shell: bash
+        run: |
+          ./scripts/build.sh --lit
+
+  lit-linux-release:
+    name: lit tests (Linux, release build)
+    runs-on: ubuntu-latest
+    container: ghcr.io/plc-lang/rust-llvm:latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run `build.sh --lit --release`
+        shell: bash
+        run: |
+          ./scripts/build.sh --lit --release

.gitignore

@@ -11,3 +11,7 @@
 *.a
 *.elf
 *.out
+
+# Garbage generated by llvm-lit
+tests/lit/**/*.txt
+tests/lit/**/Output/

scripts/build.sh

@@ -8,6 +8,7 @@ check_style=0
 build=0
 doc=0
 test=0
+lit=0
 coverage=0
 release=0
 debug=0
@@ -140,6 +141,19 @@ function run_check_style() {
     cargo fmt -- --check
 }
 
+function run_lit_test() {
+    # We need a binary as well as the stdlib and its *.so file before running lit tests
+    run_build
+    run_std_build
+    run_package_std
+
+    if [[ $release -eq 0 ]]; then
+        lit -v -DLIB=$project_location/output -DCOMPILER=$project_location/target/debug/plc tests/lit/
+    else
+        lit -v -DLIB=$project_location/output -DCOMPILER=$project_location/target/release/plc tests/lit/
+    fi
+}
+
 function run_test() {
     CARGO_OPTIONS=$(set_cargo_options)
     log "Running cargo test"
@@ -191,6 +205,8 @@ function set_offline() {
 
 function run_package_std() {
     cc=$(get_compiler)
+    OUTPUT_DIR=$project_location/output
+    make_dir "$OUTPUT_DIR"
     log "Packaging Standard functions"
     log "Removing previous output folder"
     rm -rf $OUTPUT_DIR
@@ -291,6 +307,9 @@ function run_in_container() {
     if [[ $test -ne 0 ]]; then
         params="$params --test"
     fi
+    if [[ $lit -ne 0 ]]; then
+        params="$params --lit"
+    fi
     if [[ $junit -ne 0 ]]; then
         params="$params --junit"
     fi
@@ -330,7 +349,7 @@ function run_in_container() {
 set -o errexit -o pipefail -o noclobber -o nounset
 
 OPTIONS=sorbvc
-LONGOPTS=sources,offline,release,check,check-style,build,doc,test,junit,verbose,container,linux,container-name:,coverage,package,target:
+LONGOPTS=sources,offline,release,check,check-style,build,doc,lit,test,junit,verbose,container,linux,container-name:,coverage,package,target:
 
 check_env
 # -activate quoting/enhanced mode (e.g. by writing out “--options”)
@@ -382,6 +401,9 @@ while true; do
         --test)
             test=1
             ;;
+        --lit)
+            lit=1
+            ;;
         --junit)
             junit=1
             ;;
@@ -417,11 +439,6 @@ if [[ $container -ne 0 ]]; then
     exit 0
 fi
 
-if [[ $package -ne 0 ]]; then
-    OUTPUT_DIR=$project_location/output
-    make_dir "$OUTPUT_DIR"
-fi
-
 if [[ $vendor -ne 0 ]]; then
     generate_sources
     exit 0
@@ -457,6 +474,10 @@ if [[ $test -ne 0 ]]; then
     run_test
 fi
 
+if [[ $lit -ne 0 ]]; then
+    run_lit_test
+fi
+
 if [[ $doc -ne 0 ]]; then
     run_doc
 fi

tests/lit/lit.cfg

@@ -0,0 +1,20 @@
+import lit.formats, os, os.path, sys, subprocess
+srcDir = os.path.dirname(__file__)
+config.test_format = lit.formats.ShTest(True)
+config.pipefail = False
+rustyRootDirectory = subprocess.check_output("dirname `cargo locate-project --message-format plain`", shell=True).decode("utf-8").strip()
+
+stdlibLocation = lit_config.params["LIB"]
+compilerLocation = lit_config.params["COMPILER"]
+
+# ...to make the compile command more reable we build it over multiple lines
+compile = f'{compilerLocation}'
+compile = f'{compile} -o /tmp/%basename_t.out'
+compile = f'{compile} -liec61131std -L{stdlibLocation}/lib -i "{stdlibLocation}/include/*.st"'
+compile = f'{compile} -i {rustyRootDirectory}/tests/lit/util/*.pli'
+compile = f'{compile} --linker=cc'
+print(f'Compile command: {compile}')
+
+config.substitutions.append(('%COMPILE', f'{compile}'))
+config.substitutions.append(('%RUN', f'LD_LIBRARY_PATH="{stdlibLocation}/lib" /tmp/%basename_t.out'))
+config.substitutions.append(('%CHECK', f'FileCheck-14 --check-prefixes CHECK --allow-unused-prefixes --match-full-lines'))

tests/lit/multi/enum__month_to_int/month_to_int.st

@@ -0,0 +1,26 @@
+FUNCTION main : DINT
+    printf('%d', month_to_int(MONTH#June));
+END_FUNCTION
+
+
+FUNCTION month_to_int : INT
+    VAR_INPUT
+        month_var : MONTH;
+    END_VAR
+
+    CASE month_var OF
+        MONTH.January:      month_to_int := 1;
+        MONTH.February:     month_to_int := 2;
+        MONTH.March:        month_to_int := 3;
+        MONTH.April:        month_to_int := 4;
+        MONTH.May:          month_to_int := 5;
+        MONTH.June:         month_to_int := 6;
+        MONTH.July:         month_to_int := 7;
+        MONTH.August:       month_to_int := 8;
+        MONTH.September:    month_to_int := 9;
+        MONTH.October:      month_to_int := 10;
+        MONTH.November:     month_to_int := 11;
+        MONTH.December:     month_to_int := 12;
+        else                month_to_int := 0;
+    END_CASE;
+END_FUNCTION

tests/lit/multi/enum__month_to_int/month_to_int.test

@@ -0,0 +1,2 @@
+RUN: %COMPILE %S/month_to_int.st %S/months.dt && %RUN | %CHECK %s
+CHECK:6

tests/lit/multi/enum__month_to_int/months.dt

@@ -0,0 +1,3 @@
+TYPE
+    MONTH: (January, February, March, April, May, June, July, August, September, October, November, December);
+END_TYPE

tests/lit/multi/lit.local.cfg

@@ -0,0 +1 @@
+config.suffixes = ['.test']

tests/lit/single/arithmetic/addition.st

@@ -0,0 +1,12 @@
+// RUN: (%COMPILE %s && %RUN) | %CHECK %s
+// CHECK: 10
+FUNCTION main
+    VAR
+        x : DINT;
+        y : DINT;
+    END_VAR
+
+    x := 5;
+    y := 5;
+    printf('%d$N', x + y);
+END_FUNCTION

tests/lit/single/arithmetic/sqrt.st

@@ -0,0 +1,8 @@
+// RUN: (%COMPILE %s && %RUN) | %CHECK %s
+// CHECK: 2
+FUNCTION main
+    VAR
+        res : LREAL;
+    END_VAR
+    printf('%d$N', REAL_TO_DINT(SQRT(4.0)));
+END_FUNCTION

tests/lit/single/lit.local.cfg

@@ -0,0 +1,2 @@
+# Declare any *.st file as a standlone test file
+config.suffixes = ['.st']

tests/lit/util/printf.pli

@@ -0,0 +1,9 @@
+{external}
+FUNCTION printf: DINT
+VAR_INPUT {ref}
+	str : STRING;
+END_VAR
+VAR_INPUT
+	args: ...;
+END_VAR
+END_FUNCTION