feat: argo refactored and working

* feat: a working refactor of argo

* fix: still working through dag tasks

* feat: composite steps added

* feat: Fully working deep nested argo workflows

Dockerfile

@@ -17,6 +17,6 @@ COPY . /app
 WORKDIR /app
 
 RUN uv python install && \
-    uv sync --index https://artifactory.astrazeneca.net/api/pypi/pypi-virtual/simple/ --frozen --all-extras
+    uv sync --index https://artifactory.astrazeneca.net/api/pypi/pypi-virtual/simple/  --all-extras
 
 ENV PATH="/app/.venv/bin:$PATH"

examples/11-jobs/scripts.py

@@ -0,0 +1,36 @@
+"""
+You can execute this pipeline by:
+
+    python examples/01-tasks/scripts.py
+
+The command can be anything that can be executed in a shell.
+The stdout/stderr of the execution is captured as execution log and stored in the catalog.
+
+For example:
+
+.catalog
+└── seasoned-perlman-1355
+    └── hello.execution.log
+
+"""
+
+from runnable import Job, ShellTask
+
+
+def main():
+    # If this step executes successfully, the pipeline will terminate with success
+    hello_task = ShellTask(
+        name="hello",
+        command="echo 'Hello World!'",
+        terminate_with_success=True,
+    )
+
+    job = Job(name="hello", task=hello_task)
+
+    job.execute()
+
+    return job
+
+
+if __name__ == "__main__":
+    main()

examples/11-jobs/scripts.yaml

@@ -1,20 +1,2 @@
-dag:
-  description: |
-    This is a sample pipeline with one step that
-    executes a shell command.
-
-    You can run this pipeline by:
-    runnable execute -f examples/01-tasks/scripts.yaml
-
-    For example:
-    .catalog
-    └── seasoned-perlman-1355
-        └── hello.execution.log
-
-  start_at: shell
-  steps:
-    shell:
-      type: task
-      command_type: shell
-      command: echo "hello world!!" # The path is relative to the root of the project.
-      next: success
+type: shell
+command: echo "hello world!!" # The path is relative to the root of the project.

examples/configs/argo-config-az.yaml

@@ -0,0 +1,18 @@
+pipeline-executor:
+  type: "argo" # (1)
+  config:
+    pvc_for_runnable: runnable
+    argoWorkflow:
+      metadata:
+        generateName: "argo-" # (2)
+        namespace: enterprise-mlops
+      spec:
+        serviceAccountName: "default-editor"
+        templateDefaults:
+          image: harbor.csis.astrazeneca.net/mlops/runnable:latest # (2)
+
+
+run-log-store: # (4)
+  type: chunked-fs
+  # config:
+  #   log_folder: /mnt/run_log_store

examples/configs/argo-config.yaml

@@ -1,18 +1,18 @@
 pipeline-executor:
   type: "argo" # (1)
   config:
-    image: harbor.csis.astrazeneca.net/mlops/runnable:latest # (2)
-    service_account_name: default-editor
-    persistent_volumes: # (3)
-      - name: magnus-volume
-        mount_path: /mnt
+    pvc_for_runnable: runnable
+    argoWorkflow:
+      metadata:
+        generateName: "argo-" # (2)
+        namespace: enterprise-mlops
+      spec:
+        serviceAccountName: "default-editor"
+        templateDefaults:
+          image: harbor.csis.astrazeneca.net/mlops/runnable:latest # (2)
+
 
 run-log-store: # (4)
   type: chunked-fs
   # config:
   #   log_folder: /mnt/run_log_store
-
-catalog:
-  type: file-system
-  # config:
-  #  catalog_location: /mnt/catalog

examples/configs/default.yaml

@@ -2,10 +2,10 @@ pipeline-executor:
   type: local # (1)
 
 run-log-store:
-  type: buffered # (2)
+  type: file-system
 
 catalog:
-  type: do-nothing # (3)
+  type: file-system # (3)
 
 secrets:
-  type: do-nothing # (4)
+  type: env-secrets # (4)