add new output OTLP Logs + improve the OTLP Traces/Metrics

Signed-off-by: Thomas Labarussias <issif_github@gadz.org>
falcosecurity · Jan 29, 2025 · 2fa7117 · 2fa7117
1 parent 58ec197
commit 2fa7117
Show file tree

Hide file tree

Showing 40 changed files with 579 additions and 265 deletions.
diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml
@@ -13,7 +13,7 @@ jobs:
 
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           check-latest: true
           cache: true
 

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -17,11 +17,11 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           cache: false
           check-latest: true
       - name: golangci-lint
         uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0
         with:
-          version: v1.59
+          version: v1.61
           args: --timeout=5m
diff --git a/.github/workflows/push-main.yml b/.github/workflows/push-main.yml
@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           check-latest: true
           cache: true
 

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           check-latest: true
 
       - name: Set up QEMU

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
         with:
-          go-version: '1.22'
+          go-version: '1.23'
           check-latest: true
           cache: true
       - name: Run Go tests

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -10,7 +10,7 @@ env:
   - COSIGN_YES=true
 
 snapshot:
-  name_template: 'latest'
+  version_template: 'latest'
 
 checksum:
   name_template: 'checksums.txt'

diff --git a/config.go b/config.go
@@ -19,7 +19,7 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/falcosecurity/falcosidekick/internal/pkg/utils"
-	"github.com/falcosecurity/falcosidekick/outputs/otlpmetrics"
+	otlpmetrics "github.com/falcosecurity/falcosidekick/outputs/otlp_metrics"
 	"github.com/falcosecurity/falcosidekick/types"
 )
 

diff --git a/config_example.yaml b/config_example.yaml
@@ -540,7 +540,7 @@ sumologic:
 
 otlp:
   traces:
-    # endpoint: "" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/traces
+    # endpoint: "" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/traces, if not empty, OTLP Traces output is enabled
     # protocol: "" # OTLP protocol http/json, http/protobuf, grpc (default: "" which uses SDK default: http/json)
     # timeout: "" # OTLP timeout: timeout value in milliseconds (default: "" which uses SDK default: 10000)
     # headers: "" # OTLP headers: list of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value" (default: "")
@@ -551,9 +551,18 @@ otlp:
       # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
     # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
     # checkcert: true # Set if you want to skip TLS certificate validation (default: true)
-
+  logs:
+    # endpoint: "" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/logs, if not empty, OTLP Traces output is enabled
+    # protocol: "" # OTLP protocol http/json, http/protobuf, grpc (default: "" which uses SDK default: http/json)
+    # timeout: "" # OTLP timeout: timeout value in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # OTLP headers: list of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # extraenvvars: # Extra env vars (override the other settings)
+      # OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+    # checkcert: true # Set if you want to skip TLS certificate validation (default: true)
   metrics:
-    # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics
+    # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics, if not empty, OTLP Metrics output is enabled
     # protocol: "" # OTLP transport protocol to be used for metrics data; it can be "grpc" or "http/protobuf" (default: "grpc")
     # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000)
     # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "")

diff --git a/docs/outputs/otlp_logs.md b/docs/outputs/otlp_logs.md
@@ -0,0 +1,52 @@
+# OTEL Logs
+
+- **Category**: Logs
+- **Website**: <https://opentelemetry.io/docs/concepts/signals/logs/>
+
+## Table of content
+
+- [OTEL Logs](#otel-logs)
+  - [Table of content](#table-of-content)
+  - [Configuration](#configuration)
+  - [Example of config.yaml](#example-of-configyaml)
+  - [Additional info](#additional-info)
+
+## Configuration
+
+|            Setting            |            Env var            |       Default value        |                                                             Description                                                             |
+| ----------------------------- | ----------------------------- | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
+| `otlp.logs.endpoint`        | `OTLP_LOGS_ENDPOINT`        |                            | OTLP endpoint in the form of http://{domain or ip}:4318/v1/logs                                                                   |
+| `otlp.logs.protocol`        | `OTLP_LOGS_PROTOCOL`        | `http/protobuf` (from SDK) | OTLP Protocol: `http/protobuf`, `grpc`                                                                                              |
+| `otlp.logs.timeout`         | `OTLP_LOGS_TIMEOUT`         | `10000` (from SDK)         | Timeout value in milliseconds                                                                                                       |
+| `otlp.logs.headers`         | `OTLP_LOGS_HEADERS`         |                            | List of headers to apply to all outgoing logs in the form of "some-key=some-value,other-key=other-value"                          |
+| `otlp.logs.synced`          | `OTLP_LOGS_SYNCED`          | `false`                    | Set to `true` if you want logs to be sent synchronously                                                                           |
+| `otlp.logs.minimumpriority` | `OTLP_LOGS_MINIMUMPRIORITY` | `""` (=`debug`)            | minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""` |
+| `otlp.logs.checkcert`       | `OTLP_LOGS_CHECKCERT`       | `false`                    | Set if you want to skip TLS certificate validation                                                                                  |
+| `otlp.logs.duration`        | `OTLP_LOGS_DURATION`        | `1000`                     | Artificial span duration in milliseconds (as Falco doesn't provide an ending timestamp)                                             |
+| `otlp.logs.extraenvvars`    | `OTLP_LOGS_EXTRAENVVARS`    |                            | Extra env vars (override the other settings)                                                                                        |
+
+> [!NOTE]
+For the extra Env Vars values see [standard `OTEL_*` environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/)
+
+## Example of config.yaml
+
+```yaml
+otlp:
+  logs:
+    # endpoint: "https://falcosidekick.free.beeceptor.com" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/logs, if not empty, OTLP Traces output is enabled
+    endpoint: "http://localhost:4318/v1/logs" # OTLP endpoint in the form of http://{domain or ip}:4318/v1/logs, if not empty, OTLP Traces output is enabled
+    protocol: "" # OTLP protocol: http/protobuf, grpc (default: "" which uses SDK default: "http/protobuf")
+    # timeout: "" # OTLP timeout: timeout value in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # OTLP headers: list of headers to apply to all outgoing traces in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # extraenvvars: # Extra env vars (override the other settings)
+      # OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+    # checkcert: true # Set if you want to skip TLS certificate validation (default: true)
+```
+
+## Additional info
+
+> [!WARNING]
+Because of the way the OTEL SDK is structured, the OTLP outputs don't appear in the metrics (Prometheus, Statsd, ...) 
+and the error logs just specify `OTEL` as output.
diff --git a/docs/outputs/otlp_metrics.md b/docs/outputs/otlp_metrics.md
@@ -17,42 +17,46 @@
 
 ## Configuration
 
-| Setting                        | Env var                        | Default value      | Description                                                                                                                                         |
-|--------------------------------|--------------------------------|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------|
-| `otlp.metrics.endpoint`        | `OTLP_METRICS_ENDPOINT`        |                    | OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics                                                                       |
-| `otlp.metrics.protocol`        | `OTLP_METRICS_PROTOCOL`        | `grpc`             | OTLP transport protocol to be used for metrics data; it can be `"grpc"` or `"http/protobuf"`                                                        |
-| `otlp.metrics.timeout`         | `OTLP_METRICS_TIMEOUT`         | `10000` (from SDK) | OTLP timeout for outgoing metrics in milliseconds                                                                                                   |
-| `otlp.metrics.headers`         | `OTLP_METRICS_HEADERS`         | `""`               | List of headers to apply to all outgoing metrics in the form of `some-key=some-value,other-key=other-value`                                         |
-| `otlp.metrics.extraenvvars`    | `OTLP_METRICS_EXTRAENVVARS`    | `""`               | Extra env vars (override the other settings)                                                                                                        |
-| `otlp.metrics.minimumpriority` | `OTLP_METRICS_MINIMUMPRIORITY` | `""` (=`debug`)    | Minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""`                 |
-| `otlp.metrics.checkcert`       | `OTLP_METRICS_CHECKCERT`       | `true`             | Set to false if you want to skip TLS certificate validation (only with https)                                                                       |
-| `otlp.metrics.extraattributes` | `OTLP_METRICS_EXTRAATTRIBUTES` | `""`               | Comma-separated list of fields to use as labels additionally to source, priority, rule, hostname, tags, k8s_ns_name, k8s_pod_name and custom_fields |
+|            Setting             |            Env var             |       Default value        |                                                                     Description                                                                     |
+| ------------------------------ | ------------------------------ | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `otlp.metrics.endpoint`        | `OTLP_METRICS_ENDPOINT`        |                            | OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics                                                                       |
+| `otlp.metrics.protocol`        | `OTLP_METRICS_PROTOCOL`        | `http/protobuf` (from SDK) | OTLP Protocol: `http/protobuf`, `grpc`                                                                                                              |
+| `otlp.metrics.timeout`         | `OTLP_METRICS_TIMEOUT`         | `10000` (from SDK)         | OTLP timeout for outgoing metrics in milliseconds                                                                                                   |
+| `otlp.metrics.headers`         | `OTLP_METRICS_HEADERS`         | `""`                       | List of headers to apply to all outgoing metrics in the form of `some-key=some-value,other-key=other-value`                                         |
+| `otlp.metrics.extraenvvars`    | `OTLP_METRICS_EXTRAENVVARS`    | `""`                       | Extra env vars (override the other settings)                                                                                                        |
+| `otlp.metrics.minimumpriority` | `OTLP_METRICS_MINIMUMPRIORITY` | `""` (=`debug`)            | Minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""`                 |
+| `otlp.metrics.checkcert`       | `OTLP_METRICS_CHECKCERT`       | `true`                     | Set to false if you want to skip TLS certificate validation (only with https)                                                                       |
+| `otlp.metrics.extraattributes` | `OTLP_METRICS_EXTRAATTRIBUTES` | `""`                       | Comma-separated list of fields to use as labels additionally to source, priority, rule, hostname, tags, k8s_ns_name, k8s_pod_name and custom_fields |
 
 > [!NOTE]
-For the extra Env Vars values see [standard `OTEL_*` environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/):
+For the extra Env Vars values see [standard `OTEL_*` environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/)
 
 ## Example of config.yaml
 
 ```yaml
 otlp:
   metrics:
-  # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics
-  # protocol: "" # OTLP transport protocol to be used for metrics data; it can be "grpc" or "http/protobuf" (default: "grpc")
-  # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000)
-  # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "")
-  # extraenvvars: # Extra env vars (override the other settings) (default: "")
-  # OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: 10000
-  # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
-  # minimumpriority: "" # Minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default: "")
-  # checkcert: true # Set to false if you want to skip TLS certificate validation (only with https) (default: true)
-  # extraattributes: "" # Comma-separated list of fields to use as labels additionally to source, priority, rule, hostname, tags, k8s_ns_name, k8s_pod_name and custom_fields
+    # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics, if not empty, OTLP Metrics output is enabled
+    # protocol: "" # OTLP protocol: http/protobuf, grpc (default: "" which uses SDK default: "http/protobuf")
+    # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000)
+    # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "")
+    # extraenvvars: # Extra env vars (override the other settings) (default: "")
+      # OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: 10000
+      # OTEL_EXPORTER_OTLP_TIMEOUT: 10000
+    # minimumpriority: "" # Minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default: "")
+    # checkcert: true # Set to false if you want to skip TLS certificate validation (only with https) (default: true)
+    # extraattributes: "" # Comma-separated list of fields to use as labels additionally to source, priority, rule, hostname, tags, k8s_ns_name, k8s_pod_name and custom_fields
 ```
 
 ## Additional info
 
 > [!NOTE]
 This output is used to collect metrics about Falco events and Falcosidekick inputs and outputs in OTLP metrics format.
 
+> [!WARNING]
+Because of the way the OTEL SDK is structured, the OTLP outputs don't appear in the metrics (Prometheus, Statsd, ...) 
+and the error logs just specify `OTEL` as output.
+
 ## Running a whole stack with docker-compose
 
 Below `docker-compose` file runs a stack of: