feat: use a new regexp to parse logs and keep JSON format and add add…

…itional filters configuration
sparkfabrik · May 22, 2024 · 6a2a89e · 6a2a89e
1 parent d3f75df
commit 6a2a89e
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 6 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Feat: add support for FluentBit additional fileters to decrease the amount of logs sent to the output.
+- Fix: fix the pattern used to parse the FluentBit logs to preserve the JSON format if used.
+
 ## [0.2.0] - 2024-05-09
 
 [Compare with previous version](https://github.com/sparkfabrik/terraform-helm-fluentbit/compare/0.1.0...0.2.0)

diff --git a/README.md b/README.md
@@ -25,6 +25,7 @@ It also opinionatedly configures FluentBit to send logs to CloudWatch and to exc
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_additional_exclude_from_application_log_group"></a> [additional\_exclude\_from\_application\_log\_group](#input\_additional\_exclude\_from\_application\_log\_group) | List of additional log files to exclude from the application log group. The element of this list will be prefixed with `/var/log/containers/` and suffixed with `*.log`. | `list(string)` | `[]` | no |
+| <a name="input_additional_filters"></a> [additional\_filters](#input\_additional\_filters) | Filter block(s) to add to the FluentBit configuration. The filter block(s) must be in the format of a string. | `string` | `""` | no |
 | <a name="input_additional_include_in_platform_log_group"></a> [additional\_include\_in\_platform\_log\_group](#input\_additional\_include\_in\_platform\_log\_group) | List of additional log files to include in the platform log group. The element of this list will be prefixed with `/var/log/containers/` and suffixed with `*.log`. | `list(string)` | `[]` | no |
 | <a name="input_application_log_retention_days"></a> [application\_log\_retention\_days](#input\_application\_log\_retention\_days) | The retention period for the application log group. Remember to check the valid values for the retention period in the AWS CloudWatch documentation. | `number` | `30` | no |
 | <a name="input_aws_fluentbit_role_name"></a> [aws\_fluentbit\_role\_name](#input\_aws\_fluentbit\_role\_name) | The name of the IAM role for FluentBit. | `string` | `"fluentbit"` | no |

diff --git a/files/values.yml.tftpl b/files/values.yml.tftpl
@@ -42,6 +42,13 @@ service:
         Time_Key            time
         Time_Format         %Y-%m-%dT%H:%M:%S.%LZ
 
+    [PARSER]
+        Name                custom_json
+        Format              Regex
+        Regex               ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>P|F) (?<log>.*)$
+        Time_Key            time
+        Time_Format         %Y-%m-%dT%H:%M:%S.%L%z
+
 input:
   enabled: false
 
@@ -53,8 +60,8 @@ additionalInputs: |
       Path                /var/log/containers/*.log
       Docker_Mode         On
       Docker_Mode_Flush   5
-      Docker_Mode_Parser  container_firstline
-      Parser              docker, cri
+      Docker_Mode_Parser  custom_json
+      Parser              custom_json
       DB                  /var/fluent-bit/state/flb_container.db
       Mem_Buf_Limit       50MB
       Skip_Long_Lines     On
@@ -69,8 +76,8 @@ additionalInputs: |
       Path                ${join(", ", formatlist("/var/log/containers/%s*.log", distinct(additional_include_in_platform_log_group)))}
       Docker_Mode         On
       Docker_Mode_Flush   5
-      Docker_Mode_Parser  container_firstline
-      Parser              docker, cri
+      Docker_Mode_Parser  custom_json
+      Parser              custom_json
       DB                  /var/fluent-bit/state/flb_platform.db
       Mem_Buf_Limit       50MB
       Skip_Long_Lines     On
@@ -83,7 +90,7 @@ additionalInputs: |
       Name                tail
       Tag                 fluentbit.*
       Path                /var/log/containers/${fluentbit_log_file_starts_with}*
-      Parser              docker, cri
+      Parser              custom_json
       DB                  /var/fluent-bit/state/flb_log.db
       Mem_Buf_Limit       5MB
       Skip_Long_Lines     On
@@ -97,7 +104,7 @@ additionalInputs: |
       Docker_Mode         On
       Docker_Mode_Flush   5
       Docker_Mode_Parser  cwagent_firstline
-      Parser              docker, cri
+      Parser              custom_json
       DB                  /var/fluent-bit/state/flb_cwagent.db
       Mem_Buf_Limit       5MB
       Skip_Long_Lines     On
@@ -142,6 +149,10 @@ additionalFilters: |
       Nested_under        kubernetes
       Remove_prefix       Kube.
 
+%{~ if additional_filters != "" }
+  ${additional_filters}
+%{~ endif }
+
 additionalOutputs: |
   [OUTPUT]
       Name                cloudwatch_logs

diff --git a/main.tf b/main.tf
@@ -99,6 +99,7 @@ resource "helm_release" "this" {
           fluentbit_log_retention_days                  = var.fluentbit_log_retention_days
           additional_exclude_from_application_log_group = local.exclude_from_application_log_group
           additional_include_in_platform_log_group      = local.include_in_platform_log_group
+          additional_filters                            = var.additional_filters
         }
       ),
     ],

diff --git a/variables.tf b/variables.tf
@@ -149,6 +149,12 @@ variable "additional_include_in_platform_log_group" {
   default     = []
 }
 
+variable "additional_filters" {
+  description = "Filter block(s) to add to the FluentBit configuration. The filter block(s) must be in the format of a string."
+  type        = string
+  default     = ""
+}
+
 variable "application_log_retention_days" {
   description = "The retention period for the application log group. Remember to check the valid values for the retention period in the AWS CloudWatch documentation."
   type        = number