fix: fluentbit logs (#7)

sparkfabrik · Sep 26, 2024 · 9e7a6e2 · 9e7a6e2
1 parent adec760
commit 9e7a6e2
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -42,7 +42,7 @@ It also opinionatedly configures FluentBit to send logs to CloudWatch and to exc
 | <a name="input_fluentbit_read_from_tail"></a> [fluentbit\_read\_from\_tail](#input\_fluentbit\_read\_from\_tail) | Start reading new entries. Skip entries already stored. | `string` | `"On"` | no |
 | <a name="input_fluentbit_send_fluentbit_logs_to_cloudwatch"></a> [fluentbit\_send\_fluentbit\_logs\_to\_cloudwatch](#input\_fluentbit\_send\_fluentbit\_logs\_to\_cloudwatch) | Send FluentBit logs to CloudWatch. | `bool` | `true` | no |
 | <a name="input_helm_additional_values"></a> [helm\_additional\_values](#input\_helm\_additional\_values) | Additional values to be passed to the Helm chart. | `list(string)` | `[]` | no |
-| <a name="input_helm_chart_version"></a> [helm\_chart\_version](#input\_helm\_chart\_version) | The version of the aws-for-fluent-bit Helm chart. | `string` | `"0.1.32"` | no |
+| <a name="input_helm_chart_version"></a> [helm\_chart\_version](#input\_helm\_chart\_version) | The version of the aws-for-fluent-bit Helm chart. | `string` | `"0.1.34"` | no |
 | <a name="input_helm_release_name"></a> [helm\_release\_name](#input\_helm\_release\_name) | The name of the Helm release. | `string` | `"fluentbit"` | no |
 | <a name="input_k8s_additional_labels"></a> [k8s\_additional\_labels](#input\_k8s\_additional\_labels) | Additional labels to apply to the kubernetes resources. | `map(string)` | `{}` | no |
 | <a name="input_k8s_default_labels"></a> [k8s\_default\_labels](#input\_k8s\_default\_labels) | Labels to apply to the kubernetes resources. These are opinionated labels, you can add more labels using the variable `additional_k8s_labels`. If you want to remove a label, you can override it with an empty map(string). | `map(string)` | <pre>{<br>  "managed-by": "terraform",<br>  "scope": "fluentbit"<br>}</pre> | no |

diff --git a/files/values.yml.tftpl b/files/values.yml.tftpl
@@ -75,14 +75,14 @@ additionalInputs: |
 
   [INPUT]
       Name                tail
-      Tag                 application-errors.*
+      Tag                 apperrors.*
       Exclude_Path        ${join(", ", formatlist("/var/log/containers/%s*.log", distinct(additional_exclude_from_application_log_group)))}
       Path                /var/log/containers/*.log
       Docker_Mode         On
       Docker_Mode_Flush   5
       Docker_Mode_Parser  custom_json
       Parser              custom_json
-      DB                  /var/fluent-bit/state/flb_container.db
+      DB                  /var/fluent-bit/state/flb_apperrors.db
       Mem_Buf_Limit       50MB
       Skip_Long_Lines     On
       Refresh_Interval    10
@@ -135,7 +135,7 @@ filter:
   enabled: false
 
 additionalFilters: |
-  %{~ for tag in ["application", "application-errors", "platform", "fluentbit"] ~}
+  %{~ for tag in ["application", "apperrors", "platform", "fluentbit"] ~}
   [FILTER]
       Name                kubernetes
       Match_regex         ${tag}.*
@@ -151,31 +151,35 @@ additionalFilters: |
 
   [FILTER]
       Name                nest
-      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
+      Match_regex         application.*|apperrors.*|platform.*|fluentbit.*
       Operation           lift
       Nested_under        kubernetes
       Add_prefix          Kube.
 
   [FILTER]
       Name                modify
-      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
+      Match_regex         application.*|apperrors.*|platform.*|fluentbit.*
       Remove              Kube.<Metadata_1>
       Remove              Kube.<Metadata_2>
       Remove              Kube.<Metadata_3>
 
   [FILTER]
       Name                nest
-      Match_regex         application.*|application-errors.*|platform.*|fluentbit.*
+      Match_regex         application.*|apperrors.*|platform.*|fluentbit.*
       Operation           nest
       Wildcard            Kube.*
       Nested_under        kubernetes
       Remove_prefix       Kube.
 
-  # Retrive logs with status code 4xx and 5xx
+  [FILTER]
+      Name                type_converter
+      Match               apperrors.*
+      uint_key            $log_processed['status'] response_status string
+
   [FILTER]
       Name                grep
-      Match               application-errors.*
-      Regex               $log_processed['status'] [45][0-9]{2}
+      Match               apperrors.*
+      Regex               response_status [45][0-9]{2}
 
 %{~ if additional_filters != "" }
   ${additional_filters}
@@ -195,9 +199,9 @@ additionalOutputs: |
 
   [OUTPUT]
       Name                cloudwatch_logs
-      Match               application-errors.*
+      Match               apperrors.*
       region              $${AWS_REGION}
-      log_group_name      /aws/containerinsights/$${CLUSTER_NAME}/application-errors
+      log_group_name      /aws/containerinsights/$${CLUSTER_NAME}/apperrors
       log_stream_prefix   $${HOST_NAME}-
       log_format          json
       auto_create_group   true

diff --git a/variables.tf b/variables.tf
@@ -13,7 +13,7 @@ variable "helm_release_name" {
 variable "helm_chart_version" {
   description = "The version of the aws-for-fluent-bit Helm chart."
   type        = string
-  default     = "0.1.32"
+  default     = "0.1.34"
 }
 
 variable "helm_additional_values" {