From 9b4d99a28a8b9415267bf2b77cad8da4406cf1be Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 12:32:20 +0000 Subject: [PATCH 1/8] Adding cooldown Signed-off-by: Amit Schendel --- pkg/cooldown/cooldown.go | 134 ++++++++++++++++++++++++ pkg/rulemanager/v1/rule_manager_test.go | 4 +- 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 pkg/cooldown/cooldown.go diff --git a/pkg/cooldown/cooldown.go b/pkg/cooldown/cooldown.go new file mode 100644 index 00000000..f45c8d36 --- /dev/null +++ b/pkg/cooldown/cooldown.go @@ -0,0 +1,134 @@ +package cooldown + +import ( + "container/list" + "sync" + "time" +) + +// AlertID represents the unique identifier for an alert +type AlertID string + +// CooldownConfig holds the configuration for a cooldown +type CooldownConfig struct { + Threshold int + AlertWindow time.Duration + BaseCooldown time.Duration + MaxCooldown time.Duration + CooldownIncrease float64 +} + +// Cooldown represents the cooldown mechanism for a specific alert +type Cooldown struct { + mu sync.RWMutex + lastAlertTime time.Time + currentCooldown time.Duration + alertTimes *list.List + config CooldownConfig +} + +// CooldownManager manages cooldowns for different alerts +type CooldownManager struct { + mu sync.RWMutex + cooldowns map[AlertID]*Cooldown +} + +// NewCooldownManager creates a new CooldownManager +func NewCooldownManager() *CooldownManager { + return &CooldownManager{ + cooldowns: make(map[AlertID]*Cooldown), + } +} + +// NewCooldown creates a new Cooldown with the given configuration +func NewCooldown(config CooldownConfig) *Cooldown { + return &Cooldown{ + currentCooldown: config.BaseCooldown, + alertTimes: list.New(), + config: config, + } +} + +// ConfigureCooldown sets up or updates the cooldown configuration for a specific alert +func (cm *CooldownManager) ConfigureCooldown(alertID AlertID, config CooldownConfig) { + cm.mu.Lock() + defer cm.mu.Unlock() + + if cooldown, exists := cm.cooldowns[alertID]; exists { + cooldown.mu.Lock() + cooldown.config = config + cooldown.currentCooldown = config.BaseCooldown + cooldown.mu.Unlock() + } else { + cm.cooldowns[alertID] = NewCooldown(config) + } +} + +// ShouldAlert determines if an alert should be triggered based on the cooldown mechanism +func (cm *CooldownManager) ShouldAlert(alertID AlertID) bool { + cm.mu.RLock() + cooldown, exists := cm.cooldowns[alertID] + cm.mu.RUnlock() + + if !exists { + // If no configuration exists, always allow the alert + return true + } + + return cooldown.shouldAlert() +} + +func (c *Cooldown) shouldAlert() bool { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + + // Remove alerts outside the window + for c.alertTimes.Len() > 0 { + if now.Sub(c.alertTimes.Front().Value.(time.Time)) > c.config.AlertWindow { + c.alertTimes.Remove(c.alertTimes.Front()) + } else { + break + } + } + + // Check if we're still in the cooldown period + if now.Sub(c.lastAlertTime) < c.currentCooldown { + return false + } + + // Add current alert time + c.alertTimes.PushBack(now) + + // If we've exceeded the threshold, increase the cooldown + if c.alertTimes.Len() > c.config.Threshold { + c.currentCooldown = time.Duration(float64(c.currentCooldown) * c.config.CooldownIncrease) + if c.currentCooldown > c.config.MaxCooldown { + c.currentCooldown = c.config.MaxCooldown + } + } else if c.alertTimes.Len() <= c.config.Threshold/2 { + // If we're below half the threshold, start decreasing the cooldown + c.currentCooldown = time.Duration(float64(c.currentCooldown) / c.config.CooldownIncrease) + if c.currentCooldown < c.config.BaseCooldown { + c.currentCooldown = c.config.BaseCooldown + } + } + + c.lastAlertTime = now + return true +} + +// ResetCooldown resets the cooldown for a specific alert +func (cm *CooldownManager) ResetCooldown(alertID AlertID) { + cm.mu.RLock() + cooldown, exists := cm.cooldowns[alertID] + cm.mu.RUnlock() + + if exists { + cooldown.mu.Lock() + cooldown.alertTimes.Init() // Clear the list + cooldown.currentCooldown = cooldown.config.BaseCooldown + cooldown.mu.Unlock() + } +} diff --git a/pkg/rulemanager/v1/rule_manager_test.go b/pkg/rulemanager/v1/rule_manager_test.go index cefa8559..01e2658b 100644 --- a/pkg/rulemanager/v1/rule_manager_test.go +++ b/pkg/rulemanager/v1/rule_manager_test.go @@ -27,10 +27,10 @@ func TestReportEvent(t *testing.T) { } // Create a new rule - reportEvent(utils.HardlinkEventType, e) + reportEvent(e) } -func reportEvent(eventType utils.EventType, event utils.K8sEvent) { +func reportEvent(event utils.K8sEvent) { k8sEvent := event.(*tracerhardlinktype.Event) if k8sEvent.GetNamespace() == "" || k8sEvent.GetPod() == "" { logger.L().Error("RuleManager - failed to get namespace and pod name from custom event") From c0bed6ce1032e92b42893dd2f255214cc17b86f2 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 15:53:07 +0000 Subject: [PATCH 2/8] Adding cooldown Signed-off-by: Amit Schendel --- pkg/cooldown/cooldown.go | 46 ++-- pkg/cooldown/cooldown_test.go | 224 ++++++++++++++++++ pkg/ruleengine/ruleengine_interface.go | 6 + pkg/ruleengine/ruleengine_mock.go | 9 + pkg/ruleengine/v1/failureobj.go | 5 + pkg/ruleengine/v1/helpers.go | 8 + .../v1/r0001_unexpected_process_launched.go | 5 + .../v1/r0002_unexpected_file_access.go | 15 +- .../v1/r0003_unexpected_system_call.go | 5 + .../v1/r0004_unexpected_capability_used.go | 15 +- .../v1/r0005_unexpected_domain_request.go | 5 + ...unexpected_service_account_token_access.go | 5 + .../v1/r0007_kubernetes_client_executed.go | 5 + .../v1/r0008_read_env_variables_procfs.go | 5 + pkg/ruleengine/v1/r0009_ebpf_program_load.go | 5 + .../r0010_unexpected_sensitive_file_access.go | 15 +- ...r0011_unexpected_egress_network_traffic.go | 15 +- .../v1/r1000_exec_from_malicious_source.go | 5 + .../v1/r1001_exec_binary_not_in_base_image.go | 5 + pkg/ruleengine/v1/r1002_load_kernel_module.go | 5 + .../v1/r1003_malicious_ssh_connection.go | 15 +- pkg/ruleengine/v1/r1004_exec_from_mount.go | 15 +- pkg/ruleengine/v1/r1005_fileless_execution.go | 5 + .../v1/r1006_unshare_system_call.go | 5 + pkg/ruleengine/v1/r1007_xmr_crypto_mining.go | 5 + .../v1/r1008_crypto_mining_domain.go | 5 + pkg/ruleengine/v1/r1009_crypto_mining_port.go | 15 +- ...010_symlink_created_over_sensitive_file.go | 5 + pkg/ruleengine/v1/r1011_ld_preload_hook.go | 18 +- ...12_hardlink_created_over_sensitive_file.go | 5 + pkg/rulemanager/v1/rule_manager.go | 21 +- .../node-agent/default-rule-binding.yaml | 3 +- tests/component_test.go | 220 +++++------------ 33 files changed, 550 insertions(+), 195 deletions(-) create mode 100644 pkg/cooldown/cooldown_test.go diff --git a/pkg/cooldown/cooldown.go b/pkg/cooldown/cooldown.go index f45c8d36..d8cfc34e 100644 --- a/pkg/cooldown/cooldown.go +++ b/pkg/cooldown/cooldown.go @@ -4,10 +4,9 @@ import ( "container/list" "sync" "time" -) -// AlertID represents the unique identifier for an alert -type AlertID string + "github.com/goradd/maps" +) // CooldownConfig holds the configuration for a cooldown type CooldownConfig struct { @@ -29,15 +28,12 @@ type Cooldown struct { // CooldownManager manages cooldowns for different alerts type CooldownManager struct { - mu sync.RWMutex - cooldowns map[AlertID]*Cooldown + cooldowns maps.SafeMap[string, *Cooldown] } // NewCooldownManager creates a new CooldownManager func NewCooldownManager() *CooldownManager { - return &CooldownManager{ - cooldowns: make(map[AlertID]*Cooldown), - } + return &CooldownManager{} } // NewCooldown creates a new Cooldown with the given configuration @@ -50,31 +46,27 @@ func NewCooldown(config CooldownConfig) *Cooldown { } // ConfigureCooldown sets up or updates the cooldown configuration for a specific alert -func (cm *CooldownManager) ConfigureCooldown(alertID AlertID, config CooldownConfig) { - cm.mu.Lock() - defer cm.mu.Unlock() - - if cooldown, exists := cm.cooldowns[alertID]; exists { +func (cm *CooldownManager) ConfigureCooldown(alertID string, config CooldownConfig) { + if cm.cooldowns.Has(alertID) { + cooldown := cm.cooldowns.Get(alertID) cooldown.mu.Lock() cooldown.config = config cooldown.currentCooldown = config.BaseCooldown cooldown.mu.Unlock() } else { - cm.cooldowns[alertID] = NewCooldown(config) + cm.cooldowns.Set(alertID, NewCooldown(config)) } } // ShouldAlert determines if an alert should be triggered based on the cooldown mechanism -func (cm *CooldownManager) ShouldAlert(alertID AlertID) bool { - cm.mu.RLock() - cooldown, exists := cm.cooldowns[alertID] - cm.mu.RUnlock() - - if !exists { +func (cm *CooldownManager) ShouldAlert(alertID string) bool { + if !cm.cooldowns.Has(alertID) { // If no configuration exists, always allow the alert return true } + cooldown := cm.cooldowns.Get(alertID) + return cooldown.shouldAlert() } @@ -120,15 +112,17 @@ func (c *Cooldown) shouldAlert() bool { } // ResetCooldown resets the cooldown for a specific alert -func (cm *CooldownManager) ResetCooldown(alertID AlertID) { - cm.mu.RLock() - cooldown, exists := cm.cooldowns[alertID] - cm.mu.RUnlock() - - if exists { +func (cm *CooldownManager) ResetCooldown(alertID string) { + if cm.cooldowns.Has(alertID) { + cooldown := cm.cooldowns.Get(alertID) cooldown.mu.Lock() cooldown.alertTimes.Init() // Clear the list cooldown.currentCooldown = cooldown.config.BaseCooldown cooldown.mu.Unlock() } } + +// HasCooldownConfig checks if a cooldown configuration exists for a specific alert +func (cm *CooldownManager) HasCooldownConfig(alertID string) bool { + return cm.cooldowns.Has(alertID) +} diff --git a/pkg/cooldown/cooldown_test.go b/pkg/cooldown/cooldown_test.go new file mode 100644 index 00000000..b65a99c4 --- /dev/null +++ b/pkg/cooldown/cooldown_test.go @@ -0,0 +1,224 @@ +package cooldown + +import ( + "sync" + "testing" + "time" +) + +func TestNewCooldownManager(t *testing.T) { + cm := NewCooldownManager() + if cm == nil { + t.Error("NewCooldownManager() returned nil") + } +} + +func TestConfigureCooldown(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 5, + AlertWindow: 100 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 500 * time.Millisecond, + CooldownIncrease: 2.0, + } + + cm.ConfigureCooldown("test-alert", config) + + if !cm.HasCooldownConfig("test-alert") { + t.Error("ConfigureCooldown() did not add the configuration") + } + + // Test updating existing configuration + newConfig := CooldownConfig{ + Threshold: 10, + AlertWindow: 200 * time.Millisecond, + BaseCooldown: 20 * time.Millisecond, + MaxCooldown: 1 * time.Second, + CooldownIncrease: 3.0, + } + cm.ConfigureCooldown("test-alert", newConfig) + + if !cm.HasCooldownConfig("test-alert") { + t.Error("ConfigureCooldown() did not update the configuration") + } +} + +func TestShouldAlert(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 3, + AlertWindow: 100 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 50 * time.Millisecond, + CooldownIncrease: 2.0, + } + cm.ConfigureCooldown("test-alert", config) + + // First alert should always be allowed + if !cm.ShouldAlert("test-alert") { + t.Error("First alert was not allowed") + } + + // Second alert within BaseCooldown should not be allowed + time.Sleep(5 * time.Millisecond) + if cm.ShouldAlert("test-alert") { + t.Error("Second alert within BaseCooldown was allowed") + } + + // Alert after BaseCooldown should be allowed + time.Sleep(6 * time.Millisecond) + if !cm.ShouldAlert("test-alert") { + t.Error("Alert after BaseCooldown was not allowed") + } + + // Trigger alerts to exceed threshold + for i := 0; i < 3; i++ { + time.Sleep(11 * time.Millisecond) + cm.ShouldAlert("test-alert") + } + + // Next alert should not be allowed due to increased cooldown + if cm.ShouldAlert("test-alert") { + t.Error("Alert was allowed immediately after exceeding threshold") + } + + // Wait for increased cooldown (2 * BaseCooldown) + time.Sleep(21 * time.Millisecond) + if !cm.ShouldAlert("test-alert") { + t.Error("Alert was not allowed after increased cooldown period") + } + + // Alert for unconfigured alert ID should always be allowed + if !cm.ShouldAlert("unconfigured-alert") { + t.Error("Alert for unconfigured alert ID was not allowed") + } +} + +func TestResetCooldown(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 3, + AlertWindow: 100 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 50 * time.Millisecond, + CooldownIncrease: 2.0, + } + cm.ConfigureCooldown("test-alert", config) + + // Trigger alerts to increase cooldown + for i := 0; i < 4; i++ { + cm.ShouldAlert("test-alert") + time.Sleep(11 * time.Millisecond) + } + + // Verify that cooldown is in effect + if cm.ShouldAlert("test-alert") { + t.Error("Cooldown was not in effect before reset") + } + + // Reset cooldown + cm.ResetCooldown("test-alert") + + // Allow a small delay for reset to take effect + time.Sleep(1 * time.Millisecond) + + // Alert should now be allowed + if !cm.ShouldAlert("test-alert") { + t.Error("Alert was not allowed after reset") + } + + // Resetting an unconfigured alert should not panic + cm.ResetCooldown("unconfigured-alert") +} + +func TestCooldownIncrease(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 2, + AlertWindow: 50 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 100 * time.Millisecond, + CooldownIncrease: 2.0, + } + cm.ConfigureCooldown("test-alert", config) + + // Trigger alerts to increase cooldown + for i := 0; i < 3; i++ { + time.Sleep(11 * time.Millisecond) + cm.ShouldAlert("test-alert") + } + + // Next alert should be blocked due to increased cooldown + time.Sleep(11 * time.Millisecond) + if cm.ShouldAlert("test-alert") { + t.Error("Alert was allowed despite increased cooldown") + } + + // Wait for increased cooldown and alert should be allowed + time.Sleep(11 * time.Millisecond) + if !cm.ShouldAlert("test-alert") { + t.Error("Alert was not allowed after increased cooldown period") + } +} + +func TestCooldownDecrease(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 4, + AlertWindow: 50 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 100 * time.Millisecond, + CooldownIncrease: 2.0, + } + cm.ConfigureCooldown("test-alert", config) + + // Trigger alerts to increase cooldown + for i := 0; i < 5; i++ { + time.Sleep(11 * time.Millisecond) + cm.ShouldAlert("test-alert") + } + + // Wait for alert window to pass + time.Sleep(51 * time.Millisecond) + + // Trigger a single alert + cm.ShouldAlert("test-alert") + + // Wait for base cooldown + time.Sleep(11 * time.Millisecond) + + // Alert should be allowed and cooldown should have decreased + if !cm.ShouldAlert("test-alert") { + t.Error("Alert was not allowed after cooldown should have decreased") + } +} + +func TestConcurrency(t *testing.T) { + cm := NewCooldownManager() + config := CooldownConfig{ + Threshold: 5, + AlertWindow: 100 * time.Millisecond, + BaseCooldown: 10 * time.Millisecond, + MaxCooldown: 500 * time.Millisecond, + CooldownIncrease: 2.0, + } + cm.ConfigureCooldown("test-alert", config) + + // Run 100 goroutines simultaneously + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + cm.ShouldAlert("test-alert") + }() + } + + wg.Wait() + + // Check that the cooldown has increased + if cm.ShouldAlert("test-alert") { + t.Error("Cooldown did not increase as expected under concurrent load") + } +} diff --git a/pkg/ruleengine/ruleengine_interface.go b/pkg/ruleengine/ruleengine_interface.go index d512670f..d7ab9c66 100644 --- a/pkg/ruleengine/ruleengine_interface.go +++ b/pkg/ruleengine/ruleengine_interface.go @@ -1,6 +1,7 @@ package ruleengine import ( + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/utils" @@ -71,6 +72,9 @@ type RuleEvaluator interface { // Get rule parameters GetParameters() map[string]interface{} + + // Cooldown configuration + CooldownConfig() *cooldown.CooldownConfig } // RuleSpec is an interface for rule requirements @@ -92,6 +96,8 @@ type RuleFailure interface { GetRuntimeAlertK8sDetails() apitypes.RuntimeAlertK8sDetails // Get Rule ID GetRuleId() string + // Get Failure identifier + GetFailureIdentifier() string // Set Workload Details SetWorkloadDetails(workloadDetails string) diff --git a/pkg/ruleengine/ruleengine_mock.go b/pkg/ruleengine/ruleengine_mock.go index 40e46b89..861f5a97 100644 --- a/pkg/ruleengine/ruleengine_mock.go +++ b/pkg/ruleengine/ruleengine_mock.go @@ -1,6 +1,7 @@ package ruleengine import ( + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/utils" ) @@ -62,6 +63,14 @@ func (rule *RuleMock) SetParameters(p map[string]interface{}) { rule.RuleParameters = p } +func (rule *RuleMock) CooldownConfig() *cooldown.CooldownConfig { + return nil +} + +func (rule *RuleMock) UniqueInstanceIdentifier() string { + return "" +} + var _ RuleSpec = (*RuleSpecMock)(nil) type RuleSpecMock struct { diff --git a/pkg/ruleengine/v1/failureobj.go b/pkg/ruleengine/v1/failureobj.go index e8f491b4..b2efd0ba 100644 --- a/pkg/ruleengine/v1/failureobj.go +++ b/pkg/ruleengine/v1/failureobj.go @@ -17,6 +17,7 @@ type GenericRuleFailure struct { RuleAlert apitypes.RuleAlert RuntimeAlertK8sDetails apitypes.RuntimeAlertK8sDetails RuleID string + FailureIdentifier string } func (rule *GenericRuleFailure) GetBaseRuntimeAlert() apitypes.BaseRuntimeAlert { @@ -43,6 +44,10 @@ func (rule *GenericRuleFailure) GetRuleId() string { return rule.RuleID } +func (rule *GenericRuleFailure) GetFailureIdentifier() string { + return rule.FailureIdentifier +} + func (rule *GenericRuleFailure) SetBaseRuntimeAlert(baseRuntimeAlert apitypes.BaseRuntimeAlert) { rule.BaseRuntimeAlert = baseRuntimeAlert } diff --git a/pkg/ruleengine/v1/helpers.go b/pkg/ruleengine/v1/helpers.go index 3a8b0024..887b9e75 100644 --- a/pkg/ruleengine/v1/helpers.go +++ b/pkg/ruleengine/v1/helpers.go @@ -1,6 +1,8 @@ package ruleengine import ( + "crypto/md5" + "encoding/hex" "errors" "fmt" "path/filepath" @@ -153,3 +155,9 @@ func interfaceToStringSlice(val interface{}) ([]string, bool) { } return nil, false } + +func failureIdentifireMD5(eventType string, wlid string, identifier string) string { + // Calculate the MD5 hash of the event type, whitelist ID and identifier. + hash := md5.Sum([]byte(fmt.Sprintf("%s-%s-%s", eventType, wlid, identifier))) + return hex.EncodeToString(hash[:]) +} diff --git a/pkg/ruleengine/v1/r0001_unexpected_process_launched.go b/pkg/ruleengine/v1/r0001_unexpected_process_launched.go index 35342e2d..221b0638 100644 --- a/pkg/ruleengine/v1/r0001_unexpected_process_launched.go +++ b/pkg/ruleengine/v1/r0001_unexpected_process_launched.go @@ -5,6 +5,7 @@ import ( "slices" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -155,3 +156,7 @@ func (rule *R0001UnexpectedProcessLaunched) Requirements() ruleengine.RuleSpec { EventTypes: R0001UnexpectedProcessLaunchedRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0001UnexpectedProcessLaunched) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0002_unexpected_file_access.go b/pkg/ruleengine/v1/r0002_unexpected_file_access.go index fd91852d..b0af1519 100644 --- a/pkg/ruleengine/v1/r0002_unexpected_file_access.go +++ b/pkg/ruleengine/v1/r0002_unexpected_file_access.go @@ -3,7 +3,9 @@ package ruleengine import ( "fmt" "strings" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -177,7 +179,8 @@ func (rule *R0002UnexpectedFileAccess) ProcessEvent(eventType utils.EventType, e RuntimeAlertK8sDetails: apitypes.RuntimeAlertK8sDetails{ PodName: openEvent.GetPod(), }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", openEvent.GetNamespace(), openEvent.GetPod(), openEvent.GetContainer()), openEvent.FullPath), } return &ruleFailure @@ -192,3 +195,13 @@ func (rule *R0002UnexpectedFileAccess) Requirements() ruleengine.RuleSpec { EventTypes: R0002UnexpectedFileAccessRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0002UnexpectedFileAccess) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 10, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 5, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r0003_unexpected_system_call.go b/pkg/ruleengine/v1/r0003_unexpected_system_call.go index 06c93cee..4f214125 100644 --- a/pkg/ruleengine/v1/r0003_unexpected_system_call.go +++ b/pkg/ruleengine/v1/r0003_unexpected_system_call.go @@ -3,6 +3,7 @@ package ruleengine import ( "fmt" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -123,3 +124,7 @@ func (rule *R0003UnexpectedSystemCall) Requirements() ruleengine.RuleSpec { EventTypes: R0003UnexpectedSystemCallRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0003UnexpectedSystemCall) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0004_unexpected_capability_used.go b/pkg/ruleengine/v1/r0004_unexpected_capability_used.go index 5e96899b..3759fbab 100644 --- a/pkg/ruleengine/v1/r0004_unexpected_capability_used.go +++ b/pkg/ruleengine/v1/r0004_unexpected_capability_used.go @@ -2,7 +2,9 @@ package ruleengine import ( "fmt" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -105,7 +107,8 @@ func (rule *R0004UnexpectedCapabilityUsed) ProcessEvent(eventType utils.EventTyp RuntimeAlertK8sDetails: apitypes.RuntimeAlertK8sDetails{ PodName: capEvent.GetPod(), }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", capEvent.GetNamespace(), capEvent.GetPod(), capEvent.GetContainer()), capEvent.CapName), } return &ruleFailure @@ -116,3 +119,13 @@ func (rule *R0004UnexpectedCapabilityUsed) Requirements() ruleengine.RuleSpec { EventTypes: R0004UnexpectedCapabilityUsedRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0004UnexpectedCapabilityUsed) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 5, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 5, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r0005_unexpected_domain_request.go b/pkg/ruleengine/v1/r0005_unexpected_domain_request.go index bc372fd7..02ed8315 100644 --- a/pkg/ruleengine/v1/r0005_unexpected_domain_request.go +++ b/pkg/ruleengine/v1/r0005_unexpected_domain_request.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -140,3 +141,7 @@ func (rule *R0005UnexpectedDomainRequest) Requirements() ruleengine.RuleSpec { EventTypes: R0005UnexpectedDomainRequestRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0005UnexpectedDomainRequest) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0006_unexpected_service_account_token_access.go b/pkg/ruleengine/v1/r0006_unexpected_service_account_token_access.go index b6b7fe0d..5c062ddc 100644 --- a/pkg/ruleengine/v1/r0006_unexpected_service_account_token_access.go +++ b/pkg/ruleengine/v1/r0006_unexpected_service_account_token_access.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -152,3 +153,7 @@ func (rule *R0006UnexpectedServiceAccountTokenAccess) Requirements() ruleengine. EventTypes: R0006UnexpectedServiceAccountTokenAccessRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0006UnexpectedServiceAccountTokenAccess) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0007_kubernetes_client_executed.go b/pkg/ruleengine/v1/r0007_kubernetes_client_executed.go index 39951124..ab575b8e 100644 --- a/pkg/ruleengine/v1/r0007_kubernetes_client_executed.go +++ b/pkg/ruleengine/v1/r0007_kubernetes_client_executed.go @@ -6,6 +6,7 @@ import ( "slices" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -235,3 +236,7 @@ func (rule *R0007KubernetesClientExecuted) Requirements() ruleengine.RuleSpec { EventTypes: R0007KubernetesClientExecutedDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0007KubernetesClientExecuted) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0008_read_env_variables_procfs.go b/pkg/ruleengine/v1/r0008_read_env_variables_procfs.go index ef29e017..7730634c 100644 --- a/pkg/ruleengine/v1/r0008_read_env_variables_procfs.go +++ b/pkg/ruleengine/v1/r0008_read_env_variables_procfs.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -118,3 +119,7 @@ func (rule *R0008ReadEnvironmentVariablesProcFS) Requirements() ruleengine.RuleS EventTypes: R0008ReadEnvironmentVariablesProcFSRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0008ReadEnvironmentVariablesProcFS) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0009_ebpf_program_load.go b/pkg/ruleengine/v1/r0009_ebpf_program_load.go index 3c064d38..356a77de 100644 --- a/pkg/ruleengine/v1/r0009_ebpf_program_load.go +++ b/pkg/ruleengine/v1/r0009_ebpf_program_load.go @@ -4,6 +4,7 @@ import ( "fmt" "slices" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -122,3 +123,7 @@ func (rule *R0009EbpfProgramLoad) Requirements() ruleengine.RuleSpec { EventTypes: R0009EbpfProgramLoadRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0009EbpfProgramLoad) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r0010_unexpected_sensitive_file_access.go b/pkg/ruleengine/v1/r0010_unexpected_sensitive_file_access.go index 54d35e64..1063ff72 100644 --- a/pkg/ruleengine/v1/r0010_unexpected_sensitive_file_access.go +++ b/pkg/ruleengine/v1/r0010_unexpected_sensitive_file_access.go @@ -3,7 +3,9 @@ package ruleengine import ( "fmt" "strings" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -138,7 +140,8 @@ func (rule *R0010UnexpectedSensitiveFileAccess) ProcessEvent(eventType utils.Eve PodName: openEvent.GetPod(), PodLabels: openEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", openEvent.GetNamespace(), openEvent.GetPod(), openEvent.GetContainer()), openEvent.FullPath), } return &ruleFailure @@ -149,3 +152,13 @@ func (rule *R0010UnexpectedSensitiveFileAccess) Requirements() ruleengine.RuleSp EventTypes: R0010UnexpectedSensitiveFileAccessRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R0010UnexpectedSensitiveFileAccess) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 5, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 5, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r0011_unexpected_egress_network_traffic.go b/pkg/ruleengine/v1/r0011_unexpected_egress_network_traffic.go index 3c91f107..804455e2 100644 --- a/pkg/ruleengine/v1/r0011_unexpected_egress_network_traffic.go +++ b/pkg/ruleengine/v1/r0011_unexpected_egress_network_traffic.go @@ -6,9 +6,11 @@ import ( "net" "slices" "strings" + "time" apitypes "github.com/armosec/armoapi-go/armotypes" "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -126,7 +128,8 @@ func (rule *R0011UnexpectedEgressNetworkTraffic) handleNetworkEvent(networkEvent PodName: networkEvent.GetPod(), PodLabels: networkEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", networkEvent.GetNamespace(), networkEvent.GetPod(), networkEvent.GetContainer()), endpoint), } } @@ -185,3 +188,13 @@ func isPrivateIP(ip string) bool { return false } + +func (rule *R0011UnexpectedEgressNetworkTraffic) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 20, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 5, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r1000_exec_from_malicious_source.go b/pkg/ruleengine/v1/r1000_exec_from_malicious_source.go index 2be1abf2..936a5b40 100644 --- a/pkg/ruleengine/v1/r1000_exec_from_malicious_source.go +++ b/pkg/ruleengine/v1/r1000_exec_from_malicious_source.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -120,3 +121,7 @@ func (rule *R1000ExecFromMaliciousSource) Requirements() ruleengine.RuleSpec { EventTypes: R1000ExecFromMaliciousSourceDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1000ExecFromMaliciousSource) CooldownConfig() *cooldown.CooldownConfig { + return nil +} \ No newline at end of file diff --git a/pkg/ruleengine/v1/r1001_exec_binary_not_in_base_image.go b/pkg/ruleengine/v1/r1001_exec_binary_not_in_base_image.go index f9236509..fa21c6bd 100644 --- a/pkg/ruleengine/v1/r1001_exec_binary_not_in_base_image.go +++ b/pkg/ruleengine/v1/r1001_exec_binary_not_in_base_image.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -114,3 +115,7 @@ func (rule *R1001ExecBinaryNotInBaseImage) Requirements() ruleengine.RuleSpec { EventTypes: R1001ExecBinaryNotInBaseImageRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1001ExecBinaryNotInBaseImage) CooldownConfig() *cooldown.CooldownConfig { + return nil +} \ No newline at end of file diff --git a/pkg/ruleengine/v1/r1002_load_kernel_module.go b/pkg/ruleengine/v1/r1002_load_kernel_module.go index b74dfda4..ab0f959a 100644 --- a/pkg/ruleengine/v1/r1002_load_kernel_module.go +++ b/pkg/ruleengine/v1/r1002_load_kernel_module.go @@ -3,6 +3,7 @@ package ruleengine import ( "fmt" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -106,3 +107,7 @@ func (rule *R1002LoadKernelModule) Requirements() ruleengine.RuleSpec { EventTypes: R1002LoadKernelModuleRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1002LoadKernelModule) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1003_malicious_ssh_connection.go b/pkg/ruleengine/v1/r1003_malicious_ssh_connection.go index e8205a4f..0bd055e9 100644 --- a/pkg/ruleengine/v1/r1003_malicious_ssh_connection.go +++ b/pkg/ruleengine/v1/r1003_malicious_ssh_connection.go @@ -6,9 +6,11 @@ import ( "slices" "strconv" "strings" + "time" "github.com/goradd/maps" "github.com/kubescape/go-logger/helpers" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -191,7 +193,8 @@ func (rule *R1003MaliciousSSHConnection) ProcessEvent(eventType utils.EventType, PodName: sshEvent.GetPod(), PodLabels: sshEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: fmt.Sprintf("%s-%s-%s--%s-%d", sshEvent.GetNamespace(), sshEvent.GetPod(), sshEvent.GetContainer(), sshEvent.DstIP, sshEvent.DstPort), } return &ruleFailure @@ -205,3 +208,13 @@ func (rule *R1003MaliciousSSHConnection) Requirements() ruleengine.RuleSpec { EventTypes: R1003MaliciousSSHConnectionRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1003MaliciousSSHConnection) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 5, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 30, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r1004_exec_from_mount.go b/pkg/ruleengine/v1/r1004_exec_from_mount.go index 3c06c18a..23716a51 100644 --- a/pkg/ruleengine/v1/r1004_exec_from_mount.go +++ b/pkg/ruleengine/v1/r1004_exec_from_mount.go @@ -4,7 +4,9 @@ import ( "errors" "fmt" "strings" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -109,7 +111,8 @@ func (rule *R1004ExecFromMount) ProcessEvent(eventType utils.EventType, event ut PodName: execEvent.GetPod(), PodLabels: execEvent.K8s.PodLabels, }, - RuleID: R1004ID, + RuleID: R1004ID, + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", execEvent.GetNamespace(), execEvent.GetPod(), execEvent.GetContainer()), fullPath), } return &ruleFailure @@ -128,3 +131,13 @@ func (rule *R1004ExecFromMount) Requirements() ruleengine.RuleSpec { EventTypes: R1004ExecFromMountRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1004ExecFromMount) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 5, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 5, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r1005_fileless_execution.go b/pkg/ruleengine/v1/r1005_fileless_execution.go index d1645caa..3b5e22c4 100644 --- a/pkg/ruleengine/v1/r1005_fileless_execution.go +++ b/pkg/ruleengine/v1/r1005_fileless_execution.go @@ -5,6 +5,7 @@ import ( "path/filepath" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -121,3 +122,7 @@ func (rule *R1005FilelessExecution) Requirements() ruleengine.RuleSpec { EventTypes: R1005FilelessExecutionRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1005FilelessExecution) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1006_unshare_system_call.go b/pkg/ruleengine/v1/r1006_unshare_system_call.go index 5440d673..e5b1ef24 100644 --- a/pkg/ruleengine/v1/r1006_unshare_system_call.go +++ b/pkg/ruleengine/v1/r1006_unshare_system_call.go @@ -3,6 +3,7 @@ package ruleengine import ( "fmt" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -108,3 +109,7 @@ func (rule *R1006UnshareSyscall) Requirements() ruleengine.RuleSpec { EventTypes: R1006UnshareSyscallRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1006UnshareSyscall) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1007_xmr_crypto_mining.go b/pkg/ruleengine/v1/r1007_xmr_crypto_mining.go index 4083afd9..83b3fe46 100644 --- a/pkg/ruleengine/v1/r1007_xmr_crypto_mining.go +++ b/pkg/ruleengine/v1/r1007_xmr_crypto_mining.go @@ -3,6 +3,7 @@ package ruleengine import ( "fmt" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -102,3 +103,7 @@ func (rule *R1007XMRCryptoMining) Requirements() ruleengine.RuleSpec { EventTypes: R1007XMRCryptoMiningRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1007XMRCryptoMining) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1008_crypto_mining_domain.go b/pkg/ruleengine/v1/r1008_crypto_mining_domain.go index b1a25f4b..2b7d9ce1 100644 --- a/pkg/ruleengine/v1/r1008_crypto_mining_domain.go +++ b/pkg/ruleengine/v1/r1008_crypto_mining_domain.go @@ -5,6 +5,7 @@ import ( "slices" "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -218,3 +219,7 @@ func (rule *R1008CryptoMiningDomainCommunication) Requirements() ruleengine.Rule EventTypes: R1008CryptoMiningDomainCommunicationRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1008CryptoMiningDomainCommunication) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1009_crypto_mining_port.go b/pkg/ruleengine/v1/r1009_crypto_mining_port.go index 006bbe4e..914af5ce 100644 --- a/pkg/ruleengine/v1/r1009_crypto_mining_port.go +++ b/pkg/ruleengine/v1/r1009_crypto_mining_port.go @@ -3,7 +3,9 @@ package ruleengine import ( "fmt" "slices" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -118,7 +120,8 @@ func (rule *R1009CryptoMiningRelatedPort) ProcessEvent(eventType utils.EventType PodName: networkEvent.GetPod(), PodLabels: networkEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", networkEvent.GetNamespace(), networkEvent.GetPod(), networkEvent.GetContainer()), fmt.Sprintf("%d", networkEvent.Port)), } return &ruleFailure @@ -133,3 +136,13 @@ func (rule *R1009CryptoMiningRelatedPort) Requirements() ruleengine.RuleSpec { EventTypes: R1009CryptoMiningRelatedPortRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1009CryptoMiningRelatedPort) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 2, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Second * 30, + MaxCooldown: time.Minute * 30, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r1010_symlink_created_over_sensitive_file.go b/pkg/ruleengine/v1/r1010_symlink_created_over_sensitive_file.go index 40537200..f690be0c 100644 --- a/pkg/ruleengine/v1/r1010_symlink_created_over_sensitive_file.go +++ b/pkg/ruleengine/v1/r1010_symlink_created_over_sensitive_file.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -157,3 +158,7 @@ func isSymLinkAllowed(symlinkEvent *tracersymlinktype.Event, objCache objectcach return false, nil } + +func (rule *R1010SymlinkCreatedOverSensitiveFile) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/ruleengine/v1/r1011_ld_preload_hook.go b/pkg/ruleengine/v1/r1011_ld_preload_hook.go index ea926fdd..3b539167 100644 --- a/pkg/ruleengine/v1/r1011_ld_preload_hook.go +++ b/pkg/ruleengine/v1/r1011_ld_preload_hook.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "strings" + "time" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -139,7 +141,8 @@ func (rule *R1011LdPreloadHook) handleExecEvent(execEvent *tracerexectype.Event, PodName: execEvent.GetPod(), PodLabels: execEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", execEvent.GetNamespace(), execEvent.GetPod(), execEvent.GetContainer()), ldHookVar), } return &ruleFailure @@ -174,7 +177,8 @@ func (rule *R1011LdPreloadHook) handleOpenEvent(openEvent *traceropentype.Event) PodName: openEvent.GetPod(), PodLabels: openEvent.K8s.PodLabels, }, - RuleID: rule.ID(), + RuleID: rule.ID(), + FailureIdentifier: failureIdentifireMD5(rule.ID(), fmt.Sprintf("%s-%s-%s", openEvent.GetNamespace(), openEvent.GetPod(), openEvent.GetContainer()), openEvent.Path), } return &ruleFailure @@ -212,3 +216,13 @@ func (rule *R1011LdPreloadHook) Requirements() ruleengine.RuleSpec { EventTypes: R1011LdPreloadHookRuleDescriptor.Requirements.RequiredEventTypes(), } } + +func (rule *R1011LdPreloadHook) CooldownConfig() *cooldown.CooldownConfig { + return &cooldown.CooldownConfig{ + Threshold: 5, + AlertWindow: time.Minute * 1, + BaseCooldown: time.Minute * 5, + MaxCooldown: time.Minute * 10, + CooldownIncrease: 1.5, + } +} diff --git a/pkg/ruleengine/v1/r1012_hardlink_created_over_sensitive_file.go b/pkg/ruleengine/v1/r1012_hardlink_created_over_sensitive_file.go index 3d70f34c..1eb5e820 100644 --- a/pkg/ruleengine/v1/r1012_hardlink_created_over_sensitive_file.go +++ b/pkg/ruleengine/v1/r1012_hardlink_created_over_sensitive_file.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/ruleengine" "github.com/kubescape/node-agent/pkg/utils" @@ -157,3 +158,7 @@ func isHardLinkAllowed(hardlinkEvent *tracerhardlinktype.Event, objCache objectc return false, nil } + +func (rule *R1012HardlinkCreatedOverSensitiveFile) CooldownConfig() *cooldown.CooldownConfig { + return nil +} diff --git a/pkg/rulemanager/v1/rule_manager.go b/pkg/rulemanager/v1/rule_manager.go index 56ce6156..32d83ea4 100644 --- a/pkg/rulemanager/v1/rule_manager.go +++ b/pkg/rulemanager/v1/rule_manager.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/cooldown" "github.com/kubescape/node-agent/pkg/exporters" "github.com/kubescape/node-agent/pkg/k8sclient" "github.com/kubescape/node-agent/pkg/ruleengine" @@ -64,6 +65,7 @@ type RuleManager struct { clusterName string containerIdToShimPid maps.SafeMap[string, uint32] containerIdToPid maps.SafeMap[string, uint32] + cooldownManager *cooldown.CooldownManager } var _ rulemanager.RuleManagerClient = (*RuleManager)(nil) @@ -81,6 +83,7 @@ func CreateRuleManager(ctx context.Context, cfg config.Config, k8sClient k8sclie metrics: metrics, nodeName: nodeName, clusterName: clusterName, + cooldownManager: cooldown.NewCooldownManager(), }, nil } @@ -344,10 +347,20 @@ func (rm *RuleManager) processEvent(eventType utils.EventType, event utils.K8sEv res := rule.ProcessEvent(eventType, event, rm.objectCache) if res != nil { - res = rm.enrichRuleFailure(res) - res.SetWorkloadDetails(rm.podToWlid.Get(utils.CreateK8sPodID(res.GetRuntimeAlertK8sDetails().Namespace, res.GetRuntimeAlertK8sDetails().PodName))) - rm.exporter.SendRuleAlert(res) - rm.metrics.ReportRuleAlert(rule.Name()) + if res.GetFailureIdentifier() != "" { + if !rm.cooldownManager.HasCooldownConfig(res.GetFailureIdentifier()) { + rm.cooldownManager.ConfigureCooldown(res.GetFailureIdentifier(), *rule.CooldownConfig()) + } + } + + if rm.cooldownManager.ShouldAlert(res.GetFailureIdentifier()) { + res = rm.enrichRuleFailure(res) + res.SetWorkloadDetails(rm.podToWlid.Get(utils.CreateK8sPodID(res.GetRuntimeAlertK8sDetails().Namespace, res.GetRuntimeAlertK8sDetails().PodName))) + rm.exporter.SendRuleAlert(res) + rm.metrics.ReportRuleAlert(rule.Name()) + } else { + logger.L().Debug("RuleManager - rule is in cooldown for identifier", helpers.String("identifier", res.GetFailureIdentifier())) + } } rm.metrics.ReportRuleProcessed(rule.Name()) } diff --git a/tests/chart/templates/node-agent/default-rule-binding.yaml b/tests/chart/templates/node-agent/default-rule-binding.yaml index 772fbb35..cae6fa9c 100644 --- a/tests/chart/templates/node-agent/default-rule-binding.yaml +++ b/tests/chart/templates/node-agent/default-rule-binding.yaml @@ -32,4 +32,5 @@ spec: - ruleName: "XMR Crypto Mining Detection" - ruleName: "Exec from mount" - ruleName: "Crypto Mining Related Port Communication" - - ruleName: "Crypto Mining Domain Communication" \ No newline at end of file + - ruleName: "Crypto Mining Domain Communication" + - ruleName: "LD_PRELOAD Hook" \ No newline at end of file diff --git a/tests/component_test.go b/tests/component_test.go index c5a04d2c..9da4ed86 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -715,158 +715,68 @@ func sortHTTPEndpoints(endpoints []v1beta1.HTTPEndpoint) { }) } -// func Test_10_DemoTest(t *testing.T) { -// start := time.Now() -// defer tearDownTest(t, start) - -// //testutils.IncreaseNodeAgentSniffingTime("2m") -// wl, err := testutils.NewTestWorkload("default", path.Join(utils.CurrentDir(), "resources/ping-app-role.yaml")) -// if err != nil { -// t.Errorf("Error creating role: %v", err) -// } - -// wl, err = testutils.NewTestWorkload("default", path.Join(utils.CurrentDir(), "resources/ping-app-role-binding.yaml")) -// if err != nil { -// t.Errorf("Error creating role binding: %v", err) -// } - -// wl, err = testutils.NewTestWorkload("default", path.Join(utils.CurrentDir(), "resources/ping-app-service.yaml")) -// if err != nil { -// t.Errorf("Error creating service: %v", err) -// } - -// wl, err = testutils.NewTestWorkload("default", path.Join(utils.CurrentDir(), "resources/ping-app.yaml")) -// if err != nil { -// t.Errorf("Error creating workload: %v", err) -// } -// assert.NoError(t, wl.WaitForReady(80)) -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4"}, "") -// err = wl.WaitForApplicationProfileCompletion(80) -// if err != nil { -// t.Errorf("Error waiting for application profile to be completed: %v", err) -// } -// // err = wl.WaitForNetworkNeighborhoodCompletion(80) -// // if err != nil { -// // t.Errorf("Error waiting for network neighborhood to be completed: %v", err) -// // } - -// // Do a ls command using command injection in the ping command -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;ls"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Do a cat command using command injection in the ping command -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;cat /run/secrets/kubernetes.io/serviceaccount/token"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Do an uname command using command injection in the ping command -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g'"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Download kubectl -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;curl -LO \"https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl\""}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Sleep for 10 seconds to wait for the kubectl download -// time.Sleep(10 * time.Second) - -// // Make kubectl executable -// _, _, err = wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;chmod +x kubectl"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Get the pods in the cluster -// output, _, err := wl.ExecIntoPod([]string{"sh", "-c", "ping 1.1.1.1 -c 4;./kubectl --server https://kubernetes.default --insecure-skip-tls-verify --token $(cat /run/secrets/kubernetes.io/serviceaccount/token) get pods"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// // Check that the output contains the pod-ping-app pod -// assert.Contains(t, output, "ping-app", "Expected output to contain 'ping-app'") - -// // Get the alerts and check that the alerts are generated -// alerts, err := testutils.GetAlerts(wl.Namespace) -// if err != nil { -// t.Errorf("Error getting alerts: %v", err) -// } - -// // Validate that all alerts are signaled -// expectedAlerts := map[string]bool{ -// "Unexpected process launched": false, -// "Unexpected file access": false, -// "Kubernetes Client Executed": false, -// // "Exec from malicious source": false, -// "Exec Binary Not In Base Image": false, -// "Unexpected Service Account Token Access": false, -// // "Unexpected domain request": false, -// } - -// for _, alert := range alerts { -// ruleName, ruleOk := alert.Labels["rule_name"] -// if ruleOk { -// if _, exists := expectedAlerts[ruleName]; exists { -// expectedAlerts[ruleName] = true -// } -// } -// } - -// for ruleName, signaled := range expectedAlerts { -// if !signaled { -// t.Errorf("Expected alert '%s' was not signaled", ruleName) -// } -// } -// } - -// func Test_11_DuplicationTest(t *testing.T) { -// start := time.Now() -// defer tearDownTest(t, start) - -// ns := testutils.NewRandomNamespace() -// // wl, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/deployment-multiple-containers.yaml")) -// wl, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/ping-app.yaml")) -// if err != nil { -// t.Errorf("Error creating workload: %v", err) -// } -// assert.NoError(t, wl.WaitForReady(80)) - -// err = wl.WaitForApplicationProfileCompletion(80) -// if err != nil { -// t.Errorf("Error waiting for application profile to be completed: %v", err) -// } - -// // process launched from nginx container -// _, _, err = wl.ExecIntoPod([]string{"ls", "-a"}, "ping-app") -// if err != nil { -// t.Errorf("Error executing remote command: %v", err) -// } - -// time.Sleep(20 * time.Second) - -// alerts, err := testutils.GetAlerts(wl.Namespace) -// if err != nil { -// t.Errorf("Error getting alerts: %v", err) -// } - -// // Validate that unexpected process launched alert is signaled only once -// count := 0 -// for _, alert := range alerts { -// ruleName, ruleOk := alert.Labels["rule_name"] -// if ruleOk { -// if ruleName == "Unexpected process launched" { -// count++ -// } -// } -// } - -// testutils.AssertContains(t, alerts, "Unexpected process launched", "ls", "ping-app") - -// assert.Equal(t, 1, count, "Expected 1 alert of type 'Unexpected process launched' but got %d", count) -// } +func Test_12_Cooldown(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + ns := testutils.NewRandomNamespace() + wl, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/deployment-multiple-containers.yaml")) + if err != nil { + t.Errorf("Error creating workload: %v", err) + } + assert.NoError(t, wl.WaitForReady(80)) + + assert.NoError(t, wl.WaitForApplicationProfile(80, "ready")) + assert.NoError(t, wl.WaitForNetworkNeighborhood(80, "ready")) + + // process launched from nginx container + _, _, err = wl.ExecIntoPod([]string{"ls", "-l"}, "nginx") + + // network activity from server container + _, _, err = wl.ExecIntoPod([]string{"wget", "ebpf.io", "-T", "2", "-t", "1"}, "server") + + // network activity from nginx container + _, _, err = wl.ExecIntoPod([]string{"curl", "kubernetes.io", "-m", "2"}, "nginx") + + err = wl.WaitForApplicationProfileCompletion(80) + if err != nil { + t.Errorf("Error waiting for application profile to be completed: %v", err) + } + err = wl.WaitForNetworkNeighborhoodCompletion(80) + if err != nil { + t.Errorf("Error waiting for network neighborhood to be completed: %v", err) + } + + time.Sleep(10 * time.Second) + + appProfile, _ := wl.GetApplicationProfile() + appProfileJson, _ := json.Marshal(appProfile) + + t.Logf("application profile: %v", string(appProfileJson)) + + // The cooldown for LD_PRELOAD is: + // cooldown.CooldownConfig{ + // Threshold: 5, + // AlertWindow: time.Minute * 1, + // BaseCooldown: time.Minute * 5, + // MaxCooldown: time.Minute * 10, + // CooldownIncrease: 1.5, + + // So we want to exec into the pod and run 6 times a binary with the LD_PRELOAD feature to trigger the cooldown. + // We expect to see 5 alerts and the 6th one to be ignored. + for i := 0; i < 6; i++ { + _, _, err = wl.ExecIntoPod([]string{"LD_PRELOAD=/lib/libc.so", "ls", "-l"}, "nginx") + } + + // Wait for the alert to be signaled + time.Sleep(30 * time.Second) + + alerts, err := testutils.GetAlerts(wl.Namespace) + if err != nil { + t.Errorf("Error getting alerts: %v", err) + } + + if len(alerts) != 5 { + t.Errorf("Expected 5 alerts to be generated, but got %d alerts", len(alerts)) + } +} From f61d9eda1667c2193b2e53e60b1f84165bad62a5 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 15:55:05 +0000 Subject: [PATCH 3/8] Adding fixed test name Signed-off-by: Amit Schendel --- .github/workflows/component-tests.yaml | 1 + tests/component_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index 1ba77169..6810c573 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -50,6 +50,7 @@ jobs: Test_08_ApplicationProfilePatching, Test_10_MalwareDetectionTest, Test_11_EndpointTest, + Test_12_CooldownTest, # Test_10_DemoTest # Test_11_DuplicationTest ] diff --git a/tests/component_test.go b/tests/component_test.go index 9da4ed86..876f16c8 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -715,7 +715,7 @@ func sortHTTPEndpoints(endpoints []v1beta1.HTTPEndpoint) { }) } -func Test_12_Cooldown(t *testing.T) { +func Test_12_CooldownTest(t *testing.T) { start := time.Now() defer tearDownTest(t, start) From 7e1873ec899136ce11b48b6281d127bf94510a99 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 16:00:10 +0000 Subject: [PATCH 4/8] Fixing test Signed-off-by: Amit Schendel --- tests/component_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/component_test.go b/tests/component_test.go index 876f16c8..2e0ccf0c 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -776,7 +776,17 @@ func Test_12_CooldownTest(t *testing.T) { t.Errorf("Error getting alerts: %v", err) } - if len(alerts) != 5 { - t.Errorf("Expected 5 alerts to be generated, but got %d alerts", len(alerts)) + ldpreloadAlertsCount := 0 + for _, alert := range alerts { + ruleName, ruleOk := alert.Labels["rule_name"] + if ruleOk { + if ruleName == "LD_PRELOAD Hook" { + ldpreloadAlertsCount++ + } + } + } + + if ldpreloadAlertsCount != 5 { + t.Errorf("Expected 5 alerts to be generated, but got %d alerts", ldpreloadAlertsCount) } } From fa9daef2fa47ad05e74b37841e874fe732b94b55 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 17:04:44 +0000 Subject: [PATCH 5/8] Fixing test Signed-off-by: Amit Schendel --- tests/chart/crds/runtime-rule-binding.crd.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/chart/crds/runtime-rule-binding.crd.yaml b/tests/chart/crds/runtime-rule-binding.crd.yaml index 67de8f5e..d2a111f7 100644 --- a/tests/chart/crds/runtime-rule-binding.crd.yaml +++ b/tests/chart/crds/runtime-rule-binding.crd.yaml @@ -108,6 +108,8 @@ spec: - R1007 - R1008 - R1009 + - R1010 + - R1011 type: string ruleName: enum: @@ -128,6 +130,7 @@ spec: - XMR Crypto Mining Detection - Crypto Mining Domain Communication - Crypto Mining Related Port Communication + - LD_PRELOAD Hook type: string ruleTags: items: From 2d7373f9ac1271e49723a2b4aaf34fe1c03af4e5 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 17:27:50 +0000 Subject: [PATCH 6/8] Changing test Signed-off-by: Amit Schendel --- .../chart/crds/runtime-rule-binding.crd.yaml | 1 + tests/component_test.go | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/chart/crds/runtime-rule-binding.crd.yaml b/tests/chart/crds/runtime-rule-binding.crd.yaml index d2a111f7..a3fbd2ba 100644 --- a/tests/chart/crds/runtime-rule-binding.crd.yaml +++ b/tests/chart/crds/runtime-rule-binding.crd.yaml @@ -131,6 +131,7 @@ spec: - Crypto Mining Domain Communication - Crypto Mining Related Port Communication - LD_PRELOAD Hook + - Unexpected Sensitive File Access type: string ruleTags: items: diff --git a/tests/component_test.go b/tests/component_test.go index 2e0ccf0c..c7251cdf 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -730,10 +730,7 @@ func Test_12_CooldownTest(t *testing.T) { assert.NoError(t, wl.WaitForNetworkNeighborhood(80, "ready")) // process launched from nginx container - _, _, err = wl.ExecIntoPod([]string{"ls", "-l"}, "nginx") - - // network activity from server container - _, _, err = wl.ExecIntoPod([]string{"wget", "ebpf.io", "-T", "2", "-t", "1"}, "server") + _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/hosts"}, "nginx") // network activity from nginx container _, _, err = wl.ExecIntoPod([]string{"curl", "kubernetes.io", "-m", "2"}, "nginx") @@ -754,18 +751,19 @@ func Test_12_CooldownTest(t *testing.T) { t.Logf("application profile: %v", string(appProfileJson)) - // The cooldown for LD_PRELOAD is: + // The cooldown for Unexpected Sensitive File Access is: // cooldown.CooldownConfig{ // Threshold: 5, // AlertWindow: time.Minute * 1, - // BaseCooldown: time.Minute * 5, - // MaxCooldown: time.Minute * 10, + // BaseCooldown: time.Second * 30, + // MaxCooldown: time.Minute * 5, // CooldownIncrease: 1.5, + // } - // So we want to exec into the pod and run 6 times a binary with the LD_PRELOAD feature to trigger the cooldown. + // So we want to exec into the pod and run cat 6 times on a sensitive file to trigger the cooldown. // We expect to see 5 alerts and the 6th one to be ignored. for i := 0; i < 6; i++ { - _, _, err = wl.ExecIntoPod([]string{"LD_PRELOAD=/lib/libc.so", "ls", "-l"}, "nginx") + _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/passwd"}, "nginx") } // Wait for the alert to be signaled @@ -776,17 +774,17 @@ func Test_12_CooldownTest(t *testing.T) { t.Errorf("Error getting alerts: %v", err) } - ldpreloadAlertsCount := 0 + alertsCount := 0 for _, alert := range alerts { ruleName, ruleOk := alert.Labels["rule_name"] if ruleOk { - if ruleName == "LD_PRELOAD Hook" { - ldpreloadAlertsCount++ + if ruleName == "Unexpected Sensitive File Access" { + alertsCount++ } } } - if ldpreloadAlertsCount != 5 { - t.Errorf("Expected 5 alerts to be generated, but got %d alerts", ldpreloadAlertsCount) + if alertsCount != 5 { + t.Errorf("Expected 5 alerts to be generated, but got %d alerts", alertsCount) } } From 87ba95d85809254d7217e42e0d3b12653f452e47 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Thu, 19 Sep 2024 17:47:39 +0000 Subject: [PATCH 7/8] Fixing test Signed-off-by: Amit Schendel --- tests/component_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/component_test.go b/tests/component_test.go index c7251cdf..f45f49cf 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -720,7 +720,7 @@ func Test_12_CooldownTest(t *testing.T) { defer tearDownTest(t, start) ns := testutils.NewRandomNamespace() - wl, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/deployment-multiple-containers.yaml")) + wl, err := testutils.NewTestWorkload(ns.Name, path.Join(utils.CurrentDir(), "resources/nginx-deployment.yaml")) if err != nil { t.Errorf("Error creating workload: %v", err) } @@ -730,10 +730,7 @@ func Test_12_CooldownTest(t *testing.T) { assert.NoError(t, wl.WaitForNetworkNeighborhood(80, "ready")) // process launched from nginx container - _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/hosts"}, "nginx") - - // network activity from nginx container - _, _, err = wl.ExecIntoPod([]string{"curl", "kubernetes.io", "-m", "2"}, "nginx") + _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/hosts"}, "") err = wl.WaitForApplicationProfileCompletion(80) if err != nil { @@ -763,7 +760,10 @@ func Test_12_CooldownTest(t *testing.T) { // So we want to exec into the pod and run cat 6 times on a sensitive file to trigger the cooldown. // We expect to see 5 alerts and the 6th one to be ignored. for i := 0; i < 6; i++ { - _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/passwd"}, "nginx") + _, _, err = wl.ExecIntoPod([]string{"cat", "/etc/passwd"}, "") + if err != nil { + t.Errorf("Error executing remote command: %v", err) + } } // Wait for the alert to be signaled @@ -774,6 +774,8 @@ func Test_12_CooldownTest(t *testing.T) { t.Errorf("Error getting alerts: %v", err) } + t.Logf("alerts: %v", alerts) + alertsCount := 0 for _, alert := range alerts { ruleName, ruleOk := alert.Labels["rule_name"] From 428dfe770aa5791ce4a660421e4cb8cfc98d3c26 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Fri, 20 Sep 2024 15:37:29 +0000 Subject: [PATCH 8/8] Adding things Signed-off-by: Amit Schendel --- pkg/cooldown/cooldown.go | 48 ++++++++--------- pkg/cooldown/cooldown_test.go | 82 ++++++++++++++---------------- pkg/rulemanager/v1/rule_manager.go | 2 +- 3 files changed, 64 insertions(+), 68 deletions(-) diff --git a/pkg/cooldown/cooldown.go b/pkg/cooldown/cooldown.go index d8cfc34e..131842d9 100644 --- a/pkg/cooldown/cooldown.go +++ b/pkg/cooldown/cooldown.go @@ -47,15 +47,8 @@ func NewCooldown(config CooldownConfig) *Cooldown { // ConfigureCooldown sets up or updates the cooldown configuration for a specific alert func (cm *CooldownManager) ConfigureCooldown(alertID string, config CooldownConfig) { - if cm.cooldowns.Has(alertID) { - cooldown := cm.cooldowns.Get(alertID) - cooldown.mu.Lock() - cooldown.config = config - cooldown.currentCooldown = config.BaseCooldown - cooldown.mu.Unlock() - } else { - cm.cooldowns.Set(alertID, NewCooldown(config)) - } + cooldown := NewCooldown(config) + cm.cooldowns.Set(alertID, cooldown) } // ShouldAlert determines if an alert should be triggered based on the cooldown mechanism @@ -85,29 +78,36 @@ func (c *Cooldown) shouldAlert() bool { } } - // Check if we're still in the cooldown period - if now.Sub(c.lastAlertTime) < c.currentCooldown { - return false + // If we're below the threshold, always allow the alert + if c.alertTimes.Len() < c.config.Threshold { + c.alertTimes.PushBack(now) + c.lastAlertTime = now + return true } - // Add current alert time - c.alertTimes.PushBack(now) - - // If we've exceeded the threshold, increase the cooldown - if c.alertTimes.Len() > c.config.Threshold { - c.currentCooldown = time.Duration(float64(c.currentCooldown) * c.config.CooldownIncrease) + // If we're at the threshold, allow the alert but increase the cooldown + if c.alertTimes.Len() == c.config.Threshold { + c.alertTimes.PushBack(now) + c.lastAlertTime = now + c.currentCooldown = time.Duration(float64(c.config.BaseCooldown) * c.config.CooldownIncrease) if c.currentCooldown > c.config.MaxCooldown { c.currentCooldown = c.config.MaxCooldown } - } else if c.alertTimes.Len() <= c.config.Threshold/2 { - // If we're below half the threshold, start decreasing the cooldown - c.currentCooldown = time.Duration(float64(c.currentCooldown) / c.config.CooldownIncrease) - if c.currentCooldown < c.config.BaseCooldown { - c.currentCooldown = c.config.BaseCooldown - } + return true } + // If we've exceeded the threshold, check if we're still in the cooldown period + if now.Sub(c.lastAlertTime) < c.currentCooldown { + return false + } + + // We're past the cooldown period, allow the alert and increase the cooldown further + c.alertTimes.PushBack(now) c.lastAlertTime = now + c.currentCooldown = time.Duration(float64(c.currentCooldown) * c.config.CooldownIncrease) + if c.currentCooldown > c.config.MaxCooldown { + c.currentCooldown = c.config.MaxCooldown + } return true } diff --git a/pkg/cooldown/cooldown_test.go b/pkg/cooldown/cooldown_test.go index b65a99c4..4464e639 100644 --- a/pkg/cooldown/cooldown_test.go +++ b/pkg/cooldown/cooldown_test.go @@ -1,6 +1,7 @@ package cooldown import ( + "fmt" "sync" "testing" "time" @@ -44,7 +45,7 @@ func TestConfigureCooldown(t *testing.T) { } } -func TestShouldAlert(t *testing.T) { +func TestComprehensiveShouldAlert(t *testing.T) { cm := NewCooldownManager() config := CooldownConfig{ Threshold: 3, @@ -55,43 +56,42 @@ func TestShouldAlert(t *testing.T) { } cm.ConfigureCooldown("test-alert", config) - // First alert should always be allowed - if !cm.ShouldAlert("test-alert") { - t.Error("First alert was not allowed") - } - - // Second alert within BaseCooldown should not be allowed - time.Sleep(5 * time.Millisecond) - if cm.ShouldAlert("test-alert") { - t.Error("Second alert within BaseCooldown was allowed") - } - - // Alert after BaseCooldown should be allowed - time.Sleep(6 * time.Millisecond) - if !cm.ShouldAlert("test-alert") { - t.Error("Alert after BaseCooldown was not allowed") - } - - // Trigger alerts to exceed threshold - for i := 0; i < 3; i++ { - time.Sleep(11 * time.Millisecond) - cm.ShouldAlert("test-alert") - } - - // Next alert should not be allowed due to increased cooldown - if cm.ShouldAlert("test-alert") { - t.Error("Alert was allowed immediately after exceeding threshold") - } - - // Wait for increased cooldown (2 * BaseCooldown) - time.Sleep(21 * time.Millisecond) - if !cm.ShouldAlert("test-alert") { - t.Error("Alert was not allowed after increased cooldown period") - } - - // Alert for unconfigured alert ID should always be allowed - if !cm.ShouldAlert("unconfigured-alert") { - t.Error("Alert for unconfigured alert ID was not allowed") + fmt.Println("Starting comprehensive cooldown test...") + fmt.Printf("Config: Threshold=%d, AlertWindow=%v, BaseCooldown=%v, MaxCooldown=%v, CooldownIncrease=%.1f\n\n", + config.Threshold, config.AlertWindow, config.BaseCooldown, config.MaxCooldown, config.CooldownIncrease) + + testCases := []struct { + name string + delay time.Duration + expected bool + }{ + {"First alert", 0, true}, + {"Second alert (immediate)", 0, true}, + {"Third alert (immediate)", 0, true}, + {"Fourth alert (immediate, should increase cooldown)", 0, true}, + {"Fifth alert (immediate, should be blocked)", 0, false}, + {"Sixth alert (after base cooldown)", config.BaseCooldown, false}, + {"Seventh alert (after increased cooldown)", config.BaseCooldown * 2, true}, + {"Eighth alert (immediate after cooldown)", 0, false}, + {"Ninth alert (after alert window)", config.AlertWindow, true}, + {"Tenth alert (immediate)", 0, true}, + {"Eleventh alert (immediate)", 0, true}, + } + + startTime := time.Now() + + for i, tc := range testCases { + time.Sleep(tc.delay) + result := cm.ShouldAlert("test-alert") + elapsed := time.Since(startTime) + + cooldown := cm.cooldowns.Get("test-alert") + fmt.Printf("%d. %s (at %v):\n Expected: %v, Got: %v\n Alert Count: %d, Current Cooldown: %v\n", + i+1, tc.name, elapsed.Round(time.Millisecond), tc.expected, result, cooldown.alertTimes.Len(), cooldown.currentCooldown) + + if result != tc.expected { + t.Errorf("%s: expected %v, got %v", tc.name, tc.expected, result) + } } } @@ -109,7 +109,6 @@ func TestResetCooldown(t *testing.T) { // Trigger alerts to increase cooldown for i := 0; i < 4; i++ { cm.ShouldAlert("test-alert") - time.Sleep(11 * time.Millisecond) } // Verify that cooldown is in effect @@ -145,18 +144,16 @@ func TestCooldownIncrease(t *testing.T) { // Trigger alerts to increase cooldown for i := 0; i < 3; i++ { - time.Sleep(11 * time.Millisecond) cm.ShouldAlert("test-alert") } // Next alert should be blocked due to increased cooldown - time.Sleep(11 * time.Millisecond) if cm.ShouldAlert("test-alert") { t.Error("Alert was allowed despite increased cooldown") } // Wait for increased cooldown and alert should be allowed - time.Sleep(11 * time.Millisecond) + time.Sleep(21 * time.Millisecond) if !cm.ShouldAlert("test-alert") { t.Error("Alert was not allowed after increased cooldown period") } @@ -175,7 +172,6 @@ func TestCooldownDecrease(t *testing.T) { // Trigger alerts to increase cooldown for i := 0; i < 5; i++ { - time.Sleep(11 * time.Millisecond) cm.ShouldAlert("test-alert") } diff --git a/pkg/rulemanager/v1/rule_manager.go b/pkg/rulemanager/v1/rule_manager.go index 32d83ea4..2e1615ab 100644 --- a/pkg/rulemanager/v1/rule_manager.go +++ b/pkg/rulemanager/v1/rule_manager.go @@ -359,7 +359,7 @@ func (rm *RuleManager) processEvent(eventType utils.EventType, event utils.K8sEv rm.exporter.SendRuleAlert(res) rm.metrics.ReportRuleAlert(rule.Name()) } else { - logger.L().Debug("RuleManager - rule is in cooldown for identifier", helpers.String("identifier", res.GetFailureIdentifier())) + logger.L().Info("RuleManager - rule is in cooldown for identifier", helpers.String("identifier", res.GetFailureIdentifier())) } } rm.metrics.ReportRuleProcessed(rule.Name())