Update metrics: appsec.waf.updates and appsec.waf.init

dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java

@@ -245,12 +245,6 @@ private void initializeNewWafCtx(
         currentRulesVersion = initReport.rulesetVersion;
       }
 
-      if (prevContextAndAddresses == null) {
-        WafMetricCollector.get().wafInit(Powerwaf.LIB_VERSION, currentRulesVersion);
-      } else {
-        WafMetricCollector.get().wafUpdates(currentRulesVersion);
-      }
-
       if (initReport != null) {
         log.info(
             "Created {} WAF context with rules ({} OK, {} BAD), version {}",
@@ -273,11 +267,13 @@ private void initializeNewWafCtx(
       }
     } catch (InvalidRuleSetException irse) {
       initReport = irse.ruleSetInfo;
+      sendWafMetrics(prevContextAndAddresses, false);
       throw new AppSecModuleActivationException("Error creating WAF rules", irse);
     } catch (RuntimeException | AbstractPowerwafException e) {
       if (newPwafCtx != null) {
         newPwafCtx.close();
       }
+      sendWafMetrics(prevContextAndAddresses, false);
       throw new AppSecModuleActivationException("Error creating WAF rules", e);
     } finally {
       if (initReport != null) {
@@ -287,16 +283,27 @@ private void initializeNewWafCtx(
 
     if (!this.ctxAndAddresses.compareAndSet(prevContextAndAddresses, newContextAndAddresses)) {
       newPwafCtx.close();
+      sendWafMetrics(prevContextAndAddresses, false);
       throw new AppSecModuleActivationException("Concurrent update of WAF configuration");
     }
 
+    sendWafMetrics(prevContextAndAddresses, true);
+
     if (prevContextAndAddresses != null) {
       prevContextAndAddresses.ctx.close();
     }
 
     reconf.reloadSubscriptions();
   }
 
+  private void sendWafMetrics(CtxAndAddresses prevContextAndAddresses, boolean success) {
+    if (prevContextAndAddresses == null) {
+      WafMetricCollector.get().wafInit(Powerwaf.LIB_VERSION, currentRulesVersion, success);
+    } else {
+      WafMetricCollector.get().wafUpdates(currentRulesVersion, success);
+    }
+  }
+
   private Map<String, ActionInfo> calculateEffectiveActions(
       CtxAndAddresses prevContextAndAddresses, AppSecConfig ruleConfig) {
     Map<String, ActionInfo> actionInfoMap;

dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/powerwaf/PowerWAFModuleSpecification.groovy

@@ -69,7 +69,10 @@ class PowerWAFModuleSpecification extends DDSpecification {
   Additive pwafAdditive
   PowerwafMetrics metrics
 
+  WafMetricCollector wafMetricCollector = Mock(WafMetricCollector)
+
   void setup() {
+    WafMetricCollector.INSTANCE = wafMetricCollector
     AgentTracer.forceRegister(tracer)
   }
 
@@ -205,6 +208,8 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * flow.setAction({ Flow.Action.RequestBlockingAction rba ->
       rba.statusCode == 501 &&
@@ -240,6 +245,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
     1 * flow.setAction({ Flow.Action.RequestBlockingAction rba ->
       rba.statusCode == 403 &&
         rba.blockingContentType == BlockingContentType.AUTO
@@ -281,6 +287,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * flow.setAction({ Flow.Action.RequestBlockingAction rba ->
       rba.statusCode == 403 &&
@@ -363,6 +370,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * flow.setAction({ Flow.Action.RequestBlockingAction rba ->
       rba.statusCode == 403 &&
@@ -439,7 +447,10 @@ class PowerWAFModuleSpecification extends DDSpecification {
     }
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
+    0 * _
 
     when:
     dataListener.onDataAvailable(flow, ctx, ATTACK_BUNDLE, gwCtx)
@@ -521,7 +532,10 @@ class PowerWAFModuleSpecification extends DDSpecification {
     }
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
+    0 * _
 
     when:
     dataListener.onDataAvailable(flow, ctx, ATTACK_BUNDLE, gwCtx)
@@ -596,7 +610,10 @@ class PowerWAFModuleSpecification extends DDSpecification {
     }
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
+    2 * wafMetricCollector.wafUpdates(_, true)
     2 * reconf.reloadSubscriptions()
+    0 * _
 
     when:
     dataListener.onDataAvailable(flow, ctx, ATTACK_BUNDLE, gwCtx)
@@ -971,9 +988,6 @@ class PowerWAFModuleSpecification extends DDSpecification {
     TraceSegment segment = Mock()
     TraceSegmentPostProcessor pp = service.traceSegmentPostProcessors.last()
 
-    def mockWafMetricCollector = Mock(WafMetricCollector)
-    WafMetricCollector.INSTANCE = mockWafMetricCollector
-
     when:
     dataListener.onDataAvailable(flow, ctx, db, gwCtx)
 
@@ -982,7 +996,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
       pwafAdditive = it[0].openAdditive() }
     assert !flow.blocking
     1 * ctx.increaseTimeouts()
-    1 * mockWafMetricCollector.get().wafRequestTimeout()
+    1 * wafMetricCollector.get().wafRequestTimeout()
 
     when:
     pp.processTraceSegment(segment, ctx, [])
@@ -1006,6 +1020,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
 
     then:
     thrown AppSecModule.AppSecModuleActivationException
+    0 * _
 
     when:
     cfgService.listeners['waf'].onNewSubconfig(defaultConfig['waf'], reconf)
@@ -1017,7 +1032,14 @@ class PowerWAFModuleSpecification extends DDSpecification {
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
       pwafAdditive = it[0].openAdditive() }
     1 * ctx.reportEvents(_ as Collection<AppSecEvent>)
+    1 * ctx.isAdditiveClosed()
+    1 * ctx.getWafMetrics()
+    1 * ctx.isThrottled(null)
+    1 * ctx.closeAdditive()
+    2 * tracer.activeSpan()
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
     1 * reconf.reloadSubscriptions()
+    0 * _
   }
 
   void 'rule data given through configuration'() {
@@ -1049,6 +1071,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * ctx.getOrCreateAdditive(_, true, false) >> { pwafAdditive = it[0].openAdditive() }
     2 * tracer.activeSpan()
@@ -1106,6 +1129,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then: 'no match; rule is disabled'
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
       pwafAdditive = it[0].openAdditive() }
@@ -1135,6 +1159,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     1 * ctx.getWafMetrics()
     1 * ctx.isAdditiveClosed() >> false
     1 * ctx.closeAdditive() >> {pwafAdditive.close()}
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     _ * ctx.increaseTimeouts()
     0 * _
@@ -1151,6 +1176,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then: 'now we have match'
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
       pwafAdditive = it[0].openAdditive() }
@@ -1179,6 +1205,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then: 'nothing again; we disabled the rule'
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * ctx.getOrCreateAdditive(_, true, false) >> { pwafAdditive = it[0].openAdditive() }
     1 * ctx.getWafMetrics()
@@ -1208,6 +1235,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     // no attack
     1 * ctx.getOrCreateAdditive(_, true, false) >> { pwafAdditive = it[0].openAdditive() }
@@ -1231,6 +1259,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     // no attack
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
@@ -1255,6 +1284,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     // attack found
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
@@ -1282,6 +1312,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     // no attack
     1 * ctx.getOrCreateAdditive(_, true, false) >> {
@@ -1329,7 +1360,9 @@ class PowerWAFModuleSpecification extends DDSpecification {
     pwafModule.config(cfgService)
 
     then:
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
     !pwafModule.dataSubscriptions.first().subscribedAddresses.contains(doesNotExistAddress)
+    0 * _
   }
 
   void 'bad initial configuration is given results in no subscriptions'() {
@@ -1342,6 +1375,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     then:
     thrown AppSecModule.AppSecModuleActivationException
     pwafModule.dataSubscriptions.empty
+    0 * _
   }
 
   void 'rule data not a config'() {
@@ -1353,9 +1387,8 @@ class PowerWAFModuleSpecification extends DDSpecification {
 
     then:
     thrown AppSecModule.AppSecModuleActivationException
-
-    then:
     pwafModule.ctxAndAddresses.get() == null
+    0 * _
   }
 
   void 'bad ResultWithData - empty list'() {
@@ -1493,7 +1526,18 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * ctx.isAdditiveClosed()
+    1 * ctx.getOrCreateAdditive(_ as PowerwafContext, true, false) >> {
+      pwafAdditive = it[0].openAdditive()
+    }
+    1 * ctx.getWafMetrics()
+    1 * ctx.isThrottled(null)
+    1 * ctx.reportEvents(_ as Collection<AppSecEvent>)
+    1 * ctx.closeAdditive()
+    2 * tracer.activeSpan()
+    1 * flow.isBlocking()
     0 * flow.setAction(_)
+    0 * _
 
     when:
     final ipData = new AppSecData(exclusion: [
@@ -1515,11 +1559,22 @@ class PowerWAFModuleSpecification extends DDSpecification {
     ctx.closeAdditive()
 
     then:
+    1 * wafMetricCollector.wafUpdates(_, true)
     1 * reconf.reloadSubscriptions()
     1 * flow.setAction({ Flow.Action.RequestBlockingAction rba ->
       rba.statusCode == 402 && rba.blockingContentType == BlockingContentType.AUTO
     })
+    1 * flow.isBlocking()
     1 * ctx.isAdditiveClosed() >> false
+    1 * ctx.getOrCreateAdditive(_ as PowerwafContext, true, false) >> {
+      pwafAdditive = it[0].openAdditive()
+    }
+    1 * ctx.getWafMetrics()
+    1 * ctx.isThrottled(null)
+    1 * ctx.reportEvents(_ as Collection<AppSecEvent>)
+    1 * ctx.closeAdditive()
+    2 * tracer.activeSpan()
+    0 * _
   }
 
   void 'http endpoint fingerprint support'() {
@@ -1607,6 +1662,7 @@ class PowerWAFModuleSpecification extends DDSpecification {
     then:
     1 * ctx.closeAdditive()
     1 * ctx.isAdditiveClosed() >> true
+    1 * wafMetricCollector.wafInit(Powerwaf.LIB_VERSION, _, true)
     0 * _
   }
 

internal-api/src/main/java/datadog/trace/api/telemetry/WafMetricCollector.java

@@ -55,16 +55,17 @@ private WafMetricCollector() {
    */
   private static String rulesVersion = "";
 
-  public void wafInit(final String wafVersion, final String rulesVersion) {
+  public void wafInit(final String wafVersion, final String rulesVersion, final boolean success) {
     WafMetricCollector.wafVersion = wafVersion;
     WafMetricCollector.rulesVersion = rulesVersion;
     rawMetricsQueue.offer(
-        new WafInitRawMetric(wafInitCounter.incrementAndGet(), wafVersion, rulesVersion));
+        new WafInitRawMetric(wafInitCounter.incrementAndGet(), wafVersion, rulesVersion, success));
   }
 
-  public void wafUpdates(final String rulesVersion) {
+  public void wafUpdates(final String rulesVersion, final boolean success) {
     rawMetricsQueue.offer(
-        new WafUpdatesRawMetric(wafUpdatesCounter.incrementAndGet(), wafVersion, rulesVersion));
+        new WafUpdatesRawMetric(
+            wafUpdatesCounter.incrementAndGet(), wafVersion, rulesVersion, success));
 
     // Flush request metrics to get the new version.
     if (rulesVersion != null
@@ -249,20 +250,31 @@ public WafMetric(String metricName, long counter, String... tags) {
 
   public static class WafInitRawMetric extends WafMetric {
     public WafInitRawMetric(
-        final long counter, final String wafVersion, final String rulesVersion) {
+        final long counter,
+        final String wafVersion,
+        final String rulesVersion,
+        final boolean success) {
       super(
-          "waf.init", counter, "waf_version:" + wafVersion, "event_rules_version:" + rulesVersion);
+          "waf.init",
+          counter,
+          "waf_version:" + wafVersion,
+          "event_rules_version:" + rulesVersion,
+          "success:" + success);
     }
   }
 
   public static class WafUpdatesRawMetric extends WafMetric {
     public WafUpdatesRawMetric(
-        final long counter, final String wafVersion, final String rulesVersion) {
+        final long counter,
+        final String wafVersion,
+        final String rulesVersion,
+        final boolean success) {
       super(
           "waf.updates",
           counter,
           "waf_version:" + wafVersion,
-          "event_rules_version:" + rulesVersion);
+          "event_rules_version:" + rulesVersion,
+          "success:" + success);
     }
   }