Skip to content

Commit

Permalink
Set data field of incomplete event failure entity to SDJ's data field…
Browse files Browse the repository at this point in the history 
… only

At the moment, we set all self describing json to data field of incomplete event failure entity.
However, failure entity has separate schema field and therefore we don't need to set all SDJ to data field.
It is enough to use only data field of SDJ.
  • Loading branch information
@spenes
spenes committed Apr 19, 2024
1 parent ebe793f commit f5d611a
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 50 deletions.
8 changes: 4 additions & 4 deletions .../common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ object IgluUtils {
Failure.SchemaViolation(
schemaViolation = FailureDetails.SchemaViolation.IgluError(sdj.schema, clientError),
source = "derived_contexts",
data = sdj.asJson
data = sdj.data
)
}
)
Expand Down Expand Up @@ -201,7 +201,7 @@ object IgluUtils {
Failure.SchemaViolation(
schemaViolation = FailureDetails.SchemaViolation.CriterionMismatch(sdj.schema, expectedCriterion),
source = field,
data = sdj.asJson
data = sdj.data
)
)
// Check that the SDJ holding the .data is valid
Expand All @@ -211,7 +211,7 @@ object IgluUtils {
Failure.SchemaViolation(
schemaViolation = FailureDetails.SchemaViolation.IgluError(schemaKey, clientError),
source = field,
data = sdj.asJson
data = sdj.data
)
}
// Extract .data of SelfDescribingData[Json]
Expand Down Expand Up @@ -276,7 +276,7 @@ object IgluUtils {
schemaViolation = FailureDetails.SchemaViolation
.IgluError(schemaKey, clientError): FailureDetails.SchemaViolation,
source = field,
data = json.asJson
data = sdj.data
)
}
.leftMap(NonEmptyList.one)
Expand Down
60 changes: 35 additions & 25 deletions ...cala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1241,14 +1241,16 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
JavascriptScriptEnrichment(schemaKey, script)
)
)
val invalidUe =
val invalidUeData =
"""{
"emailAddress": "hello@world.com",
"emailAddress2": "foo@bar.org",
"unallowedAdditionalField": "foo@bar.org"
}"""
val invalidUe =
s"""{
"schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
"data": {
"emailAddress": "hello@world.com",
"emailAddress2": "foo@bar.org",
"unallowedAdditionalField": "foo@bar.org"
}
"data": $invalidUeData
}"""

val parameters = Map(
Expand Down Expand Up @@ -1287,7 +1289,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
)
) &&
feJson.field("schema") == emailSentSchema.asJson &&
feJson.field("data") == jparse(invalidUe).toOption.get =>
feJson.field("data") == jparse(invalidUeData).toOption.get =>
true
case _ => false
}
Expand Down Expand Up @@ -1333,13 +1335,15 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
JavascriptScriptEnrichment(schemaKey, script)
)
)
val invalidContext =
val invalidContextData =
"""{
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}"""
val invalidContext =
s"""{
"schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
"data": {
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}
"data": $invalidContextData
}"""

val parameters = Map(
Expand Down Expand Up @@ -1383,7 +1387,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
)
) &&
feJson.field("schema") == emailSentSchema.asJson &&
feJson.field("data") == jparse(invalidContext).toOption.get =>
feJson.field("data") == jparse(invalidContextData).toOption.get =>
true
case _ => false
}
Expand Down Expand Up @@ -1428,14 +1432,17 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
JavascriptScriptEnrichment(schemaKey, script)
)
)
val invalidContext =
val invalidContextData =
"""
{
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}"""
val invalidContext =
s"""
{
"schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
"data": {
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}
"data": $invalidContextData
}"""

val parameters = Map(
Expand Down Expand Up @@ -1482,7 +1489,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
)
) &&
feJson.field("schema") == emailSentSchema.asJson &&
feJson.field("data") == jparse(invalidContext).toOption.get =>
feJson.field("data") == jparse(invalidContextData).toOption.get =>
true
case _ => false
}
Expand Down Expand Up @@ -1662,14 +1669,17 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
}

"remove an invalid enrichment context and return the enriched event if emitIncomplete is set to true" >> {
val invalidContext =
val invalidContextData =
"""
{
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}"""
val invalidContext =
s"""
{
"schema":"iglu:com.acme/email_sent/jsonschema/1-0-0",
"data": {
"foo": "hello@world.com",
"emailAddress2": "foo@bar.org"
}
"data": $invalidContextData
}"""
val script =
s"""
Expand Down Expand Up @@ -1727,7 +1737,7 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE
)
) &&
feJson.field("schema") == emailSentSchema.asJson &&
feJson.field("data") == jparse(invalidContext).toOption.get =>
feJson.field("data") == jparse(invalidContextData).toOption.get =>
true
case _ => false
}
Expand Down
44 changes: 23 additions & 21 deletions ...mon/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,12 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
"emailAddress2": "foo2@bar.org"
}
}"""
val invalidEmailSentData = s"""{
"emailAddress": "hello@world.com"
}"""
val invalidEmailSent = s"""{
"schema": "${emailSentSchema.toSchemaUri}",
"data": {
"emailAddress": "hello@world.com"
}
"data": $invalidEmailSentData
}"""
val supersedingExample1 =
s"""{
Expand Down Expand Up @@ -137,8 +138,9 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
"userId": "20d631b8-7837-49df-a73e-6da73154e6fd"
}
}"""
val noSchemaData = "{}"
val noSchema =
"""{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": {}}"""
s"""{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": $noSchemaData}"""

"extractAndValidateUnstructEvent" should {
"return None if unstruct_event field is empty" >> {
Expand Down Expand Up @@ -190,7 +192,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
}

"return a FailureDetails.SchemaViolation.CriterionMismatch if unstruct_event contains a self-describing JSON but not with the expected schema for unstructured events" >> {
val json = noSchema.toJson
val json = noSchemaData.toJson
val input = new EnrichedEvent
input.setUnstruct_event(noSchema)

Expand Down Expand Up @@ -234,7 +236,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
"return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if the JSON in .data is not self-describing" >> {
val input = new EnrichedEvent
val ue = buildUnstruct(notIglu)
val ueJson = ue.toJson
val ueJson = notIglu.toJson
input.setUnstruct_event(ue)

IgluUtils
Expand All @@ -256,7 +258,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if the JSON in .data is not a valid SDJ" >> {
val input = new EnrichedEvent
val json = invalidEmailSent.toJson
val json = invalidEmailSentData.toJson
input.setUnstruct_event(buildUnstruct(invalidEmailSent))

IgluUtils
Expand All @@ -278,7 +280,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return a FailureDetails.SchemaViolation.IgluError containing a ResolutionError if the schema of the SDJ in .data can't be resolved" >> {
val input = new EnrichedEvent
val json = noSchema.toJson
val json = noSchemaData.toJson
input.setUnstruct_event(buildUnstruct(noSchema))

IgluUtils
Expand Down Expand Up @@ -402,7 +404,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return a FailureDetails.SchemaViolation.CriterionMismatch if .contexts contains a self-describing JSON but not with the right schema" >> {
val input = new EnrichedEvent
val json = noSchema.toJson
val json = noSchemaData.toJson
input.setContexts(noSchema)

IgluUtils
Expand All @@ -424,7 +426,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
val input = new EnrichedEvent
val notArrayContexts =
s"""{"schema": "${inputContextsSchema.toSchemaUri}", "data": ${emailSent1}}"""
val json = notArrayContexts.toJson
val json = emailSent1.toJson
input.setContexts(notArrayContexts)

IgluUtils
Expand All @@ -445,7 +447,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if .data contains one invalid context" >> {
val input = new EnrichedEvent
val json = invalidEmailSent.toJson
val json = invalidEmailSentData.toJson
input.setContexts(buildInputContexts(List(invalidEmailSent)))

IgluUtils
Expand All @@ -466,7 +468,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return a FailureDetails.SchemaViolation.IgluError containing a ResolutionError if .data contains one context whose schema can't be resolved" >> {
val input = new EnrichedEvent
val json = noSchema.toJson
val json = noSchemaData.toJson
input.setContexts(buildInputContexts(List(noSchema)))

IgluUtils
Expand All @@ -487,8 +489,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return 2 expected failures for 2 invalid contexts" >> {
val input = new EnrichedEvent
val invalidEmailSentJson = invalidEmailSent.toJson
val noSchemaJson = noSchema.toJson
val invalidEmailSentJson = invalidEmailSentData.toJson
val noSchemaJson = noSchemaData.toJson
input.setContexts(buildInputContexts(List(invalidEmailSent, noSchema)))

IgluUtils
Expand Down Expand Up @@ -516,7 +518,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return an expected failure and an expected SDJ if one context is invalid and one is valid" >> {
val input = new EnrichedEvent
val noSchemaJson = noSchema.toJson
val noSchemaJson = noSchemaData.toJson
input.setContexts(buildInputContexts(List(emailSent1, noSchema)))

IgluUtils
Expand Down Expand Up @@ -582,7 +584,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"validateEnrichmentsContexts" should {
"return one expected SchemaViolation for one invalid context" >> {
val json = invalidEmailSent.toJson
val json = invalidEmailSentData.toJson
val contexts = List(
SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get
)
Expand All @@ -606,8 +608,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
}

"return two expected SchemaViolation for two invalid contexts" >> {
val invalidEmailSentJson = invalidEmailSent.toJson
val noSchemaJson = noSchema.toJson
val invalidEmailSentJson = invalidEmailSentData.toJson
val noSchemaJson = noSchemaData.toJson
val contexts = List(
SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get,
SpecHelpers.jsonStringToSDJ(noSchema).right.get
Expand Down Expand Up @@ -637,7 +639,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect
}

"return one expected SchemaViolation for one invalid context and one valid" >> {
val invalidEmailSentJson = invalidEmailSent.toJson
val invalidEmailSentJson = invalidEmailSentData.toJson
val contexts = List(
SpecHelpers.jsonStringToSDJ(invalidEmailSent).right.get,
SpecHelpers.jsonStringToSDJ(emailSent1).right.get
Expand Down Expand Up @@ -778,7 +780,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return the SchemaViolation of the invalid context in the Left and the extracted unstructured event in the Right" >> {
val input = new EnrichedEvent
val invalidEmailSentJson = invalidEmailSent.toJson
val invalidEmailSentJson = invalidEmailSentData.toJson
input.setUnstruct_event(buildUnstruct(emailSent1))
input.setContexts(buildInputContexts(List(invalidEmailSent)))

Expand Down Expand Up @@ -809,7 +811,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect

"return the SchemaViolation of the invalid unstructured event in the Left and the valid context in the Right" >> {
val input = new EnrichedEvent
val invalidEmailSentJson = invalidEmailSent.toJson
val invalidEmailSentJson = invalidEmailSentData.toJson
input.setUnstruct_event(buildUnstruct(invalidEmailSent))
input.setContexts(buildInputContexts(List(emailSent1)))

Expand Down

0 comments on commit f5d611a

Please sign in to comment.