Skip to content

Commit

Permalink
Common: Upgrade yauaa to 7.x (close #639)
Browse files Browse the repository at this point in the history
  • Loading branch information
@oguzhanunlu @istreeter
oguzhanunlu authored and istreeter committed Nov 8, 2022
1 parent 187524a commit 66dbeea
Show file tree
Hide file tree
Showing 8 changed files with 448 additions and 65 deletions.
70 changes: 36 additions & 34 deletions .../nl.basjes/yauaa_context/jsonschema/1-0-3 → .../nl.basjes/yauaa_context/jsonschema/1-0-4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,129 +5,131 @@
"vendor": "nl.basjes",
"name": "yauaa_context",
"format": "jsonschema",
"version": "1-0-3"
"version": "1-0-4"
},
"type": "object",
"properties": {
"deviceClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Handheld Game Console", "Voice", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"]
"enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Home Appliance", "Handheld Game Console", "Voice", "Car", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"]
},
"deviceName": {
"description": "Example: Google Nexus 6",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"deviceBrand": {
"description": "Example: Google",
"type": "string",
"maxLength": 50
"maxLength": 128
},
"deviceCpu": {
"type": "string",
"maxLength": 50
"maxLength": 128
},
"deviceCpuBits": {
"type": "string",
"maxLength": 20
"maxLength": 128
},
"deviceFirmwareVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"deviceVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"operatingSystemClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Desktop", "Mobile", "Cloud", "Embedded", "Game Console", "Hacker", "Anonymized", "Unknown"]
},
"operatingSystemName": {
"description": "Examples: Linux, Android.",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"operatingSystemVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"operatingSystemNameVersion": {
"type": "string",
"maxLength": 150
"maxLength": 1000
},
"operatingSystemVersionBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"layoutEngineClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Browser", "Mobile App", "Hacker", "Robot", "Unknown", "Special", "Cloud", "eReader"]
},
"layoutEngineName": {
"type": "string",
"maxLength": 100
"maxLength": 256
},
"layoutEngineVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"layoutEngineVersionMajor": {
"type": "string",
"maxLength": 20
"maxLength": 1000
},
"layoutEngineNameVersion": {
"type": "string",
"maxLength": 150
"maxLength": 1000
},
"layoutEngineNameVersionMajor": {
"type": "string",
"maxLength": 120
"maxLength": 1000
},
"layoutEngineBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentClass": {
"description": "See https://yauaa.basjes.nl/README-Output.html",
"type": "string",
"enum": ["Browser", "Browser Webview", "Mobile App", "Robot", "Robot Mobile", "Cloud Application", "Email Client", "Voice", "Special", "Testclient", "Hacker", "Unknown", "Desktop App", "eReader"]
},
"agentName": {
"description": "Example: Chrome.",
"type": "string",
"maxLength": 100
"maxLength": 256
},
"agentVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentVersionMajor": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentNameVersion": {
"type": "string",
"maxLength": 200
"maxLength": 1000
},
"agentNameVersionMajor": {
"type": "string",
"maxLength": 120
"maxLength": 1000
},
"agentBuild": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"agentLanguage": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"agentLanguageCode": {
"type": "string",
"maxLength": 20
"maxLength": 1000
},
"agentInformationEmail": {
"type": "string",
"format": "email"
"type": "string"
},
"agentInformationUrl": {
"type": "string"
Expand All @@ -147,11 +149,11 @@
},
"webviewAppVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"webviewAppNameVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"facebookCarrier": {
"type": "string"
Expand Down Expand Up @@ -196,19 +198,19 @@
},
"iECompatibilityVersion": {
"type": "string",
"maxLength": 100
"maxLength": 1000
},
"iECompatibilityVersionMajor": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"iECompatibilityNameVersion": {
"type": "string",
"maxLength": 50
"maxLength": 1000
},
"iECompatibilityNameVersionMajor": {
"type": "string",
"maxLength": 70
"maxLength": 1000
},
"carrier": {
"type": "string"
Expand Down
54 changes: 52 additions & 2 deletions ...owplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,65 @@ class YauaaEnrichmentSpec extends Specification with CatsIO {
val input = BlackBoxTesting.buildCollectorPayload(
path = "/i",
querystring = "e=pp".some,
userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0".some
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36".some,
headers = List(
"""Sec-CH-UA: "Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"""",
"""Sec-CH-UA-Arch: "x86"""",
"""Sec-CH-UA-Bitness: "64"""",
"""Sec-CH-UA-Full-Version: "106.0.5249.119"""",
"""Sec-CH-UA-Full-Version-List: "Chromium";v="106.0.5249.119", "Google Chrome";v="106.0.5249.119", "Not;A=Brand";v="99.0.0.0"""",
"""Sec-CH-UA-Mobile: ?0""",
"""Sec-CH-UA-Model: """"",
"""Sec-CH-UA-Platform: "Linux"""",
"""Sec-CH-ua-Platform-version: "6.0.1"""",
"""Sec-CH-UA-WoW64: ?0"""
)
)
val expected = Map(
"event_vendor" -> "com.snowplowanalytics.snowplow",
"event_name" -> "page_ping",
"event_format" -> "jsonschema",
"event_version" -> "1-0-0",
"event" -> "page_ping",
"derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-3","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces
"derived_contexts" -> json"""{
"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1",
"data":[{
"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-4",
"data": {
"deviceBrand":"Unknown",
"deviceName":"Linux Desktop",
"operatingSystemVersionMajor":"6",
"layoutEngineNameVersion":"Blink 106.0",
"operatingSystemNameVersion":"Linux 6.0.1",
"agentInformationEmail": "Unknown",
"networkType": "Unknown",
"operatingSystemVersionBuild":"??",
"webviewAppNameVersionMajor": "Unknown ??",
"layoutEngineNameVersionMajor":"Blink 106",
"operatingSystemName":"Linux",
"agentVersionMajor":"106",
"layoutEngineVersionMajor":"106",
"webviewAppName": "Unknown",
"deviceClass":"Desktop",
"agentNameVersionMajor":"Chrome 106",
"operatingSystemNameVersionMajor":"Linux 6",
"deviceCpuBits":"64",
"webviewAppVersionMajor": "??",
"operatingSystemClass":"Desktop",
"webviewAppVersion": "??",
"layoutEngineName":"Blink",
"agentName":"Chrome",
"agentVersion":"106.0.5249.119",
"layoutEngineClass":"Browser",
"agentNameVersion":"Chrome 106.0.5249.119",
"operatingSystemVersion":"6.0.1",
"deviceCpu":"Intel x86_64",
"agentClass":"Browser",
"layoutEngineVersion":"106.0",
"agentInformationUrl": "Unknown"
}
}]
}""".noSpaces
)
BlackBoxTesting.runTest(input, expected, Some(YauaaEnrichmentSpec.conf))
}
Expand Down
18 changes: 12 additions & 6 deletions ...la/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,23 +53,29 @@ class YauaaEnrichmentSpec extends Specification with CatsIO {
val expected = Contexts(
List(
SelfDescribingData(
SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 3)),
SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 4)),
json"""{
"deviceBrand" : "Apple",
"deviceName" : "Apple Macintosh",
"operatingSystemVersionMajor" : "10",
"operatingSystemVersionMajor" : "10.14",
"layoutEngineNameVersion" : "Gecko 81.0",
"operatingSystemNameVersion" : "Mac OS X 10.14",
"operatingSystemNameVersion" : "Mac OS 10.14",
"agentInformationEmail": "Unknown",
"networkType": "Unknown",
"layoutEngineBuild" : "20100101",
"webviewAppNameVersionMajor": "Unknown ??",
"layoutEngineNameVersionMajor" : "Gecko 81",
"operatingSystemName" : "Mac OS X",
"operatingSystemName" : "Mac OS",
"agentVersionMajor" : "81",
"layoutEngineVersionMajor" : "81",
"webviewAppName": "Unknown",
"deviceClass" : "Desktop",
"agentNameVersionMajor" : "Firefox 81",
"operatingSystemNameVersionMajor" : "Mac OS X 10",
"deviceCpuBits" : "32",
"operatingSystemNameVersionMajor" : "Mac OS 10.14",
"deviceCpuBits" : "64",
"webviewAppVersionMajor": "??",
"operatingSystemClass" : "Desktop",
"webviewAppVersion": "??",
"layoutEngineName" : "Gecko",
"agentName" : "Firefox",
"agentVersion" : "81.0",
Expand Down
8 changes: 4 additions & 4 deletions ...in/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ object EnrichmentManager {
_ <- setEventFingerprint[F](raw.parameters, registry.eventFingerprint) // This enrichment cannot fail
_ <- getCookieContexts // Execute cookie extractor enrichment
_ <- getHttpHeaderContexts // Execute header extractor enrichment
_ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version
_ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment
_ <- getCurrency[F](raw.context.timestamp, registry.currencyConversion) // Finalize the currency conversion
Expand Down Expand Up @@ -237,7 +237,7 @@ object EnrichmentManager {
_ <- getCookieContexts // Execute cookie extractor enrichment
_ <- getHttpHeaderContexts // Execute header extractor enrichment
_ <- getWeatherContext[F](registry.weather) // Fetch weather context
_ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent)
_ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version
_ <- geoLocation[F](registry.ipLookups) // Execute IP lookup enrichment
_ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment
Expand Down Expand Up @@ -693,10 +693,10 @@ object EnrichmentManager {
}
}

def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment]): EStateT[F, Unit] =
def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment], headers: List[String]): EStateT[F, Unit] =
EStateT.fromEither {
case (event, _) =>
yauaa.map(_.getYauaaContext(event.useragent)).toList.asRight
yauaa.map(_.getYauaaContext(event.useragent, headers)).toList.asRight
}

// Derive some contexts with custom SQL Query enrichment
Expand Down
Loading

0 comments on commit 66dbeea

Please sign in to comment.