diff --git a/modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-3 b/modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 similarity index 82% rename from modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-3 rename to modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 index b3bc226ee..a6244f015 100644 --- a/modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-3 +++ b/modules/common-fs2/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 @@ -5,129 +5,131 @@ "vendor": "nl.basjes", "name": "yauaa_context", "format": "jsonschema", - "version": "1-0-3" + "version": "1-0-4" }, "type": "object", "properties": { "deviceClass": { "description": "See https://yauaa.basjes.nl/README-Output.html", - "enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Handheld Game Console", "Voice", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"] + "enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Home Appliance", "Handheld Game Console", "Voice", "Car", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"] }, "deviceName": { "description": "Example: Google Nexus 6", "type": "string", - "maxLength": 100 + "maxLength": 256 }, "deviceBrand": { "description": "Example: Google", "type": "string", - "maxLength": 50 + "maxLength": 128 }, "deviceCpu": { "type": "string", - "maxLength": 50 + "maxLength": 128 }, "deviceCpuBits": { "type": "string", - "maxLength": 20 + "maxLength": 128 }, "deviceFirmwareVersion": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "deviceVersion": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "operatingSystemClass": { "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", "enum": ["Desktop", "Mobile", "Cloud", "Embedded", "Game Console", "Hacker", "Anonymized", "Unknown"] }, "operatingSystemName": { "description": "Examples: Linux, Android.", "type": "string", - "maxLength": 100 + "maxLength": 256 }, "operatingSystemVersion": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "operatingSystemNameVersion": { "type": "string", - "maxLength": 150 + "maxLength": 1000 }, "operatingSystemVersionBuild": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "layoutEngineClass": { "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", "enum": ["Browser", "Mobile App", "Hacker", "Robot", "Unknown", "Special", "Cloud", "eReader"] }, "layoutEngineName": { "type": "string", - "maxLength": 100 + "maxLength": 256 }, "layoutEngineVersion": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "layoutEngineVersionMajor": { "type": "string", - "maxLength": 20 + "maxLength": 1000 }, "layoutEngineNameVersion": { "type": "string", - "maxLength": 150 + "maxLength": 1000 }, "layoutEngineNameVersionMajor": { "type": "string", - "maxLength": 120 + "maxLength": 1000 }, "layoutEngineBuild": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "agentClass": { "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", "enum": ["Browser", "Browser Webview", "Mobile App", "Robot", "Robot Mobile", "Cloud Application", "Email Client", "Voice", "Special", "Testclient", "Hacker", "Unknown", "Desktop App", "eReader"] }, "agentName": { "description": "Example: Chrome.", "type": "string", - "maxLength": 100 + "maxLength": 256 }, "agentVersion": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "agentVersionMajor": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "agentNameVersion": { "type": "string", - "maxLength": 200 + "maxLength": 1000 }, "agentNameVersionMajor": { "type": "string", - "maxLength": 120 + "maxLength": 1000 }, "agentBuild": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "agentLanguage": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "agentLanguageCode": { "type": "string", - "maxLength": 20 + "maxLength": 1000 }, "agentInformationEmail": { - "type": "string", - "format": "email" + "type": "string" }, "agentInformationUrl": { "type": "string" @@ -147,11 +149,11 @@ }, "webviewAppVersionMajor": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "webviewAppNameVersionMajor": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "facebookCarrier": { "type": "string" @@ -196,19 +198,19 @@ }, "iECompatibilityVersion": { "type": "string", - "maxLength": 100 + "maxLength": 1000 }, "iECompatibilityVersionMajor": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "iECompatibilityNameVersion": { "type": "string", - "maxLength": 50 + "maxLength": 1000 }, "iECompatibilityNameVersionMajor": { "type": "string", - "maxLength": 70 + "maxLength": 1000 }, "carrier": { "type": "string" diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala index c7be5b986..e717fa82d 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala @@ -28,7 +28,19 @@ class YauaaEnrichmentSpec extends Specification with CatsIO { val input = BlackBoxTesting.buildCollectorPayload( path = "/i", querystring = "e=pp".some, - userAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0".some + userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36".some, + headers = List( + """Sec-CH-UA: "Chromium";v="106", "Google Chrome";v="106", "Not;A=Brand";v="99"""", + """Sec-CH-UA-Arch: "x86"""", + """Sec-CH-UA-Bitness: "64"""", + """Sec-CH-UA-Full-Version: "106.0.5249.119"""", + """Sec-CH-UA-Full-Version-List: "Chromium";v="106.0.5249.119", "Google Chrome";v="106.0.5249.119", "Not;A=Brand";v="99.0.0.0"""", + """Sec-CH-UA-Mobile: ?0""", + """Sec-CH-UA-Model: """"", + """Sec-CH-UA-Platform: "Linux"""", + """Sec-CH-ua-Platform-version: "6.0.1"""", + """Sec-CH-UA-WoW64: ?0""" + ) ) val expected = Map( "event_vendor" -> "com.snowplowanalytics.snowplow", @@ -36,7 +48,45 @@ class YauaaEnrichmentSpec extends Specification with CatsIO { "event_format" -> "jsonschema", "event_version" -> "1-0-0", "event" -> "page_ping", - "derived_contexts" -> json"""{"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-3","data":{"deviceBrand":"Unknown","deviceName":"Desktop","operatingSystemVersionMajor":"7","layoutEngineNameVersion":"Gecko 12.0","operatingSystemNameVersion":"Windows 7","layoutEngineBuild":"20100101","layoutEngineNameVersionMajor":"Gecko 12","operatingSystemName":"Windows NT","agentVersionMajor":"12","layoutEngineVersionMajor":"12","deviceClass":"Desktop","agentNameVersionMajor":"Firefox 12","operatingSystemNameVersionMajor":"Windows 7","deviceCpuBits":"64","operatingSystemClass":"Desktop","layoutEngineName":"Gecko","agentName":"Firefox","agentVersion":"12.0","layoutEngineClass":"Browser","agentNameVersion":"Firefox 12.0","operatingSystemVersion":"7","deviceCpu":"Intel x86_64","agentClass":"Browser","layoutEngineVersion":"12.0"}}]}""".noSpaces + "derived_contexts" -> json"""{ + "schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1", + "data":[{ + "schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-4", + "data": { + "deviceBrand":"Unknown", + "deviceName":"Linux Desktop", + "operatingSystemVersionMajor":"6", + "layoutEngineNameVersion":"Blink 106.0", + "operatingSystemNameVersion":"Linux 6.0.1", + "agentInformationEmail": "Unknown", + "networkType": "Unknown", + "operatingSystemVersionBuild":"??", + "webviewAppNameVersionMajor": "Unknown ??", + "layoutEngineNameVersionMajor":"Blink 106", + "operatingSystemName":"Linux", + "agentVersionMajor":"106", + "layoutEngineVersionMajor":"106", + "webviewAppName": "Unknown", + "deviceClass":"Desktop", + "agentNameVersionMajor":"Chrome 106", + "operatingSystemNameVersionMajor":"Linux 6", + "deviceCpuBits":"64", + "webviewAppVersionMajor": "??", + "operatingSystemClass":"Desktop", + "webviewAppVersion": "??", + "layoutEngineName":"Blink", + "agentName":"Chrome", + "agentVersion":"106.0.5249.119", + "layoutEngineClass":"Browser", + "agentNameVersion":"Chrome 106.0.5249.119", + "operatingSystemVersion":"6.0.1", + "deviceCpu":"Intel x86_64", + "agentClass":"Browser", + "layoutEngineVersion":"106.0", + "agentInformationUrl": "Unknown" + } + }] + }""".noSpaces ) BlackBoxTesting.runTest(input, expected, Some(YauaaEnrichmentSpec.conf)) } diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala index 6f82199b7..86d566086 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala @@ -53,23 +53,29 @@ class YauaaEnrichmentSpec extends Specification with CatsIO { val expected = Contexts( List( SelfDescribingData( - SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 3)), + SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 4)), json"""{ "deviceBrand" : "Apple", "deviceName" : "Apple Macintosh", - "operatingSystemVersionMajor" : "10", + "operatingSystemVersionMajor" : "10.14", "layoutEngineNameVersion" : "Gecko 81.0", - "operatingSystemNameVersion" : "Mac OS X 10.14", + "operatingSystemNameVersion" : "Mac OS 10.14", + "agentInformationEmail": "Unknown", + "networkType": "Unknown", "layoutEngineBuild" : "20100101", + "webviewAppNameVersionMajor": "Unknown ??", "layoutEngineNameVersionMajor" : "Gecko 81", - "operatingSystemName" : "Mac OS X", + "operatingSystemName" : "Mac OS", "agentVersionMajor" : "81", "layoutEngineVersionMajor" : "81", + "webviewAppName": "Unknown", "deviceClass" : "Desktop", "agentNameVersionMajor" : "Firefox 81", - "operatingSystemNameVersionMajor" : "Mac OS X 10", - "deviceCpuBits" : "32", + "operatingSystemNameVersionMajor" : "Mac OS 10.14", + "deviceCpuBits" : "64", + "webviewAppVersionMajor": "??", "operatingSystemClass" : "Desktop", + "webviewAppVersion": "??", "layoutEngineName" : "Gecko", "agentName" : "Firefox", "agentVersion" : "81.0", diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index 548af8692..2df27bbdf 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -209,7 +209,7 @@ object EnrichmentManager { _ <- setEventFingerprint[F](raw.parameters, registry.eventFingerprint) // This enrichment cannot fail _ <- getCookieContexts // Execute cookie extractor enrichment _ <- getHttpHeaderContexts // Execute header extractor enrichment - _ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent) + _ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent) _ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version _ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment _ <- getCurrency[F](raw.context.timestamp, registry.currencyConversion) // Finalize the currency conversion @@ -237,7 +237,7 @@ object EnrichmentManager { _ <- getCookieContexts // Execute cookie extractor enrichment _ <- getHttpHeaderContexts // Execute header extractor enrichment _ <- getWeatherContext[F](registry.weather) // Fetch weather context - _ <- getYauaaContext[F](registry.yauaa) // Runs YAUAA enrichment (gets info thanks to user agent) + _ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent) _ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version _ <- geoLocation[F](registry.ipLookups) // Execute IP lookup enrichment _ <- getJsScript[F](registry.javascriptScript) // Execute the JavaScript scripting enrichment @@ -693,10 +693,10 @@ object EnrichmentManager { } } - def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment]): EStateT[F, Unit] = + def getYauaaContext[F[_]: Applicative](yauaa: Option[YauaaEnrichment], headers: List[String]): EStateT[F, Unit] = EStateT.fromEither { case (event, _) => - yauaa.map(_.getYauaaContext(event.useragent)).toList.asRight + yauaa.map(_.getYauaaContext(event.useragent, headers)).toList.asRight } // Derive some contexts with custom SQL Query enrichment diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 01f8c0a2f..2ac17ba63 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -41,7 +41,7 @@ object YauaaEnrichment extends ParseableEnrichment { val DefaultDeviceClass = "Unknown" val DefaultResult = Map(decapitalize(UserAgent.DEVICE_CLASS) -> DefaultDeviceClass) - val outputSchema: SchemaKey = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 3)) + val outputSchema: SchemaKey = SchemaKey("nl.basjes", "yauaa_context", "jsonschema", SchemaVer.Full(1, 0, 4)) /** * Creates a YauaaConf instance from a JValue containing the configuration of the enrichment. @@ -91,23 +91,91 @@ final case class YauaaEnrichment(cacheSize: Option[Int]) extends Enrichment { * @param userAgent User agent of the event. * @return Attributes retrieved thanks to the user agent (if any), as self-describing JSON. */ - def getYauaaContext(userAgent: String): SelfDescribingData[Json] = - SelfDescribingData(YauaaEnrichment.outputSchema, parseUserAgent(userAgent).asJson) + def getYauaaContext(userAgent: String, headers: List[String]): SelfDescribingData[Json] = + SelfDescribingData(YauaaEnrichment.outputSchema, analyzeUserAgent(userAgent, headers).asJson) /** * Gets the map of attributes retrieved by YAUAA from the user agent. * @return Map with all the fields extracted by YAUAA by parsing the user agent. * If the input is null or empty, a map with just the DeviceClass set to Unknown is returned. */ - def parseUserAgent(userAgent: String): Map[String, String] = + def analyzeUserAgent(userAgent: String, headers: List[String]): Map[String, String] = userAgent match { case null | "" => YauaaEnrichment.DefaultResult case _ => - val parsedUA = uaa.parse(userAgent) + val headerMap = headers + .map(_.split(": ", 2)) + .collect { + case Array(key, value) => key -> value.replaceAll("^\\s+", "") + } + .toMap ++ Map("User-Agent" -> userAgent) + val parsedUA = uaa.parse(headerMap.asJava) parsedUA.getAvailableFieldNamesSorted.asScala .map(field => decapitalize(field) -> parsedUA.getValue(field)) .toMap - .filter(_._1 != "__SyntaxError__") + .filterKeys(validFields) } + + /** Yauaa 7.x added many new fields which are not in the 1-0-4 schema */ + private val validFields = Set( + "deviceClass", + "deviceName", + "deviceBrand", + "deviceCpu", + "deviceCpuBits", + "deviceFirmwareVersion", + "deviceVersion", + "operatingSystemClass", + "operatingSystemName", + "operatingSystemVersion", + "operatingSystemNameVersion", + "operatingSystemVersionBuild", + "layoutEngineClass", + "layoutEngineName", + "layoutEngineVersion", + "layoutEngineVersionMajor", + "layoutEngineNameVersion", + "layoutEngineNameVersionMajor", + "layoutEngineBuild", + "agentClass", + "agentName", + "agentVersion", + "agentVersionMajor", + "agentNameVersion", + "agentNameVersionMajor", + "agentBuild", + "agentLanguage", + "agentLanguageCode", + "agentInformationEmail", + "agentInformationUrl", + "agentSecurity", + "agentUuid", + "webviewAppName", + "webviewAppVersion", + "webviewAppVersionMajor", + "webviewAppNameVersionMajor", + "facebookCarrier", + "facebookDeviceClass", + "facebookDeviceName", + "facebookDeviceVersion", + "facebookFBOP", + "facebookFBSS", + "facebookOperatingSystemName", + "facebookOperatingSystemVersion", + "anonymized", + "hackerAttackVector", + "hackerToolkit", + "koboAffiliate", + "koboPlatformId", + "iECompatibilityVersion", + "iECompatibilityVersionMajor", + "iECompatibilityNameVersion", + "iECompatibilityNameVersionMajor", + "carrier", + "gSAInstallationID", + "networkType", + "operatingSystemNameVersionMajor", + "operatingSystemVersionMajor" + ) } diff --git a/modules/common/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 b/modules/common/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 new file mode 100644 index 000000000..a6244f015 --- /dev/null +++ b/modules/common/src/test/resources/iglu-client-embedded/schemas/nl.basjes/yauaa_context/jsonschema/1-0-4 @@ -0,0 +1,233 @@ +{ + "$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#", + "description": "Schema for a context generated by the YAUAA enrichment after parsing the user agent", + "self": { + "vendor": "nl.basjes", + "name": "yauaa_context", + "format": "jsonschema", + "version": "1-0-4" + }, + "type": "object", + "properties": { + "deviceClass": { + "description": "See https://yauaa.basjes.nl/README-Output.html", + "enum": ["Desktop", "Anonymized", "Unknown", "UNKNOWN", "Mobile", "Tablet", "Phone", "Watch", "Virtual Reality", "eReader", "Set-top box", "TV", "Game Console", "Home Appliance", "Handheld Game Console", "Voice", "Car", "Robot", "Robot Mobile", "Spy", "Hacker", "Augmented Reality", "Robot Imitator"] + }, + "deviceName": { + "description": "Example: Google Nexus 6", + "type": "string", + "maxLength": 256 + }, + "deviceBrand": { + "description": "Example: Google", + "type": "string", + "maxLength": 128 + }, + "deviceCpu": { + "type": "string", + "maxLength": 128 + }, + "deviceCpuBits": { + "type": "string", + "maxLength": 128 + }, + "deviceFirmwareVersion": { + "type": "string", + "maxLength": 1000 + }, + "deviceVersion": { + "type": "string", + "maxLength": 1000 + }, + "operatingSystemClass": { + "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", + "enum": ["Desktop", "Mobile", "Cloud", "Embedded", "Game Console", "Hacker", "Anonymized", "Unknown"] + }, + "operatingSystemName": { + "description": "Examples: Linux, Android.", + "type": "string", + "maxLength": 256 + }, + "operatingSystemVersion": { + "type": "string", + "maxLength": 1000 + }, + "operatingSystemNameVersion": { + "type": "string", + "maxLength": 1000 + }, + "operatingSystemVersionBuild": { + "type": "string", + "maxLength": 1000 + }, + "layoutEngineClass": { + "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", + "enum": ["Browser", "Mobile App", "Hacker", "Robot", "Unknown", "Special", "Cloud", "eReader"] + }, + "layoutEngineName": { + "type": "string", + "maxLength": 256 + }, + "layoutEngineVersion": { + "type": "string", + "maxLength": 1000 + }, + "layoutEngineVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "layoutEngineNameVersion": { + "type": "string", + "maxLength": 1000 + }, + "layoutEngineNameVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "layoutEngineBuild": { + "type": "string", + "maxLength": 1000 + }, + "agentClass": { + "description": "See https://yauaa.basjes.nl/README-Output.html", + "type": "string", + "enum": ["Browser", "Browser Webview", "Mobile App", "Robot", "Robot Mobile", "Cloud Application", "Email Client", "Voice", "Special", "Testclient", "Hacker", "Unknown", "Desktop App", "eReader"] + }, + "agentName": { + "description": "Example: Chrome.", + "type": "string", + "maxLength": 256 + }, + "agentVersion": { + "type": "string", + "maxLength": 1000 + }, + "agentVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "agentNameVersion": { + "type": "string", + "maxLength": 1000 + }, + "agentNameVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "agentBuild": { + "type": "string", + "maxLength": 1000 + }, + "agentLanguage": { + "type": "string", + "maxLength": 1000 + }, + "agentLanguageCode": { + "type": "string", + "maxLength": 1000 + }, + "agentInformationEmail": { + "type": "string" + }, + "agentInformationUrl": { + "type": "string" + }, + "agentSecurity": { + "type": "string", + "enum": ["Weak security", "Strong security", "Unknown", "Hacker", "No security"] + }, + "agentUuid": { + "type": "string" + }, + "webviewAppName": { + "type": "string" + }, + "webviewAppVersion": { + "type": "string" + }, + "webviewAppVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "webviewAppNameVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "facebookCarrier": { + "type": "string" + }, + "facebookDeviceClass": { + "type": "string", + "maxLength": 1024 + }, + "facebookDeviceName": { + "type": "string", + "maxLength": 1024 + }, + "facebookDeviceVersion": { + "type": "string" + }, + "facebookFBOP": { + "type": "string" + }, + "facebookFBSS": { + "type": "string" + }, + "facebookOperatingSystemName": { + "type": "string" + }, + "facebookOperatingSystemVersion": { + "type": "string" + }, + "anonymized": { + "type": "string" + }, + "hackerAttackVector": { + "type": "string" + }, + "hackerToolkit": { + "type": "string" + }, + "koboAffiliate": { + "type": "string" + }, + "koboPlatformId": { + "type": "string" + }, + "iECompatibilityVersion": { + "type": "string", + "maxLength": 1000 + }, + "iECompatibilityVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "iECompatibilityNameVersion": { + "type": "string", + "maxLength": 1000 + }, + "iECompatibilityNameVersionMajor": { + "type": "string", + "maxLength": 1000 + }, + "carrier": { + "type": "string" + }, + "gSAInstallationID": { + "type": "string" + }, + "networkType": { + "type": "string" + }, + "operatingSystemNameVersionMajor": { + "type": "string" + }, + "operatingSystemVersionMajor": { + "type": "string" + } + }, + "required": ["deviceClass"], + "additionalProperties": false +} diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala index 7a2ef129e..03bacebb0 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala @@ -49,8 +49,10 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { val uaPlaystation4 = "Mozilla/5.0 (PlayStation 4 1.52) AppleWebKit/536.26 (KHTML, like Gecko)" // Browsers - val uaChrome = + val uaChrome13 = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1" + val uaChrome106 = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36" val uaChromium = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Ubuntu Chromium/25.0.1364.160 Chrome/25.0.1364.160 Safari/537.22" val uaFirefox = "Mozilla/5.0 (Windows NT 6.1; rv:2.0b7pre) Gecko/20100921 Firefox/4.0b7pre" @@ -65,11 +67,11 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "YAUAA enrichment should" >> { "return default value for null" >> { - yauaaEnrichment.parseUserAgent(null) shouldEqual YauaaEnrichment.DefaultResult + yauaaEnrichment.analyzeUserAgent(null, Nil) shouldEqual YauaaEnrichment.DefaultResult } "return default value for empty user agent" >> { - yauaaEnrichment.parseUserAgent("") shouldEqual YauaaEnrichment.DefaultResult + yauaaEnrichment.analyzeUserAgent("", Nil) shouldEqual YauaaEnrichment.DefaultResult } "detect correctly DeviceClass" >> { @@ -125,8 +127,8 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { uaNexusOne -> "Android", uaIphoneX -> "iOS", uaIpad -> "iOS", - uaSafari -> "Mac OS X", - uaGoogleBot -> "Google" + uaSafari -> "Mac OS", + uaGoogleBot -> "Google Cloud" ), UserAgent.OPERATING_SYSTEM_NAME ) @@ -135,7 +137,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "detect correctly LayoutEngineClass" >> { checkYauaaParsingForField( Map( - uaChrome -> "Browser", + uaChrome13 -> "Browser", uaIE -> "Browser", uaNexusOne -> "Browser", uaIphoneX -> "Browser", @@ -167,7 +169,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "detect correctly AgentName" >> { checkYauaaParsingForField( Map( - uaChrome -> "Chrome", + uaChrome13 -> "Chrome", uaChromium -> "Chromium", uaFirefox -> "Firefox", uaIE -> "Internet Explorer", @@ -182,7 +184,20 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { ) } - "create a JSON with the schema 1-0-1 and the data" >> { + "use client hint headers to aid parsing" >> { + val clientHintHeader = + """Sec-CH-UA-Full-Version-List: "Chromium";v="106.0.5249.119", "Google Chrome";v="106.0.5249.119", "Not;A=Brand";v="99.0.0.0"""" + yauaaEnrichment.analyzeUserAgent(uaChrome106, Nil)("agentNameVersion") shouldEqual "Chrome 106" + yauaaEnrichment.analyzeUserAgent(uaChrome106, List(clientHintHeader))("agentNameVersion") shouldEqual "Chrome 106.0.5249.119" + } + + /** Resembles the case when `ua` was sent as a field in the tp2 payload */ + "Prioritize explicitly passed user agent over an http header" >> { + val headers = List("User-Agent: curl/7.54") + yauaaEnrichment.analyzeUserAgent(uaFirefox, headers)("agentName") shouldEqual "Firefox" + } + + "create a JSON with the schema 1-0-4 and the data" >> { val expected = SelfDescribingData( YauaaEnrichment.outputSchema, @@ -191,14 +206,20 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "deviceName":"Samsung SM-G960F", "layoutEngineNameVersion":"Blink 62.0", "operatingSystemNameVersion":"Android 8.0.0", + "agentInformationEmail" : "Unknown", + "networkType" : "Unknown", "operatingSystemVersionBuild":"R16NW", + "webviewAppNameVersionMajor" : "Unknown ??", "layoutEngineNameVersionMajor":"Blink 62", "operatingSystemName":"Android", "agentVersionMajor":"62", "layoutEngineVersionMajor":"62", + "webviewAppName" : "Unknown", "deviceClass":"Phone", "agentNameVersionMajor":"Chrome 62", + "webviewAppVersionMajor" : "??", "operatingSystemClass":"Mobile", + "webviewAppVersion" : "??", "layoutEngineName":"Blink", "agentName":"Chrome", "agentVersion":"62.0.3202.84", @@ -211,7 +232,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { "operatingSystemVersionMajor":"8" }""" ) - val actual = yauaaEnrichment.getYauaaContext(uaGalaxyS9) + val actual = yauaaEnrichment.getYauaaContext(uaGalaxyS9, Nil) actual shouldEqual expected val defaultJson = @@ -219,13 +240,13 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { YauaaEnrichment.outputSchema, json"""{"deviceClass":"Unknown"}""" ) - yauaaEnrichment.getYauaaContext("") shouldEqual defaultJson + yauaaEnrichment.getYauaaContext("", Nil) shouldEqual defaultJson } "never add __SyntaxError__ to the context" >> { val ua = "useragent=Mozilla/5.0 (Linux armv7l) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 OPR/25.0.1620.0 OMI/4.3.18.7.Dominik.0 VSTVB MB100 HbbTV/1.2.1 (; PANASONIC; MB100; 0.1.34.28; ;) SmartTvA/3.0.0 UID (00:09:DF:A7:74:6B/MB100/PANASONIC/0.1.34.28)" - yauaaEnrichment.parseUserAgent(ua).contains("__SyntaxError__") shouldEqual false + yauaaEnrichment.analyzeUserAgent(ua, Nil).contains("__SyntaxError__") shouldEqual false } } @@ -233,7 +254,7 @@ class YauaaEnrichmentSpec extends Specification with ValidatedMatchers { def checkYauaaParsingForField(expectedResults: Map[String, String], fieldName: String) = expectedResults.map { case (userAgent, expectedField) => - yauaaEnrichment.parseUserAgent(userAgent)(decapitalize(fieldName)) shouldEqual expectedField + yauaaEnrichment.analyzeUserAgent(userAgent, Nil)(decapitalize(fieldName)) shouldEqual expectedField }.toList "decapitalize should" >> { diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 6dba6a33b..93324044b 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -41,7 +41,8 @@ object Dependencies { val hikariCP = "5.0.1" val jaywayJsonpath = "2.7.0" val iabClient = "0.2.0" - val yauaa = "5.23" + val yauaa = "7.7.0" + val log4jToSlf4j = "2.18.0" val guava = "28.1-jre" val slf4j = "2.0.3" val log4j = "2.17.0" // CVE-2021-44228 @@ -126,6 +127,7 @@ object Dependencies { .exclude("org.slf4j", "slf4j-api") val jaywayJsonpath = "com.jayway.jsonpath" % "json-path" % V.jaywayJsonpath val yauaa = "nl.basjes.parse.useragent" % "yauaa" % V.yauaa + val log4jToSlf4j = "org.apache.logging.log4j" % "log4j-to-slf4j" % V.log4jToSlf4j val guava = "com.google.guava" % "guava" % V.guava val log4j = "org.apache.logging.log4j" % "log4j-core" % V.log4j val log4jApi = "org.apache.logging.log4j" % "log4j-api" % V.log4j @@ -241,6 +243,7 @@ object Dependencies { jaywayJsonpath, iabClient, yauaa, + log4jToSlf4j, guava, circeOptics, circeJackson,