From 4db3670622086546ffc9b37fcf1bca25490cb0c9 Mon Sep 17 00:00:00 2001 From: Xavier Vello Date: Mon, 22 Apr 2024 17:32:53 +0200 Subject: [PATCH] capture: add support for the /batch request shape (#21) --- Cargo.toml | 2 +- capture/src/api.rs | 32 ++-- capture/src/lib.rs | 4 +- capture/src/router.rs | 40 +++- capture/src/sinks/kafka.rs | 6 +- capture/src/sinks/mod.rs | 3 +- capture/src/sinks/print.rs | 3 +- capture/src/{capture.rs => v0_endpoint.rs} | 160 ++++------------ capture/src/{event.rs => v0_request.rs} | 210 +++++++++++++++++---- capture/tests/django_compat.rs | 19 +- capture/tests/requests_dump.jsonl | 2 + 11 files changed, 276 insertions(+), 205 deletions(-) rename capture/src/{capture.rs => v0_endpoint.rs} (57%) rename capture/src/{event.rs => v0_request.rs} (60%) diff --git a/Cargo.toml b/Cargo.toml index f77c1fb..ef70e64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ members = [ anyhow = "1.0" assert-json-diff = "2.0.2" async-trait = "0.1.74" -axum = { version = "0.7.5", features = ["http2", "macros"] } +axum = { version = "0.7.5", features = ["http2", "macros", "matched-path"] } axum-client-ip = "0.6.0" base64 = "0.22.0" bytes = "1" diff --git a/capture/src/api.rs b/capture/src/api.rs index b27b1a9..0938ced 100644 --- a/capture/src/api.rs +++ b/capture/src/api.rs @@ -1,19 +1,11 @@ -use crate::token::InvalidTokenReason; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::HashMap; use thiserror::Error; +use time::OffsetDateTime; +use uuid::Uuid; -#[derive(Debug, Deserialize, Serialize)] -pub struct CaptureRequest { - #[serde(alias = "$token", alias = "api_key")] - pub token: String, - - pub event: String, - pub properties: HashMap, -} +use crate::token::InvalidTokenReason; #[derive(Debug, PartialEq, Eq, Deserialize, Serialize)] pub enum CaptureResponseCode { @@ -84,3 +76,21 @@ impl IntoResponse for CaptureError { .into_response() } } + +#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] +pub struct ProcessedEvent { + pub uuid: Uuid, + pub distinct_id: String, + pub ip: String, + pub data: String, + pub now: String, + #[serde(with = "time::serde::rfc3339::option")] + pub sent_at: Option, + pub token: String, +} + +impl ProcessedEvent { + pub fn key(&self) -> String { + format!("{}:{}", self.token, self.distinct_id) + } +} diff --git a/capture/src/lib.rs b/capture/src/lib.rs index 058e994..176fc6f 100644 --- a/capture/src/lib.rs +++ b/capture/src/lib.rs @@ -1,7 +1,5 @@ pub mod api; -pub mod capture; pub mod config; -pub mod event; pub mod health; pub mod limiters; pub mod prometheus; @@ -12,3 +10,5 @@ pub mod sinks; pub mod time; pub mod token; pub mod utils; +pub mod v0_endpoint; +pub mod v0_request; diff --git a/capture/src/router.rs b/capture/src/router.rs index d02e63f..85475ce 100644 --- a/capture/src/router.rs +++ b/capture/src/router.rs @@ -10,7 +10,9 @@ use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; use crate::health::HealthRegistry; -use crate::{capture, limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource}; +use crate::{ + limiters::billing::BillingLimiter, redis::Client, sinks, time::TimeSource, v0_endpoint, +}; use crate::prometheus::{setup_metrics_recorder, track_metrics}; @@ -58,17 +60,41 @@ pub fn router< .route("/", get(index)) .route("/_readiness", get(index)) .route("/_liveness", get(move || ready(liveness.get_status()))) + .route( + "/e", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/e/", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/batch", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) + .route( + "/batch/", + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), + ) .route( "/i/v0/e", - post(capture::event) - .get(capture::event) - .options(capture::options), + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), ) .route( "/i/v0/e/", - post(capture::event) - .get(capture::event) - .options(capture::options), + post(v0_endpoint::event) + .get(v0_endpoint::event) + .options(v0_endpoint::options), ) .layer(TraceLayer::new_for_http()) .layer(cors) diff --git a/capture/src/sinks/kafka.rs b/capture/src/sinks/kafka.rs index 4a2bd94..4a48b6e 100644 --- a/capture/src/sinks/kafka.rs +++ b/capture/src/sinks/kafka.rs @@ -10,9 +10,8 @@ use tokio::task::JoinSet; use tracing::log::{debug, error, info}; use tracing::{info_span, instrument, Instrument}; -use crate::api::CaptureError; +use crate::api::{CaptureError, ProcessedEvent}; use crate::config::KafkaConfig; -use crate::event::ProcessedEvent; use crate::health::HealthHandle; use crate::limiters::overflow::OverflowLimiter; use crate::prometheus::report_dropped_events; @@ -259,9 +258,8 @@ impl Event for KafkaSink { #[cfg(test)] mod tests { - use crate::api::CaptureError; + use crate::api::{CaptureError, ProcessedEvent}; use crate::config; - use crate::event::ProcessedEvent; use crate::health::HealthRegistry; use crate::limiters::overflow::OverflowLimiter; use crate::sinks::kafka::KafkaSink; diff --git a/capture/src/sinks/mod.rs b/capture/src/sinks/mod.rs index 0747f0e..bedbcbc 100644 --- a/capture/src/sinks/mod.rs +++ b/capture/src/sinks/mod.rs @@ -1,7 +1,6 @@ use async_trait::async_trait; -use crate::api::CaptureError; -use crate::event::ProcessedEvent; +use crate::api::{CaptureError, ProcessedEvent}; pub mod kafka; pub mod print; diff --git a/capture/src/sinks/print.rs b/capture/src/sinks/print.rs index 5e71899..7845a3d 100644 --- a/capture/src/sinks/print.rs +++ b/capture/src/sinks/print.rs @@ -2,8 +2,7 @@ use async_trait::async_trait; use metrics::{counter, histogram}; use tracing::log::info; -use crate::api::CaptureError; -use crate::event::ProcessedEvent; +use crate::api::{CaptureError, ProcessedEvent}; use crate::sinks::Event; pub struct PrintSink {} diff --git a/capture/src/capture.rs b/capture/src/v0_endpoint.rs similarity index 57% rename from capture/src/capture.rs rename to capture/src/v0_endpoint.rs index 622bc75..3862995 100644 --- a/capture/src/capture.rs +++ b/capture/src/v0_endpoint.rs @@ -1,41 +1,45 @@ -use std::collections::HashSet; use std::ops::Deref; use std::sync::Arc; -use bytes::Bytes; - use axum::{debug_handler, Json}; +use bytes::Bytes; // TODO: stream this instead -use axum::extract::{Query, State}; +use axum::extract::{MatchedPath, Query, State}; use axum::http::{HeaderMap, Method}; use axum_client_ip::InsecureClientIp; use base64::Engine; use metrics::counter; - -use time::OffsetDateTime; use tracing::instrument; -use crate::event::{Compression, ProcessingContext}; use crate::limiters::billing::QuotaResource; use crate::prometheus::report_dropped_events; -use crate::token::validate_token; +use crate::v0_request::{Compression, ProcessingContext, RawRequest}; use crate::{ - api::{CaptureError, CaptureResponse, CaptureResponseCode}, - event::{EventFormData, EventQuery, ProcessedEvent, RawEvent}, + api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}, router, sinks, utils::uuid_v7, + v0_request::{EventFormData, EventQuery, RawEvent}, }; +/// Flexible endpoint that targets wide compatibility with the wide range of requests +/// currently processed by posthog-events (analytics events capture). Replay is out +/// of scope and should be processed on a separate endpoint. +/// +/// Because it must accommodate several shapes, it is inefficient in places. A v1 +/// endpoint should be created, that only accepts the BatchedRequest payload shape. + #[instrument( skip_all, fields( + path, token, batch_size, user_agent, content_encoding, content_type, version, - compression + compression, + is_historical ) )] #[debug_handler] @@ -45,11 +49,9 @@ pub async fn event( meta: Query, headers: HeaderMap, method: Method, + path: MatchedPath, body: Bytes, ) -> Result, CaptureError> { - // content-type - // user-agent - let user_agent = headers .get("user-agent") .map_or("unknown", |v| v.to_str().unwrap_or("unknown")); @@ -68,8 +70,9 @@ pub async fn event( tracing::Span::current().record("version", meta.lib_version.clone()); tracing::Span::current().record("compression", comp.as_str()); tracing::Span::current().record("method", method.as_str()); + tracing::Span::current().record("path", path.as_str().trim_end_matches('/')); - let events = match headers + let request = match headers .get("content-type") .map_or("", |v| v.to_str().unwrap_or("")) { @@ -86,41 +89,36 @@ pub async fn event( tracing::error!("failed to decode form data: {}", e); CaptureError::RequestDecodingError(String::from("missing data field")) })?; - RawEvent::from_bytes(payload.into()) + RawRequest::from_bytes(payload.into()) } ct => { tracing::Span::current().record("content_type", ct); - RawEvent::from_bytes(body) + RawRequest::from_bytes(body) } }?; + let sent_at = request.sent_at().or(meta.sent_at()); + let token = match request.extract_and_verify_token() { + Ok(token) => token, + Err(err) => { + report_dropped_events("token_shape_invalid", request.events().len() as u64); + return Err(err); + } + }; + let is_historical = request.is_historical(); // TODO: use to write to historical topic + let events = request.events(); // Takes ownership of request + + tracing::Span::current().record("token", &token); + tracing::Span::current().record("is_historical", is_historical); tracing::Span::current().record("batch_size", events.len()); if events.is_empty() { return Err(CaptureError::EmptyBatch); } - let token = extract_and_verify_token(&events).map_err(|err| { - report_dropped_events("token_shape_invalid", events.len() as u64); - err - })?; - - tracing::Span::current().record("token", &token); - counter!("capture_events_received_total").increment(events.len() as u64); - let sent_at = meta.sent_at.and_then(|value| { - let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases - if let Ok(sent_at) = OffsetDateTime::from_unix_timestamp_nanos(value_nanos) { - if sent_at.year() > 2020 { - // Could be lower if the input is in seconds - return Some(sent_at); - } - } - None - }); - let context = ProcessingContext { lib_version: meta.lib_version.clone(), sent_at, @@ -192,28 +190,6 @@ pub fn process_single_event( }) } -#[instrument(skip_all, fields(events = events.len()))] -pub fn extract_and_verify_token(events: &[RawEvent]) -> Result { - let distinct_tokens: HashSet> = HashSet::from_iter( - events - .iter() - .map(RawEvent::extract_token) - .filter(Option::is_some), - ); - - return match distinct_tokens.len() { - 0 => Err(CaptureError::NoTokenError), - 1 => match distinct_tokens.iter().last() { - Some(Some(token)) => { - validate_token(token)?; - Ok(token.clone()) - } - _ => Err(CaptureError::NoTokenError), - }, - _ => Err(CaptureError::MultipleTokensError), - }; -} - #[instrument(skip_all, fields(events = events.len()))] pub async fn process_events<'a>( sink: Arc, @@ -233,73 +209,3 @@ pub async fn process_events<'a>( sink.send_batch(events).await } } - -#[cfg(test)] -mod tests { - use crate::capture::extract_and_verify_token; - use crate::event::RawEvent; - use serde_json::json; - use std::collections::HashMap; - - #[tokio::test] - async fn all_events_have_same_token() { - let events = vec![ - RawEvent { - token: Some(String::from("hello")), - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::new(), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - RawEvent { - token: None, - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::from([(String::from("token"), json!("hello"))]), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - ]; - - let processed = extract_and_verify_token(&events); - assert_eq!(processed.is_ok(), true, "{:?}", processed); - } - - #[tokio::test] - async fn all_events_have_different_token() { - let events = vec![ - RawEvent { - token: Some(String::from("hello")), - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::new(), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - RawEvent { - token: None, - distinct_id: Some(json!("testing")), - uuid: None, - event: String::new(), - properties: HashMap::from([(String::from("token"), json!("goodbye"))]), - timestamp: None, - offset: None, - set: Default::default(), - set_once: Default::default(), - }, - ]; - - let processed = extract_and_verify_token(&events); - assert_eq!(processed.is_err(), true); - } -} diff --git a/capture/src/event.rs b/capture/src/v0_request.rs similarity index 60% rename from capture/src/event.rs rename to capture/src/v0_request.rs index ea71a3f..3d0052e 100644 --- a/capture/src/event.rs +++ b/capture/src/v0_request.rs @@ -1,15 +1,17 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::prelude::*; use bytes::{Buf, Bytes}; use flate2::read::GzDecoder; use serde::{Deserialize, Serialize}; use serde_json::Value; +use time::format_description::well_known::Iso8601; use time::OffsetDateTime; use tracing::instrument; use uuid::Uuid; use crate::api::CaptureError; +use crate::token::validate_token; #[derive(Deserialize, Default)] pub enum Compression { @@ -28,7 +30,25 @@ pub struct EventQuery { pub lib_version: Option, #[serde(alias = "_")] - pub sent_at: Option, + sent_at: Option, +} + +impl EventQuery { + /// Returns the parsed value of the sent_at timestamp if present in the query params. + /// We only support the format sent by recent posthog-js versions, in milliseconds integer. + /// Values in seconds integer (older SDKs will be ignored). + pub fn sent_at(&self) -> Option { + if let Some(value) = self.sent_at { + let value_nanos: i128 = i128::from(value) * 1_000_000; // Assuming the value is in milliseconds, latest posthog-js releases + if let Ok(sent_at) = OffsetDateTime::from_unix_timestamp_nanos(value_nanos) { + if sent_at.year() > 2020 { + // Could be lower if the input is in seconds + return Some(sent_at); + } + } + } + None + } } #[derive(Debug, Deserialize)] @@ -64,30 +84,32 @@ static GZIP_MAGIC_NUMBERS: [u8; 3] = [0x1f, 0x8b, 8]; #[derive(Deserialize)] #[serde(untagged)] -enum RawRequest { - /// Batch of events - Batch(Vec), - /// Single event +pub enum RawRequest { + /// Array of events (posthog-js) + Array(Vec), + /// Batched events (/batch) + Batch(BatchedRequest), + /// Single event (/capture) One(Box), } -impl RawRequest { - pub fn events(self) -> Vec { - match self { - RawRequest::Batch(events) => events, - RawRequest::One(event) => vec![*event], - } - } +#[derive(Deserialize)] +pub struct BatchedRequest { + #[serde(alias = "api_key")] + pub token: String, + pub historical_migration: Option, + pub sent_at: Option, + pub batch: Vec, } -impl RawEvent { - /// Takes a request payload and tries to decompress and unmarshall it into events. +impl RawRequest { + /// Takes a request payload and tries to decompress and unmarshall it. /// While posthog-js sends a compression query param, a sizable portion of requests /// fail due to it being missing when the body is compressed. /// Instead of trusting the parameter, we peek at the payload's first three bytes to /// detect gzip, fallback to uncompressed utf8 otherwise. #[instrument(skip_all)] - pub fn from_bytes(bytes: Bytes) -> Result, CaptureError> { + pub fn from_bytes(bytes: Bytes) -> Result { tracing::debug!(len = bytes.len(), "decoding new event"); let payload = if bytes.starts_with(&GZIP_MAGIC_NUMBERS) { @@ -106,9 +128,66 @@ impl RawEvent { }; tracing::debug!(json = payload, "decoded event data"); - Ok(serde_json::from_str::(&payload)?.events()) + Ok(serde_json::from_str::(&payload)?) } + pub fn events(self) -> Vec { + match self { + RawRequest::Array(events) => events, + RawRequest::One(event) => vec![*event], + RawRequest::Batch(req) => req.batch, + } + } + + pub fn extract_and_verify_token(&self) -> Result { + let token = match self { + RawRequest::Batch(req) => req.token.to_string(), + RawRequest::One(event) => event.extract_token().ok_or(CaptureError::NoTokenError)?, + RawRequest::Array(events) => extract_token(events)?, + }; + validate_token(&token)?; + Ok(token) + } + + pub fn is_historical(&self) -> bool { + match self { + RawRequest::Batch(req) => req.historical_migration.unwrap_or_default(), + _ => false, + } + } + + pub fn sent_at(&self) -> Option { + if let RawRequest::Batch(req) = &self { + if let Some(value) = &req.sent_at { + if let Ok(parsed) = OffsetDateTime::parse(value, &Iso8601::DEFAULT) { + return Some(parsed); + } + } + } + None + } +} + +#[instrument(skip_all, fields(events = events.len()))] +pub fn extract_token(events: &[RawEvent]) -> Result { + let distinct_tokens: HashSet> = HashSet::from_iter( + events + .iter() + .map(RawEvent::extract_token) + .filter(Option::is_some), + ); + + return match distinct_tokens.len() { + 0 => Err(CaptureError::NoTokenError), + 1 => match distinct_tokens.iter().last() { + Some(Some(token)) => Ok(token.clone()), + _ => Err(CaptureError::NoTokenError), + }, + _ => Err(CaptureError::MultipleTokensError), + }; +} + +impl RawEvent { pub fn extract_token(&self) -> Option { match &self.token { Some(value) => Some(value.clone()), @@ -154,26 +233,9 @@ pub struct ProcessingContext { pub client_ip: String, } -#[derive(Clone, Default, Debug, Serialize, Eq, PartialEq)] -pub struct ProcessedEvent { - pub uuid: Uuid, - pub distinct_id: String, - pub ip: String, - pub data: String, - pub now: String, - #[serde(with = "time::serde::rfc3339::option")] - pub sent_at: Option, - pub token: String, -} - -impl ProcessedEvent { - pub fn key(&self) -> String { - format!("{}:{}", self.token, self.distinct_id) - } -} - #[cfg(test)] mod tests { + use crate::token::InvalidTokenReason; use base64::Engine as _; use bytes::Bytes; use rand::distributions::Alphanumeric; @@ -181,7 +243,7 @@ mod tests { use serde_json::json; use super::CaptureError; - use super::RawEvent; + use super::RawRequest; #[test] fn decode_uncompressed_raw_event() { @@ -192,7 +254,9 @@ mod tests { .expect("payload is not base64"), ); - let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + let events = RawRequest::from_bytes(compressed_bytes) + .expect("failed to parse") + .events(); assert_eq!(1, events.len()); assert_eq!(Some("my_token1".to_string()), events[0].extract_token()); assert_eq!("my_event1".to_string(), events[0].event); @@ -212,7 +276,9 @@ mod tests { .expect("payload is not base64"), ); - let events = RawEvent::from_bytes(compressed_bytes).expect("failed to parse"); + let events = RawRequest::from_bytes(compressed_bytes) + .expect("failed to parse") + .events(); assert_eq!(1, events.len()); assert_eq!(Some("my_token2".to_string()), events[0].extract_token()); assert_eq!("my_event2".to_string(), events[0].event); @@ -227,7 +293,9 @@ mod tests { #[test] fn extract_distinct_id() { let parse_and_extract = |input: &'static str| -> Result { - let parsed = RawEvent::from_bytes(input.into()).expect("failed to parse"); + let parsed = RawRequest::from_bytes(input.into()) + .expect("failed to parse") + .events(); parsed[0].extract_distinct_id() }; // Return MissingDistinctId if not found @@ -288,10 +356,72 @@ mod tests { "distinct_id": distinct_id }]); - let parsed = RawEvent::from_bytes(input.to_string().into()).expect("failed to parse"); + let parsed = RawRequest::from_bytes(input.to_string().into()) + .expect("failed to parse") + .events(); assert_eq!( parsed[0].extract_distinct_id().expect("failed to extract"), expected_distinct_id ); } + + #[test] + fn extract_and_verify_token() { + let parse_and_extract = |input: &'static str| -> Result { + RawRequest::from_bytes(input.into()) + .expect("failed to parse") + .extract_and_verify_token() + }; + + let assert_extracted_token = |input: &'static str, expected: &str| { + let id = parse_and_extract(input).expect("failed to extract"); + assert_eq!(id, expected); + }; + + // Return NoTokenError if not found + assert!(matches!( + parse_and_extract(r#"{"event": "e"}"#), + Err(CaptureError::NoTokenError) + )); + + // Return TokenValidationError if token empty + assert!(matches!( + parse_and_extract(r#"{"api_key": "", "batch":[{"event": "e"}]}"#), + Err(CaptureError::TokenValidationError( + InvalidTokenReason::Empty + )) + )); + + // Return TokenValidationError if personal apikey + assert!(matches!( + parse_and_extract(r#"[{"event": "e", "token": "phx_hellothere"}]"#), + Err(CaptureError::TokenValidationError( + InvalidTokenReason::PersonalApiKey + )) + )); + + // Return MultipleTokensError if tokens don't match in array + assert!(matches!( + parse_and_extract( + r#"[{"event": "e", "token": "token1"},{"event": "e", "token": "token2"}]"# + ), + Err(CaptureError::MultipleTokensError) + )); + + // Return token from array if consistent + assert_extracted_token( + r#"[{"event":"e","token":"token1"},{"event":"e","token":"token1"}]"#, + "token1", + ); + + // Return token from batch if present + assert_extracted_token( + r#"{"batch":[{"event":"e","token":"token1"}],"api_key":"batched"}"#, + "batched", + ); + + // Return token from single event if present + assert_extracted_token(r#"{"event":"e","$token":"single_token"}"#, "single_token"); + assert_extracted_token(r#"{"event":"e","api_key":"single_token"}"#, "single_token"); + } } diff --git a/capture/tests/django_compat.rs b/capture/tests/django_compat.rs index 5d77899..c7ec0ad 100644 --- a/capture/tests/django_compat.rs +++ b/capture/tests/django_compat.rs @@ -4,8 +4,7 @@ use axum::http::StatusCode; use axum_test_helper::TestClient; use base64::engine::general_purpose; use base64::Engine; -use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode}; -use capture::event::ProcessedEvent; +use capture::api::{CaptureError, CaptureResponse, CaptureResponseCode, ProcessedEvent}; use capture::health::HealthRegistry; use capture::limiters::billing::BillingLimiter; use capture::redis::MockRedisClient; @@ -88,11 +87,6 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { continue; } let case: RequestDump = serde_json::from_str(&line_contents)?; - if !case.path.starts_with("/e/") { - println!("Skipping {} test case", &case.path); - continue; - } - let raw_body = general_purpose::STANDARD.decode(&case.body)?; assert_eq!( case.method, "POST", @@ -117,7 +111,7 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { ); let client = TestClient::new(app); - let mut req = client.post(&format!("/i/v0{}", case.path)).body(raw_body); + let mut req = client.post(&case.path).body(raw_body); if !case.content_encoding.is_empty() { req = req.header("Content-encoding", case.content_encoding); } @@ -164,8 +158,15 @@ async fn it_matches_django_capture_behaviour() -> anyhow::Result<()> { if let Some(expected_data) = expected.get_mut("data") { // Data is a serialized JSON map. Unmarshall both and compare them, // instead of expecting the serialized bytes to be equal - let expected_props: Value = + let mut expected_props: Value = serde_json::from_str(expected_data.as_str().expect("not str"))?; + if let Some(object) = expected_props.as_object_mut() { + // toplevel fields added by posthog-node that plugin-server will ignore anyway + object.remove("type"); + object.remove("library"); + object.remove("library_version"); + } + let found_props: Value = serde_json::from_str(&message.data)?; let match_config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict); diff --git a/capture/tests/requests_dump.jsonl b/capture/tests/requests_dump.jsonl index ec0f4df..36cf8ad 100644 --- a/capture/tests/requests_dump.jsonl +++ b/capture/tests/requests_dump.jsonl @@ -13,3 +13,5 @@ ### Compression query param mismatch, to confirm gzip autodetection {"path":"/e/?compression=gzip-js&ip=1&_=1694769302319&ver=1.78.5","method":"POST","content_encoding":"","content_type":"application/x-www-form-urlencoded","ip":"127.0.0.1","now":"2023-09-15T09:15:02.321230+00:00","body":"ZGF0YT1leUoxZFdsa0lqb2lNREU0WVRrNE1XWXROR0l5WlMwM1ltUXpMV0ppTXpBdE5qWXhOalkxTjJNek9HWmhJaXdpWlhabGJuUWlPaUlrYjNCMFgybHVJaXdpY0hKdmNHVnlkR2xsY3lJNmV5SWtiM01pT2lKTllXTWdUMU1nV0NJc0lpUnZjMTkyWlhKemFXOXVJam9pTVRBdU1UVXVNQ0lzSWlSaWNtOTNjMlZ5SWpvaVJtbHlaV1p2ZUNJc0lpUmtaWFpwWTJWZmRIbHdaU0k2SWtSbGMydDBiM0FpTENJa1kzVnljbVZ1ZEY5MWNtd2lPaUpvZEhSd09pOHZiRzlqWVd4b2IzTjBPamd3TURBdklpd2lKR2h2YzNRaU9pSnNiMk5oYkdodmMzUTZPREF3TUNJc0lpUndZWFJvYm1GdFpTSTZJaThpTENJa1luSnZkM05sY2w5MlpYSnphVzl1SWpveE1UY3NJaVJpY205M2MyVnlYMnhoYm1kMVlXZGxJam9pWlc0dFZWTWlMQ0lrYzJOeVpXVnVYMmhsYVdkb2RDSTZNVEExTWl3aUpITmpjbVZsYmw5M2FXUjBhQ0k2TVRZeU1Dd2lKSFpwWlhkd2IzSjBYMmhsYVdkb2RDSTZPVEV4TENJa2RtbGxkM0J2Y25SZmQybGtkR2dpT2pFMU5EZ3NJaVJzYVdJaU9pSjNaV0lpTENJa2JHbGlYM1psY25OcGIyNGlPaUl4TGpjNExqVWlMQ0lrYVc1elpYSjBYMmxrSWpvaU1XVTRaV1p5WkdSbE1HSTBNV2RtYkNJc0lpUjBhVzFsSWpveE5qazBOelk1TXpBeUxqTXhPQ3dpWkdsemRHbHVZM1JmYVdRaU9pSXdNVGhoT1RneFppMDBZakprTFRjNFlXVXRPVFEzWWkxaVpXUmlZV0V3TW1Fd1pqUWlMQ0lrWkdWMmFXTmxYMmxrSWpvaU1ERTRZVGs0TVdZdE5HSXlaQzAzT0dGbExUazBOMkl0WW1Wa1ltRmhNREpoTUdZMElpd2lKSEpsWm1WeWNtVnlJam9pSkdScGNtVmpkQ0lzSWlSeVpXWmxjbkpwYm1kZlpHOXRZV2x1SWpvaUpHUnBjbVZqZENJc0luUnZhMlZ1SWpvaWNHaGpYM0ZuVlVad05YZDZNa0pxUXpSU2MwWnFUVWR3VVROUVIwUnljMmsyVVRCRFJ6QkRVRFJPUVdGak1Fa2lMQ0lrYzJWemMybHZibDlwWkNJNklqQXhPR0U1T0RGbUxUUmlNbVV0TjJKa015MWlZak13TFRZMk1UZGlaalE0T0RnMk9TSXNJaVIzYVc1a2IzZGZhV1FpT2lJd01UaGhPVGd4WmkwMFlqSmxMVGRpWkRNdFltSXpNQzAyTmpFNE1UQm1aV1EyTldZaWZTd2lkR2x0WlhOMFlXMXdJam9pTWpBeU15MHdPUzB4TlZRd09Ub3hOVG93TWk0ek1UaGFJbjAlM0Q=","output":[{"uuid":"018a981f-4b2e-7bd3-bb30-6616657c38fa","distinct_id":"018a981f-4b2d-78ae-947b-bedbaa02a0f4","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-4b2e-7bd3-bb30-6616657c38fa\", \"event\": \"$opt_in\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/\", \"$host\": \"localhost:8000\", \"$pathname\": \"/\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"1e8efrdde0b41gfl\", \"$time\": 1694769302.318, \"distinct_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"timestamp\": \"2023-09-15T09:15:02.318Z\"}","now":"2023-09-15T09:15:02.321230+00:00","sent_at":"2023-09-15T09:15:02.319000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} {"path":"/e/?ip=1&_=1694769329412&ver=1.78.5","method":"POST","content_encoding":"","content_type":"text/plain","ip":"127.0.0.1","now":"2023-09-15T09:15:29.500667+00:00","body":"H4sIAAAAAAAAA+0Ya2/bNvCvGF4/bEPo6EVSyjBgbdq0HdauaZp2Q1EIFElZqmlRkehXiwL7Lftp+yU7yo7jV2KkHYoikz/Y1t3xXryX7u3H7miUi+5R13FDFoVuihhLMaJp6qAoTQUKUyzdNMTUpWn3oCvHsjBAfo+NjOasNKNKArisdCkrk8u6e/Sxe0/DT/cZ453fzzp/ABoA8VhWda4LQLhOz8U9x8KTSk9qWQHwJK9kqqcWKOQ45zI2s1IC4qGsB0aXFsFHVQXi41GlAJEZUx4dHipQQ2W6Nkeh4ziHoEati8OlOUHiCURDJlEU0AQlUiSMOR5z0uA7xk0+lq9Y8nMta6vcS8l1JfKiX1txlinIWRdgESUzWcGGVrvbyFsx+MobrktXwIoV/RHrW86yQOdn9kjNKymLOJN5PwN9XAd7V9BJLkwGQOI5ABznclLqyiyJI9ddBV9S4yAEsMoTkDORiZUCD6tX1KNhD1t4XoBeJp7HiB6LQTRVxThKs5QNLN7k1g8uAWtJ5HukF/pgkMhrkxd8ca58c3/w9NXs1D9+Tn3fO784nRZPSOmfvA5f/ypo+ScJ6CPxfDB9w1fufz0s9/h1ZL33GcLyOhZyqGMI4PeSg8dSpmoJDPuVHpVNNC9Rl8pIlIaJhyAYHCSwFCjyQ0dKzJgIrC911WdF/oGZuS+vTmHM56dYiAXyMHexxBgLLKwmRW1Ywe3V74zs7ifQaiXrYvAxS5QUMZgONxfXuYDDl/rPYztOJWuIU8UgqI/evgPUKiwu2UxpJqylIKCSTA2tBiAVDOMq54NMg3Nt6g9Zrhph9nbYGJ6s/KXIWjE+uAGfsTpOcqUgveISAn2JgBTikNYQ9vGohu8eeFyMIHhYwdTM5ByUg+heZv8NNFusFokdVxJEznbw2UGwxWTDiVs8tvH3oJhJILGV7Z6A0gbxs4RaBwgN3rTRsYJtautl2Wtc3xQ9uVIsLJWSQ6Czl/mxC0rGi0pUl+DSgy4zpopjwQxD9q9NiKZC1aiSitlLbbREhtlQLUwW8yxXkDjA2z7pdKHBXFZs5NSG/sncxM7cxE8Ha6JFPgZeXLEavAl6dX+DlCqgrtZxDHIQ14WxbQOCb65eQwoHr6G7Wa8N4Sq/SfY+mUuPVdqGKjC+ArEqZ6gGf3NwG+CacF1igTIvhJzaDF/XGG9ojDc0HqlrNU5YdaPGFr+tsYJqezunXX9jN4hfSq7NrBGNEMSiLmws1ahWUH8q1LSYow4Neh6Bj4cjSjD8lNOfOrvodZrW0hx1XEp6XhRFIaWRF5GQ2hM3xrPlsm51sGF1sM9qm4OIleX1EbpF8WVuPgObHzDL/zqBWxRfZuGC3bVybmbv72OfofkcAtBUQS7Mf8BZalviJS3UEDltvhq6z3DonG/T7SutN1zk72GRaDG7SuIMimlTROy4tKnxOmOYujYYg4lGD8B6CMqMxxf985MSTz54D94fBy/rk/fPHpen/ovHD6F6k1Pn+LFz/CJ4fp9x52kz1y06z9acIxFNhI+SxHcQIS5N0iAMQxLZQxOoOXqy90zoOqkUBKd2apinGBiAqWedsWPiZ4hGMkQMRKFAklAmJOC+szbxl9Dq7CR5B8b9RcM+aZpZO+lvT/ozf0qJZGR4cZF7Fw7bPekHoF476beTfjvp327S/+abBqiYm2bEmzIoKdUvJURJpvs9roedf/76u/NiPoPN/wPuie6vdxpCd3aa1IfUjkQAWR0wJKgXcQqFxhG2wtzR3VLbbPY0G2UmtFJBmuVB0r8g/o5mQ3uRD8Np22zaZtM2m6+6Vvq6u521ndPmmunb2/RsvhI2r1f/s01PCHsbHyqi5wY+xgGJ9m16fEJ7Dmx6KMYhjvx20dMuetpFz3+66HED2uyct8fvgMCw43soYiRCHBNCscvtlvaOLnoam9qpe9fUTWbKxxNhxnjsupo0I/L21N2ueNqpu5262xXP1orHDXaveBLHCRH1KQOZAchkhIVuwjwvsKl92WMuRrKadUBUqaQdudtOc4c7TdgfTcbD0Jtlk1mWarKj04Q91ydtp2k7TdtpbttpmlJqA24ALQAwj5pCdNpAwbvNUsO+wf24LL+NqlB97BueZkrWXH5/VX57q0XyoLOKWBQPW8N+gFf8znmlOoeds8sXw1VSWyRsixlKCM9haV8S50Kfrqcnj9g8tkmYwqNLAwBTjhNbRVhqGlcgL8hsHItRtcgOl0KB+uZ77Gq79Kj/6d2/ToySO28rAAA=","output":[{"uuid":"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aaf5-7ff0-9ffd-8f5e1f85717f\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=sessionRecordings\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"0ovdk9xlnv9fhfak\", \"$time\": 1694769326.837, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"span\", \"attr__data-attr\": \"persons-related-flags-tab\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"Feature flags\"}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 5, \"nth_of_type\": 5}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 74.26666259765625px; --lemon-tabs-slider-offset: 176.29998779296875px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 2572}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-aafa-79e8-abf4-4e68eb64c30f","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-aafa-79e8-abf4-4e68eb64c30f\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"y3x76ea6mqqi2q0a\", \"$time\": 1694769326.842, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 2567}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af3d-79d4-be4a-d729c776e0da","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af3d-79d4-be4a-d729c776e0da\", \"event\": \"$autocapture\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=featureFlags\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"ltw7rl4fhi4bgq63\", \"$time\": 1694769327.934, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"$event_type\": \"click\", \"$ce_version\": 1, \"$elements\": [{\"tag_name\": \"div\", \"classes\": [\"LemonTabs__tab-content\"], \"attr__class\": \"LemonTabs__tab-content\", \"nth_child\": 1, \"nth_of_type\": 1, \"$el_text\": \"\"}, {\"tag_name\": \"li\", \"classes\": [\"LemonTabs__tab\"], \"attr__class\": \"LemonTabs__tab\", \"attr__role\": \"tab\", \"attr__aria-selected\": \"false\", \"attr__tabindex\": \"0\", \"nth_child\": 2, \"nth_of_type\": 2}, {\"tag_name\": \"ul\", \"classes\": [\"LemonTabs__bar\"], \"attr__class\": \"LemonTabs__bar\", \"attr__role\": \"tablist\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"LemonTabs\"], \"attr__class\": \"LemonTabs\", \"attr__style\": \"--lemon-tabs-slider-width: 86.23332214355469px; --lemon-tabs-slider-offset: 367.0999755859375px;\", \"attr__data-attr\": \"persons-tabs\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"main-app-content\"], \"attr__class\": \"main-app-content\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"classes\": [\"SideBar__content\"], \"attr__class\": \"SideBar__content\", \"nth_child\": 4, \"nth_of_type\": 4}, {\"tag_name\": \"div\", \"classes\": [\"SideBar\"], \"attr__class\": \"SideBar\", \"nth_child\": 4, \"nth_of_type\": 3}, {\"tag_name\": \"div\", \"classes\": [\"h-screen\", \"flex\", \"flex-col\"], \"attr__class\": \"h-screen flex flex-col\", \"nth_child\": 1, \"nth_of_type\": 1}, {\"tag_name\": \"div\", \"attr__id\": \"root\", \"nth_child\": 3, \"nth_of_type\": 1}, {\"tag_name\": \"body\", \"attr__theme\": \"light\", \"attr__class\": \"\", \"nth_child\": 2, \"nth_of_type\": 1}], \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1475}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-af46-7832-9a69-c566751c6625","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-af46-7832-9a69-c566751c6625\", \"event\": \"$pageview\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"6yl35wdtv5v11o6c\", \"$time\": 1694769327.942, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\", \"title\": \"xavier@posthog.com \\u2022 Persons \\u2022 PostHog\"}, \"offset\": 1467}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"},{"uuid":"018a981f-b008-737a-bb40-6a6a81ba2244","distinct_id":"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc","ip":"127.0.0.1","site_url":"http://localhost:8000","data":"{\"uuid\": \"018a981f-b008-737a-bb40-6a6a81ba2244\", \"event\": \"query completed\", \"properties\": {\"$os\": \"Mac OS X\", \"$os_version\": \"10.15.0\", \"$browser\": \"Firefox\", \"$device_type\": \"Desktop\", \"$current_url\": \"http://localhost:8000/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4#activeTab=events\", \"$host\": \"localhost:8000\", \"$pathname\": \"/person/018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$browser_version\": 117, \"$browser_language\": \"en-US\", \"$screen_height\": 1052, \"$screen_width\": 1620, \"$viewport_height\": 911, \"$viewport_width\": 1548, \"$lib\": \"web\", \"$lib_version\": \"1.78.5\", \"$insert_id\": \"8guwvm82yhwyhfo6\", \"$time\": 1694769328.136, \"distinct_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"$device_id\": \"018a981f-4b2d-78ae-947b-bedbaa02a0f4\", \"$user_id\": \"pWAkITyQ3CN7332UqQxnH6p3FV8VJd7pY647EdNkxWc\", \"is_demo_project\": false, \"$groups\": {\"project\": \"018a981e-f8b2-0000-d5ed-9380ee5aad4b\", \"organization\": \"018a981e-f55c-0000-a85d-25c15e555d5d\", \"instance\": \"http://localhost:8000\"}, \"$autocapture_disabled_server_side\": false, \"$active_feature_flags\": [], \"$feature_flag_payloads\": {}, \"realm\": \"hosted-clickhouse\", \"email_service_available\": false, \"slack_service_available\": false, \"has_billing_plan\": false, \"percentage_usage.product_analytics\": 0, \"current_usage.product_analytics\": 0, \"percentage_usage.session_replay\": 0, \"current_usage.session_replay\": 0, \"percentage_usage.feature_flags\": 0, \"current_usage.feature_flags\": 0, \"$referrer\": \"$direct\", \"$referring_domain\": \"$direct\", \"query\": {\"kind\": \"EventsQuery\", \"select\": [\"*\", \"event\", \"person\", \"coalesce(properties.$current_url, properties.$screen_name) -- Url / Screen\", \"properties.$lib\", \"timestamp\"], \"personId\": \"018a981f-4c9a-0000-68ff-417481f7c5b5\", \"after\": \"-24h\"}, \"duration\": 171, \"token\": \"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I\", \"$session_id\": \"018a981f-4b2e-7bd3-bb30-6617bf488869\", \"$window_id\": \"018a981f-4b2e-7bd3-bb30-661810fed65f\"}, \"offset\": 1273}","now":"2023-09-15T09:15:29.500667+00:00","sent_at":"2023-09-15T09:15:29.412000+00:00","token":"phc_qgUFp5wz2BjC4RsFjMGpQ3PGDrsi6Q0CG0CP4NAac0I"}]} +### nodejs, default params +{"path":"/batch/","method":"POST","content_encoding":"","content_type":"application/json","ip":"127.0.0.1","now":"2024-04-17T14:40:56.918578+00:00","body":"eyJhcGlfa2V5IjoicGhjX05WWk95WWI4aFgyR0RzODBtUWQwNzJPVDBXVXJBYmRRb200WGVDVjVOZmgiLCJiYXRjaCI6W3siZGlzdGluY3RfaWQiOiJpZDEiLCJldmVudCI6InRoaXMgZXZlbnQiLCJwcm9wZXJ0aWVzIjp7IiRsaWIiOiJwb3N0aG9nLW5vZGUiLCIkbGliX3ZlcnNpb24iOiI0LjAuMCIsIiRnZW9pcF9kaXNhYmxlIjp0cnVlfSwidHlwZSI6ImNhcHR1cmUiLCJsaWJyYXJ5IjoicG9zdGhvZy1ub2RlIiwibGlicmFyeV92ZXJzaW9uIjoiNC4wLjAiLCJ0aW1lc3RhbXAiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoiLCJ1dWlkIjoiMDE4ZWVjODAtZjA0NC03YzJkLWI5NjYtMzVlZmM4ZWQ2MWQ1In0seyJkaXN0aW5jdF9pZCI6ImRpc3RpbmN0X2lkX29mX3RoZV91c2VyIiwiZXZlbnQiOiJ1c2VyIHNpZ25lZCB1cCIsInByb3BlcnRpZXMiOnsibG9naW5fdHlwZSI6ImVtYWlsIiwiaXNfZnJlZV90cmlhbCI6dHJ1ZSwiJGxpYiI6InBvc3Rob2ctbm9kZSIsIiRsaWJfdmVyc2lvbiI6IjQuMC4wIiwiJGdlb2lwX2Rpc2FibGUiOnRydWV9LCJ0eXBlIjoiY2FwdHVyZSIsImxpYnJhcnkiOiJwb3N0aG9nLW5vZGUiLCJsaWJyYXJ5X3ZlcnNpb24iOiI0LjAuMCIsInRpbWVzdGFtcCI6IjIwMjQtMDQtMTdUMTQ6NDA6NTYuOTAwWiIsInV1aWQiOiIwMThlZWM4MC1mMDQ0LTdjMmQtYjk2Ni0zNWYwNTA1OWYzNzUifV0sInNlbnRfYXQiOiIyMDI0LTA0LTE3VDE0OjQwOjU2LjkwMFoifQ==","output":[{"uuid":"018eec80-f044-7c2d-b966-35efc8ed61d5","distinct_id":"id1","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"id1\", \"event\": \"this event\", \"properties\": {\"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35efc8ed61d5\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"},{"uuid":"018eec80-f044-7c2d-b966-35f05059f375","distinct_id":"distinct_id_of_the_user","ip":"127.0.0.1","site_url":"http://127.0.0.1:8000","data":"{\"distinct_id\": \"distinct_id_of_the_user\", \"event\": \"user signed up\", \"properties\": {\"login_type\": \"email\", \"is_free_trial\": true, \"$lib\": \"posthog-node\", \"$lib_version\": \"4.0.0\", \"$geoip_disable\": true}, \"type\": \"capture\", \"library\": \"posthog-node\", \"library_version\": \"4.0.0\", \"timestamp\": \"2024-04-17T14:40:56.900Z\", \"uuid\": \"018eec80-f044-7c2d-b966-35f05059f375\"}","now":"2024-04-17T14:40:56.918578+00:00","sent_at":"2024-04-17T14:40:56.900000+00:00","token":"phc_NVZOyYb8hX2GDs80mQd072OT0WUrAbdQom4XeCV5Nfh"}]} \ No newline at end of file