From e065088cd8dfa828583afaaff33315a83bf6aefe Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:38:56 +0300 Subject: [PATCH 01/37] Add GH vault plugin bot to allowed list (#16301) --- .buildkite/pull-requests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index a75d5d5b9d3..03d61a9b238 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipeline_slug": "logstash-pull-request-pipeline", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, From c30aa1c7f52419c68d83f2809394a0fc35fac457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 17 Jul 2024 10:43:57 +0100 Subject: [PATCH 02/37] regenerate webserver test certificates (#16331) --- .../generated/intermediate-ca.crt | 54 ++++----- .../generated/intermediate-ca.key | 103 ++++++++-------- .../webserver_certs/generated/root.crt | 60 +++++----- .../webserver_certs/generated/root.key | 103 ++++++++-------- .../server_from_intermediate.chain.crt | 112 +++++++++--------- .../generated/server_from_intermediate.crt | 58 ++++----- .../generated/server_from_intermediate.jks | Bin 6099 -> 6252 bytes .../generated/server_from_intermediate.key | 103 ++++++++-------- .../generated/server_from_intermediate.p12 | Bin 5950 -> 6116 bytes .../generated/server_from_root.chain.crt | 60 +++++----- .../generated/server_from_root.crt | 60 +++++----- .../generated/server_from_root.jks | Bin 4499 -> 4652 bytes .../generated/server_from_root.key | 103 ++++++++-------- .../generated/server_from_root.p12 | Bin 4462 -> 4628 bytes 14 files changed, 410 insertions(+), 406 deletions(-) diff --git a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt index e772b2ea1fa..ca9ed7cba5a 100644 --- a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt +++ b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.crt @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzI4WhcNMjQwNzE1MTgyNzI4WjBVMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTAzWhcNMjcwNDEyMTUzNTAzWjBVMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxGDAWBgNVBAMMD2ludGVybWVkaWF0ZS1jYTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAPHXRKHl/LsaDFGnys0b4mlLlGQ6qJSC5ut8FZoa -B7kDeHsi8TiFVg/RENghFBClP9aGhWxndGSeQTDkg+KPxKovXBX1J9mP+z+ol47V -QVeCT++AvPQYmtLNLwZg0e5GVHFMnVq64LlTc1XjvzjlU751A5dEq/miMpovx7fS -oI2zxtVOYx+BF4hr+WA20mxbQoGXVM8ITDBWySM5IDhngKNAfmXNQZFffJgk+pwe -++I4KzRkLU02OEUnH63/YFkn+CoZBmg+II8cNItRVYCRifXb8tsO6OEwQQEe1RSk -fmxiiiTYCDMB5GUSowCRotKTwEXgWwTLKghzzUoCPJapsb/CzY2xp/ECtQt420CJ -aosx/fLmcuKmbhW847hYq4L8HqZvjs3i7E6wjuUAgn/um28kU1PrN9l6qTJhHKKP -GZUCC/Cou0g9q7Y9ad672mWgExo0I1vXm2H1KNYy94agmXaIgmelvPBTPREkEEL6 -MnfrZKiYOG5wjSMubg0L4JYKUMuToXvfXxe6o2rDofe/oTUXErGDYV4z7vr58dQg -TM5PtxpZCLmqELaBFydB5hEBQly8Fxh3SVpuwkcaCxOlXMkdvHEK9daYfE9jV0fu -StfckkCLdDZMHI+iSdQsqK6E9vc4tX2rnhzgKex0Byu0GUEg6XoFL2GKdUOvRZTJ -/tMxAgMBAAGjZjBkMB0GA1UdDgQWBBR6wQ0Is6YucsLFNwdZ3ANX1jrgsjAfBgNV -HSMEGDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jASBgNVHRMBAf8ECDAGAQH/AgEA -MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAgEANoo45TDx1lfglAmJ -8nt/58y7fm8trxmrSQmDAIuXZKqATNsEw/+VbIXqeC9sT2lWY9URFmmFm505JP+1 -43nM68cjfAK7ihLcGS7XwQMtQCgiQzJKeZFOx/l30lN5rRje5HHbSIF+uuwO987J -LNYJ+rS7N25lB5EQY8satuZWMVtSZ7YQCMkSJve7KjfrbE3kORjHs6jn3bdyuWAr -oApgKJgCUX3JSPZyH3T/xakMneZNkYr0CC1kNi9wL4js5I/nN2Z4er0N1Zt6PIM+ -hIJcW/Qv0BtI4bzkNkGSehm+Ofr+njuqWaaQMF7kLPTGDg/0SN63Y+X59CJQMnjF -+oW0oGm30S3okFpgXSU+DDxNPlh1j3jTMoArPvIaTceImfmsBXU62JRXQWaAr1jF -EujYbTJ147/7NmsmnskFO37Mp35JxL3ELWfMeBAL/HC11mzUbGFhtvtFqhQMWMbt -5AAR9ZUaPO8xTX6Ag8/8MXF5wFRkJBqTNn6GaX5PUrWhlAKpGuAuuR1NBY5TGUHX -NcFu+bf5JtjhgSUGB0FuZfdZvw46k30NLlT39NbXayY102XYfrAsl0X0TNqjQua1 -7j7FqYrAALQVLfYeKrE0nKaH35nU6Kk1Iy42CbB71MHqcg7DTjn0MvTKZcnQ8pxv -RQwP+BXkzwfWWEp05rlVeKS0IjQ= +AQEBBQADggIPADCCAgoCggIBALUBL8nGcBsOrfv/uI555eOiBgFYD+V3etaiFDi6 +MtZynj9oazrssevjNwP045j8a2L9apjeq9U5REs2MrYoHTEPoIeOiFPNQfiBtJt/ +DjBiXYQbrSfpkjPmRREPKkTX9lFNSlEmG3y+kNMyASvR3K4IM6Bqd+FYt/GUyhZP +LFx9ZhRAiq5uulLqVaAdAp+wdjAXeSedtCTs+20aW+NKA32+Pb8vzJp6HOmh+ZXl +orlEyKKfUYI3ODu9IoXTu6ZN8XZ4LfrDu47wswgQU4n2TOn+Qm7nwE9GIjQBjYOz +s3eUYyzE/z5oyrC6t2Z2ZEq2pCwMwf7iPXRFrgSvgLJCCSdwrB/bTxghM0vmFeoV +V2vfTp3VC/WbSW59tV++iMyMQTPw+grg0NEJCrwNnFJ1DL9iT0f902z2mknxuCM+ +2HGpt2EG+FnU4kR2gHQKt4BNGIuscchp8J85CGjheoNvuHaUktVWwqUaHUL8b9MV +dav10Obfr0+bWNIY2xsufM8sX/QP3VsfJ/nFkoImYNW5SI5g7/5DTIxIE/z8Ab1a +4JT2JCzPO8wDFyle4RVOG+2WfDBUkkjIjXlfxBwLneTPzqv5jmpPIwcpdF0c9bWw +Jwb/e4hB/bAFwZvLFr0aypJ0Zqbc/67n5HDsKx8+/XUKar1qPP6AzrfecOmBwwOH +AsE7AgMBAAGjZjBkMB0GA1UdDgQWBBT5X+u4LkGBED9seTSNYgA40tXJUTAfBgNV +HSMEGDAWgBQB2VzO86glAf1wb5+bjW8YPxZlpTASBgNVHRMBAf8ECDAGAQH/AgEA +MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAGb6iGwxKXTSgxP96 +4UMtpKYFGykBuajxr105jFSjXArhjj5/U21nIHQ3vrmAGAMY6iaaeKJ78gYSfm+O +zC5o8f4PLxtPfRJGymGUISrtIpBUTz+CVNAwQw0yrAeROizoUBhpDmUVrzlSfwgm +uH3H94jvi/IpoxUezsr1tS48G8ayaVlrPLflBhgX1NXzonA0KqGNdm5AwOx3jgZi +v5PYtz6xR1NtINnGm42A078gL9XHXmZowacGPNf7flwU9UzU1LDb1snGCxOgNq0e +ABZmd4TbEqMs1mAmn6ymzAIU2wBOStcC0is1MJ1j/NVVixETrx65UY1BTMlIojDT +bV4fbYy7jo8rtoLdnwFd+DdXna2LJ8qq+7CE35f9c+ZxLkdSfqADISwkhR+J/6TB +plRVyBc4eiPdSV27qPbjMOEHKy43+rHTsBILqIOQ0sBlMe8DgD8ZXo+PUM1Ptadn +PsuHwSflCrIz6AHgG/nORSDQf+7U79mejzi3fIjwYyYpSlBemTuW+z3x1xizN6PY +CK9zuZu8K9Z/F7GQmr3xBEtWqH2rd7TcN7fQP+zmG/v++vPlE38cQFWv5vVISxJe +U6AOJmXKNvDB4QR5CPsd6UWn/Nb1K3/MlDu5ymLq9JXm/GeokaGqHuaLLR20yN7w +/kNCmGr6r+cWAxpEZkmSBUg8/ps= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key index 17a1eadb3ed..924a050733e 100644 --- a/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key +++ b/qa/integration/fixtures/webserver_certs/generated/intermediate-ca.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEA8ddEoeX8uxoMUafKzRviaUuUZDqolILm63wVmhoHuQN4eyLx -OIVWD9EQ2CEUEKU/1oaFbGd0ZJ5BMOSD4o/Eqi9cFfUn2Y/7P6iXjtVBV4JP74C8 -9Bia0s0vBmDR7kZUcUydWrrguVNzVeO/OOVTvnUDl0Sr+aIymi/Ht9KgjbPG1U5j -H4EXiGv5YDbSbFtCgZdUzwhMMFbJIzkgOGeAo0B+Zc1BkV98mCT6nB774jgrNGQt -TTY4RScfrf9gWSf4KhkGaD4gjxw0i1FVgJGJ9dvy2w7o4TBBAR7VFKR+bGKKJNgI -MwHkZRKjAJGi0pPAReBbBMsqCHPNSgI8lqmxv8LNjbGn8QK1C3jbQIlqizH98uZy -4qZuFbzjuFirgvwepm+OzeLsTrCO5QCCf+6bbyRTU+s32XqpMmEcoo8ZlQIL8Ki7 -SD2rtj1p3rvaZaATGjQjW9ebYfUo1jL3hqCZdoiCZ6W88FM9ESQQQvoyd+tkqJg4 -bnCNIy5uDQvglgpQy5Ohe99fF7qjasOh97+hNRcSsYNhXjPu+vnx1CBMzk+3GlkI -uaoQtoEXJ0HmEQFCXLwXGHdJWm7CRxoLE6VcyR28cQr11ph8T2NXR+5K19ySQIt0 -Nkwcj6JJ1CyoroT29zi1faueHOAp7HQHK7QZQSDpegUvYYp1Q69FlMn+0zECAwEA -AQKCAgBirops3aHQ3wIj16njI/uxJ4v3kg/z+a4sbMeLyA9aqVXiORHhFFdXwOF0 -r5q2aBeye9W5tEVEQWcsgjJmom+HWifjNdP67MuC4FJnkBFHmf7wJcH0gI28csk7 -ewpuhfgy+yZCPXsxZCHt7C69EqanUqwSb40VIeAwFksScTneWHtL0t6TnmNx4U0y -u2wFlv9+TbfneuVL3D3ax5akLWfKNajd9XFJ1oLKulwglGwHlQ/cIvxxQyyW2y4M -G+a+upaC5w88Pg2fF81BBAE1oy1+Sy+/v0e7i6sav/AGqlCD/XmNKzG9TopZy88D -jmm4L14HpVK+lAAkzsv35Q3AFKJYaB7+BdxSOxdJrq9RaxuCzZU00sFxA3F7PPkh -tokxrIuyl3s2cnJkHE294s4/1VnWx8UAvdEx+XRsypp0d7JZT5Rm3Q1spOYWS+Zp -ShlHtq2qdhjbqRLq4ur2WYCGWHqxGkXbhkPcO0ezstxQV5WQ/qLbSK0jflH90dgy -OsjDB14Pm7sxVOCgNaShhXZXvzS1dShug31gLDc7Zg8ubazKIpFdAGtLCL/LEUp4 -hb9PEWjshrfi1VQMSCguZCwuazO2FnkKvpP9M8WsxkZ7invxw1A7xX5tdeCfHaJT -yPar945pZlnyt3ROKzx+QLt7yjs4igNuLHFJ7LUL8jdfo3zngQKCAQEA+PoJ0Vhy -gmjA2XVkpuI8EVe6LsyCpPu58qqQPryzBApWZbLvh7RNVEpKrRl5oQbrgzMKxz6d -1dfCdLVA2t+aPLLLuvmduTJvrsD4pFuvmqdLbAGn7J+Q3IhdkANsX3dnfUeR8ham -0fF6Wcha6gsxvof1wTu6k10uT9mvwxEwkxWiJogjGMV7njHwiyXAO+dAUCUFG1GK -pEKcwqJYEbQLBC4dbgYJus7TiQ/pakAXD02z76GzGnZXxFZHbKo2JQpW+i2zhaM8 -Rk9CF4VruMj9ASWsc1D+tV+0f4JYKWk/fwCFX0rpluASXDqmVcF94xgxMPLcTkX2 -S1SfhPLnMNM6CQKCAQEA+Kmy94zk6xRSEhdqyTGKaXYL6pNTuYLClHomg+9ifsbg -LULza4rOWVv1gzdfht4q0LSuR9C44+GEdQlJ093ql+EiKGXOkO5DzOV3D0uJKaaw -jXmfhzVyRKAnG9oCFSoCIVpyVVLTSYsCk4YBG1+xC7jR5iCjSfDoXmXDNTbhvZne -CBPXe5Y16Fu8VKGKMOCuluRywkdGtWplGKIYM/BRpqypMeyUNf5cV/QnpnHpKpvI -3xpN0hWkVPM4mDxeEmxbaTMYqLKBvWxJIKwHlZsCoVrIE/sit4PFxD33A0gd9RKF -ysG9U7j9ezxv01GGeWvLGJJ2Rm9InJaraNdD0/s56QKCAQA9szEVqRaPjMgl27q6 -xJvgLF5eVJ4Xpq45Hd2rPnYajTlF8Kb9eUgkmFiTNxn2NtHMbKzCHZldsh9lbrmI -xBrj6eclCdD1YTNeGkJD8Ni+ubCuLmbf89chX8XX95E2w09pM4BOfyCOLIqGj2lA -UM/eCLF1VojxFglxDmBJNgxUlurdI6WW8PGmjXev9AYEklVUxyfR0k5KUJuqfDLZ -CEYmngr7BhIJMlInyEKu6v0L9NSfkPHin+s2I6+FlVTfrwVI8GRpvL8mKWQWkiDU -5TIVw7FNzoegiPwhETsp4AQ1Q/rJV7Y1hofmRT6NR+sbiRY/rB++YgYcOvdhKhTy -+5WJAoIBAQCKta8dpDy2XbmCWgm5facmOTFggxp7K1P7m0AM7OZWrf2zgdtjrCR7 -eG616ZeiFiBwF7n4WZgplk1bM/k49Fu+qVlyi4f1FNRBYedOYuZ6k399C9FPjSLW -4Tevs6bhw3M4w8lviypNpfuj62aGrJSIY2rDIoWz42zb64w/DQRTzZcrjlXvnNUp -brLZytrZdRwvE/RoJ3ZsEJOgcBtsLax+e2ZOcRgkHmXIsmvDz1sv54Lf4T5z6Jqo -wJxbX6FiSYucXWJuCFWTIuqQDSaV78hB1fAyJS8yi5EwbRhzfjEn2lNDzVsdbt4H -dqkyihGnfdfnNxw/IiixIOMxMpOllkixAoIBAQC3PapCqrPeQRBkUNqXdz55MLZN -R0O/pdkzIsRYbepAEGj6Ylvt+qgLYmsGfFdlhkPAOi9PxClRr44M7MzvJDF5bQlO -y/pyzUKZJ7aXG7QNnhxg/qZRgm0qj+RsR0rbH/Bc8vuahDgHAa1JbMXkc7O50+fe -fwebVZ8hscXVOeTaDNM/XuzmYZl6/lLTmpiqKt9J1F34J4O5mcGI4rS7kdMd83Tb -zcYPTbPXxgltIHIlM8ZFR7hSzdM6B3GcddPbWAFGOI4gY9e9a4sHtU4wH8O6IYlg -BcmSuIGANVLApRHMEabaEbBR/+QAzSZ6dhBaxG/QlqHh6iKSM29mSEUoSlZm ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1AS/JxnAbDq37 +/7iOeeXjogYBWA/ld3rWohQ4ujLWcp4/aGs67LHr4zcD9OOY/Gti/WqY3qvVOURL +NjK2KB0xD6CHjohTzUH4gbSbfw4wYl2EG60n6ZIz5kURDypE1/ZRTUpRJht8vpDT +MgEr0dyuCDOganfhWLfxlMoWTyxcfWYUQIqubrpS6lWgHQKfsHYwF3knnbQk7Ptt +GlvjSgN9vj2/L8yaehzpofmV5aK5RMiin1GCNzg7vSKF07umTfF2eC36w7uO8LMI +EFOJ9kzp/kJu58BPRiI0AY2Ds7N3lGMsxP8+aMqwurdmdmRKtqQsDMH+4j10Ra4E +r4CyQgkncKwf208YITNL5hXqFVdr306d1Qv1m0lufbVfvojMjEEz8PoK4NDRCQq8 +DZxSdQy/Yk9H/dNs9ppJ8bgjPthxqbdhBvhZ1OJEdoB0CreATRiLrHHIafCfOQho +4XqDb7h2lJLVVsKlGh1C/G/TFXWr9dDm369Pm1jSGNsbLnzPLF/0D91bHyf5xZKC +JmDVuUiOYO/+Q0yMSBP8/AG9WuCU9iQszzvMAxcpXuEVThvtlnwwVJJIyI15X8Qc +C53kz86r+Y5qTyMHKXRdHPW1sCcG/3uIQf2wBcGbyxa9GsqSdGam3P+u5+Rw7Csf +Pv11Cmq9ajz+gM633nDpgcMDhwLBOwIDAQABAoICACWvipCbNUoc41KdaoKXlKqx +WEi9Wgnvt26qrkUlV2JY2OPgzLqm/2BanWDAgwaeP4mLs78CVYp+ct6p+SL99R36 +3/R5U58HFF42OzFI4d5nDItt675snhkaL3Ue0DEnOcyxp/B/W6e8UZD3T1EdT7IY +WQ9kO3N6skmlLfI5J8ZWAYPj8DvjJ/4NdihxAdPXvLPEn2AFMLWjdbHAlDMdfN6Y +VmjQ/euH0y89nyZ7ekbSsGdTfnf7IUUm2uI8NEyq28RLv17/k7RuHwhlxc7qrhbl +7lUDAYI/2u0zCZDLQGl8Rq3SkkYfMVxcc8Tsj3MS1dmqClYUKrFjtJibJdN3dkoa +/I8OUSG+OO8X1H6nd0Ht63RqBXYgR7y1+P8aGVuJYnNAdnj2iMmGulQmaXthlX/O +2lpoFJmhSZT1Ih00m29yIY8XhbKXddZvjg8zhVYmgk5KEZhN2+tF1EYTsBCtxvWh +jJ3O6TXSP+WQFByeAMl07hG/1NGa9hvgOktNCT3QjSosJWGRNj3oaZL3RwtUYOnJ +UOBzbPY00jF19/KGUMy/R8t1fOS3Z8e3qD9CLeMkr9TN0VBXFGYaMSrq+AvqTgco +uA16/ctdZcj80bwOYU6JqNqycW5S4V8t2KX4MhMf5bZ5dL13OVPhVY0ltKBEdOC3 +YBO5WvQ7l6pH5QEjWoPBAoIBAQDo1p8GDY62MMqxTUtrB/Bqz/Vsa7SgCGmSmRBB +PBfkxxARI28jTX4v528au1J5h3GZNs0gzWtbQW0yUN3hvf7KaERfBOAupg3nUO8D +731JL8jcX86UjS77PxzALlvNNONItFWZcH2VIMKJCeFRp79cVaHZtqHMqOcq4Lrc +BYjZyIcNVSMEMIO9E1ZJB0t6Gesa69OPenC0v9ywBF2h0YP9XGbCN370JfR6rJwZ +yvB29et91/agCorZBlctnRhRXYjQVqBMetpsC6tet3H/B9BfUSrQ81zeEJes/roz +aAYym89Vvav3Gi/P/NP96aEefcyG19cnTTXskeSkIkpIEG+bAoIBAQDHApYSxXKK +iPsakBno9OUA747YvyuBI2Uh7+HKGbM7sVyl7UhwduI2qpxG6xjz5+NOxJe12XQI +91+KAciuKNZ3WguxtgKy5CFLlC3syV5nuvfTWwt19qg2I+EiXT1YQ09LmhG7M1WW +xwj/QI770YlbVW8NA9pB40ojPRUB6a9QynGBvj0fTdr1IDs23Eb11s/1f5pPAfmf +ENehdZG8aTuPDKbdL++yHrg2n+v/aAWS453eUvE/evk1kv2n0hqNwJomn/e7MH0h +DFfdTHNOaT9jS8W/6TvnXTnwqCPofbVdJkwIpsyqFP99JpLv2DkjWBpnER6K2sN3 +4tlFutH3fZ7hAoIBAHhQluHg1ZLNiZ+uyj7nXlNDXJWG3lInpq4LC/w5mjsspd/F +SKOcYo1McPwGOer0kz3RtJOzsjr4hv2ODemdbPCYimPzOAjC48JY7vlAoqPT3l4J +38yhW9qWVTv1eJiEMMIb9XORTkNmJz/67ARuwxJqaXT/y4He8X0VnVkePwYSeb7o +9GwwyZJFULvx2g2hIPzLSYoFggHUdqw5xUxd4G9mdO79EEwG5NsTIKKvXCyMjQB9 +PUKbz/2vKhM27x70T0Mg0DpU7VMRgJ8z+eMH3aOzZIWOi7lwJ4BMLFlt8m3qV2v9 +08OfM3+tLLozJH+GD87J77KuUiQJi3VJBstQOusCggEAbPNOg9JS5eMeTamMdcv0 +v3FnK+RlBdsB1/WDEGjHhiuQhEqGeQOszq7SxcSZsZZHDG12fRkifHIpfN2DJ022 +NDs44b66dYONctoZ1mNb6sAiQzS57ofuCU3VSEHxfqvC8qnTgUAWYGhw1YVbgiNx +5m28UUzFpYKwqOAc6yIrV7itMZGb1tKBx+cwY9Vuh9JloYqNSSnUpi/LTVqXQaX+ +9E7B36AJwuMsiMM+qWIigwtX90pDRzSk0J67fa/515cVDYZQUsu2BWzDgaUED25x +RIRMK8edgwm32bfDiNmhe/CmeaoPrCG4yaRfly0yAtaOBdUKeoxPrQelWZZwsuS0 +IQKCAQEA1bzyZGNAf39eJ8munj703lTgANCjSqKMoHb8T9Y42ikOvrM0tH/0gOTs +43CHNI8gxeJXrjv1HpUr7kfkIX5aBvRGoG4kxS4uodQUWB80KW66esbbU2XzpuGO +uB9jjmY42EhT7wXNde9PWdu288YhVHqqIXHQMbdjfp2HjQGUnKuwWVxNSDtFSeC2 +yTM4cqkUH5tkpH/2KpGgZzEUN2MtHSt0+kojkVei/RJqRqm0/YnN9h0j8wAr2DRx +6tEQIBg6JR3dqYLmOTrTJl/6Quov65hbAR5gKDCaxw+jBp68hxs+SxOCBLIKgzSz +UhU2dLCAXnNdC7uFxDlGq9G5/e6SHw== +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/root.crt b/qa/integration/fixtures/webserver_certs/generated/root.crt index d5653f0e853..23c26078be2 100644 --- a/qa/integration/fixtures/webserver_certs/generated/root.crt +++ b/qa/integration/fixtures/webserver_certs/generated/root.crt @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFejCCA2KgAwIBAgIJAMwwXnXAAUBzMA0GCSqGSIb3DQEBCwUAMEoxCzAJBgNV -BAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVN -eUxhYjENMAsGA1UEAwwEcm9vdDAeFw0yMTEwMTkxODI3MjdaFw0yNjEwMTkxODI3 -MjdaMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9u -MQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdDCCAiIwDQYJKoZIhvcNAQEB -BQADggIPADCCAgoCggIBAMfReGxhC2tJH3y/4knTrBSrtezJb5fDs9Xy41Gdr5ng -YEQXi7JQb/MXvGst4xxX77kbBv8uo1egL6gnvZfUqbsrGS8IzhBWtRET+yaYw2ez -wZW+d9RGjsFTOGvkSpv55/VdGvtaXeMeqTbuyYZSzakNYD7AH2VDCS2q+IdF/ezm -ocOki5faj6A92yxvbHUZM1jS+Ah/v2yzJOAkrhT1XIBKyaG/g+woQxZpMS0PjpKY -K7K9XPIkU/Z2gsQkRfDQAxXTqba4Bl84/aA+aLTYqib/5WhxVuc8vjk+7g01y2qX -5aPYs3L2LToUfwPh/6leHx37d7YouPp3X3CnPyl15r20FjTFKTV6Y0he3tgmWnjC -BP0/qlCYeh1mnKFQSo688fw7K25dPhcnPctdVp5Xr6jz8GBaGgCu5hEx/QI7WzXy -qKdyyIj3gEQuHyu8RUrTUA3uruPa8TlouAu616yNNqw3EzrEjmWX+D4HRq32ofkK -FXqFxFurfzxyVJjX1P2+VP0h8fPoZJzBwfwm1hFxLpROQXVBX+b7j2S4kv/Lp156 -mV0frWXlYwTvtM+7PKONnwSIOkiTic8Zv5TcMLoW6p4RNY6b0SDzt+e90h20OeAS -R1gv7VhGfR2JYMd1C0Ke8D9i2FebX9r6T/BfRMRJBNKj6wr6OFvhz+eImNuiAaxn -AgMBAAGjYzBhMB0GA1UdDgQWBBSWO1UVkFl5LOuplY08xBLcfGyc+jAfBgNVHSME -GDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jAPBgNVHRMBAf8EBTADAQH/MA4GA1Ud -DwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAHquJcXAZlMpY3z748dmIfKGb -7F4fGgTwGJ/9PLIZdZXDRIiA/GZs9IGA+Kykv/95jJsq1ZVN7O7UoBqbTxbsddoU -EhBe3BKaJtkIuB5anRmw600lZlIFgSU1uSXyZoDBD9LnQnb13f2UQJKvzNbvoisC -f23JIVCQhGUiqVVYKF80tWlbTpnA+pIPvaWpzSI8FEjz2NGmol948aEp4OET5v/R -obWImc/X30t5Z4wO620DCNVhwx+zzomAio3dmQRnnUrVT9IwKxFmEA+0V4O4JPUz -YQMyuLuoqrvywz7JeV3GvCy+yI5uoF7u5m7e5Kh3A1G3xfbw36M3fYAQNhh/5y01 -ySimOmx6CXoX60z31RFlrXgAycLWKixkjllQTMCMvtcFOszLSpP7WoB2/l7s+9oz -iMEnjwvtUcXJ0rRHr2ENmyG8BBPKZZhKp2b/kumQnYa8Vj8FVFXGijLNF2QHyIWr -jzc9ntZyUb2LOWA2Dwtk6BxgJrKfx6N4n6itwd//mzCz0GQZEVPwlLUw0qO1qt5+ -dSP5ifsecUSsXge3nl+pZmS60Jo1/sF3QhFiFbKMRi19SqvI2X+vDwAbsS05PMdt -HpN/JLfaeabqjoYeEpCGRm7ZSIR0Os9c5qJpXo6V9kOUbQKMNwJ075pPaQEE+2Vn -IQqdInXs3f5UoI5afJQ= +MIIFhTCCA22gAwIBAgIUM0vS1bRQQCJzcfsSQBT78uhQ7lUwDQYJKoZIhvcNAQEL +BQAwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x +DjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MB4XDTI0MDcxNjE1MzUwM1oX +DTI5MDcxNjE1MzUwM1owSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD +VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290MIICIjAN +BgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAqtMGs9xm1Z6/B0FrYwxALZmzXk9E +HwS9mfmPbPUW+ZceZ8LcF9deWU+gtBwEZpV13ILhwHdlMacsnW0iriQtZKmX5/wU +Eg14tRj/vvDz3uOXv7dQ+h2VghlFvQD7iTxRt39a1NnbrKMmWQ+vzAwGOcuWthY/ +Ec9JZdYbvsBCS+RMCcQrd1E9xHjV+HYIdS18Hr5TVxnfW/Flm931i6tDsTFXvMwk +dqgCuajMvEKR3yhDx6rFxxRN9vIezbyMuVBjNhYsRVvv8FNQOKOMIadjb/vRrkMK +VgUL4WYi/liqjSCIV1rC+7v/DnvXPOCSaJgNBvB0zMJNPxSgTe8iH/LVnEHuCEu2 +MSnUVL+AApKOjiYK1Wr4h9ymllnRgq5XlsUwGnFkcaEkGO1UaHNFs4z89dr32hye +RO3T7B5ChQVvjbYK7piDDto69OW397i+yHVLxD681+KrZuKOeMZr9iFFZaFLeh0V +7l/hahhIlnZGdK5hGWIHZu+c+kfP1Q79drspE7Rr4AoMxVInxce7qV3lNMUp2v1a +0EjyFJ+iJquIBQdJTtX1rYWdFA5BBA00DGGRbIYiOheKOEnTcCsUevx4T8+2G0PM +pyCDTy7rj4pnXLr5ziSsfSww8JnxsU1IOijJ4L4b/qGCJ9JfQpslfgLBfwOJ94ZJ +mv7qjgAcR1HrV6cCAwEAAaNjMGEwHQYDVR0OBBYEFAHZXM7zqCUB/XBvn5uNbxg/ +FmWlMB8GA1UdIwQYMBaAFAHZXM7zqCUB/XBvn5uNbxg/FmWlMA8GA1UdEwEB/wQF +MAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQBslT8qF4mm +B6nKUPREE/YP8Q6nTXgZ5XhhHsBAHNTP+U2/lNjfQdFUva9mHwJRts5w59phfMpw +5gTiXJqdTzZEIEN7cdKX2aNL7id6on9j7OoJsQ81MzhUu8rOGqeCYFJFlCMXeby6 +VgInIxhIoe2RrM2bJYw8turWd/+VAa34ehvS3knvGGUTEr/8sqJ14hwifpRl+YB9 +iC1CTL0k00YXJ1IFC6sUaVya0KCRzd/J3I9XnUKOvHxSavA5QosUoWjQde+lJM+J +8sJHudaM1I2bn+8M9jqSO06/SNMdoNhbapKyvRV2DUwpMbe4WT8Ufnvbxwvw7vnA ++0E7X10KcaLn4+0tKhmNLHb1hjbaGryHO9ESSFGa7uMeiQYo0OQeuLWgRb6mOsZr +x/VEuytnK+YwR0+vXBCC7PSxuZ7G/MPv374w/+6b+rsrqz6o5JdgjTu27OISbNv9 +5IxA9+DfNvqyLvSQ72M3W6BHAE5BT9/JsTppcJ0KXc1Hkb9dJEIScO05FdsDC+SH +XvTmuXCf6G5XkrLldshntu1M2MI2Usw5ZK2RP3H25F/AOPxQDim5awMp7Feo0JYC +EXJedqqTtc1M7ub8XVvIHTaT2jNoa5uuV8ujyiaLXZqZty+UUkXUFHx4UDDWnOrz +mHyg8QosOZgiGREyjq0FJNDr9337zIfNkA== -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/root.key b/qa/integration/fixtures/webserver_certs/generated/root.key index 06006400851..29cef4f921d 100644 --- a/qa/integration/fixtures/webserver_certs/generated/root.key +++ b/qa/integration/fixtures/webserver_certs/generated/root.key @@ -1,51 +1,52 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEAx9F4bGELa0kffL/iSdOsFKu17Mlvl8Oz1fLjUZ2vmeBgRBeL -slBv8xe8ay3jHFfvuRsG/y6jV6AvqCe9l9SpuysZLwjOEFa1ERP7JpjDZ7PBlb53 -1EaOwVM4a+RKm/nn9V0a+1pd4x6pNu7JhlLNqQ1gPsAfZUMJLar4h0X97Oahw6SL -l9qPoD3bLG9sdRkzWNL4CH+/bLMk4CSuFPVcgErJob+D7ChDFmkxLQ+Okpgrsr1c -8iRT9naCxCRF8NADFdOptrgGXzj9oD5otNiqJv/laHFW5zy+OT7uDTXLapflo9iz -cvYtOhR/A+H/qV4fHft3tii4+ndfcKc/KXXmvbQWNMUpNXpjSF7e2CZaeMIE/T+q -UJh6HWacoVBKjrzx/Dsrbl0+Fyc9y11WnlevqPPwYFoaAK7mETH9AjtbNfKop3LI -iPeARC4fK7xFStNQDe6u49rxOWi4C7rXrI02rDcTOsSOZZf4PgdGrfah+QoVeoXE -W6t/PHJUmNfU/b5U/SHx8+hknMHB/CbWEXEulE5BdUFf5vuPZLiS/8unXnqZXR+t -ZeVjBO+0z7s8o42fBIg6SJOJzxm/lNwwuhbqnhE1jpvRIPO3573SHbQ54BJHWC/t -WEZ9HYlgx3ULQp7wP2LYV5tf2vpP8F9ExEkE0qPrCvo4W+HP54iY26IBrGcCAwEA -AQKCAgA4oYdtv/tKhsjjxEZQAivNF/VRjK2R56c6RcJjSTuRcuQVqE6xVNbFoL6H -6dNqHwoT/jL3c7JbjUy0bb7wHxeTsl1K1UYBn3Fly7HseHM5GtsMo5S2luCyrfFZ -srPqjONOyPpyrmPZkMEIjpiS9zjOZroWTsEXI+6gAMq7G/0hfSPvnNzrRshZ8+4A -I/wrG99Pl0swrXQFTKhw1jR4QCndOlT3AJ5U+fw5h8qmKLiRl4sa90w2XFFnkqLx -EA66d1nIf9qjrHgx2KcZGVb3mkhvEStldGna035eHxW8rLhgF3i//RSuylJ7GX/+ -C7k3uTM8fh6pg9UZN2fXN69RAtdH/8wHeGRpMVfTyFA+//ChEr8pFDudheO9elZx -ahMB9Qm+QspfJuGiA08jbkp/Kqe+vs53SXvpcWB0DCba3Y+PzKlJuQiuGJbjC0x/ -9LJjBZCB/8VyWCz2Pcg5arUyfiHz9iJIC6vfZRMwrrIvD+GQe7jTPT3bdWNeLpRQ -kVjy5AjYipblT2pSgsh0iOBwBoxqh8z+NOUgsIjcLWlMXLStQpNILuCc0qVUDfsI -LCt/BujptkezwFeicmjf4a+uavoUODDysfMxbXwyWVCPoHmcVUIj00Nib4pv4hDw -j124iEyZVy84rCUlYHb/uGpwSqE0Hw0diBdx/g6vomCMHCxLAQKCAQEA+Q2ViAWw -ekNH4wYLNDFZEhcdP9tPPP1z9OE2N6+BgpTqfiJMBa7tVMEd0HdE38GZO2Q39hpD -Lai0K8/kqjckP6CAWRoT1vgSiLVvdn2NjSKhidostZ+GZyBDbr97EBq3Q5XGGdYZ -021gQ+p27sWzwNCQfewtKeLbWWRWtgQPmrP2kwaF+xEYcfE08wy41yvypyWFmAm/ -e1KdTPiwSXjDHawUl2/Cp6S5YjvFWAMwzF7BIEkGcRqjvi4eR8+i1NEJgY/7/354 -B7/wxmPWTaqk67+8fRNSzkMR6KNafeEd1iS+TkcdF540D2kM9GJiBVBYvMboX1jF -dcNUXi6opbQqJwKCAQEAzWRQj7LkmcnDAs6HM+/UMbQ15sUHrTqDfFF85XA9/ZYA -ZHWTyLRWN5TCiRHGK6vilrVBqzYkQD3sMMepJwklhAue5SnvXLiostf61F8TzXmX -uQ0Hq2goyHGBTa9RZCD48mliuPHdjFQY3WzXGie9XvkDat8zgYtZi5ODMO27mSTc -8TnFjZ/Klp4mXgfoOzv67yTlE/DwmGQT0/HxtA+mwgB1WfKNuQqWJ3c/KKcIJ2G4 -Gf7Y/9ketF2i3pdkri1UBtnhL+/t2n9QLMjtt08hXWfrrJPfHQdAE/O4i67mbl1i -pq8OazK7YGFp//vDCCuvrfQUo+bA4TFZ8+pfIqsTwQKCAQAXfx+XaKoAru5OTdiq -wOkdWs7oomE0aNkSxlieIlG+pxwEcGnps1oyv9F3RHlb/uDdCJR7BHlICn2NEJ0L -Lg6ebh8NyPcMNfVRroRokmnTYewJnGrRa857VkDmZuxOVabqmmvJLjjbiJsnbSxx -VHej/klTh/+uzvdQ1LWVkIaEWuwtbNUhHbjHrzEGqTGbI0tOV6zl7i2BjgbHJ0NC -e0E1QHHk46dit5jB2s/AgPSOo3Xm/Ny9uPyvyZwTvFQpbdtOvO3SafUbDhHCNdMX -Gr1afIlk53jNeuhqoVa9P4rYFDlcGzt0afmRZqDRQaLry2vDy6RKso5WkOoZxJ8m -mvvLAoIBACu03xfVH6RCNQdR35+Dt4nkKVw0LVAFlZlM1eEVI3vlAaxfANXC2l31 -f/gfGiaM6yja5WCDou8wviQbOLmfzqXsuyFU7SSz/uIf57ZtjQ86grjUNv+YqKUO -0k8vsYg3JAKhvmpQva5w7uGfmZ5tOnUNVFbXrGQyvRlgiJhO3o7WM1X5zzo1+KxY -dx2nXRnNx7ZACadWrDwhf76oS6Dfe8LoyhAb92CbChzX2U7TTH7k4oJVouO+1jPX -gSMQ4FHYP3s9vhevXNQUJ4GAOGW7pwPza0lMJjxNIKueP3UqAMsXgDisI8tX/lui -HyZj8rq4YSgJOqMuJXlGQTgQdbIw6YECggEBANr5VriGXg/bynABEpz2fnYJsDPz -syxNr6262pYFhXHsuawi9RtSjndNwrUUnfEnFddSBHXzNi2BkWLXtjGdXKzF4Xuy -TTq82gXNMAxGETqC6hmof2IxyQjomoXxUvlJ5IKEoDLGRaaO9UhlG7GhsfJth5GN -hgCsij+llcgKTuJ3HJx3ThLO08wNwk4lTt4YxDdN3HcyhK4YZ1noH5nx68tjnC8u -Nd1UAPFhlPhUH1hnp9zZaFxaSU6MvnDVUIBayAMxpjQXsn9GmdG3MB13o8yAeHkP -U6MHvFri1MvZ45rd+DHresJit22GePxH/9WrYfWl37yWVSwTffh2RE0x4F8= ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCq0waz3GbVnr8H +QWtjDEAtmbNeT0QfBL2Z+Y9s9Rb5lx5nwtwX115ZT6C0HARmlXXcguHAd2Uxpyyd +bSKuJC1kqZfn/BQSDXi1GP++8PPe45e/t1D6HZWCGUW9APuJPFG3f1rU2dusoyZZ +D6/MDAY5y5a2Fj8Rz0ll1hu+wEJL5EwJxCt3UT3EeNX4dgh1LXwevlNXGd9b8WWb +3fWLq0OxMVe8zCR2qAK5qMy8QpHfKEPHqsXHFE328h7NvIy5UGM2FixFW+/wU1A4 +o4whp2Nv+9GuQwpWBQvhZiL+WKqNIIhXWsL7u/8Oe9c84JJomA0G8HTMwk0/FKBN +7yIf8tWcQe4IS7YxKdRUv4ACko6OJgrVaviH3KaWWdGCrleWxTAacWRxoSQY7VRo +c0WzjPz12vfaHJ5E7dPsHkKFBW+NtgrumIMO2jr05bf3uL7IdUvEPrzX4qtm4o54 +xmv2IUVloUt6HRXuX+FqGEiWdkZ0rmEZYgdm75z6R8/VDv12uykTtGvgCgzFUifF +x7upXeU0xSna/VrQSPIUn6Imq4gFB0lO1fWthZ0UDkEEDTQMYZFshiI6F4o4SdNw +KxR6/HhPz7YbQ8ynIINPLuuPimdcuvnOJKx9LDDwmfGxTUg6KMngvhv+oYIn0l9C +myV+AsF/A4n3hkma/uqOABxHUetXpwIDAQABAoICADul3B1JxI6rE8EQ0tKPIInL +tZoky4CZQytUvQQqwI7+eZt5qyiYB1XgPBkn/1Fitvj8qp9uEWO5IBcIWKe98ivI +HEbg+pEjKxbxV7vwnvchOYe0Fw5+7gmzkWphwXM2IMKWMb1JJS+u+GD43UW+nR3p +PQLv2a36V4K7tIxyzbN6UCLI4PSU5aCJIQ3VCTcBXunNQ4SSgC+2pug6JNdUQhrJ +Q/2kusbD+8+gZX0HMbBeeuxpcQ4GxQKJoL/uKqU6sKneLC873xhnvwdH9FJ8BHjB +0YGr3xwSPwfBhk6R31vQFEuMYbFM/86q54lWUUk8brARnncxEZAJP8B208+cF4IZ +KqHPP45r1GwWQ1G8i8IPFmLhmiSZU47NiPiYglOlBsSGuG2zPLHJUWZnetEmJgFP +5h1sBL8v6OA79Y6dMWXgs/Wdu6jOrcv9ZBEPZskqnYqZ/AQknqgIKggkvegksjFG +OoGu6slQ9QRDm09bjIEvqaVKNS5aDhmGPmOvaCpmE571KsIdWFMOIBIvJjmJ34hL +6qoLJ2XoTM5QRY7ZyA0a17uEiXZCDnd4vggbEt51CCwKHkY0+WnjzT0MgU9C4Bzo +B0a2idW8UT/yGxPz1uGFY4z/RJ8kW83MwpGX5dNPYFEq/1egZVehK56saowetPlJ +LFkoJNSA7ZetxIUZvD1BAoIBAQDuPIc9WPhT1Z17R0g/uYnQ94H8ZbpPz5sqzzGw +gXTcYDIuEVixUlDE98KDh7BswGoh9O/PSGKYAWt4FhEzmUUPGSOV8U4vnPNB/tud +quVICCaVA0qYF5RfDBZkMxawaP+0wjhiReqKMZZstEEe3RBiy2uEnkHfDXmXlTER +plF5wFTc+luS7fGx6Sl5B4rSWrhdmHAZC2Qep9SOBjNPg/BwvlKthXlDUElE/eP/ +hOOO4upQBBENKM9iRZfmNW3JU3NMRZB/SFVi4nfAcKD/k8wlp3Jf3US4E7WJxhzN +WIWfYpfEHFgxxCLBrpS2axlRe9rm8h7uWLnDspVgXl5WdnEZAoIBAQC3j7s069A5 +Kw+/MbPWMZ1tGZGKnl4uBKq3LZAz/i3/mzjhleaOEHYoam76ELLLIulblc5mjHwf +8he1An7L+ZmTo0ezwT72WHBmBAjH5Nn86BgB+3DqEhed7pSDtwz28VzV5XtwH7Ug +enloX6HqxDiHyHTyo1rcERiOq2HsHURUcu+yawl/jOadv50qln0j2WVpGeAQo4kq +NwY+OUrZGOaCtd8RshX4afNKnDU05Pk2e9eytqfoa+S0/86VhkZ5cv4pc1SaW9Hf +juUWNFeTnCb58lk7PN6W+/WQaYdn6don5yrHm3ne+ThoOXlQmbjCQ4m7P81EVJ1U +bsetbqzRBWa/AoIBAQCXkE6Byv6BZ19/NIRUdlelaDF8P8PeTrfq7lIwdsvAADss +b8C7XyEfjnhjcYneRjO/UrLWbri69/xFM9TRvOrghh/Lnu8c/38LN3bEwhqITxcs +qiFkzWuJZIlCSfZfWQ2FFCMG8nAaZx3KHotndUCdecj3YC+MFmiK9VF4ZxVineff +hi+INHUVuL63zwKUfT7Z3MdER6j0j9V8XZl7Nvda4tp6pnXVS1ix2cyZnROgreBf +85Zdht8ZKClRPiCQetTZa1BYic440NdayMzEjGTOd88eCglUCsXt1uowAedQ1Uq0 +a0v4rU1nZwgZXd2Hp4JGdzcSb8QPKqg53Q+rdkLRAoIBAEwgxBdSpWzQOxukVV1D +/nI85GuE+rJDJToLH38wr4sOqSyFqrTIhviEC7qSKlStCfL90jgdk2z8TnC3tDaM +MfFVCmu2fZJVY2grzCh69R06dJp92Xp6Pmk1hUqLJElWGJZtMiByRgBOSwPKhbCD +6J4HuNjYfto+DN0+nsH58VjEwjxQwAOyQAK0cU40U3oGs1Q/cvnzmPN1FpW/1Rbs +iM00+GqyXQJFjq0m07o4LFM4UWDA+CogdR3L6bWBlsDPvDEcr73EzunUrZXj515E +qLWImRFEUVx06ocV/R5PkjZyXP+jr4uV+JrhGzseRc3rjLen4k2wef+GYjG/cNTP +VKkCggEBALOgG98jJQlbW5GD/jBkizeDiNy99H+Cl7MzmzY2PAtpfugeMbBYceL1 +2weWpWbVpbCp/Zkksqh0kYgI45phx+eZo6tLZd2IOM3A3AbnkgVDK+1gNSb6s7fq +g0Llaz6wAHrLjTf5N5vbGJNbzez2UWCJR8boxM4+Q3pcOSFQbPv8rx09zMNa1xaL +KuvHT6xTab5q+qz4BiVtrx7ZWqakOlxO3J5chYZUsxqoH3jvE5GzY4yi52ZPyt7v +rHZbCHclS/MS5pBUUCTj0Q0qCfiT03m6nP6YgN9w1rQROe5EzqPCMfQlVKo8ayQB +tiYXTiDTadKNODupVYUpbHSfZcFvRhE= +-----END PRIVATE KEY----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt index f4c8cf7f5a5..c2360f241e6 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.chain.crt @@ -1,68 +1,68 @@ -----BEGIN CERTIFICATE----- -MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIFgDCCA2igAwIBAgIBATANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzI4WhcNMjQwNzE1MTgyNzI4WjBVMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTAzWhcNMjcwNDEyMTUzNTAzWjBVMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxGDAWBgNVBAMMD2ludGVybWVkaWF0ZS1jYTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAPHXRKHl/LsaDFGnys0b4mlLlGQ6qJSC5ut8FZoa -B7kDeHsi8TiFVg/RENghFBClP9aGhWxndGSeQTDkg+KPxKovXBX1J9mP+z+ol47V -QVeCT++AvPQYmtLNLwZg0e5GVHFMnVq64LlTc1XjvzjlU751A5dEq/miMpovx7fS -oI2zxtVOYx+BF4hr+WA20mxbQoGXVM8ITDBWySM5IDhngKNAfmXNQZFffJgk+pwe -++I4KzRkLU02OEUnH63/YFkn+CoZBmg+II8cNItRVYCRifXb8tsO6OEwQQEe1RSk -fmxiiiTYCDMB5GUSowCRotKTwEXgWwTLKghzzUoCPJapsb/CzY2xp/ECtQt420CJ -aosx/fLmcuKmbhW847hYq4L8HqZvjs3i7E6wjuUAgn/um28kU1PrN9l6qTJhHKKP -GZUCC/Cou0g9q7Y9ad672mWgExo0I1vXm2H1KNYy94agmXaIgmelvPBTPREkEEL6 -MnfrZKiYOG5wjSMubg0L4JYKUMuToXvfXxe6o2rDofe/oTUXErGDYV4z7vr58dQg -TM5PtxpZCLmqELaBFydB5hEBQly8Fxh3SVpuwkcaCxOlXMkdvHEK9daYfE9jV0fu -StfckkCLdDZMHI+iSdQsqK6E9vc4tX2rnhzgKex0Byu0GUEg6XoFL2GKdUOvRZTJ -/tMxAgMBAAGjZjBkMB0GA1UdDgQWBBR6wQ0Is6YucsLFNwdZ3ANX1jrgsjAfBgNV -HSMEGDAWgBSWO1UVkFl5LOuplY08xBLcfGyc+jASBgNVHRMBAf8ECDAGAQH/AgEA -MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQUFAAOCAgEANoo45TDx1lfglAmJ -8nt/58y7fm8trxmrSQmDAIuXZKqATNsEw/+VbIXqeC9sT2lWY9URFmmFm505JP+1 -43nM68cjfAK7ihLcGS7XwQMtQCgiQzJKeZFOx/l30lN5rRje5HHbSIF+uuwO987J -LNYJ+rS7N25lB5EQY8satuZWMVtSZ7YQCMkSJve7KjfrbE3kORjHs6jn3bdyuWAr -oApgKJgCUX3JSPZyH3T/xakMneZNkYr0CC1kNi9wL4js5I/nN2Z4er0N1Zt6PIM+ -hIJcW/Qv0BtI4bzkNkGSehm+Ofr+njuqWaaQMF7kLPTGDg/0SN63Y+X59CJQMnjF -+oW0oGm30S3okFpgXSU+DDxNPlh1j3jTMoArPvIaTceImfmsBXU62JRXQWaAr1jF -EujYbTJ147/7NmsmnskFO37Mp35JxL3ELWfMeBAL/HC11mzUbGFhtvtFqhQMWMbt -5AAR9ZUaPO8xTX6Ag8/8MXF5wFRkJBqTNn6GaX5PUrWhlAKpGuAuuR1NBY5TGUHX -NcFu+bf5JtjhgSUGB0FuZfdZvw46k30NLlT39NbXayY102XYfrAsl0X0TNqjQua1 -7j7FqYrAALQVLfYeKrE0nKaH35nU6Kk1Iy42CbB71MHqcg7DTjn0MvTKZcnQ8pxv -RQwP+BXkzwfWWEp05rlVeKS0IjQ= +AQEBBQADggIPADCCAgoCggIBALUBL8nGcBsOrfv/uI555eOiBgFYD+V3etaiFDi6 +MtZynj9oazrssevjNwP045j8a2L9apjeq9U5REs2MrYoHTEPoIeOiFPNQfiBtJt/ +DjBiXYQbrSfpkjPmRREPKkTX9lFNSlEmG3y+kNMyASvR3K4IM6Bqd+FYt/GUyhZP +LFx9ZhRAiq5uulLqVaAdAp+wdjAXeSedtCTs+20aW+NKA32+Pb8vzJp6HOmh+ZXl +orlEyKKfUYI3ODu9IoXTu6ZN8XZ4LfrDu47wswgQU4n2TOn+Qm7nwE9GIjQBjYOz +s3eUYyzE/z5oyrC6t2Z2ZEq2pCwMwf7iPXRFrgSvgLJCCSdwrB/bTxghM0vmFeoV +V2vfTp3VC/WbSW59tV++iMyMQTPw+grg0NEJCrwNnFJ1DL9iT0f902z2mknxuCM+ +2HGpt2EG+FnU4kR2gHQKt4BNGIuscchp8J85CGjheoNvuHaUktVWwqUaHUL8b9MV +dav10Obfr0+bWNIY2xsufM8sX/QP3VsfJ/nFkoImYNW5SI5g7/5DTIxIE/z8Ab1a +4JT2JCzPO8wDFyle4RVOG+2WfDBUkkjIjXlfxBwLneTPzqv5jmpPIwcpdF0c9bWw +Jwb/e4hB/bAFwZvLFr0aypJ0Zqbc/67n5HDsKx8+/XUKar1qPP6AzrfecOmBwwOH +AsE7AgMBAAGjZjBkMB0GA1UdDgQWBBT5X+u4LkGBED9seTSNYgA40tXJUTAfBgNV +HSMEGDAWgBQB2VzO86glAf1wb5+bjW8YPxZlpTASBgNVHRMBAf8ECDAGAQH/AgEA +MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAGb6iGwxKXTSgxP96 +4UMtpKYFGykBuajxr105jFSjXArhjj5/U21nIHQ3vrmAGAMY6iaaeKJ78gYSfm+O +zC5o8f4PLxtPfRJGymGUISrtIpBUTz+CVNAwQw0yrAeROizoUBhpDmUVrzlSfwgm +uH3H94jvi/IpoxUezsr1tS48G8ayaVlrPLflBhgX1NXzonA0KqGNdm5AwOx3jgZi +v5PYtz6xR1NtINnGm42A078gL9XHXmZowacGPNf7flwU9UzU1LDb1snGCxOgNq0e +ABZmd4TbEqMs1mAmn6ymzAIU2wBOStcC0is1MJ1j/NVVixETrx65UY1BTMlIojDT +bV4fbYy7jo8rtoLdnwFd+DdXna2LJ8qq+7CE35f9c+ZxLkdSfqADISwkhR+J/6TB +plRVyBc4eiPdSV27qPbjMOEHKy43+rHTsBILqIOQ0sBlMe8DgD8ZXo+PUM1Ptadn +PsuHwSflCrIz6AHgG/nORSDQf+7U79mejzi3fIjwYyYpSlBemTuW+z3x1xizN6PY +CK9zuZu8K9Z/F7GQmr3xBEtWqH2rd7TcN7fQP+zmG/v++vPlE38cQFWv5vVISxJe +U6AOJmXKNvDB4QR5CPsd6UWn/Nb1K3/MlDu5ymLq9JXm/GeokaGqHuaLLR20yN7w +/kNCmGr6r+cWAxpEZkmSBUg8/ps= -----END CERTIFICATE----- -----BEGIN CERTIFICATE----- -MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJQVDEL +MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBVMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxGDAW -BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yMTEwMTkxODI3MzFaFw0yNDA3MTUx -ODI3MzFaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz +BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yNDA3MTYxNTM1MDRaFw0yNzA0MTIx +NTM1MDRaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz Ym9uMQ4wDAYDVQQKDAVNeUxhYjEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAvkyxB/z2frDqa1F3lDqkW5L0F5L0sGZiOnzR -0kgPsPtxIjt3ntm+8SZAL7dJ4VSmH7DNo/IUtwent/ypN9b7rR11kl3EDcCoISPf -2VXqvi1cQou/M/GSZPDmZMsXW6kVWzZydOT4ub7sEbVtp+J4GhRkR/AZDLioBCme -Dw/GMg68+BHguPZL8c5oi+6YkhgLBvua2kum5vYMWkgATGneSZj6nWxbNRryFgq0 -X5qRrB7n7HQYjT/ki412EjbnfSpqfhcOAtu0RUI9XNx9QPx/s6PIxtcd3lgRfeB+ -tdPnDp4pQZGm6ySUSjZV+38+tKKCX+PF/u0gJWpQwA2x16lvIafxDNFd1DIr2bay -RC3vmmPMu58+B1hj0PILwnZcozBFMo/oii6m9lURUYuCBcHAKOyeMnroIXHEQMAD -aj9fFBGNAoB3gLT1gvHrrbkRmQ4b7pE3jDYR7kn4VUUxjIrfX5ucthpSIAqYFiRr -OmKUtTpwhPcRogOP36VoVM2bK4ShevhQdRZ5vjo2DggweIfIg0UBdLSethvct/hJ -fIrmq4JgUMwfHfW6HYR3Vn5T6Eag6+mwhlMXbg18D8r3XmzuAtMEruPJKdV3b4Ad -oZmnuYGvGBqHmY2fBMwdtnd7SJ3H3m+MPxF6/1Fks0D2JgVNGi9///nR6x4ITvjQ -joDODBkCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA +9w0BAQEFAAOCAg8AMIICCgKCAgEAsPeBqLtAWvBZmAtNciN1enkIgE7ffPe/8Qkz +ZK1so6LdrkQsE7v0HG8j86AdEaBTSiBfwG1+Rr5ZGpHwrPlf7Zl4GHb+v+gE3gkT +wponMN3wC8XAVExu0L7JnQBK50YEEI6/EfayQ9b80E6j2/FiyVj3rxt8MTwCAGdh +Fjb8Z8WSxLDelk9/2njOj3Swojar2QR+Yxs0mr7obOxB+ZonDSBJ21HK5xI5rOwa +wMSzyoHmW8Og7Z7TrGLei3ejvVy1RZ+pF6whQB823/JEdAGQnjIm7D01qf3WVgAP +a78IciejVm9hCRxUq18i7UQcmey0FcuQ4+ziFw6UOEILHN/NzR/3jiyRqOeNYDev +PfomSpT9YFF5kP2FRgjaoV4LrVUHPf066yTjztmqULSZClRKzdW4r96Tj6qiL2Ib +ZVigRwQ2mWf8RMKC9mS64eCIf4VgI1elYgW9/BRCYpV01ERTNRoIFfwm48MH7ARR +MKG92dAS70U8uV0x9jaaKYNhheLk+QLEbeJSdKF4NVY2Bd5qOX/ooZJA2wj1qvqB +tWbeXCjUSJy89FPJNCZygyyIGZT8LTwyqCNmVOrEGDjv+rzY2D8w357R9OyJydT0 +QUwd34K+tMTnlh017If8XoCaznPdruazpMCTaDi0EWjATGircWg0QiAeOVwW8VMe +BmtjGKMCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA MDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlm -aWNhdGUwHQYDVR0OBBYEFBFYhgtW76vGDcp6aW80KokkSmJgMHIGA1UdIwRrMGmA -FHrBDQizpi5ywsU3B1ncA1fWOuCyoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE +aWNhdGUwHQYDVR0OBBYEFEnxgc3u39FUWRrVmLdS1+fhF4BXMHIGA1UdIwRrMGmA +FPlf67guQYEQP2x5NI1iADjS1clRoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTALBgNVBAMM BHJvb3SCAQEwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAnwfk -vd9EAll9Bo8G2Hj5ibLDnS4hdFnT3Z8J3Okgj8jme6vKInJrNm7wepWZEqq/Sqhs -B4uYCHYN86WrOalSqON3KsT0cue6gRKXSe/zjhNc9Uk2emJsR+uPNrUrfrIlL7Z8 -IQmewQTkfbsKL+AqT3VzIdPENnEK1N9u5w0ie5vxVz2xPbtVjqmsNgiFaAP3r8pA -mIKW3Pbay8q3wMEAWXw6EHTKelj9/7QpshzgR3AM9sYnbcr+pycRZTBEwuCv5Kvm -jbUFxY8BMZPS/b0ZbPgs8rBxE0BhMt+VlgpDbskeT25/0V2pPbAcyHQth6Qker5w -DcycKMaGGQYivLfyz+9kktIOm3AGoUdCHRxAc7xmSkIhFhQVzEgAnlK0QeojntOg -c65vbgy9x6WdTkSwMBRa4d9oNPSreTIq1ygUKOAwxZo+X38q4Oo4MTQrXDNOhGbR -/ylw/avihYJszGl9lCni6kIuqCw8CN2pvQTQcvU9UJTgy6LC7CzPqFfPjBXxN65P -WV+7N0dZ9MWHb+eAoVA/ykUrp52oStAqCYLRwCxC5NjDRXR6AdXtXTC9a3T3JLkR -LW7j8CqFmbMjhuys3Y8MEysgxKAg1PFggkeN5O5uOJXWqlUF8X2ipoMSnSIIa+Q6 -4C4CAspB8k7S5LNUVzw7V4RVLG4Mi1cSCy+Sggc= +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEAQ3yk +Hll/c8HGGi4EeA0bvIUAFRfgfBkiAyvCmm/jxarC3V4WjeXaK6qYyG7ljktT3ie6 +J+qr6WnhOveaPWJ0pvBemnn4WiiUkSRKk/jEocpbksPoA1kuaPJQxwzGl7IknkhR +tNpfgCr4XrADf7s/eYv0yqZA1ZDGrAbkJ08kfpj4HnC7Xxj00G49csFhVQq/VFAk +Y5AEiUELYNmCyq2S4yvtmsfiW6dBoxZKkZDYtfmYBM55Wmsm/vRwXW6cj+B87GY0 +w2L3CL0/496IgCRRwnDQSVANLhTJ5XhSUKcNEhEWDyXM6AIrlsJEGYYzdENqzWFh +PJDoBKnv/AKQ0nOxBOo7aOZC7WwQ3Yh/dCxDsspovUnUZRC0hD7B+kZLfggIBuK7 +F1k/SXzSKw6PNvtSAbl1V6b0LMC41TzcqoHOQuxoYYiuPoR4EwA1uOzuay6ouDSL +HHPvu1irNMtiz0PsTRHvY8G3kNujLM3MfJrxEYvE0CZvqCMVfBlxFMNexE90g11s +xdIH8Fwe5hc2DuWTrYDVcB8ZVoM7VUqgrp1KHoKfI+TbIIg6mLIthMgwpoFFbBAD +YBP0hk4bOVoRnfpi77vm+1Sc9q49JGmV7h1Zrp8b0RqmE5cgLAf+mr4YxN72Av21 +MhHKbNERB8rHVaOToLOXfE5M5Vli9ffoa6Q+E0s= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt index 7329e46a571..c541934ccaf 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBVMQswCQYDVQQGEwJQVDEL +MIIGSTCCBDGgAwIBAgIBAjANBgkqhkiG9w0BAQsFADBVMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxGDAW -BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yMTEwMTkxODI3MzFaFw0yNDA3MTUx -ODI3MzFaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz +BgNVBAMMD2ludGVybWVkaWF0ZS1jYTAeFw0yNDA3MTYxNTM1MDRaFw0yNzA0MTIx +NTM1MDRaMEwxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlz Ym9uMQ4wDAYDVQQKDAVNeUxhYjEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG -9w0BAQEFAAOCAg8AMIICCgKCAgEAvkyxB/z2frDqa1F3lDqkW5L0F5L0sGZiOnzR -0kgPsPtxIjt3ntm+8SZAL7dJ4VSmH7DNo/IUtwent/ypN9b7rR11kl3EDcCoISPf -2VXqvi1cQou/M/GSZPDmZMsXW6kVWzZydOT4ub7sEbVtp+J4GhRkR/AZDLioBCme -Dw/GMg68+BHguPZL8c5oi+6YkhgLBvua2kum5vYMWkgATGneSZj6nWxbNRryFgq0 -X5qRrB7n7HQYjT/ki412EjbnfSpqfhcOAtu0RUI9XNx9QPx/s6PIxtcd3lgRfeB+ -tdPnDp4pQZGm6ySUSjZV+38+tKKCX+PF/u0gJWpQwA2x16lvIafxDNFd1DIr2bay -RC3vmmPMu58+B1hj0PILwnZcozBFMo/oii6m9lURUYuCBcHAKOyeMnroIXHEQMAD -aj9fFBGNAoB3gLT1gvHrrbkRmQ4b7pE3jDYR7kn4VUUxjIrfX5ucthpSIAqYFiRr -OmKUtTpwhPcRogOP36VoVM2bK4ShevhQdRZ5vjo2DggweIfIg0UBdLSethvct/hJ -fIrmq4JgUMwfHfW6HYR3Vn5T6Eag6+mwhlMXbg18D8r3XmzuAtMEruPJKdV3b4Ad -oZmnuYGvGBqHmY2fBMwdtnd7SJ3H3m+MPxF6/1Fks0D2JgVNGi9///nR6x4ITvjQ -joDODBkCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA +9w0BAQEFAAOCAg8AMIICCgKCAgEAsPeBqLtAWvBZmAtNciN1enkIgE7ffPe/8Qkz +ZK1so6LdrkQsE7v0HG8j86AdEaBTSiBfwG1+Rr5ZGpHwrPlf7Zl4GHb+v+gE3gkT +wponMN3wC8XAVExu0L7JnQBK50YEEI6/EfayQ9b80E6j2/FiyVj3rxt8MTwCAGdh +Fjb8Z8WSxLDelk9/2njOj3Swojar2QR+Yxs0mr7obOxB+ZonDSBJ21HK5xI5rOwa +wMSzyoHmW8Og7Z7TrGLei3ejvVy1RZ+pF6whQB823/JEdAGQnjIm7D01qf3WVgAP +a78IciejVm9hCRxUq18i7UQcmey0FcuQ4+ziFw6UOEILHN/NzR/3jiyRqOeNYDev +PfomSpT9YFF5kP2FRgjaoV4LrVUHPf066yTjztmqULSZClRKzdW4r96Tj6qiL2Ib +ZVigRwQ2mWf8RMKC9mS64eCIf4VgI1elYgW9/BRCYpV01ERTNRoIFfwm48MH7ARR +MKG92dAS70U8uV0x9jaaKYNhheLk+QLEbeJSdKF4NVY2Bd5qOX/ooZJA2wj1qvqB +tWbeXCjUSJy89FPJNCZygyyIGZT8LTwyqCNmVOrEGDjv+rzY2D8w357R9OyJydT0 +QUwd34K+tMTnlh017If8XoCaznPdruazpMCTaDi0EWjATGircWg0QiAeOVwW8VMe +BmtjGKMCAwEAAaOCASswggEnMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZA MDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlm -aWNhdGUwHQYDVR0OBBYEFBFYhgtW76vGDcp6aW80KokkSmJgMHIGA1UdIwRrMGmA -FHrBDQizpi5ywsU3B1ncA1fWOuCyoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE +aWNhdGUwHQYDVR0OBBYEFEnxgc3u39FUWRrVmLdS1+fhF4BXMHIGA1UdIwRrMGmA +FPlf67guQYEQP2x5NI1iADjS1clRoU6kTDBKMQswCQYDVQQGEwJQVDELMAkGA1UE CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTALBgNVBAMM BHJvb3SCAQEwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoG -A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAnwfk -vd9EAll9Bo8G2Hj5ibLDnS4hdFnT3Z8J3Okgj8jme6vKInJrNm7wepWZEqq/Sqhs -B4uYCHYN86WrOalSqON3KsT0cue6gRKXSe/zjhNc9Uk2emJsR+uPNrUrfrIlL7Z8 -IQmewQTkfbsKL+AqT3VzIdPENnEK1N9u5w0ie5vxVz2xPbtVjqmsNgiFaAP3r8pA -mIKW3Pbay8q3wMEAWXw6EHTKelj9/7QpshzgR3AM9sYnbcr+pycRZTBEwuCv5Kvm -jbUFxY8BMZPS/b0ZbPgs8rBxE0BhMt+VlgpDbskeT25/0V2pPbAcyHQth6Qker5w -DcycKMaGGQYivLfyz+9kktIOm3AGoUdCHRxAc7xmSkIhFhQVzEgAnlK0QeojntOg -c65vbgy9x6WdTkSwMBRa4d9oNPSreTIq1ygUKOAwxZo+X38q4Oo4MTQrXDNOhGbR -/ylw/avihYJszGl9lCni6kIuqCw8CN2pvQTQcvU9UJTgy6LC7CzPqFfPjBXxN65P -WV+7N0dZ9MWHb+eAoVA/ykUrp52oStAqCYLRwCxC5NjDRXR6AdXtXTC9a3T3JLkR -LW7j8CqFmbMjhuys3Y8MEysgxKAg1PFggkeN5O5uOJXWqlUF8X2ipoMSnSIIa+Q6 -4C4CAspB8k7S5LNUVzw7V4RVLG4Mi1cSCy+Sggc= +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEAQ3yk +Hll/c8HGGi4EeA0bvIUAFRfgfBkiAyvCmm/jxarC3V4WjeXaK6qYyG7ljktT3ie6 +J+qr6WnhOveaPWJ0pvBemnn4WiiUkSRKk/jEocpbksPoA1kuaPJQxwzGl7IknkhR +tNpfgCr4XrADf7s/eYv0yqZA1ZDGrAbkJ08kfpj4HnC7Xxj00G49csFhVQq/VFAk +Y5AEiUELYNmCyq2S4yvtmsfiW6dBoxZKkZDYtfmYBM55Wmsm/vRwXW6cj+B87GY0 +w2L3CL0/496IgCRRwnDQSVANLhTJ5XhSUKcNEhEWDyXM6AIrlsJEGYYzdENqzWFh +PJDoBKnv/AKQ0nOxBOo7aOZC7WwQ3Yh/dCxDsspovUnUZRC0hD7B+kZLfggIBuK7 +F1k/SXzSKw6PNvtSAbl1V6b0LMC41TzcqoHOQuxoYYiuPoR4EwA1uOzuay6ouDSL +HHPvu1irNMtiz0PsTRHvY8G3kNujLM3MfJrxEYvE0CZvqCMVfBlxFMNexE90g11s +xdIH8Fwe5hc2DuWTrYDVcB8ZVoM7VUqgrp1KHoKfI+TbIIg6mLIthMgwpoFFbBAD +YBP0hk4bOVoRnfpi77vm+1Sc9q49JGmV7h1Zrp8b0RqmE5cgLAf+mr4YxN72Av21 +MhHKbNERB8rHVaOToLOXfE5M5Vli9ffoa6Q+E0s= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks b/qa/integration/fixtures/webserver_certs/generated/server_from_intermediate.jks index 86a0a72c5db114596865be2642915f2cbd3c467b..bbf256203ece2a53767105cd4af3cb1a8469f313 100644 GIT binary patch literal 6252 zcma)ARahL%lEq;NG7w}SxI2Tp6Wrb1g9RTT$N&R_1P$&G2#^55J-7vj;O=h0b^n#` z-fy3FAF8U)sXF!452qUp1vwxgB7>nId^7+HOaXR_jfjqz4+SBkK#5<$P~t~06jSSe zwJ>E+pqLW>&?5hY0NR`XQDI>sBIQE?onR=S4Q%%>1u8b!5G?p_hy`XvApjBi6_7|q zKzK`tl=&lpKUoRrPmz!qfk=o{U@SC@|K5lMKt%wPqX8^n3W&DID2Oa5KyF!1zT4NI z;X!6-w>(5VAQUJ-Lwfk_H;c1s#{iUDhG7THhJa&6!US5no&d_7P%U~w;yBCQ*jf@q z!-;~!IM(ObRKV#zo&Y0D)rgW7j0>eQlI}oy+u%(N)`;@>tA*uveYxq4mS(ZkE)~tr zVgvyjFT!e$w3&>gV-+LZm-5a5b8bLNx^0@XwfAz0*ofA0?JumZ+Oa zbHVYdk^%d)VwS;(2(m%5sxaqIv6Sq}4h4T}_pOi$m4qmb3DM2yDJ9<2-+Ats+~6nY zACI-nv=+<*f4eI&kGL}V=t~pS_aAxYx_|ddCE$({{>JOdU8k8V8NS)6^{-YloVKae zW_#s7DT}`h+)UTGXr&$DtnQW)mF(#4pwl0q)`;CyhMR$hvaZZ?w)-wF_Cgy_pWnsT z{3OLD4zIa78oGQD)MeC3=|YX4AV_`GZTQ}&)@|=JxMu{v?mKe|;{Y)SE_o`)du2)o zT*PMZIoXWt##NlT9HBE&*a}*w`3ZWaY?lX}3NfA6&1-n#1hp~z{-jP-6vly_6*bXx zb@TZI$?#%C_*$Rue!1JP=T_Rs4bd+N@Y6UW9rBh$ihHI1w1`{bcq=Ab_2lNZD=0fD z)EY2Je4@QMx}18E@(ouayrcQp`{(0)sGMJkDLR!;1$)*a?p&Ykkpqe2z%7-f6-I;|jUuK|c{Y4K8QrLNU8L{ny>o?26)z{%?-Tm<=qb0$-y%F+NLLO z)yW_ypFeqSCP<~X`YXK`XntLsps;+K#*9n*qR#%M zE8(jtNPG*X3BN^|`g5?K!-t5&>j3ib9GLYbcM4V{CuMxh)S6>s_TpXMws3!7^W^Qz zxSIqp^TEwp)C%kSl*sE#k{w;1}kGRY{4?78?aX0|lM zs&a$s$Z(>&^VfyDu)eLb5MtV-b_I5hoo~NvnzPuK-AQb!3s1n?`InmK#9Xq9{rKd& zF;K)QC5dwdxdtS8h`x%zQ=&h!p7-}w4ITeT?7~Bwzdb`aL%h-Jx}@Vf;>Q|;e(8Sy z`-MR~z(@)&+W2)d7;6LLh5CZ|evP&)AwD{JQdO2UbmbS@J)ZYou&kZu5LS^G`uKs# zAc44cZrjIqn}8-~f)vb`vSk2PCuzD~bge%-ZH2Nui!BD24xso5-?fnvCaEafW?=g zT?2?F?RytOpK@Tb&P!|sO2~x`I;frX|Ly=t(vb{*Zy0@>p_8z6frXv4JAQkjOO?nU z&Dt$$oypU_s8V@bVXIAK5h_j3{Dp!9)y=u&h;us_+pN<>LAF!mr z*o*3s?=W|QOxAd2X7^hJQV)9HOgIyZb6UDw=3gd9a?)W$97f1xm_*wrc#78ADW+pi z$ni~CZ)GbwUZcxw@5#F|pJ#CGACA3%@2hrB&8q za6*&w-Ln}?ovOJLcB-Z`uMm52-jgco){3%Yp1gB5!l>7JA|eO+@VKK#zpQIukK_hq zmf_E_wcpBjRa#VcrO@l!ciWapC#h)Ci}x|KlkMNu|F-IHW=R1Yt9I#w@UtUdL;z|= zM`({)K7wb6pOu$^L{c@_l<(Ap!ofv{zG8`Djus*Xk901qFvc2UHPW>sbJnUfG>ixu4?DKNq54RW1_YO2i6Bi8qLDWuif|s%sDYy3! z7)?CW3AdvnvHY4wVZ^!R1q2g?dygzE2!TM*)-;#bYR1bHq9z+NPd8Wzc$?RJVC%~z zv^MDR^Iex`&hjt4?`8Qx;djZS*{Uj9J?KQ8p?`|=IMK{9)vPP}{=D(Ol1Jgq?20N~ zspwH%k8lTqI6Z3yt)eJ#1!n_>fn|cjV$GNa^SPo7NCY>x@IvY|4^OK+=Zt%|<84W$ zHxkq`CBFKfbSEy|^SlgE$a@!~D)h@yrRS=v^QAK82Bb_BJ<_N#{CH5UcyZ2I&)^#CWPY5RZd?}?dwHpg z^|i$xKh(G?hq9s4Bnj9fd8W}nrPN4lES~KaHia@Z-Xv&%vzJE z!~3J6=b@F$1}yO}y9OZO;-&y2cp}&$cp&)vS2jnm`{(S2;DTU|;E3Rg;Pqdu%RkxX zA8h%b7)<@|1%zA#DEK;#F1A!$0^D3&yu4gsK3*X%FciDsUqk3<`B3bnf2hw$hzS3b z$bWYb|1T?<<0u^B<5bX3Yg4f_5~LQ$h_!weIbXm_>ce*X>+o`DZJ>z#9{km$ZGBRmef&|$#&`|HPs&3Yz zoTKnqlZFjZ8ivh+0`MdixWBDx3EUN@hh8ETE|Hk)B~AN97JmtHaIVEXnpz4yBm|tJ zk^3~~54v9pHi1U2b{@oXd)1Cb-rbfpu-Ff1`k1QNr>=rgxxJ$}Qk_D*zPBfyBg8$C zUX6-q@7v3ZHYf@xjalM_T?J!tWh1iLJ0~rsL}w=y3|^6x7Cg((Gt9OLYAWy!G~nCj zUC15KPIN*7e9nC5sjX(SldDMI;B=&Y6YG1sU8m`jrh(_Q`f94y?HMl5VlMlHdxA%Z z8$b4i{Csz4DWPF4d1jJ;=!@jh9oa&*_g?JZiBmoCf(#OaoyP5Qh1b!NgIRe=IdW=U zy_s>hjkxowRK)1D+t9KXmH!>pa{%k zm6N2UYIf>>@8C)%C>KdrSSOy;;2XJlk1jJWCRKXkyjWOK^SV#Gn`M^iv@c~UokNqm zi;bdS`(5c|VT*prdR|mem;w>dDQb=rZCk}#Rk7=a&|VTk}lOvUQKoH&FdFvRIo1VutaB3DWEDw{W$s zeob$F{jlH_%3zNwVvqDY&KSp@5OTd^d!VxyaV*}qcR-OqUQ{P%3+Mi4vX2#$r`+)I+3j-GLxjDa@nH)@H(kD8 zMM)9Di31@?Sz5T%)AqOfD1YdTY3t(#YAui{eLca->~y&$E4S*3@?U{m_hdWhlF$T+ zy5LkE|%BfRXC^tYUMm%zvL$=2RSfM=2=(n8NIKIie3NN6U>%gH7hs5AlCb{afvFx>oOZZ%GonFmr_rIynK zI;OUb`aM@L$F3#T2cK>^p`O?-hOT+NT+h`jI98bSGn5i}m<0UkQ01g#%kly5{Sp0%PODkE}eu^m;C2^U-QJC3o_C z#rwz!%#oL$Mu59AX(S$1x^x`RNodEUADhv?nCC(TE9l5ILQr6(rX53HuG1TZ{NoBg zKIFlZ?8iG?v+9GkuQoUxdQ4*?~dAc$+ZtN5`lg1bCxUzmUzi$WloMMe5zGe|!=dw#l1{(#8@<~bo zh6-vojr^R5xY*6=75eZQb!CzGj-zvE6EH}S1h=xuk&lx!@aZ3o#!!e3@)7nR)(orS zECZE)n7OsWCbx2qsoPgwfCF)#U7Iv6g+kyg>0@4{i6m>-;Z2%uGB{IZX zk2$|K>yu9wmYfF6$Tm|Jft=92L>N-4Wc=^-yOq- zjgmdEikCYOB@CJAp0M!_+Mm8~XP* z_T2Wyj}-iH#hKT3fI3}z5sY(A-qth|s9J+Ghwz9Nvl!;i&t_LV){?US(m@G7(QzD2 z>;R?(ZZ!@Yyw{r!i5~d%qC%f-f4cNG(To4QsVxAKy4bo3|BM&C5sxIY-EIO9==w}| zvjw?zloDQ37$~X#NV7x#ZObFV&Mx^peL2f2iXy~#Gw#8ITgK#4I8SsdAU92i63t)4>MDvxfmdUAtS49 z>$t|7f(~4i{i5^@rx?#S?LTtgqxV;LyX~!LYPo;Chypcd+38&8YvziP83nS6t)c@r z)mAYS7JW5ChHV*%21Z429v1K}0I0DTrD1w*6~E^^gx1({2pHF;DeU<6GCC0@Ijk*c zw(~kHUk}OcX{nWsmEWhEyDO6zMX{&u{!BT%owDoM?m*)3Bb;72qCP!-_K)=m234Gh zNP(`3mJ51*ln7~NHxcSx5~XmEya}laqN{VsPFIed5q#o0bIO?Da{X_ zgJni@ec1ZzFJv{s(Y=WYKAsc-2YLbeDF|Ve&%&fX0+NZ83vb-qS{yV()WYco%Vv%o zE2GWbA&RA`I!TNZUqQt`57u#NM-@k}*U-O9fP7~qIVJ9>KTZ@Oyh{HHb3v4@?6K!p>hK$ zz@@kB6c5#&_|PQ_HYxQRh{=7?_<%BdpyPa&Dc4;6w5xqgNTuAVB*PQx>_?cG%)v2E zt#}5IJZRRS*&UgoD1Rfb-U7$}@qPgnLwBcW^(){!1Yog$XP&msuw|io86l|`V$ui_ zWdmWo;1O*KYd%>O8x^-48a(Zhj737u^D3>qYS=Ys%K{@bmL+c9cP0L9V!tNd98YL~ zmoqcbqmH1TLQ zcyhA7I)8x+(0k)Fkm>TjU!ouj<7;6lkUw)e?;wb2!H(yr-G56pNt-@2b|1rfz4Vl7p&xboas-PFfLXWKgQ1J?W~v4yr2EM0U*l`$(w|5mFxz*gtRC zN+K7d^E^wviPO|rD4P>@yx!Ef@SE*vs%e9RI7c*GZ9^U0T1i~JnIRw@6(ZJlQ1B~S zb)itxMR-DAOK-W5423VvFI+0eU$`&%$YNB~Wlk*o+k^dA>mMir}5X*>TtS-0H5~V8@1N3Ev7&SqE3n73n_9v%-d>d8o z=iA$$qRpl(P~4l9X{vdJ#eI9c#_jrgZ^oMezslyJz7O4b>9pHNl%_wYt-=~TczNsn zeJG3)qX&b!s=c!|5=1^KzY^a-f8!@bA?Q+)EY`#3CP2*T{1B!X^@}P3P50|YT1v;i zGDOMgYtv1Jz=sfpohsbA$nCRu>pv}bv(56LF{KXkEFPrzX_Qvfvb^t8=2$DC?0ULJ z#JfM!sv6|nckk_3J`Kr5_=erxknd|kqrZ$y>#PQR;&;M`)y`mk_Cvak;dcS840fcE zKHf>xb0!Sb-899b*&g(LT|?7CDvTd)rA;=e|dB z`(Z5E{^Bzv5rUQ2zixc=AD|<1W<+YBu6sbR+Mf!?kk0wGVySCYv=Cc}xg8UZCU6?; zw|9+m3Zp}&a8*94_C9j;zMT?am!aH{Vi%v;PLNIK44|!|GrXI2rG{3Ibh+%OcamLw z;vXT;dD|njOCu15H1JDGhTnb&VQE1zUvR!*pk)K;16{k$0qm54CTw_6MbF%`9$s7y zt!a}YB$HEQ^A&6`5d^D(xxm=}`hiD6L`6WMB6Dgfb4x>JsIny~_RB1`Xw@Uy7G7zo nAfo8s)mOV(K_Lj2v@1U23!JG%W!X+PKnXb>Cinc$C)s}h`2fw} delta 6065 zcmY+Gbx_m+w}$Ch8drBo>F(~76r>yJ?pQj0xF9Lru!O)OjR;61jUWvY0wUcZ-CXa? zz4Og?X3o4b&pYS*{T%R2a|1kZ$T%7*`oDi~W4?;yNgyRPNgMK`$SfKyM|DID|fa&z?Ndz}4Occ3^ z?U{W$X0Y8vlS+?%*grVlubwop%+b6hdZ@u6)V24&PiZJ7@E}a2`D_@D!o!U zTijkZB{=_7qq3qUz?(VEUr*li!obLI%nHVxNRE#HDmO5bkx#Y((A>#xs)Cm= zxksLZJR$k%l>Xyf2Z3iVZMW?gQEx_9 zuiw^+TQVv7WEt<$c6V80%djs=q8j?ZmMV;BjXCW)cFm5db^@;EmCw>T8X1_vp8R}x3vE1`{JE5y(;&b;{}3RzAn6#Crz3anD^Tdzh=uw zLb*Q1hsZH&hBWs(ZaDM=py$o$cFvi33ZsY#^jn&~k|ApP2`7eC?zRl}nJB7az1NUW z<>2n+5aRKCzB+DF!{S}waC-})FnO`$lNA%vnd;RJBzZ`J$`_o0s*(MJTg@mDIh*)^ z$-TMbAICA0+aIlt6~B|EL=-69+Y|OM9>_#ip!vQhAEYnua`An`9f8*9%2Uq4`5T?f z>bCG<0*yc^^`kB>MSa=BRf-fnlB{8O1p5|`Li`Tvm@-_Elh)LrXo_=*qk2DzEh^(X z3w%x&7^B^W9^+STLMrofSXQqqMWG~bQBjyD$F@Be@RR9Re$wYlTbkj`6ET_~yVH}Z zKodNAPRfhENt{Fds4`CN3`eM@iie)6tjTexzt*9lq?7f3`ht_>%Cpamq?kP34H_25 zkPrP7!wNB_(Om_vI*yN!uYEfj;l)teWcOJHv`l1`UJ({FevVJN=3rvq@X}*lWQ#bVe=Z-I7=ZEvRE<{d=m&oTWp+~ zUor9#!kgKAP|0W0>t$DLw7D^D3~I?;DyPm;DRj7&*^tWC-U=iKeHhPu7fNJzK;Wba z5UUKmnC}G%9dc$Y?J(BUDPP*c5w`;-e5R==+vLlbwS^>xYWaJ%;IP>HVg@JYqZQ+b z)M*8oR7YVnTlrHF0wI~NBT@DNG>lIMB0@j5$*Npv&=|+Y1>CkB-2A7H*T&ySiKZNL zbWZ2_*GT*=AZnn=CivWc`SF`94~}ILAhbi%lit%h)piHHuXnE&XX7Y`!e%B74G9P9 z`7l_yc&UjT1YFQMJL>9#XIR={s%z3@L^f^c)^Az{gchUg*;1}65_4l$5@;2dm)eOl zupfIIk1iaAkB$Si*uB3~s2=+Z)ardA;#Dm6!)^6hA##mR+WktU#m$o4Ut);|n6e8V zDgfp}@qrym7B;88f&L3vP#t=m7tx+yWLjH_C|W!?nJ}i&c+qn9}sARuxkd zTy_xXk7}&XU&cC5^4L0ktH2Oza^9;B;@7#~VoeHm=-65jcI`z{?I^H}}maJt8s znOLeYJ^X;TK60%BykUibGt@}%MyToACga^FrR2gZE3;~CUZ>dq>S+3fOz(5l^Om#X z7d?d+So`VHoT8jCX{qZHt&cY_=xJ8Jr=hWU;o^B#R;Dd}8GZ^U z5Z6X-wsNN!vV}9yvUmXk^+G}_CQYJ^#wM>B3nbtsWH~{9y9f}YM8{(4KCe_+sx1!! z@^r7?q4bd^j|)k(UY8l>XJ`2r`}%@(?M6;+-2*t)L+O+H2gC6-5*Y-1E&I-tFRLaV zui5YCHHSLCAJBS=sSSFwuWwT^cFZ69_bi>{H6)2Ngp zMK~70Ub4S5#+TEhsnEBQTu{1Z{j0?=ZHszM4U$yUg1X&gXU!>DEtgWS;Qxk~ZVl>n z`h#hy_d1h2;?`M*b!L~Lyvp*1|M>@brJl1s(6@C%E{8fBt>OULh?C! zh@T}N-|Uvfpg3Sa$CX7f2?nc0qOi-=GKOs~wkodK?=mX>t^H_b%}Hy*+?A$7Jbj*}gqQ zatzKl`Yv$0nnm0_f@H3~fM1Ne(w!>iO7PjDl3o z78$iV6zxCpx%e?AMbRLKkS8x)edg!FFUom};VB#XPJIS;5@;}`VAWA%Yvx-si!Jy_ zopvSSg1B0ErZb&x99VO&Q*h5TC;L}+e~L}QZ+DaJg+DO?Y)fDcG>1{wW*vmt>r^}^PpzErQbc0!hh6cri{--#ph7MsRvR-hsH*1cTRoZ=Cn^(^tJD(g zoDI+n#>gjVX56PHLfET@(a`!V53(G)*q9aLq0aSTs$YJAyFxq*Ums2lw#oVoe;dQM zaQ}pc2!4G;g&LtbIvEN(l{D&ftoD_f4<*J_l(HjjFE@2@AW(=b!(COvRjp&MD0s! zvn+MIBr=g6cdUM{!6%T0+*vmbMG397c11tr!xvCZs6mU5PTuGF73%IAYQxlty^V;t zQT(ilNq&sZxTOaHkIDAkA>d8qZLuE2;_E(f&P5@0@z=1RR||g|A$o2P*MJc z+|8h92|8=-@qE9|;;sdT?2NiV_w3vy20VT~phwIf0gA?G0!>lv$(H`s8&39}b zYuIlptuMI{{>)U=5x;=k3~^cV$(ZLi#apK5Y<1ZPl?zi8<|4L8G!a^BA3>G|)1oGH za$Ivd?T}NGVC2TvOBI3Srzzqg7wfrX&2D@5^=CNllHsDrWqFoW1}0WEv;{Z5%>Z+} zt6V#bk~gGBs4|>hi^e;LUr~P4q&GjkByN>=)8!PIKykbJV7vf$mK1zdCH!o}0J%K6 z7z`{GnPrdlHj1QK6D}9@U8_kbgWIsi*!%d)H6KJl7}Wj(J{NXlhGImJA0nZQ^sO0( zX|8s{_4%xV>9cE$&8jE3CvO#oQg$z|H1OW{jGb~p8x_5ZbF5w$!DZZON?hR~r}#$u zWX@l$3vCGBbN2wOJzG>rO-U|H=CNx6HgRG^r0f#bqjo79-cHGv(14n5Yn3~5qer&r zoMG!iCkyl>aG4}VGXnMhuPhgV0bf$!CE4YV*uV7mo^x(U)yMmd~nG zRL1e=UdY5y9A2tcW;$M%Y;75WLNr~K)kxCS#L8IfRM5E z>c3jFD?!wsV;}s=c4H5(LZeBHwDd>yiQF9I`$-qqyE9Tq|1RN*S@A5M0zlrmrrlBtpPmoIXm*+Z=r7Ev&+NJsqK8{80Pw4#F2pf32DN zf}n&!%d~SL4tjMm_NkvW4|Q_~={yJ=RGtUn2P@Za(NPkaiJ}ii+irPB$4ipP71^0H zm}jx&(iRshVB#?u0&14ZdSo>XZ##d*Mgo86LLy)W8?z?kh2woe@36Uy9g}^lZ8C%% zUfc&X7m{mFFq^vs9r4YZ7@X}ltZ#oQh{HR<>LJ+@mWgy~??%x(`Q61*&)far_i5{n zqw*mCSCdQ3;ZPuY*th=DoZbt)3HY4u!HKQ8>vlteLO)g#w9G&} zwA!4y?H+MOR?b911C9d`H-uhl&FWaFI%;D&owWi-5&0s-UV4RJFfBbA^%euj;G)eU z*GUdGT}-tt&ViJ5RGYZp5c?G_1bsl{@iBJ2-Jffqb3uE+I@rD;@X*H$&``>ArdO@~ z%SqVnZP#lL*kx$i}Ek6BNixb`GELKTvWhw#3E$v92wd-E&-kdpnKT&&W)54yMfTtioZY0eWK{Hb0 zdCK9f-;m()vlUS(LDGOr0|UaV%w?{Zy;Cj&t$yvuu>5W3-Tf<`y{t8#@?GRW0wln1 zlal*c$|IWbcX#~dU;vR)yq}yJ$kw%1+zt5gTt?7I+&mg z9U<=(F2{*$E$@!%-X`L@(okr@I5t#hrhEhRgakp5+GpQj1sCo@W4wNy^;#yu4Ynlk zqz81oB|~g<@)leQ1dy{O;rUwsg$ubZ?n#X8w^jv6MyH5IUZc&}2u-iSc};&TX;=-{ zaOu0rX73b8dCAh*3Q>eCvo$IXuW}|vE@GUqp{|m929n0Q{K!e2_VTO)y9mRgb)0op zBM^no#@`tM6VE=508hr{+_H1xHoZS+TU<^{F}szi$(#-(1Qe{~T;KI>Z%^VVICezb z=_nTL+7%+-TOJT4f0xxVnvVLs2-mTrQa$dPvJ+k6EtO|d(omFKh{tY@`Eb!CfclYa zsa!q=_yX&DLgk_5yGTJ^9GBSqfz}$eq69;c`THcUxQ4)KBH%Ci15?;^d6u@j_67Bw zC^WFcqpKGL1P&`$TTRPJHkNAJHx`a{r5sY(*A^Qyj&stpSGmLf4u73Hm#i<2ae%rk zzI{_6{E|o9zN~^3#p(>}5}I>S$~s-tAJY%j?-?ZvP7J8My(1xZ(_}L@slH?f9SuvM zkPvpjO3^!}b(qdjZ}U&q`&*PhRM@W{!MSxVfpQl>&Pei*LwXymJu)Co85u=09@-*!ij+*yil z^U1lN_~R9td`|c*#~Uh6s+dTi^!R~@Z8o8d3w8UPo*ZlOL6%0aGES5rgF3tV*YF#3 zR$Oy{3+r>!-_|fx&|#JyctqcA{{iIhmg+va_B}b>UYJRcBL?(APM+S4aGzb+DEto$ zrXla-Gifa)9@0uOM^9))>188&l&y+Ce-ii9fl3)-+%`n0li0i!#F|c%@Z2c_q-*5_ z@k_9z+IHjnSO>p$T{v2;^Rp1;wT zIINZm>)nZoz&Xgcn346dGrcP&CdId;9EAU<;xil#0OI^9)q~h60@5q)8ZIALVyf`u zs%}q`$X`oW+=r0IadFl}reSGR|4EE_W&+!rkuF90VETvGeflGH6EELaT24B!)T|B+ zq>OsZEqF^uCzKGF&2y%xe(GO&3;O01X#Vvan}Yx+>4rl+(U1@==LLP#{;M2DWX zIY?0;M{6=K&k5@8o8j7N!^7|%#lc>)_RBJZzT2V0g`VieH_AugURuiV3*JJ2+J}%< z-8*~_og@TA@V){t?T8=+Y1WK>SPd7Rp|X%#TaKWnZQC_vQFR#VS;j;a6qINGT|ZX+ z{8INNF>@d|5p%OoO^7s%2=Yf1bx-VdZn?QhO98?&u{2&+Z`Gmx@3W=chY>cMS!*MS z*!FgB_!r&u-TWep`XJ|n>L%e4G4l};LTEQFu!!c zEvM$lhXsx?SA^a!PvEdPx>ho1E{9IxkI*~(J;@4>7NJZIQ)<9{UhNK8O~KW%b+9Np zQ9)E%HEIq2EZT~)nljk0e~waZ&ZPX$GHjc)RITf)l3D1F-R`wVJP^4QqY>lz5l(sn}Mt%gMgjfT~ZH+)!mtFWU zZ6zOtOnwo~mtaDS{Aplx&3*`c>UKH^HH~UX&zXQ~`a$Gnn#)JSYph~^&l_%hD@$dd zTelk@On9d15L-G}(yFE|13y4!Hq00VQPGe8G-(a?n^vpqdM)YsK(Oysh1b6Pfx^xy z8;gtC$Kxs|3vt48ToYteymA=sZQ;DjZSuYFwpN*)-(}v?vjcc3N!yJ=JBX1RoXW)S zC(2Dd`e&dYV%b_WkfuY_X@`>!i`t`}A)+pQ(mzMZ!j{e`B6GHJ$isAsuGsxD@G!Ce zu(3CNS^*iQzf<_u6d6L1A4MV1)zTNJTmPqsBhoFsTn)b^LQE{8OlparOIE!!XPeE$ z@B4VA2emz=;EWO!v!K^P1{p%SduVA8k!}S<7*Y@hq`Q%n4uO&GK7QXh z*ZclFKlZct^Q?7U>+iZZ7)G%M1fYUp6dy3KIieM#uRs7yKp~7G1`S3L@iz_y!$2JW zmM{uoAez4s1rUJp_agka0sve5W6(ihZ7}3tiw9;#BV?>it5t;f=Axqyi+2EPTbFd;XoswEM00@L)5%pFT*+3iKZW{0avh#tjO zw7*RVm2R^7`^P5^ZVD*l8an$r7Rxo$T|Gj4?)Ht7P)F6q*_3b*@C&62TTWNjOpC@z zo}#0$>%?`E&ABoSpm!5vgv>XHk*h*iYF-bc#9d<)ek~z>VP36cBG}hrcB-nb;)1sr z_#-3U@`ArI7Oi&s`#T3S-T*LxFJyBgPL;AP1(*)z2M6IAA^`YHDD$hVmCJcri~8p99f@a1%5IPAQ6(Gm3F zTwXta@uIstrZkJ%gw0+4Wy34yFMcq-x=z~q!NA5Y4sqfXlCJEfcpvKG^% zj_c9q?b3=AE){oq*MG7t_0s21;pdCm1g2I8W{xnMV?78g_rJIho^UUem-6H~KmwU0 z9p<#=Vd{=z8SG8<)}26&qmyi`blBjz6`s z*hAe&6+|CY(VZ5X%Di`(@=IP|MpbDLeD^6b-TQ^mv$}!EMKl?pZuNhrci=+Gkck1% z98~X9xK_rH2Y~q1l`iWx^9{-qUlJZ%Q*kXNKev=azHew!X`z9yB3we21|B`77w323oKHD zrXlw2hP0^%8k23(Z~z3?a1?fzb6wDX>}j!7ny_j=Wc~t?Y^g<`_(hjZR-}_YCGp!5 ziP!LDoz#(Y;tos|XfV9iVDn}o^b@;9 z<(}r{?3Z$5-;JRlkAf(Lu_>@a4lRH|;U#Yi6>+QeW)N*QEbA%X=QVt+g(I`Rq`m4w z1vB$(Z?O^&_`Aclj=Ehd-9?l~8bZ)LzPv5Z55-h?&h%lVKGX14ba5Ia=G>@wlh0lE zqtyC!Zrt?SQf8j^iWv}9HWPtS%dm9H~*_iXn9gB8c9i5>p055S3Lj2a55KHPq{)U3hV#Pcjsa{n8Yy$v=ld3Gi~! zXd|*ia#C%KCKFN=Po!3le{MY;_1Z0#U@jQ>81XUlg!;m#$&?LnjoDWM3N`&mP>m|R zove}}%}bRUb8y(y_pyGuNw@&<098U*>LJPU3~QIoKwI&h4&pXpkfd?)+en+-epW}@ zf=R^dgtP_KSHgqhM;z6&Rg)!eE(}ND`j;)G^=-xPqI0RV%ZGM78~QQb%6s_u{ZFpa zB^zpRK#HS3NbXfFv&BuRt>1$;wtJl|uSnBuzNUDn&ab@n8#CR?%uilE|CQ&RuSxjR zbalf8SYMayNo5s;N@h%1JPKJV~VvG$?0$+sC)(7-Jr-6=@ z9{XR2S~j4c$3Iq@m(~M~^_+fAzPWgfkUFxx2`lZjHdDd0+Is)x`7q0me7M}84Ee*= zLR&-LBDg(6Os*+?h7XYR;}U9`LWtC3m$9dvBTW)ZUor7G6*^36I>(TUvdwQWVvr*z zh>rZ3?8rWN$#=(jBq$XYwH?v6XCJU*wBeqzDE3)z`r|h*lL)^ViGv!BRLO^h_2UZf z$lT^~q76M8;IRRM_ z=XG}xA1`@-$_-#ltdHaq?%?tk)c*0dqmhl+4j?exxFSSR0O&Vc)k!-s%`sB`#&XFq zFSo^aJF3nFVQzkZm>7OTau49eB(ma03cv*-39Ix|{NCZ&RUvqqw?(_AqY+B|mL$dv zTg_m}Qdz3Zou9T7Z>BU?7bM6Hx#Y*oD1`=qzQ6jeDLr9Lx3}&mi`WRL z)xbi@8>{pPNtb-);rHNo{yX2FJBfqX$Yk($W*lAq!LAP0C=ujFg&|uhKP9aItLnqh zl8IlYh2th(h~ zI+p01iRt7D)oY4+$@(U&o>@p!$;<(UMX-kp=YWtWLo>ZIjjsLZPV*MA8bi3=IZhaD zD&W(yzqI&aloTQF2zj@iJ4m4BCks=ROo(%0fgknmCGHN#3>i9KoX$JExR;aY7C-DU zusXF)<*mFB#V()f#|1BaswOH%2q0%WezoI;+y<&0QPbX*S>FfI67M)%{U<#55M;aK z6|okAoq`LMd$d-ljdRB`>bsh)X#HGH#4wQY}H<;o8_p zCe{f`L#|_7D0GA*#J1bXZKi`}a0+j(QYs$sT3ytGesko>mh=s*mf1?b7=h`3WV6%- zQ!-{p+P7c)j7ha(y0!>jP- z=leZROTp*N7p-OTR||`8+-dZEsii!m=j`d4giX+V(a9XU>0*9=)*bbC6C?Dlvz8=y zy^~UXykKr*D4v?GE>5IA5Si`w`f?U?N$P}3!o|tW1<2`wK9;kp79;OfE-mG5g8( zqIO5|^}J`ih@~BEYb_+$&B^kt`ZCdA*qYj-1?5)ff|^l1G=)0Ybat@*O-8OBoV%u>6+n`P6>QrJ;G@~#Ho8V*iA(& zNi|HNKErHTuex!!T3(vodvMD5sz+A4r|G5}>n!LwD@`H7@2ZKLZWs3cn{@Mus5ADl zy>a*|Ua=-3ya6>|FGzSRFuJ_xy|P3~PjwnE51QIB4#wy1_Bu(l@2y^25DZt@4h0D6 zrBIr5+slq(<~J67i?K-MMZS-2iHsDD4q=eQWdb5G{htfghGxV`n~GUB&)jA3hicx= z+Xc9C*`bAyRfSe;jrwv3s4A~1>HkFZxOC$S;BeJh^2a!CUG(?8y-*B5cfjW^w3GQU zVL4`_Kx~t0E6vQfKRLCpySLgA5`p$?c)Vdho3A@t0@;H!rN)(m!#FQ2qv~FW7xWZp z$HHm8_e8f8ji?0GqboY3*~jQE{+SpLPmUB8SjQr8V>PM8e}#UTc*9so!{!sd2kiNB zA|P|Rrht1i0bF!3e!d}0WSLkO;m|QV4NzieF*RUiWxIUtJ$6#@M{U4kmXcDAaT_~6 z|2`3d>SRmD9oH2^vi9;``(7f?l`k(A91%q@{=Cax#n-Ey^%ZEl%v8*W!ZJ6s9&SAB zU|^7TC4A*z-D;rtVLXfKNlG1Gx;eKnP03OUC9Hel5fnFgqi9DFq;|}*L>hqmiO%SY_~Hb?UWT6+aC>ptZW>Q#! zicPnd59mQX<)}0vt*f9P@-VL%2-?Xtvhy>fXG=)SE>L@yw2}bFdACNR%6N^<5w7VW z?lf_KMHBdSSI2I%WZvF;=fHakXIw`WrRJ!$T>P*F?bh$wcz4C*5CbiWkQ-_v@8J~JQ7913~ zten0OV*o;bZIbVCfyWHCpjU$EKC3szzt7nQ7)V9>t7n@vW3n<1?oB=-MuiN0*$!yf7;fyFOZqhxY?e-WROBOc`=t|g{BcPe6zwq7|E9zR}#j6#n0HENl^PkDt}&$&qVykstvZ$mS6whlrUztHXZc%E|;#+L^F zVV$%XwYI^s9cX&^y_M1oS_sxEgRK0NJf^p{zs){vIX@RS2fjwL*m9C z_MJs0y%)mi*iVzPqC7EL;*SirAk}@z49>e@Ar{u6n>d-R9C|lf8huRJcdU9aVDzw# zq|`9j=)~3&lgURW^3B$3)LD%K$}O>sw9AoF%k&`WjxKZe?EdanapNCOJ;d$bss^VYXn+aZLl>gec{{C2bMOR@{O%+{v>8p@!&WisrZ=Y5pIM~ zQS{~epK7Sna1`q>Kw-ArEHRH$~8V>t^|_}H(YXoq=V+FOpB-YbqPr=G{}JthiJ4OtpGm{zu2#MD#llS zMx|Y^m(MhdTb%sK<|fHyO*3-r#&GZ|odU>H@y8xlLFbBRs95p(;K?KbvN<(B?DdH` zUW`to3^fp-cE)243=lXyo{oep?4Ox~NbbnLas<%mLEJR|*q4^1i$5*pBM zYMR+M;SqWFg@H)GThv#_Me2DAO-zo#$knd$Z;*KFMM5O6#yVA_`+UJzdebfm)bG`} z7>C@kdBX}xRORij&+M4eKxhz((Xpe~og*map$I4)GU4~M_(6IR1@L89s1r%SSR!rn zOod4!hW2@BH?*=7`&J1jEAmec>S^4Z?YFgMuZz8Gi=p|#P*eSItz!3s`)fJcba4jh z8%I;$k-Dzvx%QMeOFxsKcd)h=5t1EM9XVx&Z$BY}r}k-4AmLpq(S*EYF{#2=r{6mq z7uFB*4;$1YFaZX&deJ`)1KP-#UmrnCYKI51R}^jz(Itw?3L$rihYzGXiq* z2*&dW9asUs{gyDdaM6< zuoM^q2L1Cd4g{d1pfQlt_o&K6Dp$M-ZiyxJ581EGad1o|=U@cYS>ZC7xhkMxeF$Ll MMaU<3{AXAEA6&_gP5=M^ literal 5950 zcmY+HWn2_c*X@U40Eh0bVd$Enq`O;6q@{Ccq-$ttX^<9)K{^GI1`&`}y1PROiR<(J z?!C{wAI?5|uXWD;y#8<`g%}VG9gd^`V1hZLRidx)(6G>okmTneB>6EMNxlz9;;#PJ z!Yo4KPW+QbfoOn#*T8=rG&n6L_W$01i-rlu2R&0xATtPu;VS}x&j4Xa+-zi{)^Ct< zVRW@Fa6l`VIMmAPtZ{kE;btR7d>A{GR*T$%EPdX`TcnSBD%D@|gYsO~cU)ee8G+pK zf!c*Xg%i2wm$6k5Oe^|l3EhQrsRUUj^n|pnjf$|L%B_QA5$wJRMBD1FA>4XDZt^y2 z>qnCYT3G@*)XMGVJln13x1=De9EG={l(k?cb0ku2>j%Rt^Mjm*DTQcqxDkcw9+Yu;x@{;=E(8)E& zjW_2rfEOm#e4n`5E3DbiEz(M5V!m8A=3F$0_VSvMlR@-P3KYp)!|T*jVovLfuyJEM z{{-|D+k>oIW+CVsBzs*GUcv_`W7?GN(2Z-ZYG3Y#caX&{hiRj5EVmeqk${o&7Q`u# zg|XgeJ^qJ-LelL~5FtVmW`&5(Qp^|l9m-#|=Em9LU-L)~=TB5Bh`EK$?0b>qMmOe4 z@5tRr4oPLpY8DeP4<+3Oi{!PD^HJU8`(o!tKDsJ^Idik9MKUw@&s5=ON8xm)g7AfE zbCy$(F}zdcq=IJZP7jstc5vo^_MldElu-@E4i6F>KN-#c^Guo|pQ%c?mU|j{_mX?p zw%+3q6J2xLj?TZ;;M#~Q>k=c2dUAl#3p-YyWMYcIEHkmP+hU7IoZ?;)(5L!Fpm09< zpvMnGtbqQ=FG`uZ)Um$ia{X7V;ETv+zE(}%I0hYh1uYcggnspqb}Ys9kYAYUVo=x* z-u-$3J;?A8$I+bri*^w*>E~78@fz*EUnV@G(%dAnG}~t6Lh1$WpB;Mmw+9&>-jg7M z*R+JbM3Ja#DGLRPaG)BK(W|On;eDk&K}rtlz|-dM%wHn_tW=%b%Jo_U0uzx1BEDm< zifw+oy4MUk;Fl=d%2g!2TyrX>nl?CRwHsTK;Z3B=qI)|nA#`}gVqDsjpVi6n_Um^; z6q!Wp0}OGo2Mk<7h?Nav>0A;g5?|Ji+kR#|6p30NCFJnj)o=D%Wh(Ox{Gtx>DKCv* z@C(He6iEwtb0#4gH4}3A8)LJt@9mZv8IE-jU8;07+KC*!ReM%Es)ApOv{kS#rLS1y*pyw0WR9boWC9Vlp2i#vIK3R3_JQi;-K( zz)v*}D>b~RDdzFT_S=D=+?qNu+zEBRzwz(3Z5g8_jjkwHULC0vv!=&t?P}ns?RO^c zt)4FAK=dhCa-><q6q4ucszVXL0a4b(qtpR1WJD1sOwzOoXew1$DGkEUtNpGy)uvCO>N9r>|4 zq%6rWXrV6kdaz?sdJC?{^)u%Xz*IO^mrJllpW5TOCrX*2H*!fN4|5XCp8gK5s%IQ_ zvGu-9e?gjCXfl`FJ#jFVcmXbtpji`-2*}zbucTMa2h&>>-*3eYJT82cw%*_*s{;3fm^{H9Fz}GdgX+N}*_eS6|e7j+Evd@vM z!dJ5)e&Dc86ndTK<>$BiVtFJ^hGZ=nax~Co4$HnXtQ=*S7FNdl`}?f$%WmhV5SLe^ zGNlSVu^^+jRGVU__X85waV4KuU9g_L9-%tp1P3DR7sEtjwD8CBa_5$%)fVrfmQ0h+ zoIC@mh+VpB4wjFrUK?vDtbtUz#dAN6-!bI<;8P*RW1CzsN z9=AdBGDTx3bYx9tEMJNizf4G(K!M&Pp^?qfN-5{j2*%dDyl%idz@_{g4n?2xpONmLrNre3>u9m?`$WfWA zMh6;Nh~ArlT_!uT&%VW(6N-hy;r=`lAMw;KtQD}xo9R!0^joSiKl=zc2#q~bpiWD8 ztqx#g!h$I`H}~nKyqyBZ9Co>)$(+UEc+@_0g1 zp-=1|hBryx(Es77P=1N6)W(t#r`a1-d&+w#LgP^vho&yZ^3@kUDT(cz zT*t3L@$05}r`ux?Pdg_uC5Mc+!F`?_nq}zGZ}-yu*y$T!CSj@Kf8z0&T53FUXBEzg z+C>81?sN!EAyPhgqbhtMn;%B*RJ6Jk%uleNpD5WIStNT9Jn=i@c!$28JijoMUq8?< z7+J~H_<0OUEvL;YR-$_(xb3M%kwuuoqEnA zan+**Qr^6tO7A_=K`P`lx*-L-5PWHeCg|}`trG7vy6)2Z0|C~Nyu;yAEgUCPy7k$_ zhD?D+aqE~@W&GsRCw)jgbrC36yX-1SR_Ny-C`}OYfSGfD-gxUbbr@%EfR0IQ{9K`U zxIAmn-H;F3+hh2+)+$`C4Wf|f`HeH0N9W1WLg5pquDkN-kxAW4c!-AS&ER{LDp7I= z7@@w-;-ma*G%w}y3 zT4J2z^ex#-+EqS@i;!phjW*Vwza!_-4ls=Ec*IyNmb~yeryIm3zDz^N;!yd> zS7UH6U~}|59p%eyGD3PkEQErykOkUC_L~^?Hat1ZQ>f(_jKI_MEQEK1Bh8Q98)HRv(gar^y_O5BoiR-E*)8r*g7h!caZhj$m2sgg;` zcbLLUXo2T+OJX~WnrlFwnlzOfY;I^!fzXgxCa<84I3a%DizqE8gWyk#E$B(_v*yJT zjxVp;{0+h4SPSe`(X~@uR&kRd@5H@m;I$YK42!vxAB)W!TPqfp5K#dgYtnMCM52~8Fn@BLJwCvvU7?DoR? zt8p?ngf$vl9+Vo7g}tSGeqncp4$k2xv%g*7=F`KD)UA&)@N*AtuMaU)%Zup=!tf`A z(ptnTex>hV$9Zj%Rp5!wgw-Tv8k}_iv)DQLdA7>!kg;gz>Z~suq4VgR#J^ZO%SI?{ z<*ib8*sJlIWtUyM2$Om>f5h;weOs~dCj{V6ea`vT>?1=aPKnQsy>^nTINi#u*2f>D zA3)GmG6!%!IfD7Bu2bEwnmE?qV~|IL65mwmRoIg`Eujcrh|vCs?M9DWQ*DXHqtRI5 zvpgJ;r3$(v167FthB4dksxPcsEos$tSXl#*MfCSNU7@;`YBf7;&*ajju&WL;xFZO0 z50x3gyTh!u`1t`RgVx^5;@49*2OnFW<_)Y&@xkTS=qpL>jjAl{RS#VwDobaNK@DuR zopffVnP9e}d(cQ-dKDdG6&DB7584+6YeAIueMA==cAXF+lzVc&GUlJJUxJ}Fm)t*h z3)EJhLGI)ed0sTTXMh8k{!4&E|zDvwZjT$lW-D7!6T85ZZb?Otm&#f1vha<6p z{}0F%A;Fg*B={7L1Rwr8yO=ot2Wzm=fd5R_f1=a>M?2{Mp`Cu3Z)W$YVRQeXoqyj| zNYoGJIB6N&D@wD!3BKi=D(-YM=xr&vldgvDSzp@+ZzIcw3S*5b zY6*jgVpIZlwy(cGAywQX7%8&8`hz2G=eSe*`2}76TIn#17u^=k4T?*8p@_bZ5~Za| zUgu(Ws%Z*pEl2RMJIffK9-TgACyho<0{ z79xKKs44aEi=1Ta(QnUq^yM#tZkvUqqA2esIWr6|yM^ZgjyI2Fl3M0M$61YGu8u@VE>Xb~@ONtm<;)z9to+yW!~)FfC8980>mQ;Dg8H;1WQ zBVqCQ@+~37I1b_^ya^_oWT^`ZU)4U~RmS(ypvx&b_w~C&eY>l-Qqk;n11Q396HAc7 zni%BO6_XXE#p;4hppn|G82-k(`~m=*+=v;4@SI~ET%xu5Xp?_3ZDSqK(=ZsnygjTYb9w=voRd z8F>e3cGz4Z7^n!CAo4v@0kqa28S}3AXlvZo@31_y#Mxym8>L4+i!$u!@g%5QzB~kz z;3e_wk&XEx!@Av;=~_ zL z=KRrEH4e??9?qXIH`7~eg8#&DkE~QSsrf9@0vvkl=~HnLhGXn(MHmr&|zf^t}|hNW8dP6Tqz>1i3i#c!lr%aZ%L8PPZa1YlV)>)+kOtLbl^D zO^1T-_+*)IP~N-AS2`;bzZyAB&P(Kf^Oia9YTl?}lDYf&Hxvz#p?`GwZL2q!_k>7H z3Q|i4^;s(#`G;d%*L^|=E&4on2&Bby*j&9x=WIBdR%_18HaHRy(oeJlWG`4BC7=xU z2B;LswEQAs)Z{@isc7}TW6bChYp_e>Gg{;Y{_{K5=4NuuFyFLkbCI^vGQiy&s+K(j%-!n zdt~UaAVB(ZG8S#VV0gvj#6>CQ*K9J|;FBQnV$AX-%-WqX`EY>C7+Xf1*JUK`TI4I} zuRVs#+8~J+wCk;1Fi`P3M9nFZ)3@wQb|^kDJx$6*WSyGmDuAg}{j$5Ao&uMM=Ib ze)1-%lk+K~rb+w#s!oik{WinYz48Mxt{WjKsMob1mK^ z+=lcuUsX1+a<@7x=(QzNp3)&NmK!5gjb*f8Q6tIT^8=})Oq|0|%d9p@&%;q0fNOQ} zYER6(CEd-AZHg8HSrR;k#4{xx?zD&Wr3NiZBZ?qq?XaFRmN`^GA|a(<9m{q?$@NKr zYk(znmdGe1&ErF`zqfF(b2)Nl z8r!jP28fPDN~u z94!(!u^1N8r)KqD#u_swDVAiuAbV_L>lqVX9Qwp^TrJM0UNMo$j=*h;NwwlpLQx!H zJC(m3M_u-k3mCz&Wo6vo%;I($9bfgObC=Xz0x-6O4x7}yKPCE@vD~eiM8<3k?F6w# zve^Z$-%gqB9pu#m8@W8+|8c(Vnd!R;iJ=E;+sARK$=ZQPJF7-;O4Z_`0>zC-1Xf)O zEVRGJIAXBR+-#P|^<_vf7)2gUNN{b8t)+Z5{l(zU3v)Xmj*gP!7&-r|VNfS`YU1-A z#EK@a{il*YT%n{bbMI(BRmY6bM(TJ-@wl~B0Gj4quNcT@2Z#RObOt8lBL|V8E2fYUefK(*J!#%EwO+2Y3N&0iJ+2|5GghcK>=e00LkEa0Ivly#Gf< z{Hr$qX3PJ~a6UK$4#vdb!bb;EV*o&;RR$D#zcl3JlzVtf_FLpDWE1EXK+m?Zu$fg- Knt1;c^Zpl@O(pgK diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt index 6ba9acb7a07..c2094a7b3a9 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_root.chain.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGRjCCBC6gAwIBAgIBAzANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIGUjCCBDqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzMyWhcNMjQwNzE1MTgyNzMyWjBMMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTA0WhcNMjcwNDEyMTUzNTA0WjBMMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxDzANBgNVBAMMBnNlcnZlcjCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBAKJZa8jP+LalBAlQ8uWWlTm8TLHWkGLFxr+rNzs23mAAfLo8FFGJ -bHQ2iIWQtjCQwHiWUrHU2eRBvL5zJ3cEWte2WF/OAkNhcZu8/JcH3jCMhbjfc3V1 -xIfRdWYEhTI+5PYaqgStNs9H24fyshMFcDubXU1pKfNXtY1sQVKfQl5XeAN5Qcif -2QSfUxNOWnvnmUdBPNOIUP3W+3pqzv+uhPt9Egj2crYPXnSnC5JZSdeya+Rz60xi -WpmSWn5/WdO6j89CVI1kYX7UXuWVhtp8QLa315cMZ9+b238SrDjzNQ+ElwQJO1Nb -oD+JfsNBJBXPy/6Tr6FOZXHOWFFvIErceImKIdG12WL6IHxPlyPpeBGTdbGCS5gy -vWKaJ8diV15Mea8rGH7D30sR2StgBtW1MJOomkarS+2oQpC78HqQrNK1Qgl3GauW -PMIKCU6TR5OPZHsb87u7iXlc+NO+tBoFtgOVK7LvN/QwQ1At4hQaLcrpEJwtjfk/ -8jrC22Ewgp1LMakAsmDjGkkOBZPL5VeNgNnUEp9/AHYfH1XmpkLUOoRmUVkVDqfs -FVcFTLLb2XDtZWtl+gJ5cXxnr87dj5X2Xowo2BMKBwwMpThmJX+i2yY420hq9uu6 -xzPjUC7NauaPnIYCIjWtaBqruXWMxX9DISED71aAVAOSqXbv2fFpKMpRAgMBAAGj -ggEzMIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC +ADCCAgoCggIBAJT0RJKyh+dR+joDaEKF++Rd8YPKmo1HllITKIak0gmgPkWb+si6 +yfi7UrnmgIeSj2CESOaPcnMzbPSajBPCtitS1YlYgB8N4OKht80ib/Bc2fYblmED ++Z9+ySnVlPVFiAglS2DsVz3QUtRmpyOYUeqvBNpSZnfnxF/InIRxVAjQ/7FZ87NP +Xa42K8J1d5WXrkE/X8j7BfZUUxbKyGDB9ys8jMoHxmnH2P/P/LcvwZPaQQkxMzqk +tHY/p+tMOdkARFmAqJ1GC+zRQExRo+ppPBMeLWvQXxtW/57giqvwFoYXdFw+GHzU +0LSK810WsBDqIJfZXcd2q/EuW5mwWt6F5c5rFGI+tzYsuE+2C3sYWK7BF+BiHs1H +mZREG6+NCCTgfXaNakLFf/alouJyfj6irZXMVnFZvFmTLwJVai86l7cUYnV1A22a +O6pweBov7od5p1UmFDbD1etVa0u0HiYbdOE5rKxM0p4CDbO7FlbUoLdlE4li12sj +iCTR3psD3TG85cOiTSv1lQGpaa1MJU8a/jH3Ms24v3uFR3U/TAy++c3p2mBqfNyI +sDyIUCBEWAY/Y9YJxiS8VlN1A5WcKSkmiou2SbCX0EUIkaDoDaOiWFJY0QmVv8nM +/Keh2tPWS+NFoBLyU0iEr+kYhRloSJ3uD1opSYURtWYd3wEU9ZJVyDEvAgMBAAGj +ggE/MIIBOzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC AQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1Ud -DgQWBBQGuAQ6NLkmWyZGLYEAYVc3+43eIDB6BgNVHSMEczBxgBSWO1UVkFl5LOup -lY08xBLcfGyc+qFOpEwwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD -VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290ggkAzDBe -dcABQHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoGA1Ud -EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAoUitYEBW -lzBMuuDMHvVrtwTcKNXeOXZ6PC78OXNxGCSK4CxTzOceyHjj0K30r2qOd4hNbq43 -J60Zf+69qIpkezG3hcCy/8Yu1lLyhxLvErCbgkvZ1MH4HkEyJOeoh0PJoIq1U3vu -SA2Ny6LDA5QOWaVME6c1L0ntueu5/BddrTGtqHvK4Td9E3Y8gQhtJHCkN2GbkRRS -vr2+efC2hNIWEfxFTP+RRKKGfn+B+YASPgLDvHpIapz65MVebJH+gIo/k57swPMJ -K3Zet53tXqUvnbh87vyGdaOpq897WISNfleQ/XJB28Q3G5qBdLiTu7oxYG42XhC1 -qIm4pZ/gGW5lauu3h9bvzODB2DV6ShnBujtzbd0vBBX38BMP2oOOQ3X2QgalGOKO -jd6yTCN6VO0ZiFD4xO2XsE1CKkRqPx9rRCKR85NCvlpyytC4KIZjcomf0miMl9e1 -G9Js2y9agiXgQ5lrlZSQ8XyvRD8PC1anCxeyn8HaBblYNwUZ8S6vXkq7K13MuPgX -3vw106gUcWjw6GwC3BOi7UNtcAYyoXn6jLJ2unpn2mqjpCfbQZ3mnAI7MbEoaSFa -1CTYh9bxYK6cN+8dOnYoXUyD27aDq6xfP9YJoNmQdYTW4Wwf5JCK23j2DtDB/qen -uyydJBuI6Me+ctWntImvjM97Teu8s+ssQaA= +DgQWBBRFf07E369WKOuyKog6EyuNDDWDnDCBhQYDVR0jBH4wfIAUAdlczvOoJQH9 +cG+fm41vGD8WZaWhTqRMMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0G +A1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdIIUM0vS +1bRQQCJzcfsSQBT78uhQ7lUwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAgEAPW9KkG+G/oRmCiB23nxOJz7R6w0S2eM938RaKkRhtXdXBmVIusr1vdFm +QvjMA3RgoQVTyFwD2PIvpFmFAYThZrxu8a9TIXqVrgHLvHgRuWVlXbkq/fsi/AEH +BNbJS7WoO1t+QxvOVXota/hUYMhes8QvRTDqql6iXdV6/mzLMxJ7vpRmnV6zaRmo +cMegAH82IEeXz61N++Cn+Tzasy2mkH/eZ+pXaPrCKhwWD08zEDSm9ury2LJzPWCT +GbHkwgJy8TwIRLCpORlccIoZED0wyeQNAuMRS3vh5w2UpHz1CAot0dF2XzPwkJaW +eJTHmU9iRmPnn9VgCSMYLadjANT1SGFeLJlYxAHmbheMKepkFddiqyCUVjIOyN79 +7Uzgl4iPSFrHSyNgeXuIEEmNb9jQlcxPzNcyjz+XPYAeh9RihvZgi0TalPJqFc57 +HKqDoUnHfb2iyN2wmMca9xKhBgQycFxsoDr+GlWehkPMf0ZIqR757v65/0JRGQuf +2VMyTYm3CsRgIuhrI2KMCeDrOeHrRAz/A9Sa+PI7Arvu0lPaTcVCAz7Nw9DnAOhR +kfC1mK+wqPYHKZduYGyBmZsGylAOaiO9KGWyzk11qI9Bk4nmkJ8KS/0rmXYAwZB3 +dbEceSt9xvY3oY0R+qZFUEF5ztHaBiggScS9duCD60iZqwScAPY= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt b/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt index 6ba9acb7a07..c2094a7b3a9 100644 --- a/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt +++ b/qa/integration/fixtures/webserver_certs/generated/server_from_root.crt @@ -1,36 +1,36 @@ -----BEGIN CERTIFICATE----- -MIIGRjCCBC6gAwIBAgIBAzANBgkqhkiG9w0BAQUFADBKMQswCQYDVQQGEwJQVDEL +MIIGUjCCBDqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBKMQswCQYDVQQGEwJQVDEL MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDTAL -BgNVBAMMBHJvb3QwHhcNMjExMDE5MTgyNzMyWhcNMjQwNzE1MTgyNzMyWjBMMQsw +BgNVBAMMBHJvb3QwHhcNMjQwNzE2MTUzNTA0WhcNMjcwNDEyMTUzNTA0WjBMMQsw CQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UE CgwFTXlMYWIxDzANBgNVBAMMBnNlcnZlcjCCAiIwDQYJKoZIhvcNAQEBBQADggIP -ADCCAgoCggIBAKJZa8jP+LalBAlQ8uWWlTm8TLHWkGLFxr+rNzs23mAAfLo8FFGJ -bHQ2iIWQtjCQwHiWUrHU2eRBvL5zJ3cEWte2WF/OAkNhcZu8/JcH3jCMhbjfc3V1 -xIfRdWYEhTI+5PYaqgStNs9H24fyshMFcDubXU1pKfNXtY1sQVKfQl5XeAN5Qcif -2QSfUxNOWnvnmUdBPNOIUP3W+3pqzv+uhPt9Egj2crYPXnSnC5JZSdeya+Rz60xi -WpmSWn5/WdO6j89CVI1kYX7UXuWVhtp8QLa315cMZ9+b238SrDjzNQ+ElwQJO1Nb -oD+JfsNBJBXPy/6Tr6FOZXHOWFFvIErceImKIdG12WL6IHxPlyPpeBGTdbGCS5gy -vWKaJ8diV15Mea8rGH7D30sR2StgBtW1MJOomkarS+2oQpC78HqQrNK1Qgl3GauW -PMIKCU6TR5OPZHsb87u7iXlc+NO+tBoFtgOVK7LvN/QwQ1At4hQaLcrpEJwtjfk/ -8jrC22Ewgp1LMakAsmDjGkkOBZPL5VeNgNnUEp9/AHYfH1XmpkLUOoRmUVkVDqfs -FVcFTLLb2XDtZWtl+gJ5cXxnr87dj5X2Xowo2BMKBwwMpThmJX+i2yY420hq9uu6 -xzPjUC7NauaPnIYCIjWtaBqruXWMxX9DISED71aAVAOSqXbv2fFpKMpRAgMBAAGj -ggEzMIIBLzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC +ADCCAgoCggIBAJT0RJKyh+dR+joDaEKF++Rd8YPKmo1HllITKIak0gmgPkWb+si6 +yfi7UrnmgIeSj2CESOaPcnMzbPSajBPCtitS1YlYgB8N4OKht80ib/Bc2fYblmED ++Z9+ySnVlPVFiAglS2DsVz3QUtRmpyOYUeqvBNpSZnfnxF/InIRxVAjQ/7FZ87NP +Xa42K8J1d5WXrkE/X8j7BfZUUxbKyGDB9ys8jMoHxmnH2P/P/LcvwZPaQQkxMzqk +tHY/p+tMOdkARFmAqJ1GC+zRQExRo+ppPBMeLWvQXxtW/57giqvwFoYXdFw+GHzU +0LSK810WsBDqIJfZXcd2q/EuW5mwWt6F5c5rFGI+tzYsuE+2C3sYWK7BF+BiHs1H +mZREG6+NCCTgfXaNakLFf/alouJyfj6irZXMVnFZvFmTLwJVai86l7cUYnV1A22a +O6pweBov7od5p1UmFDbD1etVa0u0HiYbdOE5rKxM0p4CDbO7FlbUoLdlE4li12sj +iCTR3psD3TG85cOiTSv1lQGpaa1MJU8a/jH3Ms24v3uFR3U/TAy++c3p2mBqfNyI +sDyIUCBEWAY/Y9YJxiS8VlN1A5WcKSkmiou2SbCX0EUIkaDoDaOiWFJY0QmVv8nM +/Keh2tPWS+NFoBLyU0iEr+kYhRloSJ3uD1opSYURtWYd3wEU9ZJVyDEvAgMBAAGj +ggE/MIIBOzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAzBglghkgBhvhC AQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmljYXRlMB0GA1Ud -DgQWBBQGuAQ6NLkmWyZGLYEAYVc3+43eIDB6BgNVHSMEczBxgBSWO1UVkFl5LOup -lY08xBLcfGyc+qFOpEwwSjELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYD -VQQHDAZMaXNib24xDjAMBgNVBAoMBU15TGFiMQ0wCwYDVQQDDARyb290ggkAzDBe -dcABQHMwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMBoGA1Ud -EQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQUFAAOCAgEAoUitYEBW -lzBMuuDMHvVrtwTcKNXeOXZ6PC78OXNxGCSK4CxTzOceyHjj0K30r2qOd4hNbq43 -J60Zf+69qIpkezG3hcCy/8Yu1lLyhxLvErCbgkvZ1MH4HkEyJOeoh0PJoIq1U3vu -SA2Ny6LDA5QOWaVME6c1L0ntueu5/BddrTGtqHvK4Td9E3Y8gQhtJHCkN2GbkRRS -vr2+efC2hNIWEfxFTP+RRKKGfn+B+YASPgLDvHpIapz65MVebJH+gIo/k57swPMJ -K3Zet53tXqUvnbh87vyGdaOpq897WISNfleQ/XJB28Q3G5qBdLiTu7oxYG42XhC1 -qIm4pZ/gGW5lauu3h9bvzODB2DV6ShnBujtzbd0vBBX38BMP2oOOQ3X2QgalGOKO -jd6yTCN6VO0ZiFD4xO2XsE1CKkRqPx9rRCKR85NCvlpyytC4KIZjcomf0miMl9e1 -G9Js2y9agiXgQ5lrlZSQ8XyvRD8PC1anCxeyn8HaBblYNwUZ8S6vXkq7K13MuPgX -3vw106gUcWjw6GwC3BOi7UNtcAYyoXn6jLJ2unpn2mqjpCfbQZ3mnAI7MbEoaSFa -1CTYh9bxYK6cN+8dOnYoXUyD27aDq6xfP9YJoNmQdYTW4Wwf5JCK23j2DtDB/qen -uyydJBuI6Me+ctWntImvjM97Teu8s+ssQaA= +DgQWBBRFf07E369WKOuyKog6EyuNDDWDnDCBhQYDVR0jBH4wfIAUAdlczvOoJQH9 +cG+fm41vGD8WZaWhTqRMMEoxCzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0G +A1UEBwwGTGlzYm9uMQ4wDAYDVQQKDAVNeUxhYjENMAsGA1UEAwwEcm9vdIIUM0vS +1bRQQCJzcfsSQBT78uhQ7lUwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsG +AQUFBwMBMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsF +AAOCAgEAPW9KkG+G/oRmCiB23nxOJz7R6w0S2eM938RaKkRhtXdXBmVIusr1vdFm +QvjMA3RgoQVTyFwD2PIvpFmFAYThZrxu8a9TIXqVrgHLvHgRuWVlXbkq/fsi/AEH +BNbJS7WoO1t+QxvOVXota/hUYMhes8QvRTDqql6iXdV6/mzLMxJ7vpRmnV6zaRmo +cMegAH82IEeXz61N++Cn+Tzasy2mkH/eZ+pXaPrCKhwWD08zEDSm9ury2LJzPWCT +GbHkwgJy8TwIRLCpORlccIoZED0wyeQNAuMRS3vh5w2UpHz1CAot0dF2XzPwkJaW +eJTHmU9iRmPnn9VgCSMYLadjANT1SGFeLJlYxAHmbheMKepkFddiqyCUVjIOyN79 +7Uzgl4iPSFrHSyNgeXuIEEmNb9jQlcxPzNcyjz+XPYAeh9RihvZgi0TalPJqFc57 +HKqDoUnHfb2iyN2wmMca9xKhBgQycFxsoDr+GlWehkPMf0ZIqR757v65/0JRGQuf +2VMyTYm3CsRgIuhrI2KMCeDrOeHrRAz/A9Sa+PI7Arvu0lPaTcVCAz7Nw9DnAOhR +kfC1mK+wqPYHKZduYGyBmZsGylAOaiO9KGWyzk11qI9Bk4nmkJ8KS/0rmXYAwZB3 +dbEceSt9xvY3oY0R+qZFUEF5ztHaBiggScS9duCD60iZqwScAPY= -----END CERTIFICATE----- diff --git a/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks b/qa/integration/fixtures/webserver_certs/generated/server_from_root.jks index 1596eef7b5d1c8f746867d2f31ac3484c409b2ac..11414c8e6b4e65d1bc66941eb264e538f8492343 100644 GIT binary patch literal 4652 zcma)ARag{^ww)Oo|iYv_;~VWdmCVd(BgkPZncDThvJ5v0$5 z&%NJ&zWa3V!>+aW+OK@`@0M1I$AZ_hBN4yZ?qeAP9Vk z|JB0h!$jco{0(#cZS2?t|HFEM4+P~Q@OmH!yiSPqKMX8Fh#utCf66BiE=)49=lUsC zbtMUsY>nhOjyYnaUmI{hAa)`UkP-3(8~49Ag7B~a5PEDpQ=}Zw3Ih|!iAm(X0UQG{ zGPl29HOz*I=J;YF@KkZU0Wsi-kz6W@upU`d2*VO@`d^hS4={!Sh`W(}$>8gtSE6$j&R}nh8)> zbGL-Wl8{Io1qNMo$;K_e+s4GIb001t+YMCaqx=VSI9K#RrUUBmP+>2^w-LHbrU!GB zd8Y9@Lw}~9KxzPuXUJndQWxH8aD24LRk{>kHoxPxJzAuvSTJ5z>eBh6ZinjSX47oU z>^|n3dV|DL!g(+b+nI*5ezTaF&{tGyeqxn5t2TziZvi^Wm8sEzK;&6|fQx0c;p1@) z|1@-m&IB(*q-fAHB$by@Zm@`W?oo|`R$cW@xJE4VdMUH|`Zd}j+YNn!k1BX+TQB$e zF#sG!d}71V8a%H+n^q|}evWDF9{h*dVhKO1V;`Y}>I`WmkJ_1=8nGqm0Z9`h#oVraZj0JL z=?r4GHhe7mkwnfE5T{!sb}h{6D>N$#ht5)y!}4`Go$g4e1!_ExY^;b7O}r^g!n!R9 zmRxW2<8|ja1d@gl`?~gMfXQr4=21SQT7&58c+p-Oo zbG}dUBjYk~#_#kQvjk+fx-_ozhpDd~OJ*L3NP zZ_lMEi5^r6PtgxzuX*V~I%2a%2>S!rfRgxBm^3IPPI)0tk396i@bL%zOSfDL=0O+P zVyaxylEj9R-3B9J(vCv{h6}F&?4xb9mK(w>E8YP;_Ze?S@(Uf?vLMi zx5zd~bE!;!q))P4Uz>h^NH+9l^gb{5-dG?@-|ZE5!2l}@jbjtJ(7dg!l-I(3I+);XtE{=QZhH{Rt* z_L$Af@MX5d5N2aD3Mp7{Gt0?}qmyFsZ$8SJJ`&KA$N=p$? z@heJCJw53ib4SgzQAO5u5LS8z>bYHk)E7Z+u#$fX$y%UFL*cXIl(7rkf+`m#%gJe8I%_ld5F70 zH#DQp_6*>waHX5@8j35Yu1KXG0@F?Ya!tGsMuJs2WgV|Xhm|O|T~P^j)eW))s|lTL zft8v>mLH;DteFR42KYu`=5CB$DoA7n|6t1K7=NtS2#pITQi>g~Ep5ni>?K}vg_7NW z66xO8B(x983S8vP&f)8@k+y2{!|vmPyD*8t#6!sAy~Vb9lN{9AfWCP9rm6lk;cQu~ z1@FvK{D(o|GN%@PdC%2ZXB?itr@Qr^qE6VM11&UXQ2bA$tA(N$^PX(yaCbQJ8)l+s8aOV zYo*7k=8$&RkCae_a!n?VJd@tKTzle08Y8iTWgnPa6|f>ZI!2sWQ8QCq-13Ywo{kJ{ z0Wm)*=aLel6f{2;tt`!R7pGd0PF6}PY~Vk^FLp;gy*}N?ivECuIDDr`Dd{81Ai0q0v%^1-rE-kw4 z<0Bq*c73cb${~aCxsk}{RNIemBPh&#(z{UieT?#)blNHXXvCwcd1!St$RU`CNmfX^ z+QB<7`j(k$4F2`%u-H-fr#5?2GzPx`ucH4=glI(Bh@i`OOzgTp+x#Kos)=OL^{qJH zkWi%k+-3kPr`w*vDM*FWn4G5t9B(VJ!W#7LyR-;&p)o_BOd={gHe3sSbZ?3wsi2yc za!W+18DpvH1L|>c5EjFfai2ObSLwIFzj4hs*3$AvO2-aWi#3H6tGpH5p?_+Z0B){Z zQtWJjaZV+@_MCrARNn1==2AKES7`x$Fi0JIbfm}93VPlYwqI4is2?%OfN8J8FwZPk z<#E5Q^bi%Ufd}i$VraTugO{&xKt5p)|9KYq*K$h2-S6sKu;Q6OY-83+etYFy(%VLi zCVrV-NfU-6h>lR=f!s@EM%8f1D)Q zX{*zu1OtZ#_g-DBV&kW$z}UX$ZrG9{w&n@^3sU=qU8sn}iPPMAvh1N}daR$}#CEJR zyu%D4!Kybz8YhL(>Nl*NqRK%Rx2uAbG=r~`H$jbGK+;Q|(7Q1*I#_4P$F+CPD8??# zo?6Ww=BY1*a7o;UCh0$y5nJn(C2g68`X}Y?Gb;u>OId{#useE=d`azP(1x0fxu`>x zM|(3T7d75amkmEJUvUVTclI{Z5;NL1%^JD&3P(gU)b_7%%S}#hwc-MWzXiHo20Gz( zeJSatHkN9%xkOcP#p@``2iLbhyklkWX=;37SGS$byEx!MwIQ&a9&PIKi{AZP@5)d6 zFAF1`8So2UO$g6Fat)7+7tFv2a0gfc+yGwx_9g)9zjGIW6Tsx((FNcEVfqh&oRaLoUPj6p!a-$Ca;9pL{Z9h1vVBkgNs z^=EPIUvG)De&8>v+xh=UM}PPV;k-DH`*dr(`6+6Cq9}vY( zig%)=k4Gi(yh<*kD0@|0^&{sAFth8YxblgDtM{9^(&8=#`IeZ<%m__J`}v{6Xx(5% z)zR5>7;l25i|{)~FIc^5@#y?1#MUCr_bd!_h;rBnNbT$pou zOL=PO&5bp2%6tGcueVjhJOi&C#jG&7<+MN=JQ)uTVBGifGIHzB-bkE#%(jh`Z{)to zzLXq)T@>4Md3*bF1A8}gYe4DD94*PFA)%86usNM-58{gQwxIRaZ11I)%C}lSC`{Dm zB1m@dcOTp&>O=&Gco5pyn)W-__w%RqG_$zN;)c~ZkCv0sBJN+uUjAsSYRWN6o_FDoF6#*7B$}fe z;}!6{C^R_k*z3zCX`^shj#5X+C^LD~Fr=$Ye{v8bNmde){VLxCs98&Bzxw-G5lvADh-5g+LD>Uo z3kJ6<#B%D5J7VARvZMrBHe_T!9R~Hn05lsnzrJ4vmpo~{&LYcw=ErV#MJ1q~#Q4EK zP_!k+4y4)v;G|jl?{K<^QkcVVz-AWPy_|Vyk-(BNDM#ocb_T(rglq}&4T#N>q)W>0NL!2~hgk=o)_{@W_1$v>pQK?zp zsPH??5R~wN<`j4g{VeOu2P_i)Ih*K8G$8+Eyjl}f0GG2r7ZfH?@v1NcPrW?PwYPZc zAEW%r*OI%;#o$uUOyL46xux*&wi#v>5D)!hl#$EvQ6i+_lhqf%hVUMXj2p*;Haa>@ z_4Gn`Gr!r1TC!03{pTWr9WU%L5BCQ;>(lu^=BZ?!HcIq^-E3kl)sCtpeezojD8MvM z;?Lkn6E=3?skhWGV9{j__Yx}?qf5tUZ_wTSr9fRezXX(<5nSpb!{tolQUe>n!^IvC zm*DK7-y$0DV5!96r~h@hd2;hkz|Mr-&#|jp!f0Ay2rFd_JT{(29>_ojOJ2E-!p>3a z6mXC=Zzt1coT5Y43MLnd%pc8`AQai-i3A z`e<=FFncr}9$t13fB4yJif#5~UFr=%qRk0(}#mzV9 z*0clcdBZT2_x4oxoKN{to8ExoXx57J9HL^CBHeNv6E__pr;ZWP0B{bPCg;j(PW1SM zKA|&e{KL$$h$Cr~*=IW2rF$;rWbyiQmbtHI_G;GKd-izE$2bB}Qt?c$(5)%LI}~j7 z@j5RaLzi(^il_>mu8_cj{EbR+;QZlhfb`9cRFEK2y7U~@(b6oP^@n(g=5uJ)n>-Qf zlEdmlQ`hk6xU_LS*|+by0*&?)PlGb9-+SUbYe=?uHXoxIvJg8PsTbx126Pz?==y5b zCWXArxht%BO|Z>%pRBi z4o*McM`yXS-8Y@1!>k)LnbV=(?|}|nNy>r&+?K|)U5a%{-+)7~S6^#R%FG*(%^q9i z7EFn8ly?z36W*jmQ_#fHH)*yhvq}kz ztZH|tHD8{Qu!)%R$Ph@$Nle_1I*cf=cPJUBC$7WlM}?eSYREeXFNE-)4+;o~1;AunwFr{W;Tw`6 za#KH2m=0?qVkK-9QnngP{?IvWfXUN=N!C4>_CgztbbfBVXU!0b5m-GKUin`q_Fq!< BuoVCR delta 4469 zcmV-*5sL1tB$Fe5FoF?}0s#Xsf)Pjt2`Yw2hW8Bt2LYgh5jg~c5j8M^5iu}=3H=5M zDuzgg_YDCD0ic2j?gWAf>M(){<}iW@;sy&UhDe6@4FL=a0Ro_c341Vt33o6l1_~;M zNQU?a1aiVgIVF!C2Yfnjm6@0s{bl!%zf*2}m(-R@J5?s$)!? z6swiH#n7YPLmqF>M?ff_WTTS7lG?e@8k=FN@wIhG)|X)m6T0dC*iFeSKm6`-wJ%Rj zQwsAq@+dFGRyVl)m9%t&v9R?Nj_0Ecpcpj-49nLZB{KmIg1f>V{Ms!Lf^7>zF&~~04+u(eZ#C$T0nbp<2u6$;q`B_N4EjA@ zEwK+tF(*)3{bbtAi@$!Y;Dd$>ZdN6%^6fB|Ii~S{D0%ubf_@#;xa-nSv$L}UJ=l1Q zN4A&f@DC~By$+79SkU)sk#xb!a=eer29q(f=(aRGtvqFA6$peKec4Gr$}UHccvI%} z?A@@sovol4Q_ieNw+hcr&gOY@8br7rX~C@_QxdHNis4!@yZlMd zr1b#gy^5tOCt8647S4bJ1JYqd@hGsD{2+^e!ArZw(c?^BKD#bf&I|M0*1Ogn;=hIZ zGa}DhJ^&Q}g~oox{F5FEl>78D47hiY4|QTm2}a(o0Q3gE8Hp%)>oU+hhOMmRGb{{l zeT29fn-l$YOJjubXh&x$kz{@`NZ&&8%j_l%9AtFsGi91PaLL~9Y&4iRe`M)P*@4`D z$zl;iAPa+MrFe0jSfw3-NkCFW3Q_30Gl*OkjFJsbk=X77LF z{7_pOxzZmiBRlEp_F(Rjr>a!{P~yCQ$Cg8NfZHZxr>1Yx_O}(8ghliaU*t9cD69a& zq-1q6E`E<8(zIKPFfECo1cc|{9H=jPF4i4%&#{m;5yJ_E7$D3D3V@q)E9KLcl*fy| zx8I27_>8#d`wKS&w@Mj58Yc#@s3EVNQ{K1rtq;8F#2x#H(C|%6#WxH-S6{S$Zc789-v#Y9Sxfgi{Qta{B^#4E)Wmv<_$)kFW1}S?AtY6iBu&*V9oS{^E;?lx&YPX$HcZC!*8)5QjgDOyEdgrsh z)P+%>WfxWIsRHPK{ThL_in+fk;zVEn!u!5gZB@S*RfPwjLD4@bqHa6ob7jw zdhxB=#|iVIPlQu8D{hj%WhalT??TvCzJKOlxd6crmJnAxX*eGHPJEz;=-pF)A)LjeDGx&>xCUr{ha)(G1$smP_=tKYWk%O; z;dxL367tL{F&gRdrnER>BL2I`*LF-HuZKSDa;Wmg&WzSEPxm zha=eGgTovFhlV-}@Q`Tyxr{=4iZ{v0KU{iT4qBuqhE;a%phZ_=4y#yS(w!jUsu@D# zqc72uWUbzY@C4XQ=F1D7iI1V#xD9^vNq^`&QO)uT?m z28eBcTu~ee{eDBnFQW-qW4hN$-v=wh+5!bP>OZ!|>w<#G<=2K4n%o(KzcH7S*O8T zHa0jgH8VIdFff7#ECvZGhDe6@4FLxRpn?Y-FoFje0s#Opf(H@?2`Yw2hW8Bt2LUiC z1_~;MNQUD+Edzam@^T!k~~m0s{cUP=JC4*nhe`WemBV z$dAGRZ@d)hfI^Q!SSgW2Hi5)P@Lh>X`fq zfJtZ>diK;&^_(`TGp~-VEISRO1IY}K@%oLqOujm;D(;n+WGd2TC2yX$q61tgjJk&h zo~`W_Cj?D1KfA2Zosxe<%Chof$*giy{?^@@TP$P;YN?RGVIo7#@@s_pf7ieIEWO>P zLiap$Q{c`rJPd%)K3}iLM;^kIDRDw3F59LI3}9-w&0HQ@HT-0gw4kM1w%hhk7Q1IZ zBDe`N0=xg1{MhlbFGLgy&!|>E=A(a#?<&X#XmcZQIhZYjqs}XhF9msj>FV_WfB?vT z(ods+A3zwp_xXtY>8FL=e_hZ}#24$i#nPtR$({kfUdRF0#$4Rk`#N zzv8Qw-}@Mq`$>3}%zol*ydzhU#e`d>RnS_o{*Tp@%I)mLHPue=w=LS7sx?w0PmZ?2l($1}|mlVy`H7za)GUcMzCKc7v`tB4TTA zVCoH<#eaT(>Ou5K?i?Wg-aRRZzYcruRWap(E(xH#w**|Om(!5?Py~-{RH*Mb(=gen z;ycnS)$uaclRCWM6~FH0&El-E zCSf;vX7Lh#Vcv@PZcKQK^+@Kp^)dd*e^Y&?eupake+sj{Ibf(IGn-vDixU%K6=I2A zl6(w0yC6NAY}Ooi-A6nq5d>#_F^uWBr35(N;Wac&vZqTkOr61dX`r(J@p5$)wXbcU zuA?K9C|x==!01Or`=``J_i?o}9*hq?-SdYT=wT+$J z-7D6~YC5RpgEG+hCL}Ub`xS41K@Qv!*f-gnA51KA6Pg2XA$T}4?7yvt&s3_=)P;he+f$YO_$+em3)6_A7^OzwC1=v}k30#6&5bXoWwE@yW5Gsf^Y3XP#+&(1 z8WD7ykHj6aN1c?e6MD7~We6#w_^>j=f2e>kL~lDyEc^<0;#fYsUW2n#++@sN%~aWv ziElG%A?ULavd4<*ZSWwYuRdv8)2qJC*V74@9gILd+T>bvr21_ zdcd{WAKsZpX=PB7DrcL!4YSDF;*$_KWVDYoP1po`3TR$dcswV@N_HZoTvm%ne_)cD z2hI^M$B^kCIL0U2Pe&rF^tdC{I<7ls%P3ed`2oXBdmq;}vE!xD8fQxx=@If>$5(K zxH95%@mJR=Y!4;$jBbmT7?{Izf1vBt$q;GrRjW#TLpV6VT)=9n?NdF^wFHdv&z2=g zr~Mg*d0Igqp+YDo4#*C#@-goSnPaT6o3?IhbqGpc^WtQk4GYC9V5^ESJ}@CL2?hl# z4g&%j1povT6h3xnG{ zw@dfCkAa<&em_<@cXe~74qc^B)w&%#19j|zF0n2FNZvA*(M zk#KqGEioP&6$J6h7?%0Q*t+ z%#xR~n0&k}y4rz`%6hKovoN4Lmmr+_wX#>O{i;*j!us`#R6b+NmGyDvfjti_%^tWm z2UhOs2P`9U$^JCVApfz}_(NdGtSQ_YdQB zI!H&$k00K14r zNV(aW)}>@|`11OzzRLIt$-@8|I|X$FTOPegVgRChXlT$4&z$-boxDsIh4;ijKW*Gk zuN=}Duyrwap!)+*fs8Au2RN&F{+&9zlhO6&a>0fj6g454Q&zfJ^TwVqhIU(jzU^5I zrMwo!iHX5@S7QD^QMr4(9{aEyX=Wa_)_m2fZ-cWLIniZx{E|<#Oq%)`E|*$*jc=U5 zZ=bkzlf&{%O45>8`~GzjURXV{@|PcD!Ln9~MTPnUaLFhh-j82bk30BLUK%ck^#JSlyj7`JHRvBUz3IZ!zLUFD`9=#EppwJTzWgxL ziNeAi6p<6z(0EdAE#_<^GtnAD#`XG587Z@oOvhhl13k5!_S$kkZF*AGPPw-7h{|QpN=~aiTwP$Gog%CH&M;I$ufB zx|Joyl=UXLTZ9_89wZFbp*1rT6d^rqaaBAbl*ogk#6-3>M3&k6QM{cStn2p9E3bu`l**UxW)CclFrayK zy&>yPi|!@>dCmg#M(WO5E#A1?<@KBzbSIkdUIj%22c>x1A6a7^!N{r>rcDuUaurKe zjQFbZ#2Z68wKax__|d9ty)*Y`RKv4y%WxWUhD^gp<%D#5xQZ;rqw&;2!xAfMERJ;- zSFd<(s3Dv(QPGqILF}D`(zux^;;rV$Mp59Bxrtdt+9ftakE<$93U z%fQI12Sgp#9;7v$EUlPA$L+=jUpz-w-8!r9nM(Qby*4C1dSiA^a8~Xfjnh{P_JcJ; z$Dwt2!|9idHiEOk5*p!0mcXE{#{Ehqa)W7)Sx=1%a_%|HIAyfEbGx6^zCC zS7!Zdj6m%FV>AvH5^@0;vlRr!g#CZuO~0Xdg?{>iFvcgI$ro?x;Qs}0q2BDsdGY8^ zEJVF#28n1CDi~8MsQ1K*IPxfbk$;#n#T50KGMh-dtzGydOitB6&?2m|PP`G*QN(Sh zjPBDOwfAJtr{}r4C_7yt?%lPjG099L^YVBRf@2bnu7+eV>sqn8P)~Qhd*Wbt{nIw7 zFepp*-MdYD{Krx!tRx}wCaLAl>HyHFBo+f~+tUgafOY43m7XfpF6iwGkGaGM+Yy&P z`Q-^P8^#-N|CI21YWahN2NW9f4YN;Q1IS#8Ow>OsYhY}_9nXtRFpnX;M(5+c^=GY3 zbUrkA{{esKY&qUDJD(t)^~JAO{cdD$WeXt2wFX}xx~*Q+t%|G9NFL(Tx%+gIWW)Wc z#$8WI7f6@TNcHLL6tY|k(dtxIteQb(*P64P1ud9_@Jlv^d6DdJP)7}2P1kf6koT4GNB$j;j;ZC zwHPfLYu&EMD$4Z^5(#UM>AbQi1y{V@BN-KXKGnQ#8 z55%J{FtO`AT-Nng^swCr|0ng7oo6Z2Kmldgv|zlk3Yd750hrNKi4z#}b%jMp+yQSh zI1~}t>i?^>e57&&yuxi^9MnC?>RW=t_Bt6gbxh{F9o1t##j_ldbDd7Aha7&|=B1OG zry~pV#7okEto*?tIZ9Xwc}WgnVcx)wEfr-+KUMpqvT)Q zGcWj#gWqt=%q($YMYhnb(DrpBoj5w+eO>%@EM%I;1>%V@A7y)Ysm6AvZYD_e85Vgc zGhp6`Lifhatzl|GzyHuH1#V=ksB2^zyt@ZAbe^Rnzu=v86|aB09`O1AYWjIi&y>lU z-QB-AJ%;7j46Ll49&_csqF!OXu@l1sxAa-6+#q&R_2juwI&5{+&C|7GgqWknnW@f8 zb3-nFJ>#Wvq24Whk;k4NJ}rBHqXE^iGv;kaJO0Me@9>=?WYgy$0E1p!8Hwq4DtZ9h zM5G=xn`}9lqVTEGrW48vdo*V;WO!34_&ZkQNkAdm(4s9rCUQM8wS-?QQ+IKKC_41U zwypDu2UGPCx^0nL@Z`TrEsd+BKd}beOAo%vc4iC{XvSATxhw~D5@j#y(cl^)^%HF z_}11k0bwpe`Z1~%-EUiMmM*e)=m}t|@D@Fn368@BV);@|4JwYay|ARrxFA*I* z_2ds}ogQv5dGnow)~F2_7K=0a_ADFAvhx*jtvjlGlRRAN}G*PlFBT8D);Ktk?5Ma)8|Z;)pT z9S@`5bPxzoX-jrE$c6`~(lY4w3Tj@_&)UocR(IMoyV^ESo>9u0(6hgSoQGcZ1@G$* z5`Jg1z}a}i3qwD6_h>M_u5bw1I(p5&yi7I7k$HF(8 zAVww`lu8AE?QV(}T86aBn@BFb^Te&6T7tmRy8qy)x8>(jf-R z355c3{xbI^hJE0$Dv+@3A8$Qy4By;!Kz{tGUudzpIc8Vuws=IVe|oFEgyN?pH*q_Y zjk%OlsbhKQoeM4825U|bJy5Rtk-YLJl^#ivnal!NWy4@fc<2#Oze>Vm>g6!2>E^EQUbHmvawj@ zNUq_zYnzGvX%sk4CaDuW1^>!FjqGW5YGySK+xbGNxX)^qeRkTVri`w~)M`A@T6qt5 z#@w<2x!naZl$`qf!eB=`tV1Xz6C>eT~RHwjN-CSPjzOyHKdJ0ZFl-X&A`9gwV+ z)EgN= z`()Yl*V40SaA_S!ck>5~aqWAz69e?>KBs~NW8hh6hi+AIP_krasox#_>09$=W3f8c ziKcSf6+RXA^ienG@}btAAL614ntJDG(`)u60__h~`G4dF6lC#HJoqf%rW%C|+W(~a zJn!_|cC7i66R_9GOI@d3+Fj7BKJYf0WqDPg$w-$F-lEp=`X|ik>m$KZwVlA7?3ww7 z|4~q*e$2Z#i1c>eTXYvI`h2;&CNL`DNR>LD6fq@}XuBzC_tZ~OCd&1?e&3p{493g| zaNA;F$4Si!`Na%D572@GkE&iKe$9a?m{rpgf4R2~keL2wl+%}0f_$=AQE8vD2XRRM zEXw&klHbImOC>>@LdjbozpB}JJPwJ#=`4&|2fveI#U24y^d*^GfM@t}4%aF;|8nJC zGC2S5o#ldQ5^z)Qj(RE*Nb(BQSwG*(X z@eSHu&0p*#bDcv(n8}V`HBgC-WtKT3aQF%{!E5cZ^zb0VQ*}*mJd)+jJ41XQZ3l`s z5bEuRb5VivovKjTb!KN@6S0poeg-O%B2q<@lR+qWI&|uMt8RAvA6y1DVtsncY^MW_ zin{w}p7IrK0|_(@S;_|ezTGQ=;0>k+F3mmK#0<~z5~1M&Sk^n#y}mLA{%umyYs zc>UL#1ML3hZU7g6`G2Duz!M|`;s@dW^G88OLIa@EpTLV}M9OOfz#Ow|Q%QYu!1$=@ e8T*;|;qzRW;?LPIR1Dd9aX|AZN%Vhi!~X#*r?~L| literal 4462 zcmY+HWmpv4x5a1ZZjc^eknRwHA*4fc=x&hi&XG=~q#aVa8wRBt1Qd`)I;A_V@Bg{? zzW09EXP;-Sz0cR*Iy^8!dn6z-4~&op4U;WWA@Uj*hz^9q26nzYdUx8V&2eU%&yP@!+9gbftiYg);MhAt7M^LSbn0iRz*(J-@Xd zi8YpjhhjE_O9jQU%g!V!O6{_SikkaQA^em9<3nB(mpMcHuH0M*f`w#Nr@h-XV2j?t zkP@HDEy=mW`g9Q$5;H#6evxVg#pcJjBxwpOD1b4yY9QAH(lvK5);FM*Ti4fCkI$B6 z-ff9Hh7P-sM-NluW*NI``AtaVn&wD%j#s6Hm!ej>n-t9N^Nt{+vagd1`PEjlQY5Mw zNX1h8t1cyRQY2UIX{gW+mgYq#hw;-#>RVteY~qs^ws1&$!@rl!CllX{pw=x!}Y!19jf(^8;Z7x#%;PgRc<<=jD3 zf^=OJX}15_{7ykHgCADAzVZK(nXTVT6XqJR!LV4qNTH;!1f zlSLbCbc2{U^c5~s$)h4CjCYE+ajOhOZ!Jv`)6Hd^pdMni-go$V{cC@3Wr9Q0M2Hs{46fc7nJN*WLM z&D2R(3EhX==N~_1YSF{@W+UZpMnL`f#3~0f zhc?m%kf=B49}tlg(uSdb#Iwm%t)z>QsCHkUf9t$pY>Er=Y?^OZD#jWU7ihWGkTh7a z->;_jrifVJO_UQzjq@V?I{tg|@TV$5~u=s1OM*PWN<(wRQC zPz+%VcZ9`;+ImE%=b2={IH_Bkd{ zz**4#K%4@RhonSe^qskwtS8m_8|y#A9}uxI ze|#be8J`!zAyqA(JbQ%}UT9)7;FqtAvfs3Q<%dd2osBjRi?Qk6n!EgYCf}3;&@X=T zapqWeHbTg&Pe0jhz?&!e-V4y(KBVwf(q!Y3&PgaoqBpkoY(L*AK|+-x+|JxWIL5Z% zX^Y&u$m`ahF7!Vq-FQ_z5G*^>ONE-}of|u!80f|alCJk-zNj~?FcQuFV9^a9xu}kD|As-*E4^K1T(j*e4^YRh>-+*h$Y8Iz9o%YP zsl{-AK`e;Rk};E&yoSpj-hMve5m{kFXa^;!Ybkz`GCW1H!{tUPQyyh(Et(@daNoc_ z1Lb;R)GNmnNoaBpNG;5T!@_{a%&hcbCYbxT%~P8UHuYaS2`EZ`^^cLryw(H^K6o=& zAim_Omq(~88V_c(KqRO|`b^c1(7`LFTv1*ET(v(pv|Rd2t{@Y2_D0@g(nM?E!drX2 zVDw711#~2d=0*XQGV`f+=P-SL0~l)wcAj3$9-gqMQH{SCUJmG-jkgalD*E0-t8rcN zp~hNuPO(n(JMXR@;bj<5$ht-(rr<1=G)eVvw1>MxLr{P6CbahSciv6r(s6vHSIZ=X zXp%~fe*_OwVO^9C7V_$V<|Ev_mt?sWvYrp~dS;3Sy5xwrnsjN5aEoRs_MFJi3A{{h z-QXIEzU*7xjD^Tpv9`_(&}8DWPwm>)_}9LL0|TTPD-4X>oC27_8L(#=YuUE-EYMlZy6@Z*%oGX1-f!XODiY& z#TyO%k&=0q9)=m!)w(LO_Zjv>E3^J$OSYh2CF!y-!PFM{=L&RyL-T?VlvLyUvs?SL zvNv%4uqWGCphw7m=~knClfCC5M;r;+Rf?>e2Znj}e?SO_VQ!$nFjxP|i+^hl4f}tv zhy_IYYdZfG?f)Mwk^iCPi@T$5x?2)W|IqU9x`N>v(ppvEFz-VrEi&gaR_!i}bU2d( zeL96Ox|a*tbS|4675%gBM-(dZ*U?Ahe;yFeME7mOled6wwbHO4WfMITk2%9L@d)kL zcQr9a-9i0kuuaUvvslFe$@m);oCQ+w=}$8|d;TQin0%<5>&MbED%u3!KKayWj>A+9 z8;*`J1eX(j;0^q%1#*4scZ?M!O(!j;x*Gc4MLexY!4ne;{_2oBW!pn}veuh^D|5N1 zHjh|l1GU@Z+)O5gfe9H9oG7IiluzU_@{X#lAHS(rqntg3vB~s4yXH8k&1z`wK>_^v z5{0?+(A&)2{0y}(+9Q#(XCO?$G`w|nlJQr%853+@DrH$ot6sOk?+sbRXyKNh>{^PN zCBX}B^pwU>pi^qxdssNaTD{7nzbhX)-4$uz8BZQ`**8GmV7JoEb(3oaKm=l*$Qzg7!5|x*CV>ypgq><|2p5SD zrExt}v*AC&r@6yc_k~@~kITY;D9^xfXAgES2aJ$`b+lTQ)1QJRHCI7HeqX zCIeQLh@$mv08N2&(nrY-eC94yylL?vW~(1uZ`|+TuE4Wdo7?R9UE)2p>FQebXq_)M zFFuX4U&cMGjk$>D_$J2Xy_bqjPVYpl@qc>2`HI1JzzP2lI+Kh)DWatN##9e%p~(Ri zI4*H!xWhYkV5h3UP3NFY5J|M!L9a5+I#BgXSJA1x?f|~@ySSockH#sZt(I#b=uWZh z3PCT%a~Eg#Fho}0t6qtqZlT$v;TS{Eiot#5E0YZHx*U8OL7qTJP@K5x@m;w^6wV=) zhZsKltUQ_Mh_;oJqs&LIQEmCVQ(ym}YX^sN#de060<u?fN$pUi#0#_wz_tyj(^z+CDlhB`L z81IXO;BGjL!*VVe*nSW*`w;i^XPr!X>aO(22;jJ{tiZhR%bgYS>Msu5;;_~W8&$6I zv4GB?E2OJbCq*D3Ls+mH7oMVz%ml#_vcPCWN)YnACnpNp*|?N_>aaAN3AOR8`NZO}(cLt{Znfm%(^YVb%(R(a z`@GR(@KeY6W}*+qNH&L|a*-OEt73w{P)Y#TV`r8f`>QC`^J4$ljc5AVD zV;ASA*-JY4E{HVh8)DAJ?6ctx-r8|VkMJtP9Jp^*J9sn)y^$fYaTK6?r8wFvHRKYn z($}KEC2x255&z@2&Y46om~<_)hr0ewt4G%z?Y91q^MsXdnKOy}e8w}v06cd(p_Qc= zhR~O2ZTXh3+WI$fS6i+0eLiAb#f*E96CtFOL37aaWL@u$zeGJG=zdl zo+dA{K&~Gj-*e-d6xWz=ZM>?XLq>oraL+P!uvCAaYY~I7gE|iGE5ban~qp#Rz&y);K`ZE0L+Xnj zo=nrXw&o((B9sTA`dLwA@#3&2e>=M`nM!5;8Oak%W^IR_2t$6o^5I zZ!}uiO4H}IXdDbaEWI|~<+336BterCG*lC#85m(>SjeCI+-Q;qB_}hBTzm8Wx)E)= zC;R0u91)ZD5&A$!zDtI{sTj78+m8_~Z-#pCUauyVwJv@DOS-W%qoBczwp4oQXun{& z;}^l7MxxA*)b_R9hV$o{gLka*OxwX@kI@z@SB=w`{Ss<{d}!~mNnQFsR1_A}XA~7$ z*cGDjT;wz!9q9JkxQYTJATDfFtiEb;)tW1_sTG9{r!HGsjv5%hRSy=(qc^zY!Zr^1*3L=RbQ+;y84)F=~co)PJxYyQwh9 zCG|H^2)^DN!%!s!LvYQ)E}YhgP+A28_O9ONC|)xVg7u7aGUul2B5lJ9o1Tm#6=>Vq z8Yt}7nRoEHJm6eXLuLX*zxL1ZTwdCbpEAk@a|8w~8Ds}qITIzDvda-4FT`qj&q5;- z-e3`629NeGP_(Z{1?sx8Y&N*Qz2CA+3)9zfElzcL|D)sS;y%Ymke-Bvp4X zpLP4JN=D5XoM>%=YGO$m$)#>jl-_FiV7wlXr!~@-$=o+ErO%3ge>dlvT!OS=)MS%1 z`leA~xVVynhMh;l-!1eBuX62s_MZcF*pEpZj;yAXh%MZ>in{zMLen9W{z*@2Bb41B zuep!WXN#XxmMV)%d3AWe|C`P*NqNaoNUr;%?cB2mJ&2|>l{b&x-k>L@E%9*wJ4C|E zKuZVk09XUu0p9;PO#wE4?=Ao*fa!m_3&4|ymxrDQ6AhIe4;hIP6@Wr=lDVN8^+v%a d$oVmJ*Q00{p<|wif-ymvk^d#pyyu@-_g}$9Q?>vA From 8f2dae618c6432903763f32f4378fca654829559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Thu, 18 Jul 2024 10:08:38 +0100 Subject: [PATCH 03/37] correctly handle stack overflow errors during pipeline compilation (#16323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit improves error handling when pipelines that are too big hit the Xss limit and throw a StackOverflowError. Currently the exception is printed outside of the logger, and doesn’t even show if log.format is json, leaving the user to wonder what happened. A couple of thoughts on the way this is implemented: * There should be a first barrier to handle pipelines that are too large based on the PipelineIR compilation. The barrier would use the detection of Xss to determine how big a pipeline could be. This however doesn't reduce the need to still handle a StackOverflow if it happens. * The catching of StackOverflowError could also be done on the WorkerLoop. However I'd suggest that this is unrelated to the Worker initialization itself, it just so happens that compiledPipeline.buildExecution is computed inside the WorkerLoop class for performance reasons. So I'd prefer logging to not come from the existing catch, but from a dedicated catch clause. Solves #16320 --- logstash-core/lib/logstash/java_pipeline.rb | 5 +++ .../spec/logstash/java_pipeline_spec.rb | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index 3e1aad5f1f9..9cec566ccf0 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -603,6 +603,11 @@ def init_worker_loop "Worker loop initialization error", default_logging_keys(:error => e.message, :exception => e.class, :stacktrace => e.backtrace.join("\n"))) nil + rescue Java::java.lang.StackOverflowError => se + @logger.error( + "Stack overflow error while compiling Pipeline. Please increase thread stack size using -Xss", + default_logging_keys()) + nil end end diff --git a/logstash-core/spec/logstash/java_pipeline_spec.rb b/logstash-core/spec/logstash/java_pipeline_spec.rb index 8889b87d7dd..3e743967273 100644 --- a/logstash-core/spec/logstash/java_pipeline_spec.rb +++ b/logstash-core/spec/logstash/java_pipeline_spec.rb @@ -1016,6 +1016,39 @@ def flush(options) it_behaves_like 'it flushes correctly' end end + context "Pipeline created with too many filters" do + # create pipeline with 2000 filters + # 2000 filters is more than a thread stack of size 2MB can handle + let(:config) do + <<-EOS + input { dummy_input {} } + filter { + #{" nil_flushing_filter {}\n" * 2000} + } + output { dummy_output {} } + EOS + end + let(:output) { ::LogStash::Outputs::DummyOutput.new } + + before do + allow(::LogStash::Outputs::DummyOutput).to receive(:new).with(any_args).and_return(output) + allow(LogStash::Plugin).to receive(:lookup).with("input", "dummy_input").and_return(LogStash::Inputs::DummyBlockingInput) + allow(LogStash::Plugin).to receive(:lookup).with("filter", "nil_flushing_filter").and_return(NilFlushingFilterPeriodic) + allow(LogStash::Plugin).to receive(:lookup).with("output", "dummy_output").and_return(::LogStash::Outputs::DummyOutput) + allow(LogStash::Plugin).to receive(:lookup).with("codec", "plain").and_return(LogStash::Codecs::Plain) + end + + let(:pipeline) { mock_java_pipeline_from_string(config, pipeline_settings_obj) } + + it "informs the user that a stack overflow occurred" do + allow(pipeline.logger).to receive(:error) + + pipeline.start + pipeline.shutdown + + expect(pipeline.logger).to have_received(:error).with(/Stack overflow/, anything).at_least(:once) + end + end context "Periodic Flush that intermittently returns nil" do let(:config) do <<-EOS From eff9b540df06a9d2309a6ba910bad2335ceb90a7 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Fri, 19 Jul 2024 12:34:42 -0400 Subject: [PATCH 04/37] Doc: Reposition worker-utilization in doc (#16335) --- .../monitoring/monitoring-apis.asciidoc | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/docs/static/monitoring/monitoring-apis.asciidoc b/docs/static/monitoring/monitoring-apis.asciidoc index f6b44e258d1..897507d1e22 100644 --- a/docs/static/monitoring/monitoring-apis.asciidoc +++ b/docs/static/monitoring/monitoring-apis.asciidoc @@ -499,10 +499,6 @@ Example response: "worker_concurrency" : { "current": 1.973, "lifetime": 1.721 - }, - "worker_utilization" : { - "current": 49.32, - "lifetime": 43.02 } } } @@ -540,19 +536,6 @@ Tuning a saturated pipeline to have more workers can often work to increase that A _process_ is also considered "saturated" when its top-level `worker_concurrency` flow metric approaches the _cumulative_ `pipeline.workers` across _all_ pipelines, and similarly can be addressed by tuning the <> that are saturated. -| worker_utilization | - -This is a unitless metric that indicates the percentage of available worker time being used by all plugins in a given pipeline (`duration` / (`uptime` * `pipeline.workers`). -It is useful for determining whether the pipeline has consistently-idle resources or is under resource contention. - -A _pipeline_ is considered "saturated" when its `worker_utilization` flow metric approaches 100, because it indicates that all of its workers are being kept busy. -This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline. -Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs. - -A _pipeline_ is considered "starved" when its `worker_utilization` flow metric approaches 0, because it indicates that none of its workers are being kept busy. -This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy. -Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines. - | queue_backpressure | This is a unitless metric representing the cumulative time spent by all inputs blocked pushing events into their pipeline's queue, relative to wall-clock time (`queue_push_duration_in_millis` / millisecond). It is typically most useful when looking at the stats for an <>. @@ -1002,6 +985,18 @@ A positive number indicates that the queue size-on-disk is growing, and a negati NOTE: The size of a PQ on disk includes both unacknowledged events and previously-acknowledged events from pages that contain one or more unprocessed events. This means it grows gradually as individual events are added, but shrinks in large chunks each time a whole page of processed events is reclaimed (read more: <>). + +| worker_utilization | +This is a unitless metric that indicates the percentage of available worker time being used by this individual plugin (`duration` / (`uptime` * `pipeline.workers`). +It is useful for identifying which plugins in a pipeline are using the available worker resources. + +A _pipeline_ is considered "saturated" when `worker_utilization` approaches 100, because it indicates that all of its workers are being kept busy. +This is typically an indication of either downstream back-pressure or insufficient resources allocated to the pipeline. +Tuning a saturated pipeline to have more workers can often work to increase that pipeline's throughput and decrease back-pressure to its queue, unless the pipeline is experiencing back-pressure from its outputs. + +A _pipeline_ is considered "starved" when `worker_utilization` approaches 0, because it indicates that none of its workers are being kept busy. +This is typically an indication that the inputs are not receiving or retrieving enough volume to keep the pipeline workers busy. +Tuning a starved pipeline to have fewer workers can help it to consume less memory and CPU, freeing up resources for other pipelines. |=== [discrete] From c633ad25682f3376adb0ba434d658b9eb820ac92 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Wed, 24 Jul 2024 02:22:34 -0700 Subject: [PATCH 05/37] settings: add support for observing settings after post-process hooks (#16339) Because logging configuration occurs after loading the `logstash.yml` settings, deprecation logs from `LogStash::Settings::DeprecatedAlias#set` are effectively emitted to a null logger and lost. By re-emitting after the post-process hooks, we can ensure that they make their way to the deprecation log. This change adds support for any setting that responds to `Object#observe_post_process` to receive it after all post-processing hooks have been executed. Resolves: elastic/logstash#16332 --- logstash-core/lib/logstash/settings.rb | 23 ++++++++++++++++--- .../setting_with_deprecated_alias_spec.rb | 21 +++++++++++++++++ logstash-core/spec/logstash/settings_spec.rb | 12 ++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index 4e23e34c3a9..d6238fe1cb0 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -197,6 +197,13 @@ def post_process callback.call(self) end end + + # because we cannot rely on deprecation logger being wired up when the setters + # are initially used, we re-emit setter-related deprecations after all post-processing + # hooks have been activated (and therefore after logging has been configured) + @settings.each_value do |setting| + setting.observe_post_process if setting.respond_to?(:observe_post_process) + end end def on_post_process(&block) @@ -839,9 +846,7 @@ def initialize(canonical_proxy, alias_name) end def set(value) - deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set", - :deprecated_alias => name, - :canonical_name => canonical_proxy.name)) + do_log_setter_deprecation super end @@ -856,6 +861,18 @@ def validate_value # bypass deprecation warning wrapped.validate_value if set? end + + def observe_post_process + do_log_setter_deprecation if set? + end + + private + + def do_log_setter_deprecation + deprecation_logger.deprecated(I18n.t("logstash.settings.deprecation.set", + :deprecated_alias => name, + :canonical_name => canonical_proxy.name)) + end end ## diff --git a/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb b/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb index 9ddd87a783c..481e387ba03 100644 --- a/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb +++ b/logstash-core/spec/logstash/settings/setting_with_deprecated_alias_spec.rb @@ -52,6 +52,13 @@ end include_examples '#validate_value success' + + context "#observe_post_process" do + it 'does not emit a deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end end context "when only the deprecated alias is set" do @@ -71,6 +78,13 @@ include_examples '#validate_value success' + context "#observe_post_process" do + it 're-emits the deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end + it 'validates deprecated alias' do expect { settings.get_setting(canonical_setting_name).deprecated_alias.validate_value }.to_not raise_error end @@ -107,6 +121,13 @@ end include_examples '#validate_value success' + + context "#observe_post_process" do + it 'does not emit a deprecation warning' do + expect(LogStash::Settings.deprecation_logger).to_not receive(:deprecated).with(a_string_including(deprecated_setting_name)) + settings.get_setting(deprecated_setting_name).observe_post_process + end + end end context "when both the canonical setting and deprecated alias are set" do diff --git a/logstash-core/spec/logstash/settings_spec.rb b/logstash-core/spec/logstash/settings_spec.rb index 241ac2272eb..3e0b425dd46 100644 --- a/logstash-core/spec/logstash/settings_spec.rb +++ b/logstash-core/spec/logstash/settings_spec.rb @@ -144,6 +144,18 @@ it "should preserve original settings" do expect(settings.get("foo")).to eq("bar") end + + context 'when a registered setting responds to `observe_post_process`' do + let(:observe_post_process_setting) do + LogStash::Setting::Boolean.new("this.that", true).tap { |s| allow(s).to receive(:observe_post_process) } + end + subject(:settings) do + described_class.new.tap { |s| s.register(observe_post_process_setting) } + end + it 'it sends `observe_post_process`' do + expect(observe_post_process_setting).to have_received(:observe_post_process) + end + end end context "transient settings" do From 90f303e4016e5377cb6254e0d3a85cfbbb9a32e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 24 Jul 2024 15:48:42 +0100 Subject: [PATCH 06/37] fix line used to determine ES is up (#16349) --- x-pack/qa/integration/support/helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/integration/support/helpers.rb b/x-pack/qa/integration/support/helpers.rb index 06069f3a1d3..a216d1cd8d4 100644 --- a/x-pack/qa/integration/support/helpers.rb +++ b/x-pack/qa/integration/support/helpers.rb @@ -48,7 +48,7 @@ def elasticsearch(options = {}) # Launch in the background and wait for /started/ stdout cmd = "bin/elasticsearch #{settings_arguments.join(' ')}" puts "Running elasticsearch: #{cmd}" - response = Belzebuth.run(cmd, { :directory => get_elasticsearch_path, :wait_condition => /license.*valid/, :timeout => 15 * 60 }) + response = Belzebuth.run(cmd, { :directory => get_elasticsearch_path, :wait_condition => /ClusterStateLicenseService.*license.*valid/, :timeout => 15 * 60 }) unless response.successful? raise "Could not start Elasticsearch, response: #{response}" end From 629d8fe5a82264749e1bbd7f1fba72efc2d5cf8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Mon, 29 Jul 2024 12:00:43 +0100 Subject: [PATCH 07/37] add retries to snyk buildkite job (#16343) --- .buildkite/snyk_report_pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.buildkite/snyk_report_pipeline.yml b/.buildkite/snyk_report_pipeline.yml index f4306bc1fbe..2a1476bcb5a 100644 --- a/.buildkite/snyk_report_pipeline.yml +++ b/.buildkite/snyk_report_pipeline.yml @@ -9,4 +9,7 @@ steps: # reports main, previous (ex: 7.latest) and current (ex: 8.latest) release branches to Snyk - label: ":hammer: Report to Snyk" command: - - .buildkite/scripts/snyk/report.sh \ No newline at end of file + - .buildkite/scripts/snyk/report.sh + retry: + automatic: + - limit: 3 From 03841cace3fa697a07e2099ebf6cdd4dd9324356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Tue, 30 Jul 2024 09:19:13 +0100 Subject: [PATCH 08/37] Fix 8.13.1 release notes (#16363) make a note of the fix that went to 8.13.1: #16026 Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/releasenotes.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 204454a0843..6bc41695874 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -237,7 +237,7 @@ No user-facing changes in Logstash core. [[logstash-8-13-1]] === Logstash 8.13.1 Release Notes -No user-facing changes in Logstash core. +* Inject variable reference instead of variable value in `logstash.yaml` to ensure keystore values take precedence over Environment Variables https://github.com/elastic/logstash/pull/16037[#16037] [[logstash-8-13-0]] === Logstash 8.13.0 Release Notes @@ -2318,4 +2318,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 \ No newline at end of file +Plugins align with release 7.14.0 From 09a2827802b2f86cc9bf669fdbbf0e294017a8da Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 30 Jul 2024 17:17:10 +0200 Subject: [PATCH 09/37] Update logstash_releases.json (#16347) --- ci/logstash_releases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 5657839d15e..39a6ae1a2ce 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -2,11 +2,11 @@ "releases": { "5.x": "5.6.16", "6.x": "6.8.23", - "7.x": "7.17.22", + "7.x": "7.17.23", "8.x": "8.14.3" }, "snapshots": { - "7.x": "7.17.23-SNAPSHOT", + "7.x": "7.17.24-SNAPSHOT", "8.x": "8.14.4-SNAPSHOT", "main": "8.15.0-SNAPSHOT" } From 62ef8a08476cb57deea4ca013329501168047355 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:09:26 -0700 Subject: [PATCH 10/37] [Bugfix] Resolve the array and char (single | double quote) escaped values of ${ENV} (#16365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Properly resolve the values from ENV vars if literal array string provided with ENV var. * Docker acceptance test for persisting keys and use actual values in docker container. * Review suggestion. Simplify the code by stripping whitespace before `gsub`, no need to check comma and split. Co-authored-by: João Duarte --------- Co-authored-by: João Duarte --- .../logstash/util/substitution_variables.rb | 18 +++++- qa/docker/shared_examples/xpack.rb | 56 ++++++++++++++++--- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/logstash-core/lib/logstash/util/substitution_variables.rb b/logstash-core/lib/logstash/util/substitution_variables.rb index c8d0ead762c..e5e2ede35fa 100644 --- a/logstash-core/lib/logstash/util/substitution_variables.rb +++ b/logstash-core/lib/logstash/util/substitution_variables.rb @@ -56,7 +56,8 @@ def replace_placeholders(value) end return value unless value.is_a?(String) - value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder| + is_placeholder_found = false + placeholder_value = value.gsub(SUBSTITUTION_PLACEHOLDER_REGEX) do |placeholder| # Note: Ruby docs claim[1] Regexp.last_match is thread-local and scoped to # the call, so this should be thread-safe. # @@ -65,6 +66,8 @@ def replace_placeholders(value) default = Regexp.last_match(:default) logger.debug("Replacing `#{placeholder}` with actual value") + is_placeholder_found = true + #check the secret store if it exists secret_store = SECRET_STORE.instance replacement = secret_store.nil? ? nil : secret_store.retrieveSecret(SecretStoreExt.getStoreId(name)) @@ -76,6 +79,19 @@ def replace_placeholders(value) end replacement.to_s end + + # no further action need if substitution didn't happen + return placeholder_value unless is_placeholder_found + + # ENV ${var} value may carry single quote or escaped double quote + # or single/double quoted entries in array string, needs to be refined + refined_value = placeholder_value.gsub(/[\\"\\']/, '') + if refined_value.start_with?('[') && refined_value.end_with?(']') + # remove square brackets, split by comma and cleanup leading/trailing whitespace + refined_value[1..-2].split(',').map(&:strip) + else + refined_value + end end # def replace_placeholders class << self diff --git a/qa/docker/shared_examples/xpack.rb b/qa/docker/shared_examples/xpack.rb index c1706a69fc0..82ff64ae065 100644 --- a/qa/docker/shared_examples/xpack.rb +++ b/qa/docker/shared_examples/xpack.rb @@ -2,21 +2,61 @@ before do @image = find_image(flavor) - @container = start_container(@image, {'ENV' => env}) end after do cleanup_container(@container) end - context 'when configuring xpack settings' do - let(:env) { %w(xpack.monitoring.enabled=false xpack.monitoring.elasticsearch.hosts=["http://node1:9200","http://node2:9200"]) } + describe 'when configuring xpack settings' do - it 'persists monitoring environment var keys' do - # persisting actual value of the environment keys bring the issue where keystore looses its power - # visit https://github.com/elastic/logstash/issues/15766 for details - expect(get_settings(@container)['xpack.monitoring.enabled']).to eq("${xpack.monitoring.enabled}") - expect(get_settings(@container)['xpack.monitoring.elasticsearch.hosts']).to eq("${xpack.monitoring.elasticsearch.hosts}") + context 'when persists env var keys into logstash.yml' do + let(:env) { %w(XPACK_MONITORING_ENABLED=false XPACK_MONITORING_ELASTICSEARCH_HOSTS=["http://node1:9200","http://node2:9200"]) } + + before do + @container = start_container(@image, {'ENV' => env}) + end + + it 'saves keys instead actual value which will be resolved from keystore | env later' do + settings = get_settings(@container) + expect(settings['xpack.monitoring.enabled']).to eq("${XPACK_MONITORING_ENABLED}") + expect(settings['xpack.monitoring.elasticsearch.hosts']).to eq("${XPACK_MONITORING_ELASTICSEARCH_HOSTS}") + end + end + + context 'with running with env vars' do + let(:env) { + [ + 'XPACK_MONITORING_ENABLED=true', + 'XPACK_MONITORING_ELASTICSEARCH_HOSTS="http://node1:9200"', + 'XPACK_MANAGEMENT_ENABLED=true', + 'XPACK_MANAGEMENT_PIPELINE_ID=["*"]', # double quotes intentionally placed + 'XPACK_MANAGEMENT_ELASTICSEARCH_HOSTS=["http://node3:9200", "http://node4:9200"]' + ] + } + + it 'persists var keys into logstas.yaml and uses their resolved actual values' do + container = create_container(@image, {'ENV' => env}) + + sleep(15) # wait for container run + + settings = get_settings(container) + + expect(settings['xpack.monitoring.enabled']).to eq("${XPACK_MONITORING_ENABLED}") + expect(settings['xpack.monitoring.elasticsearch.hosts']).to eq("${XPACK_MONITORING_ELASTICSEARCH_HOSTS}") + expect(settings['xpack.management.enabled']).to eq("${XPACK_MANAGEMENT_ENABLED}") + expect(settings['xpack.management.pipeline.id']).to eq("${XPACK_MANAGEMENT_PIPELINE_ID}") + expect(settings['xpack.management.elasticsearch.hosts']).to eq("${XPACK_MANAGEMENT_ELASTICSEARCH_HOSTS}") + + # get container logs + container_logs = container.logs(stdout: true) + + # check if logs contain node3 & node4 values actually resolved and used + expect(container_logs.include?('pipeline_id=>["*"]')).to be true + # note that, we are not spinning up ES nodes, so values can be in errors or in pool update logs + expect(container_logs.include?('http://node3:9200')).to be true + expect(container_logs.include?('http://node4:9200')).to be true + end end end end From 2db2a224edf8f22ec95859c1024d5aa398889f1f Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:06:48 -0400 Subject: [PATCH 11/37] Doc: Add SNMP integration to breaking changes (#16374) --- docs/static/breaking-changes.asciidoc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/static/breaking-changes.asciidoc b/docs/static/breaking-changes.asciidoc index 7a7107869e5..683a76a6749 100644 --- a/docs/static/breaking-changes.asciidoc +++ b/docs/static/breaking-changes.asciidoc @@ -12,7 +12,22 @@ Logstash 8.0.0 and later. NOTE: Migrating directly between non-consecutive major versions (6.x to 8.x) is not recommended. -We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0, +We suggest that you upgrade to {prev-major-last} before you upgrade to 8.0. + +[discrete] +[[breaking-8.15]] +=== 8.15 + +*New {ls} SNMP integration plugin* + +Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. +When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new SNMP integration plugin. + +IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in the {logstash-ref}/plugins-integrations-snmp.html[SNMP integration plugin]. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have some options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. + +[discrete] +=== Breaking changes in earlier releases See these topics for breaking changes in earlier releases: From 3d13ebe33e36942d60342c79006a1f46c5a467c7 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Fri, 9 Aug 2024 00:58:11 -0700 Subject: [PATCH 12/37] deprecate java less-than 17 (#16370) --- logstash-core/lib/logstash/runner.rb | 3 +++ logstash-core/locales/en.yml | 6 ++++++ .../src/main/java/org/logstash/util/JavaVersion.java | 2 ++ 3 files changed, 11 insertions(+) diff --git a/logstash-core/lib/logstash/runner.rb b/logstash-core/lib/logstash/runner.rb index 70041378563..876d3e1e173 100644 --- a/logstash-core/lib/logstash/runner.rb +++ b/logstash-core/lib/logstash/runner.rb @@ -321,6 +321,9 @@ def execute if JavaVersion::CURRENT < JavaVersion::JAVA_11 logger.warn I18n.t("logstash.runner.java.version", :java_home => java.lang.System.getProperty("java.home")) + elsif JavaVersion::CURRENT < JavaVersion::JAVA_17 + deprecation_logger.deprecated I18n.t("logstash.runner.java.version_17_minimum", + :java_home => java.lang.System.getProperty("java.home")) end logger.warn I18n.t("logstash.runner.java.home") if ENV["JAVA_HOME"] diff --git a/logstash-core/locales/en.yml b/logstash-core/locales/en.yml index bd9feedff21..92dcb38eebf 100644 --- a/logstash-core/locales/en.yml +++ b/logstash-core/locales/en.yml @@ -446,6 +446,12 @@ en: Running Logstash with the bundled JDK is recommended. The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), you can configure LS_JAVA_HOME to use that version instead. + version_17_minimum: >- + Starting from Logstash 9.0, the minimum required version of Java will be Java 17; + your Java version from `%{java_home}` does not meet this requirement. + Running Logstash with the bundled JDK is recommended. + The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. + If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), the version you supply with LS_JAVA_HOME must meet the minimum requirements. agent: sighup: >- SIGHUP received. diff --git a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java index 39eee187d02..505e27e6719 100644 --- a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java +++ b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java @@ -31,6 +31,8 @@ public class JavaVersion implements Comparable { public static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version")); public static final JavaVersion JAVA_11 = parse("11"); + public static final JavaVersion JAVA_17 = parse("17"); + private final List version; private JavaVersion(List version){ From e104704830e421ff18666f8fe9f16ff2fa9d4c9c Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Fri, 9 Aug 2024 09:33:01 -0700 Subject: [PATCH 13/37] Exclude substitution refinement on pipelines.yml (#16375) * Exclude substitution refinement on pipelines.yml (applies on ENV vars and logstash.yml where env2yaml saves vars) * Safety integration test for pipeline config.string contains ENV . --- logstash-core/lib/logstash/config/mixin.rb | 2 +- logstash-core/lib/logstash/settings.rb | 2 +- .../logstash/util/substitution_variables.rb | 20 ++++++++++--------- .../specs/multiple_pipeline_spec.rb | 10 ++++++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/logstash-core/lib/logstash/config/mixin.rb b/logstash-core/lib/logstash/config/mixin.rb index deea9717ba2..0d686be3f9c 100644 --- a/logstash-core/lib/logstash/config/mixin.rb +++ b/logstash-core/lib/logstash/config/mixin.rb @@ -93,7 +93,7 @@ def config_init(params) # Resolve environment variables references params.each do |name, value| - params[name.to_s] = deep_replace(value) + params[name.to_s] = deep_replace(value, true) end # Intercept codecs that have not been instantiated diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index d6238fe1cb0..d8c75f220af 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -187,7 +187,7 @@ def reset def from_yaml(yaml_path, file_name = "logstash.yml") settings = read_yaml(::File.join(yaml_path, file_name)) - self.merge(deep_replace(flatten_hash(settings)), true) + self.merge(deep_replace(flatten_hash(settings), true), true) self end diff --git a/logstash-core/lib/logstash/util/substitution_variables.rb b/logstash-core/lib/logstash/util/substitution_variables.rb index e5e2ede35fa..4319a38235f 100644 --- a/logstash-core/lib/logstash/util/substitution_variables.rb +++ b/logstash-core/lib/logstash/util/substitution_variables.rb @@ -28,19 +28,19 @@ module ::LogStash::Util::SubstitutionVariables SECRET_STORE = ::LogStash::Util::LazySingleton.new { load_secret_store } private_constant :SECRET_STORE - # Recursive method to replace substitution variable references in parameters - def deep_replace(value) + # Recursive method to replace substitution variable references in parameters and refine if required + def deep_replace(value, refine = false) if value.is_a?(Hash) value.each do |valueHashKey, valueHashValue| - value[valueHashKey.to_s] = deep_replace(valueHashValue) + value[valueHashKey.to_s] = deep_replace(valueHashValue, refine) end else if value.is_a?(Array) value.each_with_index do |single_value, i| - value[i] = deep_replace(single_value) + value[i] = deep_replace(single_value, refine) end else - return replace_placeholders(value) + return replace_placeholders(value, refine) end end end @@ -49,9 +49,11 @@ def deep_replace(value) # Process following patterns : ${VAR}, ${VAR:defaultValue} # If value matches the pattern, returns the following precedence : Secret store value, Environment entry value, default value as provided in the pattern # If the value does not match the pattern, the 'value' param returns as-is - def replace_placeholders(value) + # When setting refine to true, substituted value will be cleaned against escaped single/double quotes + # and generates array if resolved substituted value is array string + def replace_placeholders(value, refine) if value.kind_of?(::LogStash::Util::Password) - interpolated = replace_placeholders(value.value) + interpolated = replace_placeholders(value.value, refine) return ::LogStash::Util::Password.new(interpolated) end return value unless value.is_a?(String) @@ -80,8 +82,8 @@ def replace_placeholders(value) replacement.to_s end - # no further action need if substitution didn't happen - return placeholder_value unless is_placeholder_found + # no further action need if substitution didn't happen or refine isn't required + return placeholder_value unless is_placeholder_found && refine # ENV ${var} value may carry single quote or escaped double quote # or single/double quoted entries in array string, needs to be refined diff --git a/qa/integration/specs/multiple_pipeline_spec.rb b/qa/integration/specs/multiple_pipeline_spec.rb index 31496336610..cb01e56fe5f 100644 --- a/qa/integration/specs/multiple_pipeline_spec.rb +++ b/qa/integration/specs/multiple_pipeline_spec.rb @@ -34,6 +34,7 @@ let(:temporary_out_file_1) { Stud::Temporary.pathname } let(:temporary_out_file_2) { Stud::Temporary.pathname } + let(:temporary_out_file_3) { Stud::Temporary.pathname } let(:pipelines) {[ { @@ -47,6 +48,12 @@ "pipeline.workers" => 1, "pipeline.batch.size" => 1, "config.string" => "input { generator { count => 1 } } output { file { path => \"#{temporary_out_file_2}\" } }" + }, + { + "pipeline.id" => "config-string-with-env-var-pipeline", + "pipeline.workers" => 1, + "pipeline.batch.size" => 1, + "config.string" => "input { generator { count => 1 } } output { file { path => \"${TEMP_FILE_PATH}\" } }" } ]} @@ -65,6 +72,7 @@ it "executes the multiple pipelines" do logstash_service = @fixture.get_service("logstash") + logstash_service.env_variables = {'TEMP_FILE_PATH' => temporary_out_file_3} logstash_service.spawn_logstash("--path.settings", settings_dir, "--log.level=debug") try(retry_attempts) do expect(logstash_service.exited?).to be(true) @@ -74,6 +82,8 @@ expect(IO.readlines(temporary_out_file_1).size).to eq(1) expect(File.exist?(temporary_out_file_2)).to be(true) expect(IO.readlines(temporary_out_file_2).size).to eq(1) + expect(File.exist?(temporary_out_file_3)).to be(true) + expect(IO.readlines(temporary_out_file_3).size).to eq(1) end context 'effectively-empty pipelines.yml file' do From d4519711a692f290c0d9e87401fb5d679f9385ff Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:00:37 -0400 Subject: [PATCH 14/37] Doc: Forwardport 8.15.0 release notes to main (#16388) --- docs/static/releasenotes.asciidoc | 123 ++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 6bc41695874..dc4a80c8383 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -64,6 +65,128 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-15-0]] +=== Logstash 8.15.0 Release Notes + +[[snmp-ga-8.15.0]] +==== Announcing the new {ls} SNMP integration plugin + +The new `logstash-integration-snmp` plugin is available and bundled with {ls} 8.15.0 (and later) by default. +This plugin combines our classic `logstash-input-snmp` and `logstash-input-snmptrap` plugins into a single Ruby gem at v4.0.0 and later. + +The `logstash-integration-snmp` plugin combines the +`logstash-input-snmp` and `logstash-input-snmptrap` plugins into one integrated plugin that encompasses +the capabilities of both. +This integrated plugin package provides better alignment in snmp processing, better resource management, +easier package maintenance, and a smaller installation footprint. + +[[breaking-8.15.0]] +===== Breaking change with release of `logstash-integration-snmp` + +Prior to 8.15.0, {ls} bundled stand-alone versions of the `input-snmp` and `input-snmptrap` plugins by default. +When you upgrade to 8.15.0, the stand-alone plugins are replaced by the 4.0.0+ version contained in the new integration. + +IMPORTANT: Before you upgrade to {ls} 8.15.0, be aware of {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-migration[behavioral and mapping differences] between stand-alone plugins and the new versions included in `integration-snmp`. +If you need to maintain current mappings for the `input-snmptrap` plugin, you have options to {logstash-ref}/plugins-integrations-snmp.html#plugins-integrations-snmp-input-snmptrap-compat[preserve existing behavior]. + +[[featured-8.15.0]] +==== New features and enhancements + +[[wolfi-8.15.0]] +===== Wolfi-flavored Docker Image +Starting with 8.15.0, {ls} provides an extra Docker image flavor based on the Wolfi Linux distribution: https://github.com/wolfi-dev. (https://github.com/elastic/logstash/pull/16189[#16189]). +To use the image append "-wolfi" to the image name: + +``` +docker run docker.elastic.co/logstash/logstash-wolfi:8.15.0 +``` + +This new image flavor builds on top of a smaller and more secure base image, and is planned to become the default flavor in the next major version of {ls}. + +[[notable-8.15.0]] +==== Performance improvements and notable issues fixed +* Improved performance in pipeline-to-pipeline workloads by removing lock contention in event serialization https://github.com/elastic/logstash/pull/16194[#16194] +* Fixed pqcheck handling of paths with spaces on Windows https://github.com/elastic/logstash/pull/16205[#16205] + +[[core-8.15.0]] +==== Other changes to Logstash core + +* Changed rollover strategy in log4j2.properties to delete compressed files older than 7 days https://github.com/elastic/logstash/pull/16179[#16179] +* Fixed handling and reporting of compilation error when a pipeline didn't fit the JVM's ThreadStackSize https://github.com/elastic/logstash/pull/16336[#16336] +* Fixed missing deprecation logging in deprecated Logstash core settings https://github.com/elastic/logstash/pull/16348[#16348] +* Fixed innacurate accounting of `queue_size_in_bytes` metric when using `clean_consumed` option in dead_letter_queue input plugin https://github.com/elastic/logstash/pull/16195[#16195] + +[[dependencies-8.15.0]] +==== Updates to dependencies + +* Updated bundled JDK to 21 https://github.com/elastic/logstash/pull/16055[#16055] +* Updated JRuby to 9.4.8.0 https://github.com/elastic/logstash/pull/16278[#16278] + +[[docs-8.15.0]] +==== Documentation enhancements + +* Logstash {logstash-ref}/monitoring-with-ea.html[monitoring doc] improvements https://github.com/elastic/logstash/pull/16208[#16208] +* Add ecs and datastream requirement for {logstash-ref}/ea-integrations.html#es-tips[integration filter] https://github.com/elastic/logstash/pull/16268[#16268] +* Remove reference to puppet {ls} module https://github.com/elastic/logstash/pull/12356[#12356] +* Add section to describe intended usage of {logstash-ref}/jvm-settings.html#reducing-off-heap-usage[`pipeline.buffer.type`] https://github.com/elastic/logstash/pull/16083[#16083] +* Reposition {logstash-ref}/node-stats-api.html#pipeline-stats[`worker-utilization`] stat for better placement and flow https://github.com/elastic/logstash/pull/16337[#16337] +* Add {logstash-ref}/performance-troubleshooting.html[tuning guidance] based on Flow metrics https://github.com/elastic/logstash/pull/16289[#16289] + + +[[plugins-8.15.0]] +==== Plugin releases + +*De_dot Filter - 1.1.0* + +* Add support for recursively searching sub-fields with the new `recusive =>` config option https://github.com/logstash-plugins/logstash-filter-de_dot/pull/24[#24] + +*Elastic_integration Filter - 0.1.13* + +* Update default elasticsearch tree branch to 8.15 https://github.com/elastic/logstash-filter-elastic_integration/pull/156[#156] + +*Geoip Filter - 7.3.0* + +* Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases https://github.com/logstash-plugins/logstash-filter-geoip/pull/223[#223] +* Updated MaxMind dependencies. + +*Http Filter - 1.6.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-filter-http/pull/52[#52] + +*Http_poller Input - 5.6.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-input-http_poller/pull/146[#146] + +*Aws Integration - 7.1.7* + +* Remove empty temporary dirs at plugin close https://github.com/logstash-plugins/logstash-integration-aws/pull/46[#46] + +*Kafka Integration - 11.5.0* + +* Add "auto_create_topics" option to allow disabling of topic auto creation https://github.com/logstash-plugins/logstash-integration-kafka/pull/172[#172] + +*Snmp Integration - 4.0.4* + +* Introduce Snmp Integration by default, replacing Snmp Input v1.3.3 and Snmptrap Input v3.1.0 https://github.com/logstash-plugins/logstash-integration-snmp/pull/67[#67] + +*Http Output - 5.7.0* + +* Added new `ssl_enabled` setting for enabling/disabling the SSL configurations https://github.com/logstash-plugins/logstash-output-http/pull/144[#144] + +*Redis Output - 5.2.0* + +* Added support to SSL/TLS configurations https://github.com/logstash-plugins/logstash-output-redis/pull/69[#69] + * `ssl_enabled` + * `ssl_certificate_authorities` + * `ssl_certificate` + * `ssl_key` + * `ssl_verification_mode` + * `ssl_supported_protocols` + * `ssl_cipher_suites` +* Added basic support for SSL https://github.com/logstash-plugins/logstash-output-redis/pull/59[#59] +* Fixed documentation of required settings https://github.com/logstash-plugins/logstash-output-redis/pull/61[#61] + + [[logstash-8-14-3]] === Logstash 8.14.3 Release Notes From b2796afc92d27a3d55d3a74ec35f11800756eb55 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:49:34 -0700 Subject: [PATCH 15/37] Removing 8.14 from ci/branches.json as we have 8.15. (#16390) --- ci/branches.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/branches.json b/ci/branches.json index dca2ed04dea..6524ee2be1a 100644 --- a/ci/branches.json +++ b/ci/branches.json @@ -7,9 +7,6 @@ { "branch": "8.15" }, - { - "branch": "8.14" - }, { "branch": "7.17" } From 6e93b30c7fd809e148c1c1472954c1c56fbcd994 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:10:21 +0100 Subject: [PATCH 16/37] Increase Jruby -Xmx to avoid OOM during zip task in DRA (#16408) Fix: #16406 --- .buildkite/scripts/dra/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh index abdf461c1c2..6618c6314ec 100755 --- a/.buildkite/scripts/dra/common.sh +++ b/.buildkite/scripts/dra/common.sh @@ -29,7 +29,7 @@ function save_docker_tarballs { # Since we are using the system jruby, we need to make sure our jvm process # uses at least 1g of memory, If we don't do this we can get OOM issues when # installing gems. See https://github.com/elastic/logstash/issues/5179 -export JRUBY_OPTS="-J-Xmx1g" +export JRUBY_OPTS="-J-Xmx2g" # Extract the version number from the version.yml file # e.g.: 8.6.0 From ac034a14ee422148483d42a51b68f07d1a38494c Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 4 Sep 2024 11:10:29 +0200 Subject: [PATCH 17/37] Generate Dataset code with meaningful fields names (#16386) This PR is intended to help Logstash developers or users that want to better understand the code that's autogenerated to model a pipeline, assigning more meaningful names to the Datasets subclasses' fields. Updates `FieldDefinition` to receive the name of the field from construction methods, so that it can be used during the code generation phase, instead of the existing incremental `field%n`. Updates `ClassFields` to propagate the explicit field name down to the `FieldDefinitions`. Update the `DatasetCompiler` that add fields to `ClassFields` to assign a proper name to generated Dataset's fields. --- .../config/ir/compiler/ClassFields.java | 8 +++ .../config/ir/compiler/DatasetCompiler.java | 61 ++++++++++--------- .../config/ir/compiler/FieldDefinition.java | 17 ++++++ 3 files changed, 58 insertions(+), 28 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java index b34131bcd31..b4f5cf5e041 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ClassFields.java @@ -43,6 +43,10 @@ public ValueSyntaxElement add(final Object obj) { return addField(FieldDefinition.fromValue(definitions.size(), obj)); } + public ValueSyntaxElement add(final String fieldName, final Object obj) { + return addField(FieldDefinition.fromValue(fieldName, obj)); + } + /** * Adds a mutable field of the given type, that doesn't have a default value and is not * initialized by a constructor assignment. @@ -54,6 +58,10 @@ public ValueSyntaxElement add(final Class type) { return addField(FieldDefinition.mutableUnassigned(definitions.size(), type)); } + public ValueSyntaxElement add(final String fieldName, final Class type) { + return addField(FieldDefinition.mutableUnassigned(fieldName, type)); + } + /** * Add a {@link Closure} that should be executed in the constructor after field assignments * have been executed. diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java index 689fb00302d..e3db9a0105d 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; import org.jruby.RubyArray; import org.jruby.RubyHash; @@ -58,8 +59,8 @@ public static ComputeStepSyntaxElement splitDataset( final EventCondition condition) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement ifData = fields.add(new ArrayList<>()); - final ValueSyntaxElement elseData = fields.add(new ArrayList<>()); + final ValueSyntaxElement ifData = fields.add("ifData", new ArrayList<>()); + final ValueSyntaxElement elseData = fields.add("elseData", new ArrayList<>()); final ValueSyntaxElement right = fields.add(DatasetCompiler.Complement.class); final VariableDefinition event = new VariableDefinition(JrubyEventExtLibrary.RubyEvent.class, "event"); @@ -75,7 +76,7 @@ public static ComputeStepSyntaxElement splitDataset( ) ) ); - final ValueSyntaxElement conditionField = fields.add(condition); + final ValueSyntaxElement conditionField = fields.add("condition", condition); final DatasetCompiler.ComputeAndClear compute; if (parents.isEmpty()) { compute = withOutputBuffering( @@ -83,9 +84,8 @@ public static ComputeStepSyntaxElement splitDataset( Closure.wrap(clear(elseData)), ifData, fields ); } else { - final Collection parentFields = - parents.stream().map(fields::add).collect(Collectors.toList()); - final ValueSyntaxElement inputBuffer = fields.add(new ArrayList<>()); + final Collection parentFields = createParentStatementsFields(parents, fields); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", new ArrayList<>()); compute = withOutputBuffering( withInputBuffering( conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), @@ -111,20 +111,17 @@ public static ComputeStepSyntaxElement filterDataset( final AbstractFilterDelegatorExt plugin) { final ClassFields fields = new ClassFields(); - final ValueSyntaxElement outputBuffer = fields.add(new ArrayList<>()); + final ValueSyntaxElement outputBuffer = fields.add("outputBuffer", new ArrayList<>()); final Closure clear = Closure.wrap(); final Closure compute; if (parents.isEmpty()) { compute = filterBody(outputBuffer, BATCH_ARG, fields, plugin); } else { - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList() - ); + final Collection parentFields = createParentStatementsFields(parents, fields); + @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); clear.add(clearSyntax(parentFields)); - final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); + final ValueSyntaxElement inputBufferField = fields.add("inputBuffer", inputBuffer); compute = withInputBuffering( filterBody(outputBuffer, inputBufferField, fields, plugin), parentFields, inputBufferField @@ -133,6 +130,21 @@ public static ComputeStepSyntaxElement filterDataset( return prepare(withOutputBuffering(compute, clear, outputBuffer, fields)); } + private static Collection createParentStatementsFields(Collection parents, ClassFields fields) { + if (parents.size() == 1) { + return List.of(fields.add("parentStatement", parents.iterator().next())); + } + + final Collection parentFields = new ArrayList<>(); + int i = 0; + for (Dataset parent : parents) { + ValueSyntaxElement add = fields.add("parentStatement" + i, parent); + parentFields.add(add); + i++; + } + return parentFields; + } + /** *

Builds a terminal {@link Dataset} for the filters from the given parent {@link Dataset}s.

*

If the given set of parent {@link Dataset} is empty the sum is defined as the @@ -155,10 +167,7 @@ public static Dataset terminalFilterDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); final ValueSyntaxElement inputBufferField = fields.add(inputBuffer); final ValueSyntaxElement outputBufferField = fields.add(new ArrayList<>()); @@ -199,10 +208,7 @@ public static Dataset terminalOutputDataset(final Collection parents) { } final ClassFields fields = new ClassFields(); - final Collection parentFields = parents - .stream() - .map(fields::add) - .collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); final Closure compute = Closure.wrap(parentFields .stream() .map(DatasetCompiler::computeDataset) @@ -236,7 +242,7 @@ public static ComputeStepSyntaxElement outputDataset( final ClassFields fields = new ClassFields(); final Closure clearSyntax; final Closure computeSyntax; - final ValueSyntaxElement outputField = fields.add(output); + final ValueSyntaxElement outputField = fields.add("outputDelegator", output); if (parents.isEmpty()) { clearSyntax = Closure.EMPTY; computeSyntax = Closure.wrap( @@ -244,8 +250,7 @@ public static ComputeStepSyntaxElement outputDataset( invokeOutput(outputField, BATCH_ARG), unsetPluginIdForLog4j()); } else { - final Collection parentFields = - parents.stream().map(fields::add).collect(Collectors.toList()); + final Collection parentFields = createParentStatementsFields(parents, fields); @SuppressWarnings("rawtypes") final RubyArray buffer = RubyUtil.RUBY.newArray(); final Closure inlineClear; @@ -256,7 +261,7 @@ public static ComputeStepSyntaxElement outputDataset( inlineClear = Closure.EMPTY; clearSyntax = clearSyntax(parentFields); } - final ValueSyntaxElement inputBuffer = fields.add(buffer); + final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", buffer); computeSyntax = withInputBuffering( Closure.wrap( setPluginIdForLog4j(outputField), @@ -283,7 +288,7 @@ private static Closure filterBody( final ClassFields fields, final AbstractFilterDelegatorExt plugin) { - final ValueSyntaxElement filterField = fields.add(plugin); + final ValueSyntaxElement filterField = fields.add("plugin", plugin); final Closure body = Closure.wrap( setPluginIdForLog4j(filterField), buffer(outputBuffer, filterField.call("multiFilter", inputBuffer)) @@ -378,14 +383,14 @@ private static MethodLevelSyntaxElement callFilterFlush(final ClassFields fields final boolean shutdownOnly) { final MethodLevelSyntaxElement condition; final ValueSyntaxElement flushArgs; - final ValueSyntaxElement flushFinal = fields.add(flushOpts(true)); + final ValueSyntaxElement flushFinal = fields.add("shutdownFlushOptions", flushOpts(true)); if (shutdownOnly) { condition = SyntaxFactory.and(FLUSH_ARG, SHUTDOWN_ARG); flushArgs = flushFinal; } else { condition = FLUSH_ARG; flushArgs = SyntaxFactory.ternary( - SHUTDOWN_ARG, flushFinal, fields.add(flushOpts(false)) + SHUTDOWN_ARG, flushFinal, fields.add("flushOptions", flushOpts(false)) ); } return SyntaxFactory.ifCondition( diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java index b3e481e6fd4..4ed2ba3b20e 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/FieldDefinition.java @@ -46,6 +46,12 @@ public static FieldDefinition fromValue(final int index, final Object value) { ); } + public static FieldDefinition fromValue(final String fieldName, final Object value) { + return new FieldDefinition( + variableDefinition(value.getClass(), fieldName), false, null, value + ); + } + /** * Creates a mutable field with given type and without an assigned value. * @param index Index for naming @@ -58,6 +64,12 @@ public static FieldDefinition mutableUnassigned(final int index, final Class ); } + public static FieldDefinition mutableUnassigned(final String fieldName, final Class type) { + return new FieldDefinition( + variableDefinition(type, fieldName), true, null, null + ); + } + private FieldDefinition(final VariableDefinition typeDef, final boolean mutable, final SyntaxElement initializer, final Object ctorArgument) { this.def = typeDef; @@ -98,4 +110,9 @@ public String generateCode() { private static VariableDefinition variableDefinition(final Class type, final int index) { return new VariableDefinition(type, String.format("field%d", index)); } + + private static VariableDefinition variableDefinition(final Class type, final String fieldName) { + return new VariableDefinition(type, String.format("%sField", fieldName)); + } + } From b88e23702c100d411da8ce340d3c6db90dd6c0d9 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Thu, 5 Sep 2024 10:57:10 +0200 Subject: [PATCH 18/37] Implements safe evaluation of conditional expressions, logging the error without killing the pipeline (#16322) This PR protects the if statements against expression evaluation errors, cancel the event under processing and log it. This avoids to crash the pipeline which encounter a runtime error during event condition evaluation, permitting to debug the root cause reporting the offending event and removing from the current processing batch. Translates the `org.jruby.exceptions.TypeError`, `IllegalArgumentException`, `org.jruby.exceptions.ArgumentError` that could happen during `EventCodition` evaluation into a custom `ConditionalEvaluationError` which bubbles up on AST tree nodes. It's catched in the `SplitDataset` node. Updates the generation of the `SplitDataset `so that the execution of `filterEvents` method inside the compute body is try-catch guarded and defer the execution to an instance of `AbstractPipelineExt.ConditionalEvaluationListener` to handle such error. In this particular case the error management consist in just logging the offending Event. --------- Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/event-data.asciidoc | 6 + .../spec/logstash/java_pipeline_spec.rb | 139 ++++++++++++++++++ .../logstash/config/ir/CompiledPipeline.java | 23 ++- .../ir/compiler/ComputeStepSyntaxElement.java | 2 +- .../compiler/ConditionalEvaluationError.java | 20 +++ .../config/ir/compiler/DatasetCompiler.java | 57 +++++-- .../config/ir/compiler/EventCondition.java | 14 +- .../config/ir/compiler/SyntaxFactory.java | 8 + .../logstash/config/ir/compiler/Utils.java | 13 +- .../execution/AbstractPipelineExt.java | 35 ++++- .../ir/compiler/DatasetCompilerTest.java | 4 +- .../ir/cache/pipeline_reuse_baseline.conf | 4 +- 12 files changed, 299 insertions(+), 26 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java diff --git a/docs/static/event-data.asciidoc b/docs/static/event-data.asciidoc index 78713346f56..9952fc168aa 100644 --- a/docs/static/event-data.asciidoc +++ b/docs/static/event-data.asciidoc @@ -149,6 +149,12 @@ filter { } ---------------------------------- +If an expression generates an error when it is evaluated, event processing stops and a warning message is written to the log. +For example, comparing integer value `100` with string value `"100"` cannot be evaluated with certainty, and so processing stops and the error is logged. + +To capture the full content of the message at the time the error occurs, set the log level to `debug`. +Check out <> for more information about how to configure logging and available log levels. + You can specify multiple expressions in a single condition: [source,js] diff --git a/logstash-core/spec/logstash/java_pipeline_spec.rb b/logstash-core/spec/logstash/java_pipeline_spec.rb index 3e743967273..d30872e1430 100644 --- a/logstash-core/spec/logstash/java_pipeline_spec.rb +++ b/logstash-core/spec/logstash/java_pipeline_spec.rb @@ -377,6 +377,145 @@ def flush(options) end end + context "when logical expression in conditional" do + extend PipelineHelpers + + let(:settings) { LogStash::SETTINGS.clone } + + config <<-CONFIG + filter { + if [path][to][value] > 100 { + mutate { add_tag => "hit" } + } else { + mutate { add_tag => "miss" } + } + } + CONFIG + + context "raise an error when it's evaluated, should cancel the event execution and log the error" do + context "when type of evaluation doesn't have same type" do + sample_one( [{ "path" => {"to" => {"value" => "101"}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + context "when left and right operands of event condition are not comparable" do + context "comparing a non existing field" do + sample_one( [{ "path" => {"to" => "Rome"}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/:/) + end + end + + context "comparing incompatible types" do + sample_one( [{ "path" => {"to" => {"value" => [101, 102]}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/Unexpected conditional input combination of.*List.*RubyFixnum/) + end + end + end + + context "when the offending logic expression is used in a nested conditional structure" do + config <<-CONFIG + filter { + if "a" == "a" { + if "b" == "b" { + if [path][to][value] > 100 { + mutate { add_tag => "hit" } + } else { + mutate { add_tag => "miss" } + } + } + } + } + CONFIG + + sample_one( [{ "path" => {"to" => {"value" => "101"}}}] ) do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + context "when the offending condition is in the output section" do + before do + LogStash::PLUGIN_REGISTRY.add(:input, "spec_sampler_input", PipelineHelpers::SpecSamplerInput) + LogStash::PLUGIN_REGISTRY.add(:output, "spec_sampler_output", PipelineHelpers::SpecSamplerOutput) + end + + describe "given a pipeline executing an event that would trigger an evaluation error" do + let(:pipeline) do + settings.set_value("queue.drain", true) + LogStash::JavaPipeline.new( + org.logstash.config.ir.PipelineConfig.new( + LogStash::Config::Source::Local, :main, + SourceWithMetadata.new( + "config_string", "config_string", + "input { spec_sampler_input {} }\n output { if [path][to][value] > 100 { spec_sampler_output {} } }" + ), settings + ) + ) + end + let(:event) do + [LogStash::Event.new({ "path" => {"to" => {"value" => "101"}}})] + end + + let(:results) do + PipelineHelpers::SpecSamplerInput.set_event event + pipeline.run + PipelineHelpers::SpecSamplerOutput.seen + end + + after do + pipeline.close + end + + subject {results.length > 1 ? results : results.first} + + it "should raise an error without killing the pipeline" do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/no implicit conversion of nil into Integer/) + end + end + + describe "given a pipeline executing an event with invalid UTF-8 string" do + let(:pipeline) do + settings.set_value("queue.drain", true) + LogStash::JavaPipeline.new( + org.logstash.config.ir.PipelineConfig.new( + LogStash::Config::Source::Local, :main, + SourceWithMetadata.new( + "config_string", "config_string", + "input { spec_sampler_input {} }\n output { if [message] =~ /^(NOSQL|SQL):/ { spec_sampler_output {} } }" + ), settings + ) + ) + end + let(:event) do + [LogStash::Event.new({ "message" => "abrac\xC5adabra"})] + end + + let(:results) do + PipelineHelpers::SpecSamplerInput.set_event event + pipeline.run + PipelineHelpers::SpecSamplerOutput.seen + end + + after do + pipeline.close + end + + subject {results.length > 1 ? results : results.first} + + it "should raise an error without killing the pipeline" do + expect(subject).to be nil + expect(pipeline.last_error_evaluation_received).to match(/invalid byte sequence in UTF-8/) + end + end + end + end + end + context "a crashing worker terminates the pipeline and all inputs and workers" do subject { mock_java_pipeline_from_string(config, pipeline_settings_obj) } let(:config) do diff --git a/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java b/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java index 7bb7b71a256..e851e0fa899 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/CompiledPipeline.java @@ -29,6 +29,7 @@ import org.logstash.config.ir.compiler.AbstractFilterDelegatorExt; import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt; import org.logstash.config.ir.compiler.ComputeStepSyntaxElement; +import org.logstash.config.ir.compiler.ConditionalEvaluationError; import org.logstash.config.ir.compiler.Dataset; import org.logstash.config.ir.compiler.DatasetCompiler; import org.logstash.config.ir.compiler.EventCondition; @@ -39,6 +40,7 @@ import org.logstash.config.ir.graph.PluginVertex; import org.logstash.config.ir.graph.Vertex; import org.logstash.config.ir.imperative.PluginStatement; +import org.logstash.execution.AbstractPipelineExt; import org.logstash.execution.QueueBatch; import org.logstash.ext.JrubyEventExtLibrary.RubyEvent; import org.logstash.plugins.ConfigVariableExpander; @@ -96,20 +98,35 @@ public final class CompiledPipeline { */ private final RubyIntegration.PluginFactory pluginFactory; + /** + * Callback listener to manage failed condition evaluation + * */ + private final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener; + + public static final class NoopEvaluationListener implements AbstractPipelineExt.ConditionalEvaluationListener { + + @Override + public void notify(ConditionalEvaluationError err) { + // NO-OP + } + } + public CompiledPipeline( final PipelineIR pipelineIR, final RubyIntegration.PluginFactory pluginFactory) { - this(pipelineIR, pluginFactory, null); + this(pipelineIR, pluginFactory, null, new NoopEvaluationListener()); } public CompiledPipeline( final PipelineIR pipelineIR, final RubyIntegration.PluginFactory pluginFactory, - final SecretStore secretStore) + final SecretStore secretStore, + final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener) { this.pipelineIR = pipelineIR; this.pluginFactory = pluginFactory; + this.conditionalErrListener = conditionalErrListener; try (ConfigVariableExpander cve = new ConfigVariableExpander( secretStore, EnvironmentVariableProvider.defaultProvider())) { @@ -488,7 +505,7 @@ private SplitDataset split( // by requiring its else branch. if (conditional == null) { final ComputeStepSyntaxElement prepared = - DatasetCompiler.splitDataset(dependencies, condition); + DatasetCompiler.splitDataset(dependencies, condition, conditionalErrListener); conditional = prepared.instantiate(); iffs.put(vertexId, conditional); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java index 42541f2a56a..eb7e131616b 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ComputeStepSyntaxElement.java @@ -207,7 +207,7 @@ private static Path debugDir() { } /** - * @return Array of constructor arguments + * @return Map of constructor arguments, key is field's name, value is the instance of the parameter to pass. */ private Map ctorArguments() { final Map result = new HashMap<>(); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java new file mode 100644 index 00000000000..2df6681b04d --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java @@ -0,0 +1,20 @@ +package org.logstash.config.ir.compiler; + +import org.logstash.Event; + +/** + * Exception raised when an if-condition in a pipeline throws an error at runtime. + * */ +public class ConditionalEvaluationError extends RuntimeException { + private static final long serialVersionUID = -8633589068902565868L; + private final Event failedEvent; + + ConditionalEvaluationError(Throwable cause, Event failedEvent) { + super(cause); + this.failedEvent = failedEvent; + } + + public Event failedEvent() { + return failedEvent; + } +} diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java index e3db9a0105d..956bbe18f1b 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/DatasetCompiler.java @@ -32,6 +32,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.execution.AbstractPipelineExt; import org.logstash.ext.JrubyEventExtLibrary; /** @@ -56,7 +57,8 @@ private DatasetCompiler() { public static ComputeStepSyntaxElement splitDataset( final Collection parents, - final EventCondition condition) + final EventCondition condition, + final AbstractPipelineExt.ConditionalEvaluationListener conditionalErrListener) { final ClassFields fields = new ClassFields(); final ValueSyntaxElement ifData = fields.add("ifData", new ArrayList<>()); @@ -78,20 +80,36 @@ public static ComputeStepSyntaxElement splitDataset( ); final ValueSyntaxElement conditionField = fields.add("condition", condition); final DatasetCompiler.ComputeAndClear compute; + + final ValueSyntaxElement errorNotifier = fields.add(conditionalErrListener); + Closure exceptionHandlerBlock = Closure.wrap( + new SyntaxFactory.MethodCallReturnValue(SyntaxFactory.value("this"), "setDone"), + errorNotifier.call("notify", SyntaxFactory.value("ex")), + SyntaxFactory.ret(ifData)); + ValueSyntaxElement exception = SyntaxFactory.value("ex"); + if (parents.isEmpty()) { + MethodLevelSyntaxElement safeLoop = SyntaxFactory.tryBlock( + conditionalLoop(event, BATCH_ARG, conditionField, ifData, elseData), + ConditionalEvaluationError.class, exceptionHandlerBlock, exception + ); compute = withOutputBuffering( - conditionalLoop(event, BATCH_ARG, conditionField, ifData, elseData), + Closure.wrap(safeLoop), Closure.wrap(clear(elseData)), ifData, fields ); } else { - final Collection parentFields = createParentStatementsFields(parents, fields); + final Collection parentFields = + parents.stream().map(fields::add).collect(Collectors.toList()); final ValueSyntaxElement inputBuffer = fields.add("inputBuffer", new ArrayList<>()); + MethodLevelSyntaxElement safeLoop = SyntaxFactory.tryBlock( + conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), + ConditionalEvaluationError.class, exceptionHandlerBlock, exception); compute = withOutputBuffering( withInputBuffering( - conditionalLoop(event, inputBuffer, conditionField, ifData, elseData), + Closure.wrap(safeLoop), parentFields, inputBuffer ), - clearSyntax(parentFields).add(clear(elseData)), ifData, fields + clearSyntax(parentFields).add(clear(elseData)).add(clear(inputBuffer)), ifData, fields ); } return ComputeStepSyntaxElement.create( @@ -122,6 +140,7 @@ public static ComputeStepSyntaxElement filterDataset( @SuppressWarnings("rawtypes") final RubyArray inputBuffer = RubyUtil.RUBY.newArray(); clear.add(clearSyntax(parentFields)); final ValueSyntaxElement inputBufferField = fields.add("inputBuffer", inputBuffer); + compute = withInputBuffering( filterBody(outputBuffer, inputBufferField, fields, plugin), parentFields, inputBufferField @@ -145,6 +164,10 @@ private static Collection createParentStatementsFields(Colle return parentFields; } + private static boolean isContainedUnderIfStatement(Collection parents) { + return parents.size() == 1 && parents.iterator().next() instanceof SplitDataset; + } + /** *

Builds a terminal {@link Dataset} for the filters from the given parent {@link Dataset}s.

*

If the given set of parent {@link Dataset} is empty the sum is defined as the @@ -339,11 +362,15 @@ private static ComputeStepSyntaxElement prepare(final DatasetCompiler.C */ private static Closure withInputBuffering(final Closure compute, final Collection parents, final ValueSyntaxElement inputBuffer) { - return Closure.wrap( - parents.stream().map(par -> SyntaxFactory.value("org.logstash.config.ir.compiler.Utils") - .call("copyNonCancelledEvents", computeDataset(par), inputBuffer) - ).toArray(MethodLevelSyntaxElement[]::new) - ).add(compute).add(clear(inputBuffer)); + return computeNonCancelledEventsBlock(parents, inputBuffer) + .add(compute) + .add(clear(inputBuffer)); + } + + private static Closure computeNonCancelledEventsBlock(Collection parents, ValueSyntaxElement inputBuffer) { + return Closure.wrap(parents.stream().map(par -> SyntaxFactory.value("org.logstash.config.ir.compiler.Utils") + .call("copyNonCancelledEvents", computeDataset(par), inputBuffer) + ).toArray(MethodLevelSyntaxElement[]::new)); } /** @@ -495,9 +522,13 @@ public Collection compute(@SuppressWarnings("raw if (done) { return data; } - parent.compute(batch, flush, shutdown); - done = true; - return data; + try { + parent.compute(batch, flush, shutdown); + done = true; + return data; + } catch (ConditionalEvaluationError ex) { + return data; + } } @Override diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java index a68ec8a9888..62e3a3e69b5 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/EventCondition.java @@ -685,19 +685,19 @@ private static final class UnexpectedTypeException extends IllegalArgumentExcept UnexpectedTypeException(final Expression left, final Expression right) { super( String.format( - "Unexpected input types left: %s, right: %s", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) + "Unexpected conditional input combination of '%s' (left) and '%s' (right)", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) ) ); } UnexpectedTypeException(final Object inner) { - super(String.format("Unexpected input type %s", getUnexpectedTypeDetails(inner))); + super(String.format("Unexpected conditional input type '%s'", getUnexpectedTypeDetails(inner))); } UnexpectedTypeException(final Object left, final Object right) { super( String.format( - "Unexpected input type combination left %s, right %s", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) + "Unexpected conditional input combination of '%s' (left) and '%s' (right)", getUnexpectedTypeDetails(left), getUnexpectedTypeDetails(right) ) ); } @@ -709,9 +709,13 @@ private static String getUnexpectedTypeDetails(Object unexpected) { details = (expression.getSourceWithMetadata() != null) ? expression.getSourceWithMetadata().toString() : expression.toString(); } else { - details = unexpected.toString(); + if (unexpected == null) { + details = ""; + } else { + details = unexpected.toString(); + } } - return String.format("%s:%s", unexpected.getClass(), details); + return String.format("%s:%s", unexpected == null ? "" : unexpected.getClass(), details); } } } diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java index f4076560283..c8134719470 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/SyntaxFactory.java @@ -88,6 +88,14 @@ public static MethodLevelSyntaxElement forLoop(final VariableDefinition element, ); } + public static MethodLevelSyntaxElement tryBlock(final Closure body, final Class exceptionToCatch, Closure exceptionHandlerBlock, + ValueSyntaxElement exception) { + return () -> SyntaxFactory.join( + "try {\n", body.generateCode(), + "\n} catch (" + exceptionToCatch.getCanonicalName() + " " + exception.generateCode() + ") { " + exceptionHandlerBlock.generateCode() + " }" + ); + } + public static MethodLevelSyntaxElement ifCondition(final MethodLevelSyntaxElement condition, final Closure body) { return ifCondition(condition, body, Closure.EMPTY); diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java index 3bf64beb058..cb6341058d8 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/Utils.java @@ -20,6 +20,8 @@ package org.logstash.config.ir.compiler; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.logstash.ext.JrubyEventExtLibrary; import java.util.Collection; @@ -29,6 +31,7 @@ * Static utility methods that replace common blocks of generated code in the Java execution. */ public class Utils { + private static final Logger logger = LogManager.getLogger(Utils.class); @SuppressWarnings({"unchecked", "rawtypes"}) // has field1.compute(batchArg, flushArg, shutdownArg) passed as input @@ -44,7 +47,15 @@ public static void copyNonCancelledEvents(Collection input, EventCondition filter, List fulfilled, List unfulfilled) { for (JrubyEventExtLibrary.RubyEvent e : input) { - if (filter.fulfilled(e)) { + boolean isFulfilled; + try { + isFulfilled = filter.fulfilled(e); + } catch (org.jruby.exceptions.TypeError | IllegalArgumentException | org.jruby.exceptions.ArgumentError ex) { + // in case of error evaluation of an if condition, cancel the event + e.getEvent().cancel(); + throw new ConditionalEvaluationError(ex, e.getEvent()); + } + if (isFulfilled) { fulfilled.add(e); } else { unfulfilled.add(e); diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index 82cd216e64e..0ac584f1595 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -21,6 +21,8 @@ package org.logstash.execution; import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; @@ -76,6 +78,7 @@ import org.logstash.config.ir.PipelineIR; import org.logstash.config.ir.compiler.AbstractFilterDelegatorExt; import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt; +import org.logstash.config.ir.compiler.ConditionalEvaluationError; import org.logstash.execution.queue.QueueWriter; import org.logstash.ext.JRubyAbstractQueueWriteClientExt; import org.logstash.ext.JRubyWrappedWriteClientExt; @@ -163,10 +166,33 @@ public class AbstractPipelineExt extends RubyBasicObject { private @SuppressWarnings("rawtypes") RubyArray filters; private @SuppressWarnings("rawtypes") RubyArray outputs; + private String lastErrorEvaluationReceived = ""; + public AbstractPipelineExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + public interface ConditionalEvaluationListener { + void notify(ConditionalEvaluationError err); + } + + public final class LogErrorEvaluationListener implements ConditionalEvaluationListener { + @Override + public void notify(ConditionalEvaluationError err) { + lastErrorEvaluationReceived = err.getCause().getMessage(); + LOGGER.warn("{}. Event was dropped, enable debug logging to see the event's payload.", lastErrorEvaluationReceived); + LOGGER.debug("Event generating the fault: {}", err.failedEvent().toMap().toString()); + + // logs the exception at debug level + try (StringWriter sw = new StringWriter(); PrintWriter pw = new PrintWriter(sw)) { + err.printStackTrace(pw); + LOGGER.debug("{}", sw); + } catch (IOException ioex) { + LOGGER.warn("Invalid operation on closing internal resources", ioex); + } + } + } + @JRubyMethod(required = 4) public AbstractPipelineExt initialize(final ThreadContext context, final IRubyObject[] args) throws IncompleteSourceWithMetadataException, NoSuchAlgorithmException { @@ -183,7 +209,8 @@ public AbstractPipelineExt initialize(final ThreadContext context, final IRubyOb ).initialize(context, args[3], this, dlqWriter(context)), RubyUtil.FILTER_DELEGATOR_CLASS ), - getSecretStore(context) + getSecretStore(context), + new LogErrorEvaluationListener() ); inputs = RubyArray.newArray(context.runtime, lirExecution.inputs()); filters = RubyArray.newArray(context.runtime, lirExecution.filters()); @@ -828,4 +855,10 @@ public IRubyObject isShutdownRequested(final ThreadContext context) { // shutdown_requested? MUST be implemented in the concrete implementation of this class. throw new IllegalStateException("Pipeline implementation does not provide `shutdown_requested?`, which is a Logstash internal error."); } + + @VisibleForTesting + @JRubyMethod(name = "last_error_evaluation_received") + public final RubyString getLastErrorEvaluationReceived(final ThreadContext context) { + return RubyString.newString(context.runtime, lastErrorEvaluationReceived); + } } diff --git a/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java b/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java index 4881035a090..c38f4e29e81 100644 --- a/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java +++ b/logstash-core/src/test/java/org/logstash/config/ir/compiler/DatasetCompilerTest.java @@ -26,6 +26,7 @@ import org.logstash.Event; import org.logstash.FieldReference; import org.logstash.RubyUtil; +import org.logstash.config.ir.CompiledPipeline; import org.logstash.config.ir.PipelineTestUtil; import org.logstash.ext.JrubyEventExtLibrary; @@ -54,7 +55,8 @@ public void compilesOutputDataset() { public void compilesSplitDataset() { final FieldReference key = FieldReference.from("foo"); final SplitDataset left = DatasetCompiler.splitDataset( - Collections.emptyList(), event -> event.getEvent().includes(key) + Collections.emptyList(), event -> event.getEvent().includes(key), + new CompiledPipeline.NoopEvaluationListener() ).instantiate(); final Event trueEvent = new Event(); trueEvent.setField(key, "val"); diff --git a/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf b/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf index 4aa09f7dcf8..a612588e9dd 100644 --- a/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf +++ b/logstash-core/src/test/resources/org/logstash/config/ir/cache/pipeline_reuse_baseline.conf @@ -17,6 +17,7 @@ filter { if [a] { if [a] { if [a] { + noop {} noop {} if [a] { noop {} } } @@ -25,7 +26,8 @@ filter { } output { if [a] { noop {} } + else { noop {} } stdout {} stdout {} - stdout {} +# stdout {} } From 285d13a5156f0b9ec66796854cc59de1cecec682 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:10:52 +0100 Subject: [PATCH 19/37] Update logstash_releases.json (#16426) --- ci/logstash_releases.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 39a6ae1a2ce..8eae04e6bba 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -3,11 +3,11 @@ "5.x": "5.6.16", "6.x": "6.8.23", "7.x": "7.17.23", - "8.x": "8.14.3" + "8.x": "8.15.1" }, "snapshots": { "7.x": "7.17.24-SNAPSHOT", - "8.x": "8.14.4-SNAPSHOT", - "main": "8.15.0-SNAPSHOT" + "8.x": "8.15.2-SNAPSHOT", + "main": "8.16.0-SNAPSHOT" } } From 58b6a0ac77432df1d3cc0ab4bcd0f29a54f5adbf Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:12:58 +0100 Subject: [PATCH 20/37] Release notes for 8.15.1 (#16405) (#16427) * Update release notes for 8.15.1 * update release note --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: Kaise Cheng (cherry picked from commit 2fca7e39e87c20fcfcd934e984720173ce3417e8) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/static/releasenotes.asciidoc | 36 ++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index dc4a80c8383..fa8b56b79a5 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -65,6 +66,39 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-15-1]] +=== Logstash 8.15.1 Release Notes + +[[notable-8.15.1]] +==== Performance improvements and notable issues fixed + +* Fixed issue where `${VAR}` incorrectly treated array as a string https://github.com/elastic/logstash/pull/16375[#16375] + +[[core-8.15.1]] +==== Other changes to Logstash core + +* Emit warning message for java version less than 17. Starting from Logstash 9.0, the minimum required version of Java will be Java 17 https://github.com/elastic/logstash/pull/16370[#16370] + +[[plugins-8.15.1]] +==== Plugin releases + +*Elasticsearch Input - 4.20.4* + +* Fixed issue where the `index` parameter was being ignored when using `response_type => aggregations` https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/209[#209] + +*Redis Input - 3.7.1* + +* [Doc] Added `threads` option https://github.com/logstash-plugins/logstash-input-redis/pull/95[#95] + +*Aws Integration - 7.1.8* + +* Added support to all storage classes for s3 output https://github.com/logstash-plugins/logstash-integration-aws/pull/49[#49] + +*Kafka Integration - 11.5.1* + +* [Doc] Added only headers with UTF-8 encoded values are supported in `decorate_events` https://github.com/logstash-plugins/logstash-integration-kafka/pull/174[#174] + + [[logstash-8-15-0]] === Logstash 8.15.0 Release Notes @@ -2441,4 +2475,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 +Plugins align with release 7.14.0 \ No newline at end of file From bb7ecc203f698a56f341fa538bdc1cd4da15b28c Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Fri, 6 Sep 2024 12:09:58 +0200 Subject: [PATCH 21/37] Fix ConditionalEvaluationError to do not include the event that errored in its serialiaxed form, because it's not expected that this class is ever serialized. (#16429) Make inner field of ConditionalEvaluationError transient to be avoided during serialization. --- .../config/ir/compiler/ConditionalEvaluationError.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java index 2df6681b04d..0dd464fc255 100644 --- a/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java +++ b/logstash-core/src/main/java/org/logstash/config/ir/compiler/ConditionalEvaluationError.java @@ -7,7 +7,10 @@ * */ public class ConditionalEvaluationError extends RuntimeException { private static final long serialVersionUID = -8633589068902565868L; - private final Event failedEvent; + + // This class is serializable because of inheritance from Throwable, however it's not expected + // to be ever transmitted on wire on stored in some binary storage. + private final transient Event failedEvent; ConditionalEvaluationError(Throwable cause, Event failedEvent) { super(cause); From 69f0fa54ca07cb3f822846745fdbdd1504175cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Mon, 9 Sep 2024 11:33:44 +0100 Subject: [PATCH 22/37] use gnu tar compatible minitar to generate tar artifact (#16432) Using VERSION_QUALIFIER when building the tarball distribution will fail since Ruby's TarWriter implements the older POSIX88 version of tar and paths will be longer than 100 characters. For the long paths being used in Logstash's plugins, mainly due to nested folders from jar-dependencies, we need the tarball to follow either the 2001 ustar format or gnu tar, which is implemented by the minitar gem. --- Gemfile.template | 1 + rakelib/artifacts.rake | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile.template b/Gemfile.template index 9989269fb72..66eea71b5f2 100644 --- a/Gemfile.template +++ b/Gemfile.template @@ -14,6 +14,7 @@ gem "logstash-output-elasticsearch", ">= 11.14.0" gem "polyglot", require: false gem "treetop", require: false gem "faraday", "~> 1", :require => false # due elasticsearch-transport (elastic-transport) depending faraday '~> 1' +gem "minitar", :group => :build gem "childprocess", "~> 4", :group => :build gem "fpm", "~> 1", ">= 1.14.1", :group => :build # compound due to bugfix https://github.com/jordansissel/fpm/pull/1856 gem "gems", "~> 1", :group => :build diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 7d5d7e4ea92..7e0c553a520 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -492,6 +492,7 @@ namespace "artifact" do require "zlib" require 'rubygems' require 'rubygems/package' + require 'minitar' ensure_logstash_version_constant_defined tarpath = "build/logstash#{tar_suffix}-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}#{platform}.tar.gz" if File.exist?(tarpath) && ENV['SKIP_PREPARE'] == "1" && !source_modified_since?(File.mtime(tarpath)) @@ -500,7 +501,7 @@ namespace "artifact" do end puts("[artifact:tar] building #{tarpath}") gz = Zlib::GzipWriter.new(File.new(tarpath, "wb"), Zlib::BEST_COMPRESSION) - Gem::Package::TarWriter.new(gz) do |tar| + Minitar::Writer.open(gz) do |tar| files(exclude_paths).each do |path| write_to_tar(tar, path, "logstash-#{LOGSTASH_VERSION}#{PACKAGE_SUFFIX}/#{path}") end @@ -520,11 +521,11 @@ namespace "artifact" do def write_to_tar(tar, path, path_in_tar) stat = File.lstat(path) if stat.directory? - tar.mkdir(path_in_tar, stat.mode) + tar.mkdir(path_in_tar, :mode => stat.mode) elsif stat.symlink? - tar.add_symlink(path_in_tar, File.readlink(path), stat.mode) + tar.symlink(path_in_tar, File.readlink(path), :mode => stat.mode) else - tar.add_file_simple(path_in_tar, stat.mode, stat.size) do |io| + tar.add_file_simple(path_in_tar, :mode => stat.mode, :size => stat.size) do |io| File.open(path, 'rb') do |fd| chunk = nil size = 0 From f60e98717308ba610452cc74daf9dfdbbb93eddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Mon, 9 Sep 2024 13:46:00 +0100 Subject: [PATCH 23/37] bump to 9.0.0 and adapt CI accordingly (#16428) --- ci/branches.json | 3 +++ ci/logstash_releases.json | 5 +++-- versions.yml | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/branches.json b/ci/branches.json index 6524ee2be1a..470e36cc2c3 100644 --- a/ci/branches.json +++ b/ci/branches.json @@ -4,6 +4,9 @@ { "branch": "main" }, + { + "branch": "8.x" + }, { "branch": "8.15" }, diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 8eae04e6bba..2dabec5daf1 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -7,7 +7,8 @@ }, "snapshots": { "7.x": "7.17.24-SNAPSHOT", - "8.x": "8.15.2-SNAPSHOT", - "main": "8.16.0-SNAPSHOT" + "8.15": "8.15.2-SNAPSHOT", + "8.x": "8.16.0-SNAPSHOT", + "main": "9.0.0-SNAPSHOT" } } diff --git a/versions.yml b/versions.yml index 4626e47ca9b..f8249cb1965 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 8.16.0 -logstash-core: 8.16.0 +logstash: 9.0.0 +logstash-core: 9.0.0 logstash-core-plugin-api: 2.1.16 bundled_jdk: From 17dba9f829a2514aba295ed7a8fa21655b55c86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Tue, 10 Sep 2024 10:55:34 +0100 Subject: [PATCH 24/37] account for the 8.x in DRA publishing task (#16436) the current DRA publishing task computes the branch from the version contained in the version.yml This is done by taking the major.minor and confirming that a branch exists with that name. However this pattern won't be applicable for 8.x, as that branch currently points to 8.16.0 and there is no 8.16 branch. This commit falls back to reading the buildkite injected BUILDKITE_BRANCH variable. --- .buildkite/scripts/dra/publish.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh index 598da84bbb7..67c6ce895d9 100755 --- a/.buildkite/scripts/dra/publish.sh +++ b/.buildkite/scripts/dra/publish.sh @@ -15,8 +15,9 @@ RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\. if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then RELEASE_BRANCH=$RELEASE_VER else - RELEASE_BRANCH=main + RELEASE_BRANCH="${BUILDKITE_BRANCH:="main"}" fi +echo "RELEASE BRANCH: $RELEASE_BRANCH" if [ -n "$VERSION_QUALIFIER_OPT" ]; then # Qualifier is passed from CI as optional field and specify the version postfix From 701108f88b3c16a08fb501a71d812b804a79fe68 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:20:50 +0100 Subject: [PATCH 25/37] update ci release 7.17.24 (#16439) --- ci/logstash_releases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 2dabec5daf1..a5aff98f558 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -2,11 +2,11 @@ "releases": { "5.x": "5.6.16", "6.x": "6.8.23", - "7.x": "7.17.23", + "7.x": "7.17.24", "8.x": "8.15.1" }, "snapshots": { - "7.x": "7.17.24-SNAPSHOT", + "7.x": "7.17.25-SNAPSHOT", "8.15": "8.15.2-SNAPSHOT", "8.x": "8.16.0-SNAPSHOT", "main": "9.0.0-SNAPSHOT" From 5195332bc6a758198cae70fea7d88dfddf0fa15a Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:14:14 -0700 Subject: [PATCH 26/37] Logstash Health Report Tests Buildkite pipeline setup. (#16416) --- .buildkite/health_report_tests_pipeline.yml | 17 ++++++ .../scripts/health-report-tests/main.sh | 9 ++++ catalog-info.yaml | 54 +++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 .buildkite/health_report_tests_pipeline.yml create mode 100644 .buildkite/scripts/health-report-tests/main.sh diff --git a/.buildkite/health_report_tests_pipeline.yml b/.buildkite/health_report_tests_pipeline.yml new file mode 100644 index 00000000000..84d9bf9d4b1 --- /dev/null +++ b/.buildkite/health_report_tests_pipeline.yml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" + cpu: "2" + memory: "4Gi" + ephemeralStorage: "64Gi" + +steps: + # todo: add a step for 8.latest when Health Report MVP is landed + - label: ":logstash: Health Report Integration tests on main branch." + command: | + source .buildkite/scripts/common/vm-agent.sh + ./.buildkite/scripts/health-report-tests/main.sh + retry: + automatic: + - limit: 3 diff --git a/.buildkite/scripts/health-report-tests/main.sh b/.buildkite/scripts/health-report-tests/main.sh new file mode 100644 index 00000000000..c31a4b120e3 --- /dev/null +++ b/.buildkite/scripts/health-report-tests/main.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +# TODO: +# if branch is specified with X.Y, pull branches from ACTIVE_BRANCHES_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/branches.json", parse and use +# build Logstash from specificed (ex: 8.x -> translates to 8.latest, 8.16) branch, defaults to main +# install requirements of the python package and run main.py + + diff --git a/catalog-info.yaml b/catalog-info.yaml index 54a397fd90b..b5330636ab1 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -32,6 +32,7 @@ spec: - resource:logstash-linux-jdk-matrix-pipeline - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline + - resource:logstash-health-report-tests-pipeline # *********************************** # Declare serverless IT pipeline @@ -642,4 +643,57 @@ spec: # ******************************* # SECTION END: Benchmark pipeline +# ******************************* + +# *********************************** +# Declare Health Report Tests pipeline +# *********************************** +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-health-report-tests-pipeline + description: Buildkite pipeline for the Logstash Health Report Tests + links: + - title: ':logstash Logstash Health Report Tests (Daily, Auto) pipeline' + url: https://buildkite.com/elastic/logstash-health-report-tests-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-health-report-tests-pipeline + description: ':logstash: Logstash Health Report tests :pipeline:' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/health_report_tests_pipeline.yml" + maximum_timeout_in_minutes: 60 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + schedules: + Daily Health Report tests on main branch: + branch: main + cronline: 30 20 * * * + message: Daily trigger of Health Report Tests Pipeline + +# ******************************* +# SECTION END: Health Report Tests pipeline # ******************************* \ No newline at end of file From 5452cccf7658554f79a7f70c7e04c9ab9a830bac Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Thu, 12 Sep 2024 19:45:23 +0100 Subject: [PATCH 27/37] [CI] benchmark dashboard and pipeline for testing against multiple versions (#16421) - add becnhmark dashboard and related save objects - add one buildkite pipeline to test against multiple versions - remove null field in json - add `FLOG_FILE_CNT`, `VAULT_PATH`, `TAGS` --- .buildkite/benchmark_versions_pipeline.yml | 11 ++ .buildkite/scripts/benchmark/README.md | 21 ++ .buildkite/scripts/benchmark/all.sh | 18 ++ .buildkite/scripts/benchmark/config/uuid | 1 + .buildkite/scripts/benchmark/main.sh | 71 +++++-- .../scripts/benchmark/save-objects/README.md | 14 ++ .../save-objects/benchmark_objects.ndjson | 31 +++ .buildkite/scripts/benchmark/setup/alias | 6 + .../benchmark/setup/benchmark_summary_v2 | 179 ++++++++++++++++++ .buildkite/scripts/benchmark/util.sh | 9 + catalog-info.yaml | 50 ++++- 11 files changed, 397 insertions(+), 14 deletions(-) create mode 100644 .buildkite/benchmark_versions_pipeline.yml create mode 100644 .buildkite/scripts/benchmark/README.md create mode 100755 .buildkite/scripts/benchmark/all.sh create mode 100644 .buildkite/scripts/benchmark/config/uuid create mode 100644 .buildkite/scripts/benchmark/save-objects/README.md create mode 100644 .buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson create mode 100644 .buildkite/scripts/benchmark/setup/alias create mode 100644 .buildkite/scripts/benchmark/setup/benchmark_summary_v2 diff --git a/.buildkite/benchmark_versions_pipeline.yml b/.buildkite/benchmark_versions_pipeline.yml new file mode 100644 index 00000000000..d60166ac441 --- /dev/null +++ b/.buildkite/benchmark_versions_pipeline.yml @@ -0,0 +1,11 @@ +agents: + provider: gcp + imageProject: elastic-images-prod + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 100 + diskType: pd-ssd + +steps: + - label: "Benchmark multiple versions" + command: .buildkite/scripts/benchmark/all.sh \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/README.md b/.buildkite/scripts/benchmark/README.md new file mode 100644 index 00000000000..330e7b266b6 --- /dev/null +++ b/.buildkite/scripts/benchmark/README.md @@ -0,0 +1,21 @@ +## Steps to set up GCP instance to run benchmark script +- Create an instance "n2-standard-16" with Ubuntu image +- Install docker + - `sudo snap install docker` + - `sudo usermod -a -G docker $USER` +- Install jq +- Install vault + - `sudo snap install vault` + - `vault login --method github` + - `vault kv get -format json secret/ci/elastic-logstash/benchmark` +- Setup Elasticsearch index mapping and alias with `setup/*` +- Import Kibana dashboard with `save-objects/*` +- Run the benchmark script + - Send data to your own Elasticsearch. Customise `VAULT_PATH="secret/ci/elastic-logstash/your/path"` + - Run the script `main.sh` + - or run in background `nohup bash -x main.sh > log.log 2>&1 &` + +## Notes +- Benchmarks should only be compared using the same hardware setup. +- Please do not send the test metrics to the benchmark cluster. You can set `VAULT_PATH` to send data and metrics to your own server. +- Run `all.sh` as calibration which gives you a baseline of performance in different versions. \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/all.sh b/.buildkite/scripts/benchmark/all.sh new file mode 100755 index 00000000000..282ed3d488a --- /dev/null +++ b/.buildkite/scripts/benchmark/all.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -eo pipefail + +# ******************************************************* +# Run benchmark for versions that have flow metrics +# Usage: +# nohup bash -x all.sh > log.log 2>&1 & +# Accept env vars: +# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string +# ******************************************************* + +IFS=',' +STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" +read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" + +for V in "${STACK_VERSIONS[@]}" ; do + LS_VERSION="$V" "main.sh" +done \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/config/uuid b/.buildkite/scripts/benchmark/config/uuid new file mode 100644 index 00000000000..b6ac241f753 --- /dev/null +++ b/.buildkite/scripts/benchmark/config/uuid @@ -0,0 +1 @@ +f74f1a28-25e9-494f-ba41-ca9f13d4446d \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/main.sh b/.buildkite/scripts/benchmark/main.sh index 9d218d772d4..9534ce3fc9a 100755 --- a/.buildkite/scripts/benchmark/main.sh +++ b/.buildkite/scripts/benchmark/main.sh @@ -36,6 +36,9 @@ source "$SCRIPT_PATH/util.sh" ## MEM=4 # number of GB for Logstash container ## QTYPE=memory # queue type to test {persisted|memory|all} ## FB_CNT=4 # number of filebeats to use in benchmark +## FLOG_FILE_CNT=4 # number of files to generate for ingestion +## VAULT_PATH=secret/path # vault path point to Elasticsearch credentials. The default value points to benchmark cluster. +## TAGS=test,other # tags with "," separator. usage() { echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" echo "Example: $0 4 {persisted|memory|all} 2 2" @@ -82,13 +85,17 @@ parse_args() { read -ra MULTIPLIERS <<< "$MULTIPLIERS" BATCH_SIZES="${BATCH_SIZES:-500}" read -ra BATCH_SIZES <<< "$BATCH_SIZES" + # tags to json array + read -ra TAG_ARRAY <<< "$TAGS" + JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') + JSON_TAGS="[$JSON_TAGS]" IFS=' ' echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" } get_secret() { - VAULT_PATH=secret/ci/elastic-logstash/benchmark + VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} VAULT_DATA=$(vault kv get -format json $VAULT_PATH) BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') @@ -102,10 +109,11 @@ get_secret() { pull_images() { echo "--- Pull docker images" - # pull the latest snapshot logstash image if [[ -n "$LS_VERSION" ]]; then - docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" + # pull image if it doesn't exist in local + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" else + # pull the latest snapshot logstash image # select the SNAPSHOT artifact with the highest semantic version number LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') @@ -125,12 +133,15 @@ pull_images() { } generate_logs() { + FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} + SINGLE_SIZE=524288000 + TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" FLOG_PATH="$SCRIPT_PATH/flog" mkdir -p $FLOG_PATH - if [[ ! -e "$FLOG_PATH/log4.log" ]]; then - echo "--- Generate logs in background. log: 5, size: 500mb" - docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b 2621440000 -p 524288000 + if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then + echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" + docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE fi } @@ -138,7 +149,7 @@ check_logs() { echo "--- Check log generation" local cnt=0 - until [[ -e "$FLOG_PATH/log4.log" || $cnt -gt 600 ]]; do + until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do echo "wait 30s" && sleep 30 cnt=$((cnt + 30)) done @@ -152,6 +163,7 @@ start_logstash() { cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml + cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ @@ -160,6 +172,7 @@ start_logstash() { -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ + -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ docker.elastic.co/logstash/logstash:$LS_VERSION } @@ -211,13 +224,39 @@ aggregate_stats() { send_summary() { echo "--- Send summary to Elasticsearch" + # build json + local timestamp timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") + SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" + not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" + not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" + not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" + not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" + not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" + not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" + not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" + not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" + not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" + not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" + not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" + not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" + SUMMARY="$SUMMARY}" + tee summary.json << EOF {"index": {}} -{"timestamp": "$timestamp", "version": "$LS_VERSION", "cpu": "$CPU", "mem": "$MEM", "workers": "$WORKER", "batch_size": "$BATCH_SIZE", "queue_type": "$QTYPE", "total_events_out": "$TOTAL_EVENTS_OUT", "max_eps_1m": "$MAX_EPS_1M", "max_eps_5m": "$MAX_EPS_5M", "max_worker_utilization": "$MAX_WORKER_UTIL", "max_worker_concurrency": "$MAX_WORKER_CONCURR", "avg_cpu_percentage": "$AVG_CPU_PERCENT", "avg_heap": "$AVG_HEAP", "avg_non_heap": "$AVG_NON_HEAP", "avg_virtual_memory": "$AVG_VIRTUAL_MEM", "max_queue_events": "$MAX_Q_EVENT_CNT", "max_queue_bytes_size": "$MAX_Q_SIZE"} +$SUMMARY EOF - curl -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json" - echo "" + + # send to ES + local resp + local err_status + resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") + echo "$resp" + err_status=$(echo "$resp" | jq -r ".errors") + if [[ "$err_status" == "true" ]]; then + echo "Failed to send summary" + exit 1 + fi } # $1: snapshot index @@ -225,7 +264,7 @@ node_stats() { NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json # curl inside container because docker on mac cannot resolve localhost to host network interface - docker exec -it ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null + docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null } # $1: index @@ -302,6 +341,13 @@ stop_pipeline() { # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 } +clean_up() { + # stop log generation if it has not done yet + [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + # remove image + docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION +} + main() { parse_args "$@" get_secret @@ -317,8 +363,7 @@ main() { worker fi - # stop log generation if it has not done yet - [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + clean_up } main "$@" diff --git a/.buildkite/scripts/benchmark/save-objects/README.md b/.buildkite/scripts/benchmark/save-objects/README.md new file mode 100644 index 00000000000..26420761827 --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/README.md @@ -0,0 +1,14 @@ +benchmark_objects.ndjson contains the following resources + +- Dashboards + - daily snapshot + - released versions +- Data Views + - benchmark + - runtime fields + - | Fields Name | Type | Comment | + |--------------|---------------------------------------------------------------------------------------|--------------------------------------------------| + | versions_num | long | convert semantic versioning to number for graph sorting | + | release | boolean | `true` for released version. `false` for snapshot version. It is for graph filtering. | + +To import objects to Kibana, navigate to Stack Management > Save Objects and click Import \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson new file mode 100644 index 00000000000..e68a8fc77a4 --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson @@ -0,0 +1,31 @@ +{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.endsWith(\\\"-SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzcyNiw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":100},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Release EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:29.581Z","id":"2c017667-7403-402c-b027-3cc790945a27","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:29.581Z","version":"WzEzOTMsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:13.413Z","id":"9546a73b-5937-4d3d-850f-30625d6e1405","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:13.413Z","version":"WzEzOTEsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"2786f33f-2ad1-4722-8706-9cba9613662e","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:00.945Z","id":"6fba7dff-2564-450b-b885-8c6cf97fa727","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"2786f33f-2ad1-4722-8706-9cba9613662e","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:00.945Z","version":"WzEzODksN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"368034aa-1121-464d-8aed-8a4312274fa8","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ][R] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:50.960Z","id":"6295535b-736d-4704-99d3-1cc0c4bb4cb4","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"368034aa-1121-464d-8aed-8a4312274fa8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:50.960Z","version":"WzEzODcsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"0586bc7c-4662-41bd-8e6d-2337477ca621","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":"release : true "},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[MQ][R] memory by worker ","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:40.740Z","id":"a577558f-f9a6-4c86-b562-7618b2c71db0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"0586bc7c-4662-41bd-8e6d-2337477ca621","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:40.740Z","version":"WzEzODUsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"d556c1f1-16ab-45de-adda-366de4703635","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[PQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:52:31.089Z","id":"1872e84f-56f2-4a17-9426-3d25d6e03931","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"d556c1f1-16ab-45de-adda-366de4703635","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:52:31.089Z","version":"WzEzODQsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"a22f82f6-c7b8-4e06-b2f5-69e25c5aefc4","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:52.744Z","id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"a22f82f6-c7b8-4e06-b2f5-69e25c5aefc4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:52.744Z","version":"WzEzODAsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_queue_events","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_queue_events"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"48ba7011-43c1-48fc-bc36-858bcec74547","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"events in queue"}},"title":"[PQ][R] events in queue by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:43.703Z","id":"d79b8742-6284-4842-ac44-65640d090179","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"48ba7011-43c1-48fc-bc36-858bcec74547","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:43.703Z","version":"WzEzNzgsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"b16664e5-7dda-45f4-853a-0ae2a0e0a692","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ][R] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:33.831Z","id":"64d0f8a7-afe3-4443-88e1-7551d666fee8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"b16664e5-7dda-45f4-853a-0ae2a0e0a692","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:33.831Z","version":"WzEzNzcsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9f57d841-9266-4f58-af56-7fa5e40d8da7","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":"release : true "},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ][R] memory by worker ","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:51:25.955Z","id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9f57d841-9266-4f58-af56-7fa5e40d8da7","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:51:25.955Z","version":"WzEzNzYsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"release","index":"0e2f84c5-7f78-4388-ad55-1230425efb24","key":"release","negate":false,"params":{"query":false},"type":"phrase"},"query":{"match_phrase":{"release":false}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Daily EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"0e2f84c5-7f78-4388-ad55-1230425efb24","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"columns":{"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"2c32a023-462c-4205-b462-6a87182732e4":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"ba0999ad-bf15-429c-ba76-93b1ff919cea","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","2c32a023-462c-4205-b462-6a87182732e4"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[MQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"ba0999ad-bf15-429c-ba76-93b1ff919cea","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"8e8a8743-f0a4-47ed-a4f0-21e21971e264","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"8cbdba7b-e921-42b9-b55b-da567717e259","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"8e8a8743-f0a4-47ed-a4f0-21e21971e264","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Nyw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_queue_events","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_queue_events"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"738fa511-8000-4f6a-b892-4a767bd733ab","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"events in queue"}},"title":"[PQ] events in queue by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"7a2c9e96-8e48-4546-b736-065757a52c47","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"738fa511-8000-4f6a-b892-4a767bd733ab","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Myw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"6dab2bf1-e8cd-4d30-99d6-4e8038c8d19b","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[PQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"daa3722a-cab7-49ba-b5bb-033e81133096","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"6dab2bf1-e8cd-4d30-99d6-4e8038c8d19b","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"columns":{"0ad565fc-0a55-492b-ba69-1be57f501e13":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"962f0beb-0dd8-4449-a652-878d0d2c63e8","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"4d1f7bef-c184-4457-b159-465128d2d70b","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"962f0beb-0dd8-4449-a652-878d0d2c63e8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Miw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Niw3XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzEyNTEsN10="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzEyNDUsN10="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"07700ebe-b8c9-498f-9a91-30efb0c48784","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e0f2401-369e-4818-8a1f-52b64ff175df","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"29ebb6b1-bee6-4f15-b3dc-aac083d5d2f5","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e0f2401-369e-4818-8a1f-52b64ff175df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"26ec195e-e4a9-4417-be97-6d1fbc6b63c0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0OSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"da1edddb-a868-406c-b3a3-e02cf1069e50","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[PQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"bb65f9af-1246-4e75-8b61-dea93d920da8","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"da1edddb-a868-406c-b3a3-e02cf1069e50","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1MCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"8171fdc1-2ff4-4f45-8796-dffd31749c63","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"max 5m_eps"}},"title":"[MQ] EPS by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"5c7f9518-5dac-4a3f-b8e1-6ece4135a8be","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1MSw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e1e2474-ea84-47f5-bb34-0bda229814df","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"max 5m_eps"}},"title":"[PQ] EPS by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f24fc69c-7714-40d3-b34e-3d90fa9b7023","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e1e2474-ea84-47f5-bb34-0bda229814df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1Miw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"637adce1-2cf3-4869-a35d-ead85c340ab9","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[PQ] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"fc548f47-c5ab-491f-a234-4f24ce439d55","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"637adce1-2cf3-4869-a35d-ead85c340ab9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1Myw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"26183784-8280-42c4-ae7c-71c4e9a5908c","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b3c44ade-7f2f-497a-bc9b-f4b53695d975","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"26183784-8280-42c4-ae7c-71c4e9a5908c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc1NCw3XQ=="} +{"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"904fa8c5-7ce1-45ec-bc0d-a52929cabaa6":{"columnOrder":["3dfc74bb-0629-4809-8416-282f6924ecf6","2e3311e8-099f-48bc-81e8-f8dd643cee72","c62ffb65-c24f-49fe-b8ed-683257e60808"],"columns":{"2e3311e8-099f-48bc-81e8-f8dd643cee72":{"dataType":"number","isBucketed":false,"label":"Moving average of Last value of 5m_num","operationType":"moving_average","params":{"window":5},"references":["c62ffb65-c24f-49fe-b8ed-683257e60808"],"scale":"ratio"},"3dfc74bb-0629-4809-8416-282f6924ecf6":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"c62ffb65-c24f-49fe-b8ed-683257e60808":{"dataType":"number","filter":{"language":"kuery","query":"\"5m_num\": *"},"isBucketed":false,"label":"Last value of 5m_num","operationType":"last_value","params":{"showArrayValues":true,"sortField":"timestamp"},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"None","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["2e3311e8-099f-48bc-81e8-f8dd643cee72"],"colorMapping":{"assignments":[],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"904fa8c5-7ce1-45ec-bc0d-a52929cabaa6","layerType":"data","position":"top","seriesType":"bar","showGridlines":false,"xAccessor":"3dfc74bb-0629-4809-8416-282f6924ecf6"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"bar","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide"}},"title":"logstash_5m_eps","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-07-08T16:56:26.170Z","id":"af3a2cb0-138f-4b4a-aaf8-49ff4325386d","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-904fa8c5-7ce1-45ec-bc0d-a52929cabaa6","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-07-08T16:56:26.170Z","version":"WzQzLDRd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":30,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/setup/alias b/.buildkite/scripts/benchmark/setup/alias new file mode 100644 index 00000000000..efa658a4d22 --- /dev/null +++ b/.buildkite/scripts/benchmark/setup/alias @@ -0,0 +1,6 @@ +POST /_aliases +{ + "actions": [ + { "add": { "index": "benchmark_summary_v2", "alias": "benchmark_summary" } } + ] +} \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 b/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 new file mode 100644 index 00000000000..57c80f8c7b3 --- /dev/null +++ b/.buildkite/scripts/benchmark/setup/benchmark_summary_v2 @@ -0,0 +1,179 @@ +PUT /benchmark_summary_v2/_mapping +{ + "properties": { + "avg_cpu_percentage": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_heap": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_non_heap": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "avg_virtual_memory": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "batch_size": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "cpu": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_eps_1m": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_eps_5m": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_queue_bytes_size": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_queue_events": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_worker_concurrency": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "max_worker_utilization": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "mem": { + "type": "float", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "queue_type": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tag": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "timestamp": { + "type": "date" + }, + "total_events_out": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "version": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "workers": { + "type": "integer", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "tags" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } +} diff --git a/.buildkite/scripts/benchmark/util.sh b/.buildkite/scripts/benchmark/util.sh index d7e714e0ba6..ac036d59e68 100755 --- a/.buildkite/scripts/benchmark/util.sh +++ b/.buildkite/scripts/benchmark/util.sh @@ -30,3 +30,12 @@ jqavg() { jqmax() { jq -r "$1 | select(. != null)" $2 | jq -s . | jq 'max' } + +# return true if $1 is non empty and not "null" +not_empty() { + if [[ -n "$1" && "$1" != "null" ]]; then + return 0 + else + return 1 + fi +} \ No newline at end of file diff --git a/catalog-info.yaml b/catalog-info.yaml index b5330636ab1..9f1dfa94dd6 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -614,7 +614,7 @@ spec: kind: Pipeline metadata: name: logstash-benchmark-pipeline - description: ':logstash: The Benchmark pipeline' + description: ':logstash: The Benchmark pipeline for snapshot version' spec: repository: elastic/logstash pipeline_file: ".buildkite/benchmark_pipeline.yml" @@ -645,6 +645,54 @@ spec: # SECTION END: Benchmark pipeline # ******************************* +# *********************************** +# SECTION START: Benchmark multiple Logstash versions +# *********************************** +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-benchmark-versions-pipeline + description: Buildkite pipeline for benchmarking multiple versions + links: + - title: 'Logstash benchmark pipeline for multiple versions' + url: https://buildkite.com/elastic/logstash-benchmark-versions-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-benchmark-versions-pipeline + description: ':logstash: The Benchmark pipeline for multiple versions' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/benchmark_versions_pipeline.yml" + maximum_timeout_in_minutes: 480 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ******************************* +# SECTION END: Benchmark multiple Logstash versions +# ******************************* + # *********************************** # Declare Health Report Tests pipeline # *********************************** From 2ebf2658ff86678125b04c8826958b468ee0da1f Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Thu, 12 Sep 2024 13:24:53 -0700 Subject: [PATCH 28/37] Make health report test runner script executable. (#16446) --- .buildkite/scripts/health-report-tests/main.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .buildkite/scripts/health-report-tests/main.sh diff --git a/.buildkite/scripts/health-report-tests/main.sh b/.buildkite/scripts/health-report-tests/main.sh old mode 100644 new mode 100755 From 1ec37b7c4186d25f2c5f3ab7d4f3afe7e37e91f2 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Fri, 13 Sep 2024 17:33:16 +0200 Subject: [PATCH 29/37] Drop JDK 11 support (#16443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a user runs Logstash with a hosted JDK and not the one bundled with Logstash distribution, like setting a specific LS_JAVA_HOME, which is minor than JDK 17 then Logstash refuses to start. Has to provide at least a JDK 17 or unset the LS_JAVA_HOME and let Logstash uses the bundled JDK. Updates the jvm.options and JvmOptionsParser to remove support for JDK 11. If the options parser identifies that the running JVM is less than 17, it refuses to start. --------- Co-authored-by: João Duarte --- .buildkite/linux_jdk_matrix_pipeline.yml | 6 ------ .buildkite/windows_jdk_matrix_pipeline.yml | 6 ------ config/jvm.options | 10 +--------- docs/static/jvm.asciidoc | 14 +++++++------- docs/static/running-logstash-windows.asciidoc | 2 -- logstash-core/lib/logstash/runner.rb | 5 +---- logstash-core/locales/en.yml | 8 +------- .../main/java/org/logstash/util/JavaVersion.java | 1 - .../java/org/logstash/launchers/JavaVersion.java | 4 ++-- .../org/logstash/launchers/JvmOptionsParser.java | 6 +++--- 10 files changed, 15 insertions(+), 47 deletions(-) diff --git a/.buildkite/linux_jdk_matrix_pipeline.yml b/.buildkite/linux_jdk_matrix_pipeline.yml index eb42bd5ee07..65d80d6b42a 100644 --- a/.buildkite/linux_jdk_matrix_pipeline.yml +++ b/.buildkite/linux_jdk_matrix_pipeline.yml @@ -60,20 +60,14 @@ steps: value: "adoptiumjdk_21" - label: "Adoptium JDK 17 (Eclipse Temurin)" value: "adoptiumjdk_17" - - label: "Adoptium JDK 11 (Eclipse Temurin)" - value: "adoptiumjdk_11" - label: "OpenJDK 21" value: "openjdk_21" - label: "OpenJDK 17" value: "openjdk_17" - - label: "OpenJDK 11" - value: "openjdk_11" - label: "Zulu 21" value: "zulu_21" - label: "Zulu 17" value: "zulu_17" - - label: "Zulu 11" - value: "zulu_11" - wait: ~ if: build.source != "schedule" && build.source != "trigger_job" diff --git a/.buildkite/windows_jdk_matrix_pipeline.yml b/.buildkite/windows_jdk_matrix_pipeline.yml index 2c336e1418b..a54e7eca5fb 100644 --- a/.buildkite/windows_jdk_matrix_pipeline.yml +++ b/.buildkite/windows_jdk_matrix_pipeline.yml @@ -33,20 +33,14 @@ steps: value: "adoptiumjdk_21" - label: "Adoptium JDK 17 (Eclipse Temurin)" value: "adoptiumjdk_17" - - label: "Adoptium JDK 11 (Eclipse Temurin)" - value: "adoptiumjdk_11" - label: "OpenJDK 21" value: "openjdk_21" - label: "OpenJDK 17" value: "openjdk_17" - - label: "OpenJDK 11" - value: "openjdk_11" - label: "Zulu 21" value: "zulu_21" - label: "Zulu 17" value: "zulu_17" - - label: "Zulu 11" - value: "zulu_11" - wait: ~ if: build.source != "schedule" && build.source != "trigger_job" diff --git a/config/jvm.options b/config/jvm.options index b729fce9179..f84e904ce57 100644 --- a/config/jvm.options +++ b/config/jvm.options @@ -16,10 +16,6 @@ ## ################################################################ -## GC configuration -11-13:-XX:+UseConcMarkSweepGC -11-13:-XX:CMSInitiatingOccupancyFraction=75 -11-13:-XX:+UseCMSInitiatingOccupancyOnly ## Locale # Set the locale language @@ -59,11 +55,7 @@ #-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof ## GC logging -#-Xlog:gc*,gc+age=trace,safepoint:file=@loggc@:utctime,pid,tags:filecount=32,filesize=64m - -# log GC status to a file with time stamps -# ensure the directory exists -#-Xloggc:${LS_GC_LOG_FILE} +#-Xlog:gc*,gc+age=trace,safepoint:file=${LS_GC_LOG_FILE}:utctime,pid,tags:filecount=32,filesize=64m # Entropy source for randomness -Djava.security.egd=file:/dev/urandom diff --git a/docs/static/jvm.asciidoc b/docs/static/jvm.asciidoc index 88765d9077f..51d735b55ff 100644 --- a/docs/static/jvm.asciidoc +++ b/docs/static/jvm.asciidoc @@ -4,8 +4,8 @@ {ls} requires one of these versions: -* Java 11 * Java 17 (default). Check out <> for settings info. +* Java 21 Use the http://www.oracle.com/technetwork/java/javase/downloads/index.html[official @@ -17,13 +17,13 @@ for the official word on supported versions across releases. [[bundled-jdk]] .Bundled JDK [NOTE] -===== +===== {ls} offers architecture-specific https://www.elastic.co/downloads/logstash[downloads] that include -Adoptium Eclipse Temurin 17, the latest long term support (LTS) release of the JDK. +Adoptium Eclipse Temurin 17, a long term support (LTS) release of the JDK. Use the LS_JAVA_HOME environment variable if you want to use a JDK other than the -version that is bundled. +version that is bundled. If you have the LS_JAVA_HOME environment variable set to use a custom JDK, Logstash will continue to use the JDK version you have specified, even after you upgrade. ===== @@ -40,9 +40,9 @@ On systems with Java installed, this command produces output similar to the foll [source,shell] ----- -java version "11.0.1" 2018-10-16 LTS -Java(TM) SE Runtime Environment 18.9 (build 11.0.1+13-LTS) -Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.1+13-LTS, mixed mode) +openjdk version "17.0.12" 2024-07-16 +OpenJDK Runtime Environment Temurin-17.0.12+7 (build 17.0.12+7) +OpenJDK 64-Bit Server VM Temurin-17.0.12+7 (build 17.0.12+7, mixed mode) ----- [float] diff --git a/docs/static/running-logstash-windows.asciidoc b/docs/static/running-logstash-windows.asciidoc index 7a3819c58e6..36a98233e99 100644 --- a/docs/static/running-logstash-windows.asciidoc +++ b/docs/static/running-logstash-windows.asciidoc @@ -46,8 +46,6 @@ Java(TM) SE Runtime Environment 18.9 (build 11.0.3+12-LTS) Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.3+12-LTS, mixed mode) ----- -NOTE: As of the publication of this document, please review this https://github.com/elastic/logstash/issues/10496[known issue that impacts Java 11] before proceeding. - Once you have <> and validated JVM pre-requisites, you may proceed. NOTE: For the examples listed below, we are running Windows Server 2016, Java 11.0.3, diff --git a/logstash-core/lib/logstash/runner.rb b/logstash-core/lib/logstash/runner.rb index 876d3e1e173..bdd66cbfda8 100644 --- a/logstash-core/lib/logstash/runner.rb +++ b/logstash-core/lib/logstash/runner.rb @@ -318,10 +318,7 @@ def execute deprecation_logger.deprecated msg end - if JavaVersion::CURRENT < JavaVersion::JAVA_11 - logger.warn I18n.t("logstash.runner.java.version", - :java_home => java.lang.System.getProperty("java.home")) - elsif JavaVersion::CURRENT < JavaVersion::JAVA_17 + if JavaVersion::CURRENT < JavaVersion::JAVA_17 deprecation_logger.deprecated I18n.t("logstash.runner.java.version_17_minimum", :java_home => java.lang.System.getProperty("java.home")) end diff --git a/logstash-core/locales/en.yml b/logstash-core/locales/en.yml index 92dcb38eebf..10d7a88227c 100644 --- a/logstash-core/locales/en.yml +++ b/logstash-core/locales/en.yml @@ -440,14 +440,8 @@ en: Running Logstash with the bundled JDK is recommended. The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), you can configure LS_JAVA_HOME to use that version instead. - version: >- - Starting from Logstash 8.0, the minimum required version of Java is Java 11; your Java version from - %{java_home} does not meet this requirement. Please reconfigure your version of Java to one that is supported. - Running Logstash with the bundled JDK is recommended. - The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. - If you have compelling reasons for using your own JDK (organizational-specific compliance requirements, for example), you can configure LS_JAVA_HOME to use that version instead. version_17_minimum: >- - Starting from Logstash 9.0, the minimum required version of Java will be Java 17; + Starting from Logstash 9.0, the minimum required version of Java is Java 17; your Java version from `%{java_home}` does not meet this requirement. Running Logstash with the bundled JDK is recommended. The bundled JDK has been verified to work with each specific version of Logstash, and generally provides best performance and reliability. diff --git a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java index 505e27e6719..c1d619e9857 100644 --- a/logstash-core/src/main/java/org/logstash/util/JavaVersion.java +++ b/logstash-core/src/main/java/org/logstash/util/JavaVersion.java @@ -30,7 +30,6 @@ public class JavaVersion implements Comparable { public static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version")); - public static final JavaVersion JAVA_11 = parse("11"); public static final JavaVersion JAVA_17 = parse("17"); private final List version; diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java index 0e4d0cf5152..7aa3f484d23 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JavaVersion.java @@ -30,10 +30,10 @@ public class JavaVersion implements Comparable { public static final JavaVersion CURRENT = parse(System.getProperty("java.specification.version")); - public static final JavaVersion JAVA_11 = parse("11"); + public static final JavaVersion JAVA_17 = parse("17"); private final List version; - private JavaVersion(List version){ + private JavaVersion(List version) { this.version = version; } diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java index 84799da1ca5..d19b5f547e7 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java @@ -114,11 +114,11 @@ public static void main(final String[] args) { handleJvmOptions(args, System.getenv("LS_JAVA_OPTS")); } - static void bailOnOldJava(){ - if (JavaVersion.CURRENT.compareTo(JavaVersion.JAVA_11) < 0) { + static void bailOnOldJava() { + if (JavaVersion.CURRENT.compareTo(JavaVersion.JAVA_17) < 0) { final String message = String.format( Locale.ROOT, - "The minimum required Java version is 11; your Java version from [%s] does not meet this requirement", + "The minimum required Java version is 17; your Java version from [%s] does not meet this requirement", System.getProperty("java.home") ); System.err.println(message); From 4e82655cd50a48e4d37988ebc2633b0f3206929b Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:57:51 +0100 Subject: [PATCH 30/37] remove ingest-converter (#16453) Removed the tool Ingest Converter --- .fossa.yml | 4 - bin/ingest-convert.bat | 10 - bin/ingest-convert.sh | 4 - docs/index.asciidoc | 2 - docs/static/filebeat-modules.asciidoc | 4 - docs/static/ingest-convert.asciidoc | 90 ------- rakelib/artifacts.rake | 1 - settings.gradle | 3 +- tools/ingest-converter/build.gradle | 63 ----- tools/ingest-converter/gradle.properties | 2 - .../main/java/org/logstash/ingest/Append.java | 37 --- .../java/org/logstash/ingest/Convert.java | 37 --- .../main/java/org/logstash/ingest/Date.java | 37 --- .../main/java/org/logstash/ingest/GeoIp.java | 34 --- .../main/java/org/logstash/ingest/Grok.java | 37 --- .../main/java/org/logstash/ingest/Gsub.java | 34 --- .../org/logstash/ingest/IngestAppend.java | 72 ------ .../org/logstash/ingest/IngestConvert.java | 66 ----- .../org/logstash/ingest/IngestConverter.java | 231 ------------------ .../java/org/logstash/ingest/IngestDate.java | 97 -------- .../java/org/logstash/ingest/IngestGeoIp.java | 85 ------- .../java/org/logstash/ingest/IngestGrok.java | 99 -------- .../java/org/logstash/ingest/IngestGsub.java | 67 ----- .../java/org/logstash/ingest/IngestJson.java | 79 ------ .../org/logstash/ingest/IngestLowercase.java | 67 ----- .../org/logstash/ingest/IngestPipeline.java | 136 ----------- .../org/logstash/ingest/IngestRename.java | 66 ----- .../java/org/logstash/ingest/IngestSet.java | 80 ------ .../main/java/org/logstash/ingest/JsUtil.java | 210 ---------------- .../main/java/org/logstash/ingest/Json.java | 36 --- .../java/org/logstash/ingest/Lowercase.java | 37 --- .../java/org/logstash/ingest/Pipeline.java | 37 --- .../main/java/org/logstash/ingest/Rename.java | 33 --- .../main/java/org/logstash/ingest/Set.java | 36 --- .../java/org/logstash/ingest/AppendTest.java | 39 --- .../java/org/logstash/ingest/ConvertTest.java | 39 --- .../java/org/logstash/ingest/DateTest.java | 39 --- .../java/org/logstash/ingest/GeoIpTest.java | 39 --- .../java/org/logstash/ingest/GrokTest.java | 39 --- .../java/org/logstash/ingest/GsubTest.java | 39 --- .../java/org/logstash/ingest/IngestTest.java | 102 -------- .../java/org/logstash/ingest/JsonTest.java | 39 --- .../org/logstash/ingest/LowercaseTest.java | 39 --- .../org/logstash/ingest/PipelineTest.java | 55 ----- .../java/org/logstash/ingest/RenameTest.java | 39 --- .../java/org/logstash/ingest/SetTest.java | 39 --- .../org/logstash/ingest/ingestAppend.json | 11 - .../logstash/ingest/ingestAppendScalar.json | 11 - .../logstash/ingest/ingestComplexCase1.json | 52 ---- .../logstash/ingest/ingestComplexCase2.json | 41 ---- .../logstash/ingest/ingestComplexCase3.json | 35 --- .../logstash/ingest/ingestComplexCase4.json | 41 ---- .../org/logstash/ingest/ingestConvert.json | 10 - .../logstash/ingest/ingestConvertBoolean.json | 10 - .../logstash/ingest/ingestConvertString.json | 10 - .../org/logstash/ingest/ingestDate.json | 12 - .../ingest/ingestDateExtraFields.json | 14 -- .../ingest/ingestDotsInAppendField.json | 11 - .../ingest/ingestDotsInConvertField.json | 11 - .../ingest/ingestDotsInDateField.json | 14 -- .../ingest/ingestDotsInGeoIpField.json | 13 - .../ingest/ingestDotsInJsonField.json | 11 - .../ingest/ingestDotsInRenameField.json | 11 - .../logstash/ingest/ingestDotsInSetField.json | 11 - .../logstash/ingest/ingestGeoIpSimple.json | 16 -- .../org/logstash/ingest/ingestGrok.json | 15 -- .../ingestGrokMultiplePatternDefinitions.json | 19 -- .../ingest/ingestGrokPatternDefinition.json | 18 -- .../org/logstash/ingest/ingestGsubSimple.json | 12 - .../org/logstash/ingest/ingestJson.json | 10 - .../ingest/ingestJsonExtraFields.json | 11 - .../logstash/ingest/ingestLowercaseDots.json | 10 - .../ingest/ingestLowercaseSimple.json | 10 - .../org/logstash/ingest/ingestRename.json | 11 - .../org/logstash/ingest/ingestSet.json | 12 - .../org/logstash/ingest/ingestSetNumber.json | 11 - .../org/logstash/ingest/logstashAppend.conf | 15 -- .../logstash/ingest/logstashAppendScalar.conf | 12 - .../logstash/ingest/logstashComplexCase1.conf | 46 ---- .../logstash/ingest/logstashComplexCase2.conf | 33 --- .../logstash/ingest/logstashComplexCase3.conf | 31 --- .../logstash/ingest/logstashComplexCase4.conf | 36 --- .../org/logstash/ingest/logstashConvert.conf | 12 - .../ingest/logstashConvertBoolean.conf | 12 - .../ingest/logstashConvertString.conf | 12 - .../org/logstash/ingest/logstashDate.conf | 15 -- .../ingest/logstashDateExtraFields.conf | 17 -- .../ingest/logstashDotsInAppendField.conf | 15 -- .../ingest/logstashDotsInConvertField.conf | 12 - .../ingest/logstashDotsInDateField.conf | 17 -- .../ingest/logstashDotsInGeoIpField.conf | 15 -- .../ingest/logstashDotsInJsonField.conf | 11 - .../ingest/logstashDotsInRenameField.conf | 12 - .../ingest/logstashDotsInSetField.conf | 12 - .../logstash/ingest/logstashGeoIpSimple.conf | 15 -- .../org/logstash/ingest/logstashGrok.conf | 15 -- ...ogstashGrokMultiplePatternDefinitions.conf | 19 -- .../ingest/logstashGrokPatternDefinition.conf | 18 -- .../logstash/ingest/logstashGsubSimple.conf | 12 - .../org/logstash/ingest/logstashJson.conf | 10 - .../ingest/logstashJsonExtraFields.conf | 11 - .../ingest/logstashLowercaseDots.conf | 10 - .../ingest/logstashLowercaseSimple.conf | 10 - .../org/logstash/ingest/logstashRename.conf | 12 - .../org/logstash/ingest/logstashSet.conf | 12 - .../logstash/ingest/logstashSetNumber.conf | 12 - 106 files changed, 1 insertion(+), 3454 deletions(-) delete mode 100644 bin/ingest-convert.bat delete mode 100755 bin/ingest-convert.sh delete mode 100644 docs/static/ingest-convert.asciidoc delete mode 100644 tools/ingest-converter/build.gradle delete mode 100644 tools/ingest-converter/gradle.properties delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java delete mode 100644 tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java delete mode 100644 tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf delete mode 100644 tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf diff --git a/.fossa.yml b/.fossa.yml index 55e69801bbc..60fd562b6da 100755 --- a/.fossa.yml +++ b/.fossa.yml @@ -21,10 +21,6 @@ analyze: type: gradle target: 'dependencies-report:' path: . - - name: ingest-converter - type: gradle - target: 'ingest-converter:' - path: . - name: logstash-core type: gradle target: 'logstash-core:' diff --git a/bin/ingest-convert.bat b/bin/ingest-convert.bat deleted file mode 100644 index e9067f42869..00000000000 --- a/bin/ingest-convert.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -cd /d "%~dp0\.." -for /f %%i in ('cd') do set RESULT=%%i - -"%JAVACMD%" -cp "!RESULT!\tools\ingest-converter\build\libs\ingest-converter.jar;*" ^ - org.logstash.ingest.Pipeline %* - -endlocal diff --git a/bin/ingest-convert.sh b/bin/ingest-convert.sh deleted file mode 100755 index 1205cebc5c3..00000000000 --- a/bin/ingest-convert.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -java -cp "$(cd `dirname $0`/..; pwd)"'/tools/ingest-converter/build/libs/ingest-converter.jar:*' \ - org.logstash.ingest.Pipeline "$@" diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 34dad2ee375..adbd2ce6a61 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -116,8 +116,6 @@ include::static/managing-multiline-events.asciidoc[] include::static/glob-support.asciidoc[] -include::static/ingest-convert.asciidoc[] - include::static/field-reference.asciidoc[] //The `field-reference.asciidoc` file (included above) contains a diff --git a/docs/static/filebeat-modules.asciidoc b/docs/static/filebeat-modules.asciidoc index 404e21cf22f..257bc8623a3 100644 --- a/docs/static/filebeat-modules.asciidoc +++ b/docs/static/filebeat-modules.asciidoc @@ -134,10 +134,6 @@ For a full example, see <>. //* <> //* <> // -//TIP: {ls} provides an <> -//to help you migrate ingest pipeline definitions to {ls} configs. The tool does -//not currently support all the processors that are available for ingest node, but -//it's a good starting point. // //[[parsing-apache2]] //==== Apache 2 Logs diff --git a/docs/static/ingest-convert.asciidoc b/docs/static/ingest-convert.asciidoc deleted file mode 100644 index b263faebde4..00000000000 --- a/docs/static/ingest-convert.asciidoc +++ /dev/null @@ -1,90 +0,0 @@ -[[ingest-converter]] -=== Converting Ingest Node Pipelines - -After implementing {ref}/ingest.html[ingest] pipelines to parse your data, you -might decide that you want to take advantage of the richer transformation -capabilities in Logstash. For example, you may need to use Logstash instead of -ingest pipelines if you want to: - -* Ingest from more inputs. Logstash can natively ingest data from many other -sources like TCP, UDP, syslog, and relational databases. - -* Use multiple outputs. Ingest node was designed to only support Elasticsearch -as an output, but you may want to use more than one output. For example, you may -want to archive your incoming data to S3 as well as indexing it in -Elasticsearch. - -* Take advantage of the richer transformation capabilities in Logstash, such as -external lookups. - -* Use the persistent queue feature to handle spikes when ingesting data (from -Beats and other sources). - -To make it easier for you to migrate your configurations, Logstash provides an -ingest pipeline conversion tool. The conversion tool takes the ingest pipeline -definition as input and, when possible, creates the equivalent Logstash -configuration as output. - -See <> for a full list of tool limitations. - -[[ingest-converter-run]] -==== Running the tool - -You'll find the conversion tool in the `bin` directory of your Logstash -installation. See <> to find the location of `bin` on your system. - -To run the conversion tool, use the following command: - -[source,shell] ------ -bin/ingest-convert.sh --input INPUT_FILE_URI --output OUTPUT_FILE_URI [--append-stdio] ------ - -Where: - -* `INPUT_FILE_URI` is a file URI that specifies the full path to the JSON file -that defines the ingest node pipeline. - -* `OUTPUT_FILE_URI` is the file URI of the Logstash DSL file that will be -generated by the tool. - -* `--append-stdio` is an optional flag that adds stdin and stdout sections to -the config instead of adding the default Elasticsearch output. - -This command expects a file URI, so make sure you use forward slashes and -specify the full path to the file. - -For example: - -[source,text] ------ -bin/ingest-convert.sh --input file:///tmp/ingest/apache.json --output file:///tmp/ingest/apache.conf ------ - - -[[ingest-converter-limitations]] -==== Limitations - -* Painless script conversion is not supported. - -* Only a subset of available processors are -<> for conversion. For -processors that are not supported, the tool produces a warning and continues -with a best-effort conversion. - -[[ingest-converter-supported-processors]] -==== Supported Processors - -The following ingest node processors are currently supported for conversion by -the tool: - -* Append -* Convert -* Date -* GeoIP -* Grok -* Gsub -* Json -* Lowercase -* Rename -* Set diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 7e0c553a520..f4a8657c4e4 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -50,7 +50,6 @@ namespace "artifact" do "logstash-core-plugin-api/*.gemspec", "patterns/**/*", - "tools/ingest-converter/build/libs/ingest-converter.jar", "vendor/??*/**/*", # To include ruby-maven's hidden ".mvn" directory, we need to # do add the line below. This directory contains a file called diff --git a/settings.gradle b/settings.gradle index 4415c1417c6..aa27de37971 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,10 +1,9 @@ rootProject.name = "logstash" -include ':logstash-core', 'logstash-core-benchmarks', 'ingest-converter', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report' +include ':logstash-core', 'logstash-core-benchmarks', 'benchmark-cli', 'jvm-options-parser', 'logstash-integration-tests', 'dependencies-report' project(':logstash-core').projectDir = new File('./logstash-core') project(':logstash-core-benchmarks').projectDir = new File('./logstash-core/benchmarks') project(':logstash-integration-tests').projectDir = new File('./qa/integration') -project(':ingest-converter').projectDir = new File('./tools/ingest-converter') project(':benchmark-cli').projectDir = new File('./tools/benchmark-cli') project(':dependencies-report').projectDir = new File('./tools/dependencies-report') project(':jvm-options-parser').projectDir = new File('./tools/jvm-options-parser') diff --git a/tools/ingest-converter/build.gradle b/tools/ingest-converter/build.gradle deleted file mode 100644 index 62e4437a6ab..00000000000 --- a/tools/ingest-converter/build.gradle +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.yaml.snakeyaml.Yaml - -// fetch version from Logstash's main versions.yml file -def versionMap = (Map) (new Yaml()).load(new File("$projectDir/../../versions.yml").text) - -description = """Ingest JSON to Logstash Grok Config Converter""" -version = versionMap['logstash-core'] -String jacksonDatabindVersion = versionMap['jackson-databind'] - -repositories { - mavenCentral() -} - -buildscript { - repositories { - mavenCentral() - gradlePluginPortal() - } - dependencies { - classpath "org.yaml:snakeyaml:${snakeYamlVersion}" - classpath "com.github.johnrengelman:shadow:${shadowGradlePluginVersion}" - } -} - -dependencies { - implementation 'net.sf.jopt-simple:jopt-simple:4.6' - implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}" - testImplementation "junit:junit:4.13.2" - testImplementation 'commons-io:commons-io:2.16.1' -} - -javadoc { - enabled = true -} - -apply plugin: 'com.github.johnrengelman.shadow' - -shadowJar { - archiveBaseName = 'ingest-converter' - archiveClassifier = null - archiveVersion = '' -} - -assemble.dependsOn shadowJar diff --git a/tools/ingest-converter/gradle.properties b/tools/ingest-converter/gradle.properties deleted file mode 100644 index 7f7a1fae05e..00000000000 --- a/tools/ingest-converter/gradle.properties +++ /dev/null @@ -1,2 +0,0 @@ -isDistributedArtifact=false - diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java deleted file mode 100644 index 131805f22da..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Append.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Append DSL to Logstash mutate Transpiler. - */ -public final class Append { - - private Append() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_append_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java deleted file mode 100644 index 802eb4e391b..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Convert.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Convert DSL to Logstash Date Transpiler. - */ -public final class Convert { - - private Convert() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_convert_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java deleted file mode 100644 index 91237ff07d0..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Date.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Date DSL to Logstash Date Transpiler. - */ -public final class Date { - - private Date() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_date"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java deleted file mode 100644 index 949ca243f63..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/GeoIp.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public final class GeoIp { - - private GeoIp() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_geoip"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java deleted file mode 100644 index 44d9f189010..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Grok.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest JSON DSL to Logstash Grok Transpiler. - */ -public final class Grok { - - private Grok() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_grok"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java deleted file mode 100644 index 29ee3d01f1e..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Gsub.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public final class Gsub { - - private Gsub() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_to_logstash_gsub"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java deleted file mode 100644 index eee5d8dfbdd..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestAppend.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestAppend { - - /** - * Converts Ingest Append JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestAppend::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", appendHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String appendHash(Map processor) { - Map append_json = processor.get("append"); - Object value = append_json.get("value"); - Object value_contents; - if (value instanceof List) { - value_contents = IngestConverter.createArray((List) value); - } else { - value_contents = IngestConverter.quoteString((String) value); - } - Object mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) append_json.get("field"))), - (String) value_contents); - return IngestConverter.createField("add_field", IngestConverter.wrapInCurly((String) mutate_contents)); - } - - public static boolean has_append(Map processor) { - return processor.containsKey("append"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java deleted file mode 100644 index 6c1128264ac..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConvert.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestConvert { - - /** - * Converts Ingest Convert JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestConvert::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", convertHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String convertHash(Map processor) { - Map convert_json = processor.get("convert"); - - Object mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) convert_json.get("field"))), - IngestConverter.quoteString((String) convert_json.get("type"))); - return IngestConverter.createField("convert", IngestConverter.wrapInCurly((String) mutate_contents)); - } - - public static boolean has_convert(Map processor) { - return processor.containsKey("convert"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java deleted file mode 100644 index 16bc91e3c3f..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestConverter.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.logstash.ingest; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class IngestConverter { - - /** - * Translates the JSON naming pattern (`name.qualifier.sub`) into the LS pattern - * [name][qualifier][sub] for all applicable tokens in the given string. - * This function correctly identifies and omits renaming of string literals. - * @param content to replace naming pattern in - * @returns {string} with Json naming translated into grok naming - */ - public static String dotsToSquareBrackets(String content) { - final Pattern pattern = Pattern.compile("\\(\\?:%\\{.*\\|-\\)"); - final Matcher matcher = pattern.matcher(content); - List tokens = new ArrayList<>(); - String right = content; - while (matcher.find()) { - final int start = matcher.start(); - final int end = matcher.end(); - final String matchContent = content.substring(start, end); - right = content.substring(end); - tokens.add(tokenDotsToSquareBrackets(content.substring(0, start))); - tokens.add(matchContent); - } - tokens.add(tokenDotsToSquareBrackets(right)); - return String.join("", tokens); - } - - private static String tokenDotsToSquareBrackets(String content) { - //Break out if this is not a naming pattern we convert - final String adjusted; - if (Pattern.compile("([\\w_]+\\.)+[\\w_]+").matcher(content).find()) { - adjusted = content.replaceAll("(\\w*)\\.(\\w*)", "$1][$2") - .replaceAll("\\[(\\w+)(}|$)", "[$1]$2") - .replaceAll("\\{(\\w+):(\\w+)]", "{$1:[$2]") - .replaceAll("^(\\w+)]\\[", "[$1]["); - } else { - adjusted = content; - } - return adjusted; - } - - public static String quoteString(String content) { - return "\"" + content.replace("\"", "\\\"") + "\""; - } - - public static String wrapInCurly(String content) { - return "{\n" + content + "\n}"; - } - - public static String createField(String fieldName, String content) { - return fieldName + " => " + content; - } - - public static String createHash(String fieldName, String content) { - return fieldName + " " + wrapInCurly(content); - } - - /** - * All hash fields in LS start on a new line. - * @param fields Array of Strings of Serialized Hash Fields - * @returns {string} Joined Serialization of Hash Fields - */ - public static String joinHashFields(String... fields) { - return String.join("\n", fields); - } - - /** - * Fixes indentation in LS string. - * @param content LS string to fix indentation in, that has no indentation intentionally with - * all lines starting on a token without preceding spaces. - * @return LS string indented by 3 spaces per level - */ - public static String fixIndent(String content) { - final String[] lines = content.split("\n"); - int count = 0; - for (int i = 0; i < lines.length; i++) { - if (Pattern.compile("(\\{|\\[)$").matcher(lines[i]).find()) { - lines[i] = indent(lines[i], count); - ++count; - } else if (Pattern.compile("(\\}|\\])$").matcher(lines[i]).find()) { - --count; - lines[i] = indent(lines[i], count); - // Only indent line if previous line ended on relevant control char. - } else if (i > 0 && Pattern.compile("(=>\\s+\".+\"|,|\\{|\\}|\\[|\\])$").matcher(lines[i - 1]).find()) { - lines[i] = indent(lines[i], count); - } - } - - return String.join("\n", lines); - } - - private static String indent(String content, int shifts) { - StringBuilder spacing = new StringBuilder(); - for (int i = 0; i < shifts * 3; i++) { - spacing.append(" "); - } - return spacing.append(content).toString(); - } - - /** - * Converts Ingest/JSON style pattern array to LS pattern array, performing necessary variable - * name and quote escaping adjustments. - * @param patterns Pattern Array in JSON formatting - * @return Pattern array in LS formatting - */ - public static String createPatternArray(String... patterns) { - final String body = Arrays.stream(patterns) - .map(IngestConverter::dotsToSquareBrackets) - .map(IngestConverter::quoteString) - .collect(Collectors.joining(",\n")); - return "[\n" + body + "\n]"; - } - - public static String createArray(List ingestArray) { - final String body = ingestArray.stream() - .map(IngestConverter::quoteString) - .collect(Collectors.joining(",\n")); - return "[\n" + body + "\n]"; - } - - - /** - * Converts Ingest/JSON style pattern array to LS pattern array or string if the given array - * contains a single element only, performing necessary variable name and quote escaping - * adjustments. - * @param patterns Pattern Array in JSON formatting - * @return Pattern array or string in LS formatting - */ - public static String createPatternArrayOrField(String... patterns) { - return patterns.length == 1 - ? quoteString(dotsToSquareBrackets(patterns[0])) - : createPatternArray(patterns); - } - - public static String filterHash(String contents) { - return fixIndent(createHash("filter", contents)); - } - - public static String filtersToFile(String... filters) { - return String.join("\n\n", filters) + "\n"; - } - - /** - * Does it have an on_failure field? - * @param processor Json - * @param name Name of the processor - * @return true if has on failure - */ - @SuppressWarnings("rawtypes") - public static boolean hasOnFailure(Map processor, String name) { - final List onFailure = (List) processor.get(name).get("on_failure"); - return onFailure != null && !onFailure.isEmpty(); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public static List> getOnFailure(Map processor, String name) { - return (List>) processor.get(name).get("on_failure"); - } - - /** - * Creates an if clause with the tag name - * @param tag String tag name to find in [tags] field - * @param onFailurePipeline The on failure pipeline converted to LS to tack on in the conditional - * @return a string representing a conditional logic - */ - public static String createTagConditional(String tag, String onFailurePipeline) { - return "if " + quoteString(tag) + " in [tags] {\n" + - onFailurePipeline + "\n" + - "}"; - } - - public static String getElasticsearchOutput() { - return fixIndent("output {\n" + - "elasticsearch {\n" + - "hosts => \"localhost\"\n" + - "}\n" + - "}"); - } - - public static String getStdinInput() { - return fixIndent("input {\n" + - "stdin {\n" + - "}\n" + - "}"); - } - - public static String getStdoutOutput() { - return fixIndent("output {\n" + - "stdout {\n" + - "codec => \"rubydebug\"\n" + - "}\n" + - "}"); - } - - public static String appendIoPlugins(List filtersPipeline, boolean appendStdio) { - // TODO create unique list to join all - String filtersPipelineStr = String.join("\n", filtersPipeline); - if (appendStdio) { - return String.join("\n", IngestConverter.getStdinInput(), filtersPipelineStr, IngestConverter.getStdoutOutput()); - } else { - return String.join("\n", filtersPipelineStr, IngestConverter.getElasticsearchOutput()); - } - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java deleted file mode 100644 index 6aea685e815..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestDate.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestDate { - - /** - * Converts Ingest Date JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestDate::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("date", dateHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String dateHash(Map processor) { - Map date_json = processor.get("date"); - List formats = (List) date_json.get("formats"); - - final String firstElem = IngestConverter.dotsToSquareBrackets((String) date_json.get("field")); - List match_contents = new ArrayList<>(); - match_contents.add(firstElem); - for (String f : formats) { - match_contents.add(f); - } - String date_contents = IngestConverter.createField( - "match", - IngestConverter.createPatternArray(match_contents.toArray(new String[0]))); - if (JsUtil.isNotEmpty((String) date_json.get("target_field"))) { - String target = IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) date_json.get("target_field")) - ) - ); - date_contents = IngestConverter.joinHashFields(date_contents, target); - } - if (JsUtil.isNotEmpty((String) date_json.get("timezone"))) { - String timezone = IngestConverter.createField( - "timezone", - IngestConverter.quoteString((String) date_json.get("timezone")) - ); - date_contents = IngestConverter.joinHashFields(date_contents, timezone); - } - if (JsUtil.isNotEmpty((String) date_json.get("locale"))) { - String locale = IngestConverter.createField( - "locale", - IngestConverter.quoteString((String) date_json.get("locale")) - ); - date_contents = IngestConverter.joinHashFields(date_contents, locale); - } - return date_contents; - } - - public static boolean has_date(Map processor) { - return processor.containsKey("date"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java deleted file mode 100644 index 3394ba52d39..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGeoIp.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGeoIp { - - - /** - * Converts Ingest Date JSON to LS Date filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGeoIp::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("geoip", geoIpHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String geoIpHash(Map processor) { - Map geoip_data = processor.get("geoip"); - final String sourceField = IngestConverter.createField( - "source", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) geoip_data.get("field")) - ) - ); - - final String targetField = IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) geoip_data.get("target_field")) - ) - ); - - if (geoip_data.containsKey("properties")) { - String fields = IngestConverter.createField( - "fields", - IngestConverter.createPatternArray(((List) geoip_data.get("properties")).toArray(new String[0]) - )); - return IngestConverter.joinHashFields(sourceField, targetField, fields); - } else { - return IngestConverter.joinHashFields(sourceField, targetField); - } - } - - public static boolean has_geoip(Map processor) { - return processor.containsKey("geoip"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java deleted file mode 100644 index 9c8c8067b69..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGrok.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGrok { - - /** - * Converts Ingest JSON to LS Grok. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGrok::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("grok", grokHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String grokHash(Map processor) { - Map grok_data = processor.get("grok"); - String grok_contents = createHashField("match", - IngestConverter.createField( - IngestConverter.quoteString((String) grok_data.get("field")), - IngestConverter.createPatternArrayOrField(((List) grok_data.get("patterns")).toArray(new String[0])) - )); - if (grok_data.containsKey("pattern_definitions")) { - grok_contents = IngestConverter.joinHashFields( - grok_contents, - createPatternDefinitionHash((Map) grok_data.get("pattern_definitions")) - ); - } - return grok_contents; - } - - private static String createHashField(String name, String content) { - return IngestConverter.createField(name, IngestConverter.wrapInCurly(content)); - } - - private static String createPatternDefinitionHash(Map definitions) { - List content = new ArrayList<>(); - for(Map.Entry entry : definitions.entrySet()) { - content.add(IngestConverter.createField( - IngestConverter.quoteString(entry.getKey()), - IngestConverter.quoteString(entry.getValue()))); - } - - final String patternDefs = content.stream().map(IngestConverter::dotsToSquareBrackets) - .collect(Collectors.joining("\n")); - - return createHashField( - "pattern_definitions", - patternDefs - ); - } - - public static boolean has_grok(Map processor) { - return processor.containsKey(get_name()); - } - - public static String get_name() { - return "grok"; - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java deleted file mode 100644 index 1ff3e3b51f3..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestGsub.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestGsub { - - /** - * Converts Ingest JSON to LS Grok. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestGsub::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", gsubHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String gsubHash(Map processor) { - Map gsub_data = processor.get("gsub"); - final String body = String.join(", ", - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) gsub_data.get("field"))), - IngestConverter.quoteString((String) gsub_data.get("pattern")), - IngestConverter.quoteString((String) gsub_data.get("replacement"))); - - return IngestConverter.createField("gsub", "[\n" + body + "\n]"); - } - - public static boolean has_gsub(Map processor) { - return processor.containsKey("gsub"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java deleted file mode 100644 index 77a9b37df62..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestJson.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestJson { - - /** - * Converts Ingest json processor to LS json filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestJson::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("json", jsonHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String jsonHash(Map processor) { - Map json_data = processor.get("json"); - - List parts = new ArrayList(); - parts.add(IngestConverter.createField("source", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) json_data.get("field")) - ) - )); - - if (json_data.containsKey("target_field")) { - parts.add(IngestConverter.createField( - "target", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) json_data.get("target_field")) - ) - )); - } - return IngestConverter.joinHashFields(parts.toArray(new String[0])); - } - - public static boolean has_json(Map processor) { - return processor.containsKey("json"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java deleted file mode 100644 index 09b3c703a3c..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestLowercase.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestLowercase { - - /** - * Converts Ingest Lowercase JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestLowercase::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", lowercaseHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String lowercaseHash(Map processor) { - Map lowercase_data = processor.get("lowercase"); - return IngestConverter.createField( - "lowercase", - IngestConverter.quoteString( - IngestConverter.dotsToSquareBrackets((String) lowercase_data.get("field")) - ) - ); - } - - public static boolean has_lowercase(Map processor) { - return processor.containsKey("lowercase"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java deleted file mode 100644 index 1084f680c8c..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestPipeline.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestPipeline { - - /** - * Converts Ingest JSON to LS. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestPipeline::mapProcessor).collect(Collectors.toList()); - - String logstash_pipeline = IngestConverter.filterHash( - IngestConverter.joinHashFields(filters_pipeline.toArray(new String[0]))); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(Collections.singletonList(logstash_pipeline), appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - List filter_blocks = new ArrayList<>(); - if (IngestGrok.has_grok(processor)) { - filter_blocks.add(IngestConverter.createHash(IngestGrok.get_name(), IngestGrok.grokHash(processor))); - - if (IngestConverter.hasOnFailure(processor, IngestGrok.get_name())) { - filter_blocks.add( - handle_on_failure_pipeline( - IngestConverter.getOnFailure(processor, IngestGrok.get_name()), - "_grokparsefailure" - ) - ); - } - } - boolean processed = false; - if (IngestDate.has_date(processor)) { - filter_blocks.add( - IngestConverter.createHash("date", IngestDate.dateHash(processor)) - ); - processed = true; - } - if (IngestGeoIp.has_geoip(processor)) { - filter_blocks.add( - IngestConverter.createHash("geoip", IngestGeoIp.geoIpHash(processor)) - ); - processed = true; - } - if (IngestConvert.has_convert(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestConvert.convertHash(processor)) - ); - processed = true; - } - if (IngestGsub.has_gsub(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestGsub.gsubHash(processor)) - ); - processed = true; - } - if (IngestAppend.has_append(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestAppend.appendHash(processor)) - ); - processed = true; - } - if (IngestJson.has_json(processor)) { - filter_blocks.add( - IngestConverter.createHash("json", IngestJson.jsonHash(processor)) - ); - processed = true; - } - if (IngestRename.has_rename(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestRename.renameHash(processor)) - ); - processed = true; - } - if (IngestLowercase.has_lowercase(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestLowercase.lowercaseHash(processor)) - ); - processed = true; - } - if (IngestSet.has_set(processor)) { - filter_blocks.add( - IngestConverter.createHash("mutate", IngestSet.setHash(processor)) - ); - processed = true; - } - if (!processed) { - System.out.println("WARN Found unrecognized processor named: " + processor.keySet().iterator().next()); - } - return IngestConverter.joinHashFields(filter_blocks.toArray(new String[0])); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String handle_on_failure_pipeline(List on_failure_json, String tag_name) { - final List mapped = on_failure_json.stream().map(IngestPipeline::mapProcessor).collect(Collectors.toList()); - return IngestConverter.createTagConditional(tag_name, - IngestConverter.joinHashFields(mapped.toArray(new String[0])) - ); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java deleted file mode 100644 index a21c81f53a8..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestRename.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestRename { - - /** - * Converts Ingest Rename JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestRename::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", renameHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String renameHash(Map processor) { - Map rename_json = processor.get("rename"); - final String mutateContents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) rename_json.get("field"))), - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) rename_json.get("target_field"))) - ); - return IngestConverter.createField("rename", IngestConverter.wrapInCurly(mutateContents)); - } - - public static boolean has_rename(Map processor) { - return processor.containsKey("rename"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java deleted file mode 100644 index 044d787ed87..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/IngestSet.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.logstash.ingest; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IngestSet { - - /** - * Converts Ingest Set JSON to LS mutate filter. - */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public static String toLogstash(String json, boolean appendStdio) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - TypeReference> typeRef = new TypeReference>() {}; - final HashMap jsonDefinition = mapper.readValue(json, typeRef); - final List processors = (List) jsonDefinition.get("processors"); - List filters_pipeline = processors.stream().map(IngestSet::mapProcessor).collect(Collectors.toList()); - - return IngestConverter.filtersToFile( - IngestConverter.appendIoPlugins(filters_pipeline, appendStdio)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private static String mapProcessor(Map processor) { - return IngestConverter.filterHash(IngestConverter.createHash("mutate", setHash(processor))); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - static String setHash(Map processor) { - Map set_json = processor.get("set"); - final Object value = set_json.get("value"); - final Object value_contents; - if (value instanceof String) { - value_contents = IngestConverter.quoteString((String) value); - } else { - value_contents = value; - } - if (set_json.containsKey("if") && set_json.get("if") != null) { - String painless_condition = (String) set_json.get("if"); - if (!painless_condition.isEmpty()) { - System.out.println("WARN Found in 'set' processor an 'if' painless condition not translated: " + painless_condition); - } - - } - - String mutate_contents = IngestConverter.createField( - IngestConverter.quoteString(IngestConverter.dotsToSquareBrackets((String) set_json.get("field"))), - value_contents.toString()); - return IngestConverter.createField("add_field", IngestConverter.wrapInCurly(mutate_contents)); - } - - public static boolean has_set(Map processor) { - return processor.containsKey("set"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java deleted file mode 100644 index 806c2f36f27..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/JsUtil.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import javax.script.Invocable; -import javax.script.ScriptEngine; -import javax.script.ScriptEngineManager; -import javax.script.ScriptException; -import joptsimple.OptionException; -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; - -final class JsUtil { - - /** - * Script names used by the converter in correct load order. - */ - - private static final String[] SCRIPTS = { - "shared", "date", "grok", "geoip", "gsub", "pipeline", "convert", "append", "json", - "rename", "lowercase", "set" - }; - - private JsUtil() { - // Utility Class - } - - /** - * Sets up a {@link ScriptEngine} with all Ingest to LS DSL Converter JS scripts loaded. - * @return {@link ScriptEngine} for Ingest to LS DSL Converter - */ - public static ScriptEngine engine() { - final ScriptEngine engine = - new ScriptEngineManager().getEngineByName("nashorn"); - try { - for (final String file : SCRIPTS) { - add(engine, String.format("/ingest-%s.js", file)); - } - } catch (final IOException | ScriptException ex) { - throw new IllegalStateException(ex); - } - return engine; - } - - /** - * Converts the given files from ingest to LS conf using the javascript function - * @param args CLI Arguments - * @param jsFunc JS function to call - * @throws ScriptException - * @throws NoSuchMethodException - */ - public static void convert(final String[] args, final String jsFunc) - throws ScriptException, NoSuchMethodException { - final OptionParser parser = new OptionParser(); - final OptionSpec input = parser.accepts( - "input", - "Input JSON file location URI. Only supports 'file://' as URI schema." - ).withRequiredArg().ofType(URI.class).required().forHelp(); - final OptionSpec output = parser.accepts( - "output", - "Output Logstash DSL file location URI. Only supports 'file://' as URI schema." - ).withRequiredArg().ofType(URI.class).required().forHelp(); - final OptionSpec appendStdio = parser.accepts( - "append-stdio", - "Flag to append stdin and stdout as outputs instead of the default ES output." - ).forHelp(); - try { - final OptionSet options; - try { - options = parser.parse(args); - } catch (final OptionException ex) { - parser.printHelpOn(System.out); - throw ex; - } - switch (jsFunc) { - case "ingest_append_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestAppend.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_convert_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestConvert.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_date": - Files.write( - Paths.get(options.valueOf(output)), - IngestDate.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_geoip": - Files.write( - Paths.get(options.valueOf(output)), - IngestGeoIp.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_grok": - Files.write( - Paths.get(options.valueOf(output)), - IngestGrok.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_to_logstash_gsub": - Files.write( - Paths.get(options.valueOf(output)), - IngestGsub.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_json_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestJson.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_lowercase_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestLowercase.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_rename_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestRename.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_set_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestSet.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - case "ingest_pipeline_to_logstash": - Files.write( - Paths.get(options.valueOf(output)), - IngestPipeline.toLogstash(input(options.valueOf(input)), options.has(appendStdio)).getBytes(StandardCharsets.UTF_8) - ); - break; - - default: { - throw new IllegalArgumentException("Can't recognize " + jsFunc + " processor"); - } - } - - } catch (final IOException ex) { - throw new IllegalStateException(ex); - } - } - - /** - * Retrieves the input Ingest JSON from a given {@link URI}. - * @param uri {@link URI} of Ingest JSON - * @return Json String - * @throws IOException On failure to load Ingest JSON - */ - private static String input(final URI uri) throws IOException { - if ("file".equals(uri.getScheme())) { - return new String( - Files.readAllBytes(Paths.get(uri)), StandardCharsets.UTF_8 - ); - } - throw new IllegalArgumentException("--input must be of schema file://"); - } - - private static void add(final ScriptEngine engine, final String file) - throws IOException, ScriptException { - try (final Reader reader = - new InputStreamReader(JsUtil.class.getResourceAsStream(file))) { - engine.eval(reader); - } - } - - /*** - * Not empty check with nullability - * @param s string to check - * @return true iff s in not null and not empty - */ - static boolean isNotEmpty(String s) { - return s != null && !s.isEmpty(); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java deleted file mode 100644 index 21b10d856ce..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Json.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest JSON processor DSL to Logstash json Transpiler. - */ -public class Json { - private Json() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_json_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java deleted file mode 100644 index 77ccb8c64ec..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Lowercase.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Lowercase DSL to Logstash mutate Transpiler. - */ -public final class Lowercase { - - private Lowercase() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_lowercase_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java deleted file mode 100644 index e888852f749..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Pipeline.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Full DSL to Logstash DSL Transpiler. - */ -public final class Pipeline { - - private Pipeline() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_pipeline_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java deleted file mode 100644 index dac5129a0ce..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Rename.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -public class Rename { - private Rename() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_rename_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java b/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java deleted file mode 100644 index 5872958692d..00000000000 --- a/tools/ingest-converter/src/main/java/org/logstash/ingest/Set.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import javax.script.ScriptException; - -/** - * Ingest Set DSL to Logstash mutate Transpiler. - */ -public class Set { - private Set() { - // Utility Wrapper for JS Script. - } - - public static void main(final String... args) throws ScriptException, NoSuchMethodException { - JsUtil.convert(args, "ingest_set_to_logstash"); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java deleted file mode 100644 index 0fa5cd43d9d..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/AppendTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class AppendTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Append", "DotsInAppendField", "AppendScalar"); - } - - @Test - public void convertsAppendProcessorCorrectly() throws Exception { - assertCorrectConversion(Append.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java deleted file mode 100644 index e2197e116c6..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/ConvertTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class ConvertTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Convert", "DotsInConvertField", "ConvertBoolean", "ConvertString"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Convert.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java deleted file mode 100644 index 49a475d2a8d..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/DateTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class DateTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Date", "DateExtraFields", "DotsInDateField"); - } - - @Test - public void convertsDateFieldCorrectly() throws Exception { - assertCorrectConversion(Date.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java deleted file mode 100644 index fb83ad3be1e..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GeoIpTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GeoIpTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("GeoIpSimple", "DotsInGeoIpField"); - } - - @Test - public void convertsGeoIpFieldCorrectly() throws Exception { - assertCorrectConversion(GeoIp.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java deleted file mode 100644 index d423baf5f80..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GrokTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GrokTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Grok", "GrokPatternDefinition", "GrokMultiplePatternDefinitions"); - } - - @Test - public void convertsGrokFieldCorrectly() throws Exception { - assertCorrectConversion(Grok.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java deleted file mode 100644 index 26f422c136a..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/GsubTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Collections; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class GsubTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Collections.singletonList("GsubSimple"); - } - - @Test - public void convertsGsubCorrectly() throws Exception { - assertCorrectConversion(Gsub.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java deleted file mode 100644 index d8983abe14a..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/IngestTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.regex.Pattern; -import org.apache.commons.io.IOUtils; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.runners.Parameterized.Parameter; - -/** - * Base class for ingest migration tests - */ -@RunWith(Parameterized.class) -public abstract class IngestTest { - - /** - * Used to normalize line endings since static reference result files have Unix line endings. - */ - private static final Pattern CR_LF = - Pattern.compile("\\r\\n"); - - /** - * Used to normalize line endings since static reference result files have Unix line endings. - */ - private static final Pattern CARRIAGE_RETURN = Pattern.compile("\\r"); - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - @Parameter - public String testCase; - - protected final void assertCorrectConversion(final Class clazz) throws Exception { - final URL append = getResultPath(temp); - clazz.getMethod("main", String[].class).invoke( - null, - (Object) new String[]{ - String.format("--input=%s", resourcePath(String.format("ingest%s.json", testCase))), - String.format("--output=%s", append) - } - ); - assertThat( - utf8File(append), is(utf8File(resourcePath(String.format("logstash%s.conf", testCase)))) - ); - } - - /** - * Reads a file, normalizes line endings to Unix line endings and returns the whole content - * as a String. - * @param path Url to read - * @return String content of the URL - * @throws IOException On failure to read from given URL - */ - private static String utf8File(final URL path) throws IOException { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (final InputStream input = path.openStream()) { - IOUtils.copy(input, baos); - } - return CARRIAGE_RETURN.matcher( - CR_LF.matcher( - baos.toString(StandardCharsets.UTF_8.name()) - ).replaceAll("\n") - ).replaceAll("\n"); - } - - private static URL resourcePath(final String name) { - return IngestTest.class.getResource(name); - } - - private static URL getResultPath(TemporaryFolder temp) throws IOException { - return temp.newFolder().toPath().resolve("converted").toUri().toURL(); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java deleted file mode 100644 index cb7bf65b087..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/JsonTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class JsonTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Json", "DotsInJsonField", "JsonExtraFields"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Json.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java deleted file mode 100644 index e6231dc6080..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/LowercaseTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class LowercaseTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("LowercaseSimple", "LowercaseDots"); - } - - @Test - public void convertsAppendProcessorCorrectly() throws Exception { - assertCorrectConversion(Lowercase.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java deleted file mode 100644 index 830f5c466a0..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/PipelineTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.ArrayList; -import java.util.Collection; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class PipelineTest extends IngestTest { - - @Parameters - public static Iterable data() { - final Collection cases = new ArrayList<>(); - cases.add("ComplexCase1"); - cases.add("ComplexCase2"); - cases.add("ComplexCase3"); - cases.add("ComplexCase4"); - GeoIpTest.data().forEach(cases::add); - DateTest.data().forEach(cases::add); - GrokTest.data().forEach(cases::add); - ConvertTest.data().forEach(cases::add); - GsubTest.data().forEach(cases::add); - AppendTest.data().forEach(cases::add); - JsonTest.data().forEach(cases::add); - RenameTest.data().forEach(cases::add); - LowercaseTest.data().forEach(cases::add); - SetTest.data().forEach(cases::add); - return cases; - } - - @Test - public void convertsComplexCaseCorrectly() throws Exception { - assertCorrectConversion(Pipeline.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java deleted file mode 100644 index 2ecd074eb2e..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/RenameTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class RenameTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Rename", "DotsInRenameField"); - } - - @Test - public void convertsConvertProcessorCorrectly() throws Exception { - assertCorrectConversion(Rename.class); - } -} diff --git a/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java b/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java deleted file mode 100644 index 8682b616cd6..00000000000 --- a/tools/ingest-converter/src/test/java/org/logstash/ingest/SetTest.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.logstash.ingest; - -import java.util.Arrays; -import org.junit.Test; - -import static org.junit.runners.Parameterized.Parameters; - -public final class SetTest extends IngestTest { - - @Parameters - public static Iterable data() { - return Arrays.asList("Set", "DotsInSetField", "SetNumber"); - } - - @Test - public void convertsSetProcessorCorrectly() throws Exception { - assertCorrectConversion(Set.class); - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json deleted file mode 100644 index 471d4337cd2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppend.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "client", - "value": ["host1", "host2"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json deleted file mode 100644 index 34ad59e5e7d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestAppendScalar.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "foo", - "value": "bar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json deleted file mode 100644 index 370fd72d604..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase1.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{COMBINEDAPACHELOG}" - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "client.ip", - "target_field": "geo" - } - }, - { - "convert": { - "field" : "bytes", - "type": "integer" - } - }, - { - "append": { - "field" : "response_code", - "value": ["200", "400", "503"] - } - }, - { - "json": { - "field": "string_source" - } - }, - { - "rename": { - "field": "foo", - "target_field": "foobar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json deleted file mode 100644 index 054f6e3fbb2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase2.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{COMBINEDAPACHELOG}" - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "client.ip", - "target_field": "client.geo" - } - }, - { - "geoip": { - "field": "source.ip", - "target_field": "source.geo" - } - }, - { - "convert": { - "field" : "[client][bytes]", - "type": "integer" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json deleted file mode 100644 index d059b3536de..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase3.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{COMBINEDAPACHELOG}"], - "on_failure" : [ - { - "set" : { - "field" : "error", - "value" : "field does not exist" - } - } - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "clientip", - "target_field": "geo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json deleted file mode 100644 index c63bda67333..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestComplexCase4.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": ["%{COMBINEDAPACHELOG}"], - "on_failure" : [ - { - "set" : { - "field" : "error", - "value" : "field does not exist" - } - }, - { - "convert": { - "field" : "client.ip", - "type": "integer" - } - } - ] - } - }, - { - "date": { - "field": "timestamp", - "target_field": "@timestamp", - "formats": [ - "dd/MMM/YYYY:HH:mm:ss Z" - ], - "locale": "en" - } - }, - { - "geoip": { - "field": "clientip", - "target_field": "geo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json deleted file mode 100644 index 11b730dcdad..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvert.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "bytes", - "type": "integer" - } - } -] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json deleted file mode 100644 index 3904f5ab88d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertBoolean.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "delete", - "type": "boolean" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json deleted file mode 100644 index e61614c5648..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestConvertString.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "processors": [ - { - "convert": { - "field" : "blah", - "type": "string" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json deleted file mode 100644 index b93104bc28b..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDate.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json deleted file mode 100644 index 8a342f95a47..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDateExtraFields.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"], - "timezone" : "Europe/Amsterdam", - "locale": "en" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json deleted file mode 100644 index 70989b3546d..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInAppendField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "append": { - "field" : "client.ip", - "value": ["127.0.0.1", "127.0.0.2"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json deleted file mode 100644 index c8ac73b1b0a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInConvertField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "Pipeline to parse Apache logs", - "processors": [ - { - "convert": { - "field" : "client.bytes", - "type": "float" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json deleted file mode 100644 index 4a9a3576604..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInDateField.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "description" : "...", - "processors" : [ - { - "date" : { - "field" : "initial_date", - "target_field" : "apache.timestamp", - "formats" : ["dd/MM/yyyy hh:mm:ss", "dd/MM/yyyy"], - "timezone" : "Europe/Amsterdam", - "locale": "en" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json deleted file mode 100644 index 4a3b7fe18d3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInGeoIpField.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "description" : "Add geoip info", - "processors" : [ - { - "geoip" : { - "field" : "ip", - "target_field" : "apache.geo", - "database_file" : "GeoLite2-Country.mmdb.gz", - "properties": ["continent_name", "country_iso_code"] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json deleted file mode 100644 index ed4f41f1f34..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInJsonField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "[foo][string_source]", - "target_field": "[bar][json_target]" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json deleted file mode 100644 index 33aa582a655..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInRenameField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleRename", - "processors": [ - { - "rename": { - "field": "foo.bar", - "target_field": "foo.baz" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json deleted file mode 100644 index 7112b7131b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestDotsInSetField.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "foo.bar", - "value": "baz" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json deleted file mode 100644 index 2554b92c474..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGeoIpSimple.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "description": "Add geoip info", - "processors": [ - { - "geoip": { - "field": "ip", - "target_field": "geo", - "database_file": "GeoLite2-Country.mmdb.gz", - "properties": [ - "continent_name", - "country_iso_code" - ] - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json deleted file mode 100644 index 36db975a177..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrok.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "description": "Pipeline for parsing apache error logs", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"%{WORD:apache2.access.method} %{DATA:apache2.access.url} HTTP/%{NUMBER:apache2.access.http_version}\" %{NUMBER:apache2.access.response_code} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( \"%{DATA:apache2.access.referrer}\")?( \"%{DATA:apache2.access.agent}\")?", - "%{IPORHOST:apache2.access.remote_ip} - %{DATA:apache2.access.user_name} \\[%{HTTPDATE:apache2.access.time}\\] \"-\" %{NUMBER:apache2.access.response_code} -" - ], - "ignore_missing": true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json deleted file mode 100644 index e3a2ce60eda..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokMultiplePatternDefinitions.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "description":"Syslog", - "processors":[ - { - "grok":{ - "field":"message", - "patterns":[ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}", - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" - ], - "pattern_definitions":{ - "GREEDYMULTILINE":"(.|\\n)*", - "AUDIT_TYPE": "^type=%{NOTSPACE:auditd.log.record_type}" - }, - "ignore_missing":true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json deleted file mode 100644 index bb123eed1b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGrokPatternDefinition.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "description":"Syslog", - "processors":[ - { - "grok":{ - "field":"message", - "patterns":[ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}", - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" - ], - "pattern_definitions":{ - "GREEDYMULTILINE":"(.|\\n)*" - }, - "ignore_missing":true - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json deleted file mode 100644 index 14835baa028..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestGsubSimple.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "ExampleGsub", - "processors": [ - { - "gsub": { - "field": "field1", - "pattern": "\\.", - "replacement": "_" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json deleted file mode 100644 index 54a8adf6609..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJson.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "string_source" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json deleted file mode 100644 index e95bedc274a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestJsonExtraFields.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleJson", - "processors": [ - { - "json": { - "field": "string_source", - "target_field": "json_target" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json deleted file mode 100644 index 2f375b36ef3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseDots.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleLowercase", - "processors": [ - { - "lowercase": { - "field": "foo.bar" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json deleted file mode 100644 index 8ae738f1961..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestLowercaseSimple.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "description": "ExampleLowercase", - "processors": [ - { - "lowercase": { - "field": "foo" - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json deleted file mode 100644 index 7a2d89ed4ed..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestRename.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "ExampleRename", - "processors": [ - { - "rename": { - "field": "foo", - "target_field": "foobar" - } - } - ] -} \ No newline at end of file diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json deleted file mode 100644 index 3ae6f3c3912..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSet.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "field1", - "value": "bar" - } - } - ] -} - diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json b/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json deleted file mode 100644 index 70558bd5255..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/ingestSetNumber.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "description": "SetExample", - "processors": [ - { - "set": { - "field": "field1", - "value": 5344.4 - } - } - ] -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf deleted file mode 100644 index 40372d15cc4..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppend.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - mutate { - add_field => { - "client" => [ - "host1", - "host2" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf deleted file mode 100644 index f40d36e3810..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashAppendScalar.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "foo" => "bar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf deleted file mode 100644 index be27a7bc6f2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase1.conf +++ /dev/null @@ -1,46 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "[client][ip]" - target => "geo" - } - mutate { - convert => { - "bytes" => "integer" - } - } - mutate { - add_field => { - "response_code" => [ - "200", - "400", - "503" - ] - } - } - json { - source => "string_source" - } - mutate { - rename => { - "foo" => "foobar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf deleted file mode 100644 index 20b94771d0a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase2.conf +++ /dev/null @@ -1,33 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "[client][ip]" - target => "[client][geo]" - } - geoip { - source => "[source][ip]" - target => "[source][geo]" - } - mutate { - convert => { - "[client][bytes]" => "integer" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf deleted file mode 100644 index 20ae4380e4b..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase3.conf +++ /dev/null @@ -1,31 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - if "_grokparsefailure" in [tags] { - mutate { - add_field => { - "error" => "field does not exist" - } - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "clientip" - target => "geo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf deleted file mode 100644 index 4da212f20bd..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashComplexCase4.conf +++ /dev/null @@ -1,36 +0,0 @@ -filter { - grok { - match => { - "message" => "%{COMBINEDAPACHELOG}" - } - } - if "_grokparsefailure" in [tags] { - mutate { - add_field => { - "error" => "field does not exist" - } - } - mutate { - convert => { - "[client][ip]" => "integer" - } - } - } - date { - match => [ - "timestamp", - "dd/MMM/YYYY:HH:mm:ss Z" - ] - target => "@timestamp" - locale => "en" - } - geoip { - source => "clientip" - target => "geo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf deleted file mode 100644 index 1771fe204c8..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvert.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "bytes" => "integer" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf deleted file mode 100644 index 0d98c5a4720..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertBoolean.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "delete" => "boolean" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf deleted file mode 100644 index 6a872cf7a7e..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashConvertString.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "blah" => "string" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf deleted file mode 100644 index 0533a7f5296..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDate.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "timestamp" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf deleted file mode 100644 index 52cb3e471a3..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDateExtraFields.conf +++ /dev/null @@ -1,17 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "timestamp" - timezone => "Europe/Amsterdam" - locale => "en" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf deleted file mode 100644 index 8ee17721312..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInAppendField.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - mutate { - add_field => { - "[client][ip]" => [ - "127.0.0.1", - "127.0.0.2" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf deleted file mode 100644 index 8bbe66ce5b7..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInConvertField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - convert => { - "[client][bytes]" => "float" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf deleted file mode 100644 index 881d9e47b06..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInDateField.conf +++ /dev/null @@ -1,17 +0,0 @@ -filter { - date { - match => [ - "initial_date", - "dd/MM/yyyy hh:mm:ss", - "dd/MM/yyyy" - ] - target => "[apache][timestamp]" - timezone => "Europe/Amsterdam" - locale => "en" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf deleted file mode 100644 index fbcd2b2026a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInGeoIpField.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - geoip { - source => "ip" - target => "[apache][geo]" - fields => [ - "continent_name", - "country_iso_code" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf deleted file mode 100644 index 57032dfcb95..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInJsonField.conf +++ /dev/null @@ -1,11 +0,0 @@ -filter { - json { - source => "[foo][string_source]" - target => "[bar][json_target]" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf deleted file mode 100644 index aedde2200af..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInRenameField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - rename => { - "[foo][bar]" => "[foo][baz]" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf deleted file mode 100644 index 3b63d8f0fa2..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashDotsInSetField.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "[foo][bar]" => "baz" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf deleted file mode 100644 index 218e672c5b5..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGeoIpSimple.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - geoip { - source => "ip" - target => "geo" - fields => [ - "continent_name", - "country_iso_code" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf deleted file mode 100644 index ddf6150c10a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrok.conf +++ /dev/null @@ -1,15 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"%{WORD:[apache2][access][method]} %{DATA:[apache2][access][url]} HTTP/%{NUMBER:[apache2][access][http_version]}\" %{NUMBER:[apache2][access][response_code]} (?:%{NUMBER:apache2.access.body_sent.bytes}|-)( \"%{DATA:[apache2][access][referrer]}\")?( \"%{DATA:[apache2][access][agent]}\")?", - "%{IPORHOST:[apache2][access][remote_ip]} - %{DATA:[apache2][access][user_name]} \[%{HTTPDATE:[apache2][access][time]}\] \"-\" %{NUMBER:[apache2][access][response_code]} -" - ] - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf deleted file mode 100644 index e89065b7138..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokMultiplePatternDefinitions.conf +++ /dev/null @@ -1,19 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}", - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{GREEDYMULTILINE:[system][syslog][message]}" - ] - } - pattern_definitions => { - "GREEDYMULTILINE" => "(.|\n)*" - "AUDIT_TYPE" => "^type=%{NOTSPACE:[auditd][log][record_type]}" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf deleted file mode 100644 index 0a24c2676bb..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGrokPatternDefinition.conf +++ /dev/null @@ -1,18 +0,0 @@ -filter { - grok { - match => { - "message" => [ - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{SYSLOGHOST:[system][syslog][hostname]} %{DATA:[system][syslog][program]}(?:\[%{POSINT:[system][syslog][pid]}\])?: %{GREEDYMULTILINE:[system][syslog][message]}", - "%{SYSLOGTIMESTAMP:[system][syslog][timestamp]} %{GREEDYMULTILINE:[system][syslog][message]}" - ] - } - pattern_definitions => { - "GREEDYMULTILINE" => "(.|\n)*" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf deleted file mode 100644 index bc4092a8070..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashGsubSimple.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - gsub => [ - "field1", "\.", "_" - ] - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf deleted file mode 100644 index 1936d3c3453..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJson.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - json { - source => "string_source" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf deleted file mode 100644 index dfe7c2d48d0..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashJsonExtraFields.conf +++ /dev/null @@ -1,11 +0,0 @@ -filter { - json { - source => "string_source" - target => "json_target" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf deleted file mode 100644 index 9f070e3678a..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseDots.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - mutate { - lowercase => "[foo][bar]" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf deleted file mode 100644 index 49229ab26a6..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashLowercaseSimple.conf +++ /dev/null @@ -1,10 +0,0 @@ -filter { - mutate { - lowercase => "foo" - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf deleted file mode 100644 index daf5725ec25..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashRename.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - rename => { - "foo" => "foobar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf deleted file mode 100644 index 204cb54240f..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSet.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "field1" => "bar" - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} diff --git a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf b/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf deleted file mode 100644 index 36a98a3f567..00000000000 --- a/tools/ingest-converter/src/test/resources/org/logstash/ingest/logstashSetNumber.conf +++ /dev/null @@ -1,12 +0,0 @@ -filter { - mutate { - add_field => { - "field1" => 5344.4 - } - } -} -output { - elasticsearch { - hosts => "localhost" - } -} From 7c64c7394bf47e8b5316710876ed55350df46d61 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Tue, 17 Sep 2024 06:46:19 -0700 Subject: [PATCH 31/37] Fixes the issue where LS wipes out all quotes from docker env variables. (#16456) * Fixes the issue where LS wipes out all quotes from docker env variables. This is an issue when running LS on docker with CONFIG_STRING, needs to keep quotes with env variable. * Add a docker acceptance integration test. --- .../logstash/util/substitution_variables.rb | 22 ++++++- .../util/substitution_variables_spec.rb | 63 +++++++++++++++++++ .../shared_examples/container_options.rb | 36 +++++++++++ qa/docker/shared_examples/xpack.rb | 2 +- 4 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 logstash-core/spec/logstash/util/substitution_variables_spec.rb diff --git a/logstash-core/lib/logstash/util/substitution_variables.rb b/logstash-core/lib/logstash/util/substitution_variables.rb index 4319a38235f..61b938503e5 100644 --- a/logstash-core/lib/logstash/util/substitution_variables.rb +++ b/logstash-core/lib/logstash/util/substitution_variables.rb @@ -87,15 +87,33 @@ def replace_placeholders(value, refine) # ENV ${var} value may carry single quote or escaped double quote # or single/double quoted entries in array string, needs to be refined - refined_value = placeholder_value.gsub(/[\\"\\']/, '') + refined_value = strip_enclosing_char(strip_enclosing_char(placeholder_value, "'"), '"') if refined_value.start_with?('[') && refined_value.end_with?(']') # remove square brackets, split by comma and cleanup leading/trailing whitespace - refined_value[1..-2].split(',').map(&:strip) + refined_array = refined_value[1..-2].split(',').map(&:strip) + refined_array.each_with_index do |str, index| + refined_array[index] = strip_enclosing_char(strip_enclosing_char(str, "'"), '"') + end + refined_array else refined_value end end # def replace_placeholders + private + + # removes removed_char of string_value if string_value is wrapped with removed_char + def strip_enclosing_char(string_value, remove_char) + return string_value unless string_value.is_a?(String) + return string_value if string_value.empty? + + if string_value.start_with?(remove_char) && string_value.end_with?(remove_char) + string_value[1..-2] # Remove the first and last characters + else + string_value + end + end + class << self private diff --git a/logstash-core/spec/logstash/util/substitution_variables_spec.rb b/logstash-core/spec/logstash/util/substitution_variables_spec.rb new file mode 100644 index 00000000000..13c5c93c43b --- /dev/null +++ b/logstash-core/spec/logstash/util/substitution_variables_spec.rb @@ -0,0 +1,63 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "spec_helper" +require "logstash/util/substitution_variables" + +describe LogStash::Util::SubstitutionVariables do + + subject { Class.new { extend LogStash::Util::SubstitutionVariables } } + + context "ENV or Keystore ${VAR} with single/double quotes" do + # single or double quotes come from ENV/Keystore ${VAR} value + let(:xpack_monitoring_host) { '"http://node1:9200"' } + let(:xpack_monitoring_hosts) { "'[\"http://node3:9200\", \"http://node4:9200\"]'" } + let(:xpack_management_pipeline_id) { '"*"' } + let(:config_string) { + "'input { + stdin { } + beats { port => 5040 } + } + output { + elasticsearch { + hosts => [\"https://es:9200\"] + user => \"elastic\" + password => 'changeme' + } + }'" + } + + # this happens mostly when running LS with docker + it "stripes out quotes" do + expect(subject.send(:strip_enclosing_char, xpack_monitoring_host, '"')).to eql('http://node1:9200') + expect(subject.send(:strip_enclosing_char, xpack_monitoring_hosts, "'")).to eql('["http://node3:9200", "http://node4:9200"]') + expect(subject.send(:strip_enclosing_char, xpack_management_pipeline_id, '"')).to eql('*') + # make sure we keep the hosts, user and password param enclosed quotes + expect(subject.send(:strip_enclosing_char, config_string, "'")).to eql('input { + stdin { } + beats { port => 5040 } + } + output { + elasticsearch { + hosts => ["https://es:9200"] + user => "elastic" + password => \'changeme\' + } + }') + end + end +end \ No newline at end of file diff --git a/qa/docker/shared_examples/container_options.rb b/qa/docker/shared_examples/container_options.rb index 1d5108bff94..48c51d382b9 100644 --- a/qa/docker/shared_examples/container_options.rb +++ b/qa/docker/shared_examples/container_options.rb @@ -56,4 +56,40 @@ expect(get_settings(@container)['pipeline.unsafe_shutdown']).to be_truthy end end + + context 'when setting config.string' do + let(:options) { + { + 'ENV' => [ + 'USER=kimchy', + 'CONFIG_STRING=input { + beats { port => 5040 } + } + output { + elasticsearch { + hosts => ["https://es:9200"] + user => "${USER}" + password => \'changeme\' + } + }' + ] + } + } + + it "persists ${CONFIG_STRING} key in logstash.yml, resolves when running and spins up without issue" do + settings = get_settings(@container) + expect(settings['config.string']).to eq("${CONFIG_STRING}") + + pipeline_config = get_pipeline_stats(@container) + input_plugins = pipeline_config.dig('plugins', 'inputs') + expect(input_plugins[0].dig('name')).to eql('beats') + + output_plugins = pipeline_config.dig('plugins', 'outputs') + expect(output_plugins[0].dig('name')).to eql('elasticsearch') + + # check if logs contain the ES request with the resolved ${USER} + container_logs = @container.logs(stdout: true) + expect(container_logs.include?('https://kimchy:xxxxxx@es:9200')).to be true + end + end end diff --git a/qa/docker/shared_examples/xpack.rb b/qa/docker/shared_examples/xpack.rb index 82ff64ae065..254e0354ca9 100644 --- a/qa/docker/shared_examples/xpack.rb +++ b/qa/docker/shared_examples/xpack.rb @@ -35,7 +35,7 @@ ] } - it 'persists var keys into logstas.yaml and uses their resolved actual values' do + it 'persists var keys into logstash.yml and uses their resolved actual values' do container = create_container(@image, {'ENV' => env}) sleep(15) # wait for container run From 3e98cb16253b6135ed2cbd894ead3bed439f0185 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 17 Sep 2024 22:29:02 +0100 Subject: [PATCH 32/37] [CI] fix benchmark marathon (#16447) - split main.sh to core.sh and main.sh - rename all.sh to marathon.sh - fix vault expiry issue for long running task - fix unable to call main function - update save object runtime field `release` to return true when `version` contains "SNAPSHOT" --- ...ne.yml => benchmark_marathon_pipeline.yml} | 4 +- .buildkite/scripts/benchmark/all.sh | 18 - .buildkite/scripts/benchmark/core.sh | 315 +++++++++++++++++ .buildkite/scripts/benchmark/main.sh | 316 +----------------- .buildkite/scripts/benchmark/marathon.sh | 41 +++ .../benchmark/save-objects/CHANGELOG.md | 5 + .../save-objects/benchmark_objects.ndjson | 6 +- catalog-info.yaml | 20 +- 8 files changed, 379 insertions(+), 346 deletions(-) rename .buildkite/{benchmark_versions_pipeline.yml => benchmark_marathon_pipeline.yml} (68%) delete mode 100755 .buildkite/scripts/benchmark/all.sh create mode 100755 .buildkite/scripts/benchmark/core.sh create mode 100755 .buildkite/scripts/benchmark/marathon.sh create mode 100644 .buildkite/scripts/benchmark/save-objects/CHANGELOG.md diff --git a/.buildkite/benchmark_versions_pipeline.yml b/.buildkite/benchmark_marathon_pipeline.yml similarity index 68% rename from .buildkite/benchmark_versions_pipeline.yml rename to .buildkite/benchmark_marathon_pipeline.yml index d60166ac441..d2f8b657726 100644 --- a/.buildkite/benchmark_versions_pipeline.yml +++ b/.buildkite/benchmark_marathon_pipeline.yml @@ -7,5 +7,5 @@ agents: diskType: pd-ssd steps: - - label: "Benchmark multiple versions" - command: .buildkite/scripts/benchmark/all.sh \ No newline at end of file + - label: "Benchmark Marathon" + command: .buildkite/scripts/benchmark/marathon.sh \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/all.sh b/.buildkite/scripts/benchmark/all.sh deleted file mode 100755 index 282ed3d488a..00000000000 --- a/.buildkite/scripts/benchmark/all.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# ******************************************************* -# Run benchmark for versions that have flow metrics -# Usage: -# nohup bash -x all.sh > log.log 2>&1 & -# Accept env vars: -# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string -# ******************************************************* - -IFS=',' -STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" -read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" - -for V in "${STACK_VERSIONS[@]}" ; do - LS_VERSION="$V" "main.sh" -done \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/core.sh b/.buildkite/scripts/benchmark/core.sh new file mode 100755 index 00000000000..a7bb733713b --- /dev/null +++ b/.buildkite/scripts/benchmark/core.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -eo pipefail + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +CONFIG_PATH="$SCRIPT_PATH/config" +source "$SCRIPT_PATH/util.sh" + +usage() { + echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" + echo "Example: $0 4 {persisted|memory|all} 2 2" + exit 1 +} + +parse_args() { + while [[ "$#" -gt 0 ]]; do + if [ -z "$FB_CNT" ]; then + FB_CNT=$1 + elif [ -z "$QTYPE" ]; then + case $1 in + all | persisted | memory) + QTYPE=$1 + ;; + *) + echo "Error: wrong queue type $1" + usage + ;; + esac + elif [ -z "$CPU" ]; then + CPU=$1 + elif [ -z "$MEM" ]; then + MEM=$1 + else + echo "Error: Too many arguments" + usage + fi + shift + done + + # set default value + # number of filebeat + FB_CNT=${FB_CNT:-4} + # all | persisted | memory + QTYPE=${QTYPE:-all} + CPU=${CPU:-4} + MEM=${MEM:-4} + XMX=$((MEM / 2)) + + IFS=',' + # worker multiplier: 1,2,4 + MULTIPLIERS="${MULTIPLIERS:-1,2,4}" + read -ra MULTIPLIERS <<< "$MULTIPLIERS" + BATCH_SIZES="${BATCH_SIZES:-500}" + read -ra BATCH_SIZES <<< "$BATCH_SIZES" + # tags to json array + read -ra TAG_ARRAY <<< "$TAGS" + JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') + JSON_TAGS="[$JSON_TAGS]" + + IFS=' ' + echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" +} + +get_secret() { + VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} + VAULT_DATA=$(vault kv get -format json $VAULT_PATH) + BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') + BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') + BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') + + MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') + MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') + MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') +} + +pull_images() { + echo "--- Pull docker images" + + if [[ -n "$LS_VERSION" ]]; then + # pull image if it doesn't exist in local + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" + else + # pull the latest snapshot logstash image + # select the SNAPSHOT artifact with the highest semantic version number + LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) + BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') + ARCH=$(arch) + IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" + IMAGE_FILENAME="$LS_VERSION.tar.gz" + + echo "Download $LS_VERSION from $IMAGE_URL" + [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" + fi + + # pull filebeat image + FB_DEFAULT_VERSION="8.13.4" + FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} + docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" +} + +generate_logs() { + FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} + SINGLE_SIZE=524288000 + TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" + FLOG_PATH="$SCRIPT_PATH/flog" + mkdir -p $FLOG_PATH + + if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then + echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" + docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE + fi +} + +check_logs() { + echo "--- Check log generation" + + local cnt=0 + until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do + echo "wait 30s" && sleep 30 + cnt=$((cnt + 30)) + done + + ls -lah $FLOG_PATH +} + +start_logstash() { + LS_CONFIG_PATH=$SCRIPT_PATH/ls/config + mkdir -p $LS_CONFIG_PATH + + cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml + cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml + cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid + + LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} + docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ + -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ + -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ + -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ + -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ + -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ + -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ + docker.elastic.co/logstash/logstash:$LS_VERSION +} + +start_filebeat() { + for ((i = 0; i < FB_CNT; i++)); do + FB_PATH="$SCRIPT_PATH/fb${i}" + mkdir -p $FB_PATH + + cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml + + docker run -d --name=fb$i --net=host --user=root \ + -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ + -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ + docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false + done +} + +capture_stats() { + CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) + local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) + local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) + local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) + local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) + local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) + local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) + local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) + local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) + local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) + TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) + printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ + $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT +} + +aggregate_stats() { + local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" + MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) + MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) + MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) + MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) + MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) + MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) + + AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) + AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) + AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) + AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) +} + +send_summary() { + echo "--- Send summary to Elasticsearch" + + # build json + local timestamp + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") + SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" + not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" + not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" + not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" + not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" + not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" + not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" + not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" + not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" + not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" + not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" + not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" + not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" + SUMMARY="$SUMMARY}" + + tee summary.json << EOF +{"index": {}} +$SUMMARY +EOF + + # send to ES + local resp + local err_status + resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") + echo "$resp" + err_status=$(echo "$resp" | jq -r ".errors") + if [[ "$err_status" == "true" ]]; then + echo "Failed to send summary" + exit 1 + fi +} + +# $1: snapshot index +node_stats() { + NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json + + # curl inside container because docker on mac cannot resolve localhost to host network interface + docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null +} + +# $1: index +snapshot() { + node_stats $1 + capture_stats +} + +create_directory() { + NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 + mkdir -p "$SCRIPT_PATH/$NS_DIR" +} + +queue() { + for QTYPE in "persisted" "memory"; do + worker + done +} + +worker() { + for m in "${MULTIPLIERS[@]}"; do + WORKER=$((CPU * m)) + batch + done +} + +batch() { + for BATCH_SIZE in "${BATCH_SIZES[@]}"; do + run_pipeline + stop_pipeline + done +} + +run_pipeline() { + echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" + + start_logstash + start_filebeat + docker ps + + echo "(0) sleep 3m" && sleep 180 + snapshot "0" + + for i in {1..8}; do + echo "($i) sleep 30s" && sleep 30 + snapshot "$i" + + # print docker log when ingestion rate is zero + # remove '.' in number and return max val + [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && + docker logs fb0 && + docker logs ls + done + + aggregate_stats + send_summary +} + +stop_pipeline() { + echo "--- Stop Pipeline" + + for ((i = 0; i < FB_CNT; i++)); do + docker stop fb$i + docker rm fb$i + done + + docker stop ls + docker rm ls + + curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default + echo " data stream deleted " + + # TODO: clean page caches, reduce memory fragmentation + # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 +} + +clean_up() { + # stop log generation if it has not done yet + [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + # remove image + docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION +} diff --git a/.buildkite/scripts/benchmark/main.sh b/.buildkite/scripts/benchmark/main.sh index 9534ce3fc9a..3d615fcb41b 100755 --- a/.buildkite/scripts/benchmark/main.sh +++ b/.buildkite/scripts/benchmark/main.sh @@ -15,9 +15,8 @@ set -eo pipefail # - The script sends a summary of EPS and resource usage to index `benchmark_summary` # ******************************************************* -SCRIPT_PATH="$(cd "$(dirname "$0")"; pwd)" -CONFIG_PATH="$SCRIPT_PATH/config" -source "$SCRIPT_PATH/util.sh" +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" ## usage: ## main.sh FB_CNT QTYPE CPU MEM @@ -39,315 +38,6 @@ source "$SCRIPT_PATH/util.sh" ## FLOG_FILE_CNT=4 # number of files to generate for ingestion ## VAULT_PATH=secret/path # vault path point to Elasticsearch credentials. The default value points to benchmark cluster. ## TAGS=test,other # tags with "," separator. -usage() { - echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" - echo "Example: $0 4 {persisted|memory|all} 2 2" - exit 1 -} - -parse_args() { - while [[ "$#" -gt 0 ]]; do - if [ -z "$FB_CNT" ]; then - FB_CNT=$1 - elif [ -z "$QTYPE" ]; then - case $1 in - all | persisted | memory) - QTYPE=$1 - ;; - *) - echo "Error: wrong queue type $1" - usage - ;; - esac - elif [ -z "$CPU" ]; then - CPU=$1 - elif [ -z "$MEM" ]; then - MEM=$1 - else - echo "Error: Too many arguments" - usage - fi - shift - done - - # set default value - # number of filebeat - FB_CNT=${FB_CNT:-4} - # all | persisted | memory - QTYPE=${QTYPE:-all} - CPU=${CPU:-4} - MEM=${MEM:-4} - XMX=$((MEM / 2)) - - IFS=',' - # worker multiplier: 1,2,4 - MULTIPLIERS="${MULTIPLIERS:-1,2,4}" - read -ra MULTIPLIERS <<< "$MULTIPLIERS" - BATCH_SIZES="${BATCH_SIZES:-500}" - read -ra BATCH_SIZES <<< "$BATCH_SIZES" - # tags to json array - read -ra TAG_ARRAY <<< "$TAGS" - JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') - JSON_TAGS="[$JSON_TAGS]" - - IFS=' ' - echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" -} - -get_secret() { - VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} - VAULT_DATA=$(vault kv get -format json $VAULT_PATH) - BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') - BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') - BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') - - MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') - MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') - MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') -} - -pull_images() { - echo "--- Pull docker images" - - if [[ -n "$LS_VERSION" ]]; then - # pull image if it doesn't exist in local - [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" - else - # pull the latest snapshot logstash image - # select the SNAPSHOT artifact with the highest semantic version number - LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) - BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') - ARCH=$(arch) - IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" - IMAGE_FILENAME="$LS_VERSION.tar.gz" - - echo "Download $LS_VERSION from $IMAGE_URL" - [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" - [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" - fi - - # pull filebeat image - FB_DEFAULT_VERSION="8.13.4" - FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} - docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" -} - -generate_logs() { - FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} - SINGLE_SIZE=524288000 - TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" - FLOG_PATH="$SCRIPT_PATH/flog" - mkdir -p $FLOG_PATH - - if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then - echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" - docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE - fi -} - -check_logs() { - echo "--- Check log generation" - - local cnt=0 - until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do - echo "wait 30s" && sleep 30 - cnt=$((cnt + 30)) - done - - ls -lah $FLOG_PATH -} - -start_logstash() { - LS_CONFIG_PATH=$SCRIPT_PATH/ls/config - mkdir -p $LS_CONFIG_PATH - - cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml - cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml - cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid - - LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} - docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ - -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ - -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ - -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ - -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ - -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ - -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ - docker.elastic.co/logstash/logstash:$LS_VERSION -} - -start_filebeat() { - for ((i = 0; i < FB_CNT; i++)); do - FB_PATH="$SCRIPT_PATH/fb${i}" - mkdir -p $FB_PATH - - cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml - - docker run -d --name=fb$i --net=host --user=root \ - -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ - -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ - docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false - done -} - -capture_stats() { - CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) - local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) - local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) - local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) - local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) - local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) - local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) - local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) - local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) - local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) - TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) - printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ - $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT -} - -aggregate_stats() { - local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" - MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) - MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) - MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) - MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) - MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) - MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) - - AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) - AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) - AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) - AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) -} - -send_summary() { - echo "--- Send summary to Elasticsearch" - - # build json - local timestamp - timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") - SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" - not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" - not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" - not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" - not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" - not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" - not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" - not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" - not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" - not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" - not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" - not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" - not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" - SUMMARY="$SUMMARY}" - - tee summary.json << EOF -{"index": {}} -$SUMMARY -EOF - - # send to ES - local resp - local err_status - resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") - echo "$resp" - err_status=$(echo "$resp" | jq -r ".errors") - if [[ "$err_status" == "true" ]]; then - echo "Failed to send summary" - exit 1 - fi -} - -# $1: snapshot index -node_stats() { - NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json - - # curl inside container because docker on mac cannot resolve localhost to host network interface - docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null -} - -# $1: index -snapshot() { - node_stats $1 - capture_stats -} - -create_directory() { - NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 - mkdir -p "$SCRIPT_PATH/$NS_DIR" -} - -queue() { - for QTYPE in "persisted" "memory"; do - worker - done -} - -worker() { - for m in "${MULTIPLIERS[@]}"; do - WORKER=$((CPU * m)) - batch - done -} - -batch() { - for BATCH_SIZE in "${BATCH_SIZES[@]}"; do - run_pipeline - stop_pipeline - done -} - -run_pipeline() { - echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" - - start_logstash - start_filebeat - docker ps - - echo "(0) sleep 3m" && sleep 180 - snapshot "0" - - for i in {1..8}; do - echo "($i) sleep 30s" && sleep 30 - snapshot "$i" - - # print docker log when ingestion rate is zero - # remove '.' in number and return max val - [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && - docker logs fb0 && - docker logs ls - done - - aggregate_stats - send_summary -} - -stop_pipeline() { - echo "--- Stop Pipeline" - - for ((i = 0; i < FB_CNT; i++)); do - docker stop fb$i - docker rm fb$i - done - - docker stop ls - docker rm ls - - curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default - echo " data stream deleted " - - # TODO: clean page caches, reduce memory fragmentation - # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 -} - -clean_up() { - # stop log generation if it has not done yet - [[ -n $(docker ps | grep flog) ]] && docker stop flog || true - # remove image - docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION -} - main() { parse_args "$@" get_secret @@ -366,4 +56,4 @@ main() { clean_up } -main "$@" +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/marathon.sh b/.buildkite/scripts/benchmark/marathon.sh new file mode 100755 index 00000000000..4878a2d3959 --- /dev/null +++ b/.buildkite/scripts/benchmark/marathon.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -eo pipefail + +# ******************************************************* +# Run benchmark for versions that have flow metrics +# When the hardware changes, run the marathon task to establish a new baseline. +# Usage: +# nohup bash -x all.sh > log.log 2>&1 & +# Accept env vars: +# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string +# ******************************************************* + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" + +parse_stack_versions() { + IFS=',' + STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" + read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" +} + +main() { + parse_stack_versions + parse_args "$@" + get_secret + generate_logs + check_logs + + for V in "${STACK_VERSIONS[@]}" ; do + LS_VERSION="$V" + pull_images + create_directory + if [[ $QTYPE == "all" ]]; then + queue + else + worker + fi + done +} + +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md new file mode 100644 index 00000000000..335acb0ac0c --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md @@ -0,0 +1,5 @@ +## 20240912 +Updated runtime field `release` to return `true` when `version` contains "SNAPSHOT" + +## 20240912 +Initial dashboards \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson index e68a8fc77a4..ef4511357d3 100644 --- a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson +++ b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson @@ -1,4 +1,4 @@ -{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.endsWith(\\\"-SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzcyNiw3XQ=="} +{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.contains(\\\"SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-12T20:34:35.037Z","updated_by":"u_4132377203_cloud","version":"WzE1NjYsOF0="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":100},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Release EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:29.581Z","id":"2c017667-7403-402c-b027-3cc790945a27","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:29.581Z","version":"WzEzOTMsN10="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:13.413Z","id":"9546a73b-5937-4d3d-850f-30625d6e1405","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:13.413Z","version":"WzEzOTEsN10="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"2786f33f-2ad1-4722-8706-9cba9613662e","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:00.945Z","id":"6fba7dff-2564-450b-b885-8c6cf97fa727","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"2786f33f-2ad1-4722-8706-9cba9613662e","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:00.945Z","version":"WzEzODksN10="} @@ -17,8 +17,8 @@ {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"columns":{"0ad565fc-0a55-492b-ba69-1be57f501e13":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"962f0beb-0dd8-4449-a652-878d0d2c63e8","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"4d1f7bef-c184-4457-b159-465128d2d70b","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"962f0beb-0dd8-4449-a652-878d0d2c63e8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOSw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Miw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Niw3XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzEyNTEsN10="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzEyNDUsN10="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzE0ODksOF0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzE0OTAsOF0="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"07700ebe-b8c9-498f-9a91-30efb0c48784","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MCw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e0f2401-369e-4818-8a1f-52b64ff175df","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"29ebb6b1-bee6-4f15-b3dc-aac083d5d2f5","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e0f2401-369e-4818-8a1f-52b64ff175df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NSw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"26ec195e-e4a9-4417-be97-6d1fbc6b63c0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0OSw3XQ=="} diff --git a/catalog-info.yaml b/catalog-info.yaml index 9f1dfa94dd6..c5367ea6691 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -614,7 +614,7 @@ spec: kind: Pipeline metadata: name: logstash-benchmark-pipeline - description: ':logstash: The Benchmark pipeline for snapshot version' + description: ':running: The Benchmark pipeline for snapshot version' spec: repository: elastic/logstash pipeline_file: ".buildkite/benchmark_pipeline.yml" @@ -646,18 +646,18 @@ spec: # ******************************* # *********************************** -# SECTION START: Benchmark multiple Logstash versions +# SECTION START: Benchmark Marathon # *********************************** --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: logstash-benchmark-versions-pipeline - description: Buildkite pipeline for benchmarking multiple versions + name: logstash-benchmark-marathon-pipeline + description: Buildkite pipeline for benchmarking multi-version links: - - title: 'Logstash benchmark pipeline for multiple versions' - url: https://buildkite.com/elastic/logstash-benchmark-versions-pipeline + - title: 'Logstash Benchmark Marathon' + url: https://buildkite.com/elastic/logstash-benchmark-marathon-pipeline spec: type: buildkite-pipeline owner: group:logstash @@ -666,11 +666,11 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: logstash-benchmark-versions-pipeline - description: ':logstash: The Benchmark pipeline for multiple versions' + name: logstash-benchmark-marathon-pipeline + description: ':running: The Benchmark Marathon for multi-version' spec: repository: elastic/logstash - pipeline_file: ".buildkite/benchmark_versions_pipeline.yml" + pipeline_file: ".buildkite/benchmark_marathon_pipeline.yml" maximum_timeout_in_minutes: 480 provider_settings: trigger_mode: none # don't trigger jobs from github activity @@ -690,7 +690,7 @@ spec: access_level: READ_ONLY # ******************************* -# SECTION END: Benchmark multiple Logstash versions +# SECTION END: Benchmark Marathon # ******************************* # *********************************** From b54caf3fd8e907c526ab2b8897ce4de4656c2fd5 Mon Sep 17 00:00:00 2001 From: Luca Belluccini Date: Thu, 19 Sep 2024 19:32:14 +0200 Subject: [PATCH 33/37] Known issue for 8.15.1 related to env vars references (#16455) --- docs/static/releasenotes.asciidoc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index fa8b56b79a5..81c1db94cef 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -69,6 +69,15 @@ This section summarizes the changes in the following releases: [[logstash-8-15-1]] === Logstash 8.15.1 Release Notes +[[known-issues-8-15-1]] +==== Known issues + +* **{ls} may fail to start under some circumstances.** Single and double quotes are stripped from a pipeline configuration if the configuration includes environment or keystore variable references. +If this situation occurs, {ls} may fail to start or some plugins may use a malformed configuration. +Check out issue https://github.com/elastic/logstash/issues/16437[#16437] for details. ++ +Workaround: Downgrade to {ls} 8.15.0, or temporarily avoid using environment and keystore variable references. + [[notable-8.15.1]] ==== Performance improvements and notable issues fixed @@ -2475,4 +2484,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 \ No newline at end of file +Plugins align with release 7.14.0 From 0ef4c7da3280dca2175d00886d3e19f23e2deb9b Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Fri, 20 Sep 2024 19:58:20 +0100 Subject: [PATCH 34/37] [ci] fix wrong queue type in benchmark marathon (#16465) --- .buildkite/scripts/benchmark/marathon.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/scripts/benchmark/marathon.sh b/.buildkite/scripts/benchmark/marathon.sh index 4878a2d3959..eecbc7d8e24 100755 --- a/.buildkite/scripts/benchmark/marathon.sh +++ b/.buildkite/scripts/benchmark/marathon.sh @@ -26,8 +26,11 @@ main() { generate_logs check_logs + USER_QTYPE="$QTYPE" + for V in "${STACK_VERSIONS[@]}" ; do LS_VERSION="$V" + QTYPE="$USER_QTYPE" pull_images create_directory if [[ $QTYPE == "all" ]]; then From 51cca7320e5c54865ab3fe2d4101496bd69cacca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 25 Sep 2024 13:15:14 +0100 Subject: [PATCH 35/37] bump .ruby_version to jruby-9.4.8.0 (#16477) --- .ruby-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ruby-version b/.ruby-version index 57fc792f86a..dac625a8779 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -jruby-9.3.10.0 +jruby-9.4.8.0 From 5c57adebb9707e761886b9aa90db21feebd85838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Duarte?= Date: Wed, 25 Sep 2024 14:38:52 +0100 Subject: [PATCH 36/37] simplify snyk scanning (#16475) * remove docker image scanning as that's handled by infosec * run buildkite job on a docker image instead of vm (no need to test docker any more) --- .buildkite/scripts/snyk/report.sh | 73 ----------------------------- .buildkite/snyk_report_pipeline.yml | 9 ++-- 2 files changed, 4 insertions(+), 78 deletions(-) diff --git a/.buildkite/scripts/snyk/report.sh b/.buildkite/scripts/snyk/report.sh index 7dab7c5e357..0d666f777cc 100755 --- a/.buildkite/scripts/snyk/report.sh +++ b/.buildkite/scripts/snyk/report.sh @@ -4,12 +4,6 @@ set -e TARGET_BRANCHES=("main") -install_java() { - # TODO: let's think about using BK agent which has Java installed - # Current caveat is Logstash BK agent doesn't support docker operatioins in it - sudo apt update && sudo apt install -y openjdk-17-jdk && sudo apt install -y openjdk-17-jre -} - # Resolves the branches we are going to track resolve_latest_branches() { source .buildkite/scripts/snyk/resolve_stack_version.sh @@ -55,7 +49,6 @@ report() { ./snyk monitor --prune-repeated-subdependencies --all-projects --org=logstash --remote-repo-url="$REMOTE_REPO_URL" --target-reference="$REMOTE_REPO_URL" --detection-depth=6 --exclude=qa,tools,devtools,requirements.txt --project-tags=branch="$TARGET_BRANCH",git_head="$GIT_HEAD" || : } -install_java resolve_latest_branches download_auth_snyk @@ -72,69 +65,3 @@ do echo "$TARGET_BRANCH branch doesn't exist." fi done - -# Scan Logstash docker images and report -REPOSITORY_BASE_URL="docker.elastic.co/logstash/" - -report_docker_image() { - image=$1 - project_name=$2 - platform=$3 - echo "Reporting $image to Snyk started..." - docker pull "$image" - if [[ $platform != null ]]; then - ./snyk container monitor "$image" --org=logstash --platform="$platform" --project-name="$project_name" --project-tags=version="$version" || : - else - ./snyk container monitor "$image" --org=logstash --project-name="$project_name" --project-tags=version="$version" || : - fi -} - -report_docker_images() { - version=$1 - echo "Version value: $version" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT" - snyk_project_name="logstash-$version-SNAPSHOT" - report_docker_image "$image" "$snyk_project_name" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT" - snyk_project_name="logstash-oss-$version-SNAPSHOT" - report_docker_image "$image" "$snyk_project_name" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-arm64" - snyk_project_name="logstash-$version-SNAPSHOT-arm64" - report_docker_image "$image" "$snyk_project_name" "linux/arm64" - - image=$REPOSITORY_BASE_URL"logstash:$version-SNAPSHOT-amd64" - snyk_project_name="logstash-$version-SNAPSHOT-amd64" - report_docker_image "$image" "$snyk_project_name" "linux/amd64" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-arm64" - snyk_project_name="logstash-oss-$version-SNAPSHOT-arm64" - report_docker_image "$image" "$snyk_project_name" "linux/arm64" - - image=$REPOSITORY_BASE_URL"logstash-oss:$version-SNAPSHOT-amd64" - snyk_project_name="logstash-oss-$version-SNAPSHOT-amd64" - report_docker_image "$image" "$snyk_project_name" "linux/amd64" -} - -resolve_version_and_report_docker_images() { - git reset --hard HEAD # reset if any generated files appeared - git checkout "$1" - - # parse version (ex: 8.8.2 from 8.8 branch, or 8.9.0 from main branch) - versions_file="$PWD/versions.yml" - version=$(awk '/logstash:/ { print $2 }' "$versions_file") - report_docker_images "$version" -} - -# resolve docker artifact and report -#for TARGET_BRANCH in "${TARGET_BRANCHES[@]}" -#do -# if git show-ref --quiet refs/heads/"$TARGET_BRANCH"; then -# echo "Using $TARGET_BRANCH branch for docker images." -# resolve_version_and_report_docker_images "$TARGET_BRANCH" -# else -# echo "$TARGET_BRANCH branch doesn't exist." -# fi -#done \ No newline at end of file diff --git a/.buildkite/snyk_report_pipeline.yml b/.buildkite/snyk_report_pipeline.yml index 2a1476bcb5a..4f541256ae6 100644 --- a/.buildkite/snyk_report_pipeline.yml +++ b/.buildkite/snyk_report_pipeline.yml @@ -1,9 +1,8 @@ agents: - provider: gcp - imageProject: elastic-images-prod - image: family/platform-ingest-logstash-ubuntu-2204 - machineType: "n2-standard-4" - diskSizeGb: 120 + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" + cpu: "2" + memory: "4Gi" + ephemeralStorage: "64Gi" steps: # reports main, previous (ex: 7.latest) and current (ex: 8.latest) release branches to Snyk From f35e10d79251b4ce3a5a0aa0fbb43c2e96205ba1 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Thu, 26 Sep 2024 18:25:32 +0200 Subject: [PATCH 37/37] Updated CI releases inventory after 8.15.2 (#16474) --- ci/logstash_releases.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index a5aff98f558..0e63fc37ab0 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -3,11 +3,11 @@ "5.x": "5.6.16", "6.x": "6.8.23", "7.x": "7.17.24", - "8.x": "8.15.1" + "8.x": "8.15.2" }, "snapshots": { "7.x": "7.17.25-SNAPSHOT", - "8.15": "8.15.2-SNAPSHOT", + "8.15": "8.15.3-SNAPSHOT", "8.x": "8.16.0-SNAPSHOT", "main": "9.0.0-SNAPSHOT" }