From 86308888773f522e22f5be6dec3fbf53c94ff1ee Mon Sep 17 00:00:00 2001 From: extreme4all <> Date: Wed, 3 Jan 2024 19:26:31 +0100 Subject: [PATCH] fixed data in kafka --- docker-compose.yml | 2 +- kafka_setup/kafka_data/kafka_data.zip | Bin 0 -> 1908 bytes .../kafka_data/kafka_data/kafka_data.json | 877 ------------------ kafka_setup/setup_kafka.py | 3 +- src/app/schemas/highscores.py | 108 --- src/app/schemas/report.py | 18 +- src/main.py | 9 +- 7 files changed, 19 insertions(+), 998 deletions(-) create mode 100644 kafka_setup/kafka_data/kafka_data.zip delete mode 100644 kafka_setup/kafka_data/kafka_data/kafka_data.json delete mode 100644 src/app/schemas/highscores.py diff --git a/docker-compose.yml b/docker-compose.yml index 22e2370..22ca863 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -75,7 +75,7 @@ services: worker: container_name: worker - image: bot-detector/highscore_worker + image: bot-detector/report-worker build: context: . dockerfile: Dockerfile diff --git a/kafka_setup/kafka_data/kafka_data.zip b/kafka_setup/kafka_data/kafka_data.zip new file mode 100644 index 0000000000000000000000000000000000000000..401c9317bb24e445291de587f7856929ffda0d94 GIT binary patch literal 1908 zcmV-)2aEVnO9KQH000080B)%wSal^MWAp|90ODW(01p5F0Bd1pYhhnxVRT_GYIARH z?VM3l<2De--|tgoc4?FfB0cL#LB*urBeCSgw=6(awGMT3m})bQQ@V z#C9%uxMR>FJI+t*|Ngbk@9#$S>+h$3_3NaPrK(o4I{9{V^4;S^Ra04!lW+bPkm-Qy zS(z7ixj8(|RWZ)>IAIZ|VC>WQR;fylgi#C!9-D!f6EN`C^FRQ8yIShWMZhQxMJT|? zA}!YGe4NRZyj{KctNN@KaxszBa{QpG`O^uS!{vSXJOuyp>38E|t^Zh`BV}Mv{<+S} zg)H>pn(NpMjb}2={=FU{Pr%y-G3jDGm#Z&P#dUta3*RF3b{`C94^ z=^Y5!7UG}Oy+?=rAnT*y%LalhSb)z2L9W+C6by#s2z;al$;N;O$znJe`ABjwFOvQ;*;)fU#6hGT08z}OnoP;3nW5L;s) z!`7e4f2f&cN4;`dZ9Jr}=cc+{7@%1Tst@2asU_ z2iy!ZqZI5n!-PaYy@L>wVT!9*=unUlj{ycnkl@M_FkFjM{Ct2t5ipC-0quz(!cPZ6 z9LF)fHX+1}@Mxb9=Zi8gu4h&Kza3(pKpO%;OyD~*AdXv!2~PQNsyafL8)9y$)^^Wu z&7P~C(+oT7IX{MB?E_-E%)@S%*phTm;3D4)iS>u7_-VL3#ItX765Mx5|3<{mKdLlfiwX?Qo zjL$z@j4ot#x0)$yUHA6JTAjb)5&8|XUq04v%mwmL^RW>4!3>9(CY@zF-VNK|}m#pnr`L3wQuw!6ICk zK!RzISat|IMEGVVZ_@j;+JfS*F%Q@B(vlh5=3zY(E>sOwY)P>O#5}=I$xy{H4yd7u zd5kYBlVTC#hwGAJ^(ntS%cuEOYPBdf^#H*p0ka1P4!BW1xE>%Ml8+wSP8hz-3wf$9 z$5sHtBy9lP&LkcZz$75<-U-5lv$ErsI=|tUQG61BZ;XqCwk2KQQPI^m%--pUA7znM z>F8#r7WRGTO#s6I2rv{@9sq~X!R$G}#|(@Fb;xk7g(qscB)WaU&hQouY)Pxfk{BD% zma`Yb_4~qQY;PGRB(dxyLx`i$atG=V;-4#3%4&3~w%M!OQAszjrisX-2b1WRi(x$# zev5}SUF2g-L<~1ez-FEB=)?4Kh?BrliFWA_pQ$39XJ>LMJM8`$){b2eTV!hcj(Cc% z2-r<5oZZB=7IQ(Y4PtyXv~|}Q_YlSVZvD}H6ThYT$90DY56WKiMA+THMtR`M#FSa= z*mej<96mGyLJo%Dtvi2gZ4>QU;u(PtzW1GAuMKCPqHeGI&4=CjbFR|*r~0Nw=~i`} zB@?%;>$qXY^_$~kUDq>)Uu{lUA#P`|i2)_!Bg8#}__|OJ=T%;0n=E1o3ABjeHoW8P?=JK;X1)+m>~F}V9yUtESoOIUbBcI{BT`M;;xIj zL-Zw%MctRzehB_$omO|&G)%a_(=bg!%a^awxFy2yv+Oy-UP8jBsOy)I80=2nM5a}# z)W@y(g#!k^?W(!JfT`d7QWjejqmhEcFv0UH%)#|kE+wI3oPBFL2)`}BJr0XXFdycy z9L$gs%;y_Z_Kk5q&Dd=|pWD9$b3;W3yZ&DDr9a_-Y>9edyXCM?A|}KsC43UG#}}sT zED;lGDFqBELE>bO1iz8976uR=H^Raw8vBi~mzFr1690|AJ;Sec){Jy(IBSq@C&ZkH z;cV0$A>KK}@Ba7)P)h*<6aW+e000O8ZmA+zbtNNX^acO`;$Q#(4*&oF000000RR91 u00000003)YW@}+zWMOn+E^2dcZcs}F1^@s60096206hQz03ine0002sRg}&E literal 0 HcmV?d00001 diff --git a/kafka_setup/kafka_data/kafka_data/kafka_data.json b/kafka_setup/kafka_data/kafka_data/kafka_data.json deleted file mode 100644 index c9d763b..0000000 --- a/kafka_setup/kafka_data/kafka_data/kafka_data.json +++ /dev/null @@ -1,877 +0,0 @@ -[ - { - "reporter": "Cyborger1", - "reported": "Shpinki", - "region_id": 14652, - "x_coord": 3682, - "y_coord": 3851, - "z_coord": 0, - "ts": 1704223737, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13592, - "equip_amulet_id": null, - "equip_torso_id": 13596, - "equip_legs_id": 13598, - "equip_boots_id": 13602, - "equip_cape_id": 13594, - "equip_hands_id": 13600, - "equip_weapon_id": 1381, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Loveskippy", - "region_id": 14652, - "x_coord": 3685, - "y_coord": 3870, - "z_coord": 0, - "ts": 1704223743, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 27446, - "equip_amulet_id": 10132, - "equip_torso_id": 27452, - "equip_legs_id": 27455, - "equip_boots_id": 27461, - "equip_cape_id": 27449, - "equip_hands_id": 27458, - "equip_weapon_id": 7409, - "equip_shield_id": 13660 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Loveskippy", - "region_id": 14652, - "x_coord": 3666, - "y_coord": 3866, - "z_coord": 0, - "ts": 1704223777, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 27446, - "equip_amulet_id": 10132, - "equip_torso_id": 27452, - "equip_legs_id": 27455, - "equip_boots_id": 27461, - "equip_cape_id": 27449, - "equip_hands_id": 27458, - "equip_weapon_id": 7409, - "equip_shield_id": 13660 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Spooned Tuna", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224138, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "IM Hale", - "region_id": 4922, - "x_coord": 1249, - "y_coord": 3758, - "z_coord": 0, - "ts": 1704224088, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": 9810, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "NewdogsNtown", - "region_id": 12850, - "x_coord": 3232, - "y_coord": 3211, - "z_coord": 0, - "ts": 1704224293, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1710, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Smooth Biggs", - "region_id": 12851, - "x_coord": 3214, - "y_coord": 3274, - "z_coord": 0, - "ts": 1704224470, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 1167, - "equip_amulet_id": 1725, - "equip_torso_id": 1135, - "equip_legs_id": 1099, - "equip_boots_id": 4097, - "equip_cape_id": 10498, - "equip_hands_id": 1065, - "equip_weapon_id": 8880, - "equip_shield_id": 4156 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "SimpinShrimp", - "region_id": 12595, - "x_coord": 3180, - "y_coord": 3288, - "z_coord": 0, - "ts": 1704225257, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 1147, - "equip_amulet_id": 552, - "equip_torso_id": 1135, - "equip_legs_id": 1099, - "equip_boots_id": 1061, - "equip_cape_id": 10499, - "equip_hands_id": 1065, - "equip_weapon_id": 853, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Shpinki", - "region_id": 14651, - "x_coord": 3682, - "y_coord": 3837, - "z_coord": 0, - "ts": 1704223741, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13592, - "equip_amulet_id": null, - "equip_torso_id": 13596, - "equip_legs_id": 13598, - "equip_boots_id": 13602, - "equip_cape_id": 13594, - "equip_hands_id": 13600, - "equip_weapon_id": 1381, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Enkanooda", - "region_id": 14652, - "x_coord": 3687, - "y_coord": 3861, - "z_coord": 0, - "ts": 1704223741, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 25071, - "equip_amulet_id": null, - "equip_torso_id": 25077, - "equip_legs_id": 25080, - "equip_boots_id": 25086, - "equip_cape_id": 13123, - "equip_hands_id": 25083, - "equip_weapon_id": 7409, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Merkthon", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704223832, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13592, - "equip_amulet_id": 1704, - "equip_torso_id": 13596, - "equip_legs_id": 13598, - "equip_boots_id": 13602, - "equip_cape_id": 13594, - "equip_hands_id": 13600, - "equip_weapon_id": 772, - "equip_shield_id": 12954 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Spooned Tuna", - "region_id": 4922, - "x_coord": 1249, - "y_coord": 3739, - "z_coord": 0, - "ts": 1704224027, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Ribet", - "region_id": 4922, - "x_coord": 1248, - "y_coord": 3728, - "z_coord": 0, - "ts": 1704224027, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 11851, - "equip_amulet_id": null, - "equip_torso_id": 11855, - "equip_legs_id": 11857, - "equip_boots_id": 11861, - "equip_cape_id": 9810, - "equip_hands_id": 11859, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Shpinki", - "region_id": 4922, - "x_coord": 1247, - "y_coord": 3727, - "z_coord": 0, - "ts": 1704224027, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13592, - "equip_amulet_id": null, - "equip_torso_id": 13596, - "equip_legs_id": 13598, - "equip_boots_id": 13602, - "equip_cape_id": 13594, - "equip_hands_id": 13600, - "equip_weapon_id": 1381, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "TittyJugs", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224064, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 11851, - "equip_amulet_id": 11968, - "equip_torso_id": 11855, - "equip_legs_id": 11857, - "equip_boots_id": 11861, - "equip_cape_id": 11853, - "equip_hands_id": 11859, - "equip_weapon_id": 7409, - "equip_shield_id": 13660 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Enkanooda", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224138, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 25071, - "equip_amulet_id": null, - "equip_torso_id": 25077, - "equip_legs_id": 25080, - "equip_boots_id": 25086, - "equip_cape_id": 13123, - "equip_hands_id": 25083, - "equip_weapon_id": 7409, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "V1BER", - "region_id": 4922, - "x_coord": 1249, - "y_coord": 3758, - "z_coord": 0, - "ts": 1704224088, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 11865, - "equip_amulet_id": 19547, - "equip_torso_id": 27238, - "equip_legs_id": 27241, - "equip_boots_id": 13237, - "equip_cape_id": 22109, - "equip_hands_id": 7462, - "equip_weapon_id": 20997, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "TheTavar", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224138, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13580, - "equip_amulet_id": null, - "equip_torso_id": 13584, - "equip_legs_id": 13586, - "equip_boots_id": 13590, - "equip_cape_id": 13582, - "equip_hands_id": 13588, - "equip_weapon_id": 20730, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "ToxicDigiRat", - "region_id": 12850, - "x_coord": 3225, - "y_coord": 3215, - "z_coord": 0, - "ts": 1704224293, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "NineBoots", - "region_id": 12850, - "x_coord": 3222, - "y_coord": 3219, - "z_coord": 0, - "ts": 1704224293, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 1017, - "equip_amulet_id": 1478, - "equip_torso_id": 581, - "equip_legs_id": 428, - "equip_boots_id": 89, - "equip_cape_id": 11853, - "equip_hands_id": 1059, - "equip_weapon_id": 1387, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Kendra Thoma", - "region_id": 12850, - "x_coord": 3220, - "y_coord": 3262, - "z_coord": 0, - "ts": 1704224310, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Theboss12237", - "region_id": 12595, - "x_coord": 3180, - "y_coord": 3296, - "z_coord": 0, - "ts": 1704224432, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1706, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": 11972, - "equip_weapon_id": 6317, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Proper Bot", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704223832, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 13647, - "equip_amulet_id": 11111, - "equip_torso_id": 13643, - "equip_legs_id": 13641, - "equip_boots_id": 13645, - "equip_cape_id": 9799, - "equip_hands_id": 11859, - "equip_weapon_id": 9084, - "equip_shield_id": 13660 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "DonaldDegen", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224027, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 10828, - "equip_amulet_id": 1725, - "equip_torso_id": 1127, - "equip_legs_id": 1079, - "equip_boots_id": 3105, - "equip_cape_id": 13121, - "equip_hands_id": 7462, - "equip_weapon_id": 4587, - "equip_shield_id": 12954 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Waifu", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224138, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 10828, - "equip_amulet_id": 1704, - "equip_torso_id": 2503, - "equip_legs_id": 2497, - "equip_boots_id": 3105, - "equip_cape_id": 6570, - "equip_hands_id": 7462, - "equip_weapon_id": 4587, - "equip_shield_id": 12954 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Ribet", - "region_id": 4922, - "x_coord": 1247, - "y_coord": 3737, - "z_coord": 0, - "ts": 1704224146, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 11851, - "equip_amulet_id": null, - "equip_torso_id": 11855, - "equip_legs_id": 11857, - "equip_boots_id": 11861, - "equip_cape_id": 9810, - "equip_hands_id": 11859, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Goated12722", - "region_id": 12595, - "x_coord": 3183, - "y_coord": 3296, - "z_coord": 0, - "ts": 1704224519, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1706, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": 11972, - "equip_weapon_id": 6317, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Theboss12237", - "region_id": 12595, - "x_coord": 3183, - "y_coord": 3292, - "z_coord": 0, - "ts": 1704224746, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1706, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": 11972, - "equip_weapon_id": 6317, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "SnowGrind", - "region_id": 12851, - "x_coord": 3209, - "y_coord": 3278, - "z_coord": 0, - "ts": 1704224770, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 11865, - "equip_amulet_id": 1725, - "equip_torso_id": 544, - "equip_legs_id": 542, - "equip_boots_id": 11840, - "equip_cape_id": 22114, - "equip_hands_id": 22981, - "equip_weapon_id": 27660, - "equip_shield_id": 12954 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Waifu", - "region_id": 4922, - "x_coord": 1253, - "y_coord": 3741, - "z_coord": 0, - "ts": 1704224027, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 10828, - "equip_amulet_id": 1704, - "equip_torso_id": 2503, - "equip_legs_id": 2497, - "equip_boots_id": 3105, - "equip_cape_id": 6570, - "equip_hands_id": 7462, - "equip_weapon_id": 4587, - "equip_shield_id": 12954 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "GIM Quark", - "region_id": 11571, - "x_coord": 2936, - "y_coord": 3281, - "z_coord": 0, - "ts": 1704224003, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": null, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": null, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "bearpoof", - "region_id": 11571, - "x_coord": 2931, - "y_coord": 3286, - "z_coord": 0, - "ts": 1704224024, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 24872, - "equip_amulet_id": 13393, - "equip_torso_id": 24874, - "equip_legs_id": 24876, - "equip_boots_id": 24878, - "equip_cape_id": 9781, - "equip_hands_id": 13676, - "equip_weapon_id": 24880, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "bigd4cilly", - "region_id": 12595, - "x_coord": 3190, - "y_coord": 3274, - "z_coord": 0, - "ts": 1704224319, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": 1155, - "equip_amulet_id": null, - "equip_torso_id": 1103, - "equip_legs_id": null, - "equip_boots_id": 1061, - "equip_cape_id": null, - "equip_hands_id": null, - "equip_weapon_id": 1265, - "equip_shield_id": 1189 - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Goated12722", - "region_id": 12595, - "x_coord": 3183, - "y_coord": 3300, - "z_coord": 0, - "ts": 1704224751, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1706, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": 11972, - "equip_weapon_id": 6317, - "equip_shield_id": null - }, - "equip_ge_value": 0 - }, - { - "reporter": "Cyborger1", - "reported": "Theboss12237", - "region_id": 12595, - "x_coord": 3174, - "y_coord": 3293, - "z_coord": 0, - "ts": 1704225178, - "manual_detect": 0, - "on_members_world": 1, - "on_pvp_world": 0, - "world_number": 324, - "equipment": { - "equip_head_id": null, - "equip_amulet_id": 1706, - "equip_torso_id": null, - "equip_legs_id": null, - "equip_boots_id": null, - "equip_cape_id": null, - "equip_hands_id": 11972, - "equip_weapon_id": 6317, - "equip_shield_id": null - }, - "equip_ge_value": 0 - } -] \ No newline at end of file diff --git a/kafka_setup/setup_kafka.py b/kafka_setup/setup_kafka.py index 8f0faf6..da08dc0 100644 --- a/kafka_setup/setup_kafka.py +++ b/kafka_setup/setup_kafka.py @@ -51,7 +51,7 @@ def create_topics(): def extract_zip(extract_to: str): current_dir = "./kafka_data" # Get the current working directory - + print(os.listdir(current_dir)) # Find zip file in the current directory zip_files = [file for file in os.listdir(current_dir) if file.endswith(".zip")] @@ -107,6 +107,7 @@ def send_messages(producer: KafkaProducer, send_queue: Queue, topic: str = "scra if send_queue.qsize() % 100 == 0: print(f"{send_queue.qsize()=}") message = send_queue.get() + print(message) producer.send(topic=topic, value=message) send_queue.task_done() diff --git a/src/app/schemas/highscores.py b/src/app/schemas/highscores.py deleted file mode 100644 index 6b757ab..0000000 --- a/src/app/schemas/highscores.py +++ /dev/null @@ -1,108 +0,0 @@ -from datetime import date, datetime -from typing import Optional - -from pydantic import BaseModel, ConfigDict - - -class playerHiscoreData(BaseModel): - model_config = ConfigDict(from_attributes=True) - - id: Optional[int] = None - timestamp: datetime = datetime.utcnow() - ts_date: Optional[date] = None - Player_id: int - total: int - attack: int - defence: int - strength: int - hitpoints: int - ranged: int - prayer: int - magic: int - cooking: int - woodcutting: int - fletching: int - fishing: int - firemaking: int - crafting: int - smithing: int - mining: int - herblore: int - agility: int - thieving: int - slayer: int - farming: int - runecraft: int - hunter: int - construction: int - league: int - bounty_hunter_hunter: int - bounty_hunter_rogue: int - cs_all: int - cs_beginner: int - cs_easy: int - cs_medium: int - cs_hard: int - cs_elite: int - cs_master: int - lms_rank: int - soul_wars_zeal: int - abyssal_sire: int - alchemical_hydra: int - barrows_chests: int - bryophyta: int - callisto: int - cerberus: int - chambers_of_xeric: int - chambers_of_xeric_challenge_mode: int - chaos_elemental: int - chaos_fanatic: int - commander_zilyana: int - corporeal_beast: int - crazy_archaeologist: int - dagannoth_prime: int - dagannoth_rex: int - dagannoth_supreme: int - deranged_archaeologist: int - general_graardor: int - giant_mole: int - grotesque_guardians: int - hespori: int - kalphite_queen: int - king_black_dragon: int - kraken: int - kreearra: int - kril_tsutsaroth: int - mimic: int - nightmare: int - nex: int = 0 - phosanis_nightmare: int - obor: int - phantom_muspah: int = 0 - sarachnis: int - scorpia: int - skotizo: int - tempoross: int = 0 - the_gauntlet: int - the_corrupted_gauntlet: int - theatre_of_blood: int - theatre_of_blood_hard: int = 0 - thermonuclear_smoke_devil: int - tombs_of_amascut: int = 0 - tombs_of_amascut_expert: int = 0 - tzkal_zuk: int - tztok_jad: int - venenatis: int - vetion: int - vorkath: int - wintertodt: int - zalcano: int - zulrah: int - rifts_closed: int = 0 - artio: int = 0 - calvarion: int = 0 - duke_sucellus: int = 0 - spindel: int = 0 - the_leviathan: int = 0 - the_whisperer: int = 0 - vardorvis: int = 0 diff --git a/src/app/schemas/report.py b/src/app/schemas/report.py index a48ed2f..ae7d7d2 100644 --- a/src/app/schemas/report.py +++ b/src/app/schemas/report.py @@ -4,15 +4,15 @@ class Equipment(BaseModel): - equip_head_id: Optional[int] - equip_amulet_id: Optional[int] - equip_torso_id: Optional[int] - equip_legs_id: Optional[int] - equip_boots_id: Optional[int] - equip_cape_id: Optional[int] - equip_hands_id: Optional[int] - equip_weapon_id: Optional[int] - equip_shield_id: Optional[int] + equip_head_id: Optional[int] = None + equip_amulet_id: Optional[int] = None + equip_torso_id: Optional[int] = None + equip_legs_id: Optional[int] = None + equip_boots_id: Optional[int] = None + equip_cape_id: Optional[int] = None + equip_hands_id: Optional[int] = None + equip_weapon_id: Optional[int] = None + equip_shield_id: Optional[int] = None class Report(BaseModel): diff --git a/src/main.py b/src/main.py index 1b16a09..e54e4ac 100644 --- a/src/main.py +++ b/src/main.py @@ -43,6 +43,9 @@ async def process_data(receive_queue: Queue, error_queue: Queue): # Get a message from the chosen queue message: dict = await receive_queue.get() + print(message) + await asyncio.sleep(1) + continue try: # Acquire an asynchronous database session @@ -72,9 +75,11 @@ async def process_data(receive_queue: Queue, error_queue: Queue): async def main(): + TOPIC = "report" + GROUP = "report-worker" # get kafka engine consumer = await kafka.kafka_consumer( - topic="report", group="report-worker", bootstrap_servers=[settings.KAFKA_HOST] + topic=TOPIC, group=GROUP, bootstrap_servers=[settings.KAFKA_HOST] ) producer = await kafka.kafka_producer(bootstrap_servers=[settings.KAFKA_HOST]) @@ -87,7 +92,7 @@ async def main(): ) ) asyncio.create_task( - kafka.send_messages(topic="scraper", producer=producer, send_queue=send_queue) + kafka.send_messages(topic=TOPIC, producer=producer, send_queue=send_queue) ) asyncio.create_task( process_data(receive_queue=receive_queue, error_queue=send_queue)