From abaff8c71f0af51719a4bc9ae4454a035417f2e8 Mon Sep 17 00:00:00 2001 From: Jacob Cable Date: Fri, 27 Sep 2024 17:21:46 +0100 Subject: [PATCH] chore(firestore-bigquery-export): temporarily disable GCS --- firestore-bigquery-export/README.md | 4 - firestore-bigquery-export/extension.yaml | 38 +++---- .../__snapshots__/config.test.ts.snap | 1 + ...restore-bigquery-change-tracker-1.1.37.tgz | Bin 0 -> 14193 bytes .../functions/package-lock.json | 5 +- .../functions/package.json | 2 +- .../functions/src/cloud_storage_backups.ts | 16 +-- .../functions/src/config.ts | 4 +- .../functions/src/index.ts | 55 ++++----- .../functions/stress_test/count.js | 33 ++++++ .../functions/stress_test/main.js | 104 ++++++++++++++++++ .../functions/stress_test/worker.js | 59 ++++++++++ .../scripts/gen-schema-view/package-lock.json | 2 +- .../scripts/import/package-lock.json | 2 +- 14 files changed, 262 insertions(+), 63 deletions(-) create mode 100644 firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz create mode 100644 firestore-bigquery-export/functions/stress_test/count.js create mode 100644 firestore-bigquery-export/functions/stress_test/main.js create mode 100644 firestore-bigquery-export/functions/stress_test/worker.js diff --git a/firestore-bigquery-export/README.md b/firestore-bigquery-export/README.md index b77bdfc69..6e5c6612d 100644 --- a/firestore-bigquery-export/README.md +++ b/firestore-bigquery-export/README.md @@ -162,10 +162,6 @@ essential for the script to insert data into an already partitioned table.) * Maximum number of enqueue attempts: This parameter will set the maximum number of attempts to enqueue a document to cloud tasks for export to BigQuery. If the maximum number of attempts is reached, the failed export will be handled according to the `LOG_FAILED_EXPORTS` parameter. -* Backup to GCS: If enabled, failed BigQuery updates will be written to a GCS bucket. - -* Backup GCS Bucket Name: This (optional) parameter will allow you to specify a GCS bucket for which failed BigQuery updates will be written to, if this feature is enabled. - **Cloud Functions:** diff --git a/firestore-bigquery-export/extension.yaml b/firestore-bigquery-export/extension.yaml index 23694257f..0fbaf7561 100644 --- a/firestore-bigquery-export/extension.yaml +++ b/firestore-bigquery-export/extension.yaml @@ -425,25 +425,25 @@ params: validationErrorMessage: Please select an integer between 1 and 10 default: 3 - - param: BACKUP_TO_GCS - label: Backup to GCS - description: >- - If enabled, failed BigQuery updates will be written to a GCS bucket. - type: select - options: - - label: Yes - value: yes - - label: No - value: no - default: no - required: true - - - param: BACKUP_GCS_BUCKET - label: Backup GCS Bucket Name - description: >- - This (optional) parameter will allow you to specify a GCS bucket for which - failed BigQuery updates will be written to, if this feature is enabled. - type: string + # - param: BACKUP_TO_GCS + # label: Backup to GCS + # description: >- + # If enabled, failed BigQuery updates will be written to a GCS bucket. + # type: select + # options: + # - label: Yes + # value: yes + # - label: No + # value: no + # default: no + # required: true + + # - param: BACKUP_GCS_BUCKET + # label: Backup GCS Bucket Name + # description: >- + # This (optional) parameter will allow you to specify a GCS bucket for which + # failed BigQuery updates will be written to, if this feature is enabled. + # type: string events: - type: firebase.extensions.firestore-counter.v1.onStart diff --git a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap index 73a6a8cc1..b9cb5f541 100644 --- a/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap @@ -5,6 +5,7 @@ Object { "backupBucketName": "undefined.appspot.com", "backupCollectionId": undefined, "backupDir": "_firestore-bigquery-export", + "backupToGCS": false, "bqProjectId": undefined, "clustering": Array [ "data", diff --git a/firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz b/firestore-bigquery-export/functions/firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz new file mode 100644 index 0000000000000000000000000000000000000000..44c71a95366d6d134b9781af0fed5332dedf7aee GIT binary patch literal 14193 zcmV-%H;%|3iwFP!00002|LuMKR~tF9=={uIq4?%x+F8@)BOh;I*(?KQhP?@7fy}%) z*<5JcGVV#c+iA5ST;l)zohp5+)omNd%;qheITL6}rBX>M)t5@+sQ)p#pxrb%@17?Y zpC+`p?)HbY|MA^8j!MeH-`RI}{8?F9dGYKS@qJ%eS$Veha)o@i{`|$t>gtOZE6>Px zE3405tgVvoR=)cKekQDpitkod?qQHCD=S3(`;GjxCX5nR7D>NsZ7v<2|4#d57}G(L z(YHlDrbT%j(9h$%C|QTJPERR&lgE>kwmRg7Tt?}HZjiE=P%>?AE{TfaWkRox=%-0i zP{x-L*&?#AjLuV9DZR{-n5=9r_4ACCWE>S`QYLwpWEZEaWQ!EMe$WcLW?8GfxfHSM ztWO4$tPdZF2YhhDzjjDJ&j!gwhs1e*GNM_zAMfO8?{mUnbmOQTzKx1##Eg%58t+DB zWPF*n$;}cWgmj;hV_Hte1h)$#PrLY;kKl%kXc@&(8IdX3;{9)mW>3iuv>^R-!b(~s z*@fMfW4Xgi$rky?@!>%TbCg{qgX@47hq`rp(`bx6)wY#L8LeN1gM_9rBVWFd_a6kT zWIzHNjYnHs7VI|GgP($ylOn@a;EwPrN#lN0#ELio_t7IU0$I8rGX(FMfVp~+vq_Se z^z(Ew%B=1@lRX#MJdICr0tHNpO)b#w>X-|zxq$77w@S81Mz5Tm5WY434CHFYX|WbT z!hVU;Bv!S_7KyH+#O@(OKP`QIZdE<-o5Ylhh^=7OWrY9uH}&8f<*~}uZQxuZ>dSd!{k-SKY#f`TCGi|#V}&8=|$AP{-V#wVl+BbxY z#Ty)EMd&8eO$&sC9$HZJA9igRv%oXfBM$ssg}>)X7PLC8c6-yueBT!NC}A`#D9h7J zCAJ|q-n2h-i21glC!dWvmDSC#Q?O18v)HRXh1#Py03F69qsL9;#$%q@)zOU#;EZF9*_ zR|5;)4__tq(^(NiZ)|Sx{sULzH6b=_3ChsM=UL+@_*7(3h%zAn18z(9T&w-a zi7Ax#a1@P$;4!ZL$6*$YsEx)3dH=!fEip-9@v3l;q_8Rkz(*P{Emc1@nGtG;COENH z+s;-r=h9#jcGCs99h^SqvRzt$^ixChw7ANS!MFcod|f0L!;-A6to}fL%JYkqlGm?ymI#)=PWm)sG$xZQrUh#L z+vBJ|q_U(#exU^e`^;LnLISAJ5=E$hKPA`sgp8tV)La1rV60@2q?FLleL4o_MEdz? zoF-A$r{pRrhuDvR5(13h1dRN=jFOCiL1TPP@_|{AM5RC)4x6$ZvyE=|>gp8Os1DG@!C1q6Nr^>jX zqwala6#Ze8T~K~PoLrAhe}Y-`CXcJm1W$!PooF3${s}%xWdeT3Ll10OJz$8z8MY*k zZ&afxT3F&_fP#oPx5+4vX-IxdF1Q^E{@fssZ#ewvCOPI}ko>Ouo1WE-RAB3V@(oN7 z#U2=Zlmmm`y#w2*j!Xa+<0`Mm=D6TbJ}v;M$0fh(?rO&+kaa&jSatqQ>_5XOi&MH6 zB`J+hiYQ|dTH9|c8q7HVSDvl9=l{#qXAkH9w`c#6T1-LDBSuX#kNmbA#iJx^S#?>N z7u5Fh+SR+N7Bob4OI8S@@z_;1R9TWGWfG;y|I+R8SSpLEQ~?dzA)_Un+AO#s$wihI z^j*ej5DOwClo{jHN<&Ns(IhRk9iqsunD#qS!yLf}9G?T1Ug?-44=-rBC;w~*mM8B# zD*MCjqKLq;@jgBe@k7uS+FVLY0tn(EypvBdZq`}RrC5JIi|J?m9gut#4)UTG^@joA z?3hh1qk-jwDf1Hd_8Q+iFVVw5?Mwn!b7UjNa=F#ZC*NA)o zdItp^bR6|C48tP7>R8`UDLIX486_#(AT%6N#-a_`y|g^=QNKgbf2dU5+ZNX<;Jh8-SiPt=!51 z1L}dAV~~6`0w!wQpoL{&KOc>fGH9E|k$TJ3^x;VUPdxt!p*UIvko(^c)AKLKY`!aslb-B{y zhPv^uMlIo>uhbLYb|i$1?4_n zQ|+M8`(H6{x*tzN4(o(d?I^kumJHYe+yS--`2Q*-HDBjE?hXwxq7xwR26+wRD=`pe zQP8|8fZ-tb2PDd_37ecV`e_2>MB4L-AOn!Hk*HTmnv$q2>1bTS%oJ3sQ{1%2{q=M4 z`Z`a3#dY88WIWX<#AHiG4_Q^tKZ(Fe$?ijmm+37ORl{+S!_L`{RgdBMr?=|kmoH>o z3BW7jb~w zfSI5xXt0mTct1-@w|~RAt%(_U{bvr>{uz&oVve{}1CNS}6I4$)*>MEiYl*+t24$#@ zegR>FY*i2f?fI=$NkH86K)W$QK|7#t>flUt@M25z!8ca~=x9ssyNk`ZNPG#X1N>O;p{A1j?EvpaFGvMAWOqUxc`N>vBi? zyuAuYo`?|be1~uuKVG;5&Ba?=%?b`OV~n3XsqMM4oS!@)HyDt7I>rwb;_A9>GWF1M zYJW1e#5#5#RO(_fdsVgl03Ny~U+qQz5+_c(1(Q=r_8>BVPS=p*i%XuDO zSLX6Y>>uR0zR4WYD5eG5AUCA7lV>H(%4OJ{tqsx=eojEB3;e28nR+jSl|=XlGj1Ko zk0AhZBci%pa*`7^8KVHwkJ1z+C@F!mj*$IOga=TQly7w(Q#QMQ74pijur_)1$|}W= z4PhfdG;@J@w!RxVgzM{iO!EOzKTg0E+G^LGX*%G{oqcw3W>YV(6Emon(^Ou#;8oP6ks(Ewe1t+aIQr$`#}%5gxW7w!$AXJLc1GGIvkA$Nd>tqw_rjJ}~PHHHL)4s5D*^uUg21!Tcv3l@x4lyjgYRm~JQh;F0_FjP}HgRdqcD z)S6QhxLHusiJ~FYk$xrQE>>$*(j;hM#Z+l>6&PuaK4{@$4%lA7&FpgOVO^*ZYKV3H z+S{gv%G26|zjACE@M^drnm8EVM#={zn0KO&6xAMJAl#ONx0rxAM;B2^6))T-!zdk) zh%nmEvzWv*jjn;8aSk}9g^&`ih7d}Ib|Y~tN)Y`#E0b&zr70MPmc`ZGNmeE))-U84 z5)+}GBQBhmozs3kq9nSE5)@KG56e0QiDxe__N8`>(;ieKj3syxRV?il!a@~Pr0Kr2 zxEU*x*au=GM~n|1rz2 zxH9IiS43CBSn7JO9gI8PVEW?P%T(uI56)$8@Y3^2JnlVcS{ZH6tqo``j@Rr0X)hxx zHhs1uewhR7%ZcK{ZH7(b8qkKIi+=2Y`XpCA?NUK&5)6E-PA-}n2Irs(u#;MR-K&p| zmdZ(T<692?6~^6XVqrV|OkGu}_qk6gFuQCrI;RC}y8^Lg9m3$6Ws*txQpasni22B- zrZ=p^Lq)CD*87*VlFJEUbO7-%GlqAht;`@oLKJYsI46%Oh?P9tR4n|2K{RDDNaVHB z1+2wt+Rp`_I?;=XOA`T=LeQoIyvVNz_1kGpv0u0)Y-T7O5;~??3JO9DL`mN@@_UTx zQ++I=M6SIgNp_ikjJPQwDUtOtl347XM2t;F;@&o+23k=8{$McvgSJw`sI}qPG~ibl zn=-hVQ&F=$Cq>q@HmbmDt=Tmk(}pKm4HplqkU4)1KeKpIioE>V)6TncUe{cd8XbM9hDB0F;GJ z5OFob&`aPlYf0jqvh4ekL@77u@N^!{Y+G^2J1ZscMFXBE>Wk)^v)8VV9XBoC=GE&tyDIDWcFOYY_JCgrvoNUc z8?5B+8m?MZ_sp%bLs^2inIftt;Q<Ht3#B|goW}i(WZ&jq z)GJ%LXo$x5I$j&a@d3TsL4+F3A|2v|%N}`*Sy1U+z9tM5}$`+_F#M~$uSfd z(FY5UQ@b|4#!wx{Nwc}(=~MDfq{!ktT%~MG2KvsglOaSz;tumk8mkOd+Av}Vq55*# zwSy|fcs?7SRxn5|tPC{OO1|87l}f%`cT1ia89Vn~qrV>(i%ACGN&phWQR+*1SY@tO za>6K%&ExDO=Ru}{pJ98}67((7ojJ3MmKT0Lv#=f=ez-8>aH-+&c0!*dyHE|9SFhhA zGuNcgJ@Io-{_F+dTLjbjWbdjBpk(YiImy^-GU8vSWO^Tlq1yY^xM;B`KDarW+yI~% ze9X?UvLHK~`VLX;oFIcDp;??>A7CV&MFq*%W}*La#QvKi03Y_Ev-d5U^^5BYos8%;^bUxc>kZ{DzXXxzAU?m<;_*5qwe#InH;Y$}Mt?j-NApaH4_UfljrfMO)i z(rZ~!h0ZVuEgz65O?B{z+@B1+h_za1v!-cU_wH;e!QlNmSDo2YRau0luC55*YE?xi zKLx6|OTX@dd>(Se7%ha2Sg2`H?Q2ra^NF&HUz)gc+B zF%#iz=`|T7pWW^2sDZ&kdSLtC>k#4UmCO=mrvJjbp5K%0%3M07SrZ&zWEs-0l~tT==Gz zi~jDLe(?=|>&;%b(T$6K8`TSe)%y3r6sGOXX1U6CJykZ5Jq*cqGw6hy1TH zIk&nwe&@VF3-WrIW(cyu$V1jm8Dpjvnl<2gx?MOki$-j&VcjNg%q?@q7?}O(`DXWI1&@g7pChrnv6ul&Fi2-tXe(pSR5V zjLeBz4;!0iC4AHDAgEu+4?ffDxZ2a^dv_G;u0djHy~US)pPQ?Ct;djT+kxCii#{a<4M{16B*x|PiTCPml{yCfouaPn`FG_4qnC_C)vdr z&#v7x>dH4G*R7w1fxb90-$>H@A~=)n)ioyL;-OA@=^$K-gTJ5&b-qCEfV*`E>$b>q zENV(>&{ymKcF4OpJ9oGi8G^#Z``tx<#%>Sh-3ZjYNLWaC`r&FBV|&JLetXcLfyq48 z+`-M{eF1ly-DaYEacVrGWaD4i;~V^~Zl-49HD5({xW3~c$mVBe_5k|=hVWkF_#WQ? zxaq&Lnt}N+s4a8R_a1qHa`vi_vV+@>h{t}T0*NTZt4yC@(O9^8R6`FVA0k|LRJ+znKYw%YB`0em-!WQf)lQEcMfPl@oD>S^{ zy||m&#qE&S5bjKU6B&O&(-`iuf(#J%H44uv(k1RVnl(q24fF22Xu@c*ERu6-WBubB zvxmCTH6uAkt$9bySL2-qZ5|q5W6ahXt*_*c*b!_%Ai4AKWO@q;tZpcNk)}bIaSSI4 z+YVV+7eEN>>Ei%D$~QcrBH?;)hLQ3u{jb)@u?TR!iEvnNnVvQ z-&hT4;5JLFUKm(ptyLuf<7+L}&RV-CF<@c5t81%H1aQoox5B(0&$*6&2L<0}AJ*2{ ze;c%$j{Zd`?mPFb?+w`BVZAgS`nSNpx@M{ezF}w&7QnT&BiiA+yo7$H{R!TBw2?BP z%U@;3j^Q8D+2Nafix5n;e|#gGPa$1qz}j}gOi4wDW)2SKpH(!q@8VXt%I4_D1bA~J zO#;ko2(vlisSU-Y9oDWLk4CogtvWKXWN$wM0N~D3z_GCYHc_=*PUb_*1Np#lM^#+N z(!A%hF~e(>!mo%>Ln2fCst29%4d#w|D@ocD)zi$*DD43_U*t%ovkkIoEo3DWiL91Q{xo8r7>SX;r?gt4woJ9>9b$S?vM#qm(RcI z%1Opb8pW=lQirbhyK+|8{!p%{iRE+~>EwH%kWIfgtzwZ{S<~;uzIB}wHXi1m2C6mA z-LHnVy65fqU>ZhIgJMld<(a^GA9rBAp+l^_FB(TH9$_SqS1@5E(1X@bWZp;BzCd zxsiw+vw}@nnU7R@TEIkJyM3gF137|buU$nFGU5zx16SDT+wG$h3}f0q_~~?SzxR6g z%<2InfA6`6)05xc_Ka@a88Gwx-&?;&z2lR^qnQpb*`9aQ4(m_7SV%XQ7Ey)A4 z7)^PyVR5Rpyc@7J*KVm<;`3Mb8Yi9bq`-VwVj(_h2powL;H1F8vp>iQF#DMCLdEQj zcXaqGFwkv$j%mZZpgyxLe?|h{W7m!Nw}#gW$%>Kz>&9+5Y_LQaU)2+m7Kqd>^UY+E4f5d303bjLhaSCx^in0BX7ZM;I0=|5;`)%O ze3O{_0__$ylvj`!Pzf4SYi;AczAN99dfXg1$Z=k1+w34jL%gcK=_8O2b-It~sr;5y zLKSA9*i^i^q^!?1F_eLEjki7b7^RvcuKSXG(9jJ?Q-1mi+~914%rL_u&mdhfj&Esp zaA!?Pbx?NTXfXdc4syZ=!skUjUV?9;s8>qJu?RY{awB<1Q3+x7em6$Y93#8HP%6vL z5cZ)HJBJ*|qR?kZ_VNP7RcUhQU<|UV%paN=!pjJ8l%&@kGRm`Z2*2b7xuy|hjJYVH zY?4MrQeMwRteJtq%@wK(z{n#d+2g!)dzVRxEwa*A3Goul9Eu~7ILELw+|Iy-tmg-i z;NDeXv7*X1rIP#Ntfl%VFE}rLfcK*bTt(4{7dsx@24DUh zhIo7kZm%P{xsgyRti!t&5r2Voz}jK)X6%g0N50U3B^7T4)H$v?xyZeQ(#`|AuFVC`%pY_orw~5^`y8To!7<-QKYzc6M`Er@? zXLh-r50i}Z!>Cn#6B_bYfyt2D3(>~G0{8XMkvfX%6WI9j&nt4**AI+@I-7e2wi50;A0#`=!fnqe;dKD_8bz9%GHs75A zI0;j=UMC+VmR^F`jjPy$9Mvt85i98^8h23CjzP5I4^#8ejEg*;aBYA%v%-h;F2|DM=kdO*QREB%Ib{mLBni$v!c#Y zfH!APP9?dM@l>562kxOAw8S@&qpNrT0s-+gDs_+}Jygx)%5VyeTNZ~OP<_J=gdoCWmr(Rq@I``c(DL9E6_ewoBH zHab?6JWSLBS2VVOKK-=Lb~fYK)=>aW&8 zspN1cFuB4KZ#AEQyqe0&?(phUnkhK8eLKO{g7)&l zo<>`WyR8^U?pu!YQCB{`EO*CPZ|~|BG5W@MPxBpB4|i~tF{5xEbj@6+Go)s7j{13a z$sh2h7=p*$Fu%A8>K)EGO~;6#!ei)O&YU$Lc*4pQ+7U@m9Lf{S!T)2pd zG2pPw_m2;S0jAv!FL_L&kAXNt`U=BYa45CI2-)PEn}{HD_3FyptK`j3Y!lb=9m`mbk}A*l`b_$ z5eE&8rJb$HGwTeI4e-s^VFY~DcF&wFQGAl0__$OxU%I4~_XP!uesU4n$7mBe+J3p< zYhpm0feXe*68}v5F&%jDn(?Tanp7}3$5FwJs{RSR-J(O`e^wlFAa>wlA?Th-cQ&1n zj8J&uy>T^6J)`vggDbO&vkXrYErWA|g>g<)WW*YYs!5ts#OEg+^vHX#R5s6!r&ebtX0p( z9$C_{tLxta2N_Q1M4`=%LXCia2Tb%?%~#I+)Tr9I?gb4HL2ieHGz#7e8eD5^%{by_ zcwg!;sXk9-W9E5Dsuoijfu3h-Wl)9PBm}-K(+~xKtzT2Mu(cq4nMZ zm6HZ7eCA4PG+x|+2^ppqsRPvDQiaNsVr!tPp~-TiMrK<%l;rUuI#J;9WHIJsE>7;q zvI3?{E){fQ2M6vZoGg*^5Gn)+$`@@O-{^3BpDE9)tnznDAr#&Mi#RX&p5(@-o8;R$ z7qmQ}SGH8qHXW+wD>#**;YGdeIHnE)`p)Fc)izbPG06U|YGK8ayKA87QCGha?T<5` zZPs+pgMZ9RyKcy~x98eJdV#K+&2>(J#DWEgkSun}Rt$Ob$Oet*%nH_Ud8yddo}dm; z8Lr39`_xq@sh?x=7iICeXY`80omrWf6&N($1W)J{b2(avC#p}FFfk2iR89)Wr5@o` z9K1z|>@b{r%-%D%Y2XzxDh$KA`c;`raORm9&5*FnSF;=h?Jz&-1h8xwn1DnC0?EMi z#bdaZY8_oN9H1H|$Zd~SNCGc?w6(><=8TfoDWm;7i&+cqRLW>99U^Ezf3NF);5}uN z(K#(zHoPp#@nmX@8Z!@2X6E8u# zK&uZWdkV!X+eS&6XBSC1iD{)ED!t!J%Qa%+HKs-68!j6Bt38_@fJ&J$K>I$9yu~j| zF?o0g1rfL`8%u8izeE_wTZHh1%ZN~C1OE?Bz1G3I*RQvKeBBc^I)t_)MVrnN%xbqw zPKK0GL+@aifd<#ldBhSX_wiW~U7d-n4Y28!d;_wE5VW8O0{V_mj`k0Jg5MNQ#**?; z%LOE=oWdY%h)2DDzS}?Q?V1psX>ovI>1h(rhEth$2a+nE4XU!X98fw)w?pixNE4cO zzM_Hx>y8!k(6I$S(fn z`?1i4^h&3g%7@(%UAWFG!!Qa8M+VLCnumO6o@WdamJ3gEnIvX5_6|?A@Xpitl<(0y z(|7Ek_TwVlvLHs+@!IpGospl{}p!qvWW z)Psxzsv#3o$Hk{#sf-xm$LTeMxQGx-h>(FKxP5UOy@Tc#49UmOJcohID+kcPIy$68 zQU|HUkJg#zo5`v-$a|&D*%kybDF!F7iv6==s;e6o=dg;a6V}g1qr^;@hZB`B5s?GC zV+XER>O~fkeLk1gJng@Edw6sr>@i0AJx1s_?+<}&G-Jv(6J^&%doE%nBcLCRjYTpU z(}K~MGVGA|5gKEfcAAN(dHmuuv{rgO0rz+1+8*VsT!!S1z-a9!S41cm4!tZAu+I`1~?74;E3yN0XjGb=T{>JZhszn#%aQbucx$wBcPun z$nDyZ`CeGJpQn%qE6KC2WWTH%h;7&I%I2&KsnA&$q5~EfzHvZ*m7yPIIAG3(I^^A3 zp0#iB^=fe8uQojm+_^s~3TyQumtKg{_EGX@4C2;DjDMIlq6)9@!8&8TTF-qleI)}v zF#TII{bm|2%=EuKMt?5^_%<2+orm7Y=rM!UZtr#Pr1uT+e1}~|gHo7{?jn$LDmVi9 zf5I8ExSDdiBd6OXKhtzf3;i4n^kClXQAd%$N0^stEO#Q&iQ%5Q?S*j+!(}znQJesFV)YxnpewpU%8p4lCqxjw|7M6Q!9~H4IzD>7jUQf{XgRE!F%!m zHT!>7U#`EX{&ycCkeeo|tf0NU|q;UGy%S_Jw*Tx)*9B#Y_i!+|W` z+9J!V#%&~%sOqClTyav1Myl-iQ>vog&2>CunwyG^tg+2=RUx z2q;p0@!+HuChU-%n%}Cui&8teU6EfAeyd5n$pY8$`D!)ZEqG)H1_J{NAdy>NgU<)f z@Z767{U%f@=2E5j1}lWSbPx71yc8 z(RG?fv1?W2x|E|~X(*Zta~zx9b$bwv(6dWAZKTMy;SP{o+nw*>a^NWzISx^h6Gqq^-PGTcVdrEesZBN+3Zwr2jrAz zd;3SnC#Sz`zkb&X9^dHFsmWsxf9V|s>WvHX<8L-LcLbjw-*{NSl26zy91i>XC!P6C1^! zjt<|w^#EL8JrhCvhP5o&C1;OWp8W(h4VlS|m5|XK%1raYYz#nag)OUQ6vxM?XDbtg z`y`b-7c@+LC6w3o-qk-VL;3X7S>-aVJvbMh1@09&+dlYBZ8#3G4ZKK!x9V?@Gg)9& z+FUXsw&jI=5Q8)#b)%mPX%GBQf6Uo-kEh!m&bP06!rj*ycaNtWLDf@E+V-(Hg5)7Y zM$zZM&v0<@@^+`Hykqkg$)oR?GuE*(ENr`HsJ=aKoT0Zo!#$@s*H&<^*<2_XwA=O> zf3K8)$tn@(KMwZ~)}j@V~#TthCx~I3Oh?(N%R}%ms)~9zVadP5hB_{(jAaOe&RqoY6-va}D~|q{_oi z@K#5udSz(`8=RAxPjJkzd^8o$V+JrlY^SH-2q#A1U0eBP;Mf`C+rE zkvM3e{^|!q^J{Fx1xjqRhK~0E`9cCO0dJ;otG~+s zA+G!0x&JKwzw%;j&E@~A&(~KT`2V;6{)>_7(b~1u3fAte7Ry~+EqUKk+p1;kr@D$f z!LzK4f7Lh85pLK*|B2hrJIV$UnA_3Y*4UPR?)}e6nWXo~2%7i*^JlQ2~zSfczqRd7<93)!5)of^kzDd8b%_j!WYEFtTAPC}MAp`uw?#fm82u~egZbN(Y=bcZzHl=a zy#AMhZBL2mA~)55B;v72hQW#j#h^@45a*O--^`llAfMz@kiT+nV&7tT%KOU7T#ABv%ukW$^>ubjre{KelHW^0mD z{kJW15!DT4!Qt>B!72-2335Mp%i-K$vx`YSDMLsB?rb?oszw&wbz!SrgV1yaMeV=% zgKP4kbk9Pbj}m;%jr2jrxr|Psf^_lX`-)hPtb6Nhkk)n`e<^6(sv^Xb!g}9@4fNx( zGbd73ZP(?vop#9cl@;?@_@!Bgxc9pexKa@e`0R9~ahsS?82nv6thTA*P5jii(!O=c zvCPNIYva@=*~OBRcT!GfX!XJgK*d`JGKCO_RF^`CmwHnO3D}tJhk9`RSs=#?8PE2< z{dOM@pB#-i;%!}VsimN}qaa1sCFR2x8P(as1GF&Ni_-Kw66vt6hEyG>jAX)`6lGS2 zBok+}T)X@k%L$mucnuFmXHCof@r_&wugDpRCwL=SeVl?tzSWwBXXZ%MLxf_C(?&V8 z^6fu6_mEfr!z^)--HrR7?y3pSu>Y^ER`UNmUtfRrp#Ocd`k%9YD%#x*5hZLVO~4sF zt$$`$J}tgHin8c}2jq625}v}3(WvMTC9rLtlsg~cizbXH{bd?*w+2Wf=Ttq5PLd37 z!GSalDLO&1;s$tn3Es7W_VXf!jGc@YC71R20ICBNQgmWQRKNiyUJH;hxMt8qpO1Y@ z%J&9A0xLj9US@-PE?69T#x0v*^aKjTfz=umKnchVgcRSyvGO;=L6=mr66(SD-juSq zv8Za@VDT&q`^!TbxIc~mi+{u4S)To|_W$+Om6uif|FZ}G?;r958@z?VavP-ecbUpj zT@Pkmm8WJ|iU-SFOJ5e@0(e=iRnT$Hk}@x@TN{FFv@qxW25DU+rMS`7niQ${GMt=; z{e0AwU3PV!q0t~t3fdQ7TOM3EZO(#nC#k%0-NJHcP$Z)w8UU)QNYsSu{;V&*LcqudKH>}ozf-0tJ}&4uDyZ2z zM@N)j&{GhM zpO^b93A?Q5cTZ2DCObXt4wBEHQLrxily+t3GS=l-;eaN`kbO$^OhY^Y5S1rEXaU`y zN}7wmmB@<)OL*OtkXAG)hj{@5`bU(JU$5E!UT5rMLjHOse-%OsEaAWLNfcoO(1=Gv zKpcTwTVtBVH0vkSm}Fbdk(faH|9Kg{5N#{&8cFTg&?kHjyXjs`l9;mYPrZZQ(e_F2^mymz-ka^yU-o;y9*3j2O`afK-kG2@Y9r@jBXe|)lk@YCt;;m*4^y@M0PE?ipLFBt)$l#D0EIOlPQ7{e$X z3gn4+X$1$Oq+0IUdBPJGN-;sCu8", "license": "Apache-2.0", "dependencies": { - "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.37", + "@firebaseextensions/firestore-bigquery-change-tracker": "file:firebaseextensions-firestore-bigquery-change-tracker-1.1.37.tgz", "@google-cloud/bigquery": "^7.6.0", "@types/chai": "^4.1.6", "@types/express-serve-static-core": "4.17.30", diff --git a/firestore-bigquery-export/functions/src/cloud_storage_backups.ts b/firestore-bigquery-export/functions/src/cloud_storage_backups.ts index 7ffa5c712..bb1a11e57 100644 --- a/firestore-bigquery-export/functions/src/cloud_storage_backups.ts +++ b/firestore-bigquery-export/functions/src/cloud_storage_backups.ts @@ -21,8 +21,8 @@ import * as path from "path"; import * as fs from "fs"; import { promisify } from "util"; -// Promisify the writeFile function to use async/await -const writeFileAsync = promisify(fs.writeFile); +// TODO: we dont need to promisify in node 18+ +const writeFile = promisify(fs.writeFile); // Initialize Google Cloud Storage client const storage = new Storage(); @@ -51,11 +51,8 @@ export async function backupToGCS( context: functions.EventContext; } ) { - // Create a timestamp for the backup file - const timestamp = new Date().toISOString(); - // Define the filename using documentId and timestamp to ensure uniqueness - const fileName = `${dirName}/${documentId}_${timestamp}.csv`; + const fileName = `${dirName}/${documentId}_${context.eventId}.csv`; // Create a CSV string from the event data const csvData = ` @@ -68,8 +65,11 @@ timestamp,event_id,document_name,operation,data,old_data,document_id try { // Write the CSV data to a temporary local file - const tempFilePath = path.join("/tmp", `${documentId}_${timestamp}.csv`); - await writeFileAsync(tempFilePath, csvData, "utf8"); + const tempFilePath = path.join( + "/tmp", + `${documentId}_${context.eventId}.csv` + ); + await writeFile(tempFilePath, csvData, "utf8"); // Upload the file to Google Cloud Storage await storage.bucket(bucketName).upload(tempFilePath, { diff --git a/firestore-bigquery-export/functions/src/config.ts b/firestore-bigquery-export/functions/src/config.ts index b92f5bdbe..15134fc69 100644 --- a/firestore-bigquery-export/functions/src/config.ts +++ b/firestore-bigquery-export/functions/src/config.ts @@ -63,7 +63,9 @@ export default { process.env.MAX_DISPATCHES_PER_SECOND || "10" ), kmsKeyName: process.env.KMS_KEY_NAME, - maxEnqueueAttempts: parseInt(process.env.MAX_ENQUEUE_ATTEMPTS || "3"), + maxEnqueueAttempts: isNaN(parseInt(process.env.MAX_ENQUEUE_ATTEMPTS)) + ? 3 + : parseInt(process.env.MAX_ENQUEUE_ATTEMPTS), // backup bucket defaults to default firebase cloud storage bucket backupToGCS: process.env.BACKUP_TO_GCS === "yes" ? true : false, backupBucketName: diff --git a/firestore-bigquery-export/functions/src/index.ts b/firestore-bigquery-export/functions/src/index.ts index 5ce946193..70e82c2ce 100644 --- a/firestore-bigquery-export/functions/src/index.ts +++ b/firestore-bigquery-export/functions/src/index.ts @@ -111,16 +111,16 @@ export const syncBigQuery = functions.tasks oldData, }); - if (config.backupToGCS) { - // Backup to Google Cloud Storage as a last resort. - await backupToGCS(config.backupBucketName, config.backupDir, { - changeType, - documentId, - serializedData: data, - serializedOldData: oldData, - context, - }); - } + // if (config.backupToGCS) { + // // Backup to Google Cloud Storage as a last resort. + // await backupToGCS(config.backupBucketName, config.backupDir, { + // changeType, + // documentId, + // serializedData: data, + // serializedOldData: oldData, + // context, + // }); + // } throw err; } @@ -130,9 +130,8 @@ export const syncBigQuery = functions.tasks /** * Cloud Function triggered on Firestore document changes to export data to BigQuery. */ -export const fsexportbigquery = functions - .runWith({ failurePolicy: true }) - .firestore.database(config.databaseId) +export const fsexportbigquery = functions.firestore + .database(config.databaseId) .document(config.collectionPath) .onWrite(async (change, context) => { // Start logging the function execution. @@ -189,8 +188,12 @@ export const fsexportbigquery = functions context ); } catch (err) { + functions.logger.warn( + "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.", + err + ); // Handle enqueue errors with retries and backup to GCS. - await handleEnqueueError( + await attemptToEnqueue( err, context, changeType, @@ -232,7 +235,7 @@ async function recordEventToBigQuery( }; // Record the event in the Firestore Event History Tracker and BigQuery. - eventTracker.record([event]); + await eventTracker.record([event]); } /** @@ -245,7 +248,7 @@ async function recordEventToBigQuery( * @param serializedData - The serialized new data of the document. * @param serializedOldData - The serialized old data of the document. */ -async function handleEnqueueError( +async function attemptToEnqueue( err: Error, context: functions.EventContext, changeType: string, @@ -316,16 +319,16 @@ async function handleEnqueueError( ); } - if (config.backupToGCS) { - // Backup to Google Cloud Storage as a last resort. - await backupToGCS(config.backupBucketName, config.backupDir, { - changeType, - documentId, - serializedData, - serializedOldData, - context, - }); - } + // if (config.backupToGCS) { + // // Backup to Google Cloud Storage as a last resort. + // await backupToGCS(config.backupBucketName, config.backupDir, { + // changeType, + // documentId, + // serializedData, + // serializedOldData, + // context, + // }); + // } } } diff --git a/firestore-bigquery-export/functions/stress_test/count.js b/firestore-bigquery-export/functions/stress_test/count.js new file mode 100644 index 000000000..6f9ea64d9 --- /dev/null +++ b/firestore-bigquery-export/functions/stress_test/count.js @@ -0,0 +1,33 @@ +const admin = require("firebase-admin"); + +// Initialize Firebase Admin with your credentials +// Make sure you've already set up your Firebase Admin SDK +admin.initializeApp({ + projectId: "vertex-testing-1efc3", +}); + +const firestore = admin.firestore(); + +async function countDocuments(collectionPath) { + try { + const collectionRef = firestore.collection(collectionPath); + + // Perform an aggregate query to count the documents + const snapshot = await collectionRef.count().get(); + + // Access the count from the snapshot + const docCount = snapshot.data().count; + + console.log( + `Number of documents in collection '${collectionPath}':`, + docCount + ); + return docCount; + } catch (error) { + console.error("Error counting documents:", error); + throw error; + } +} + +// Call the function and pass the collection path +countDocuments("posts_2"); diff --git a/firestore-bigquery-export/functions/stress_test/main.js b/firestore-bigquery-export/functions/stress_test/main.js new file mode 100644 index 000000000..dc1289ac8 --- /dev/null +++ b/firestore-bigquery-export/functions/stress_test/main.js @@ -0,0 +1,104 @@ +const { Worker } = require("worker_threads"); +const { performance } = require("perf_hooks"); +const path = require("path"); + +const totalDocs = 10000000; // Total number of documents to write +const maxThreads = 20; // Maximum number of worker threads +const batchSize = 500; // Documents per batch +const rampUpDelay = 2000; // 5 seconds delay between ramp-ups +const rampUps = 20; // Number of ramp-ups (planned) + +const docsPerRampUp = Math.ceil(totalDocs / rampUps); // Documents per ramp-up + +// Start measuring total execution time +const totalStartTime = performance.now(); + +const workerJsPath = path.resolve(__dirname, "worker.js"); + +// Function to spawn worker threads for a specific ramp-up +const spawnWorkers = async (activeThreads, startDoc, docsPerRampUp) => { + console.log(`Spawning ${activeThreads} worker(s)...`); + let promises = []; + const docsPerThread = Math.ceil(docsPerRampUp / activeThreads); + + for (let i = 0; i < activeThreads; i++) { + const docsForThisThread = Math.min(docsPerThread, docsPerRampUp); + const start = startDoc + i * docsPerThread; + const end = Math.min(start + docsForThisThread, startDoc + docsPerRampUp); + + promises.push( + new Promise((resolve, reject) => { + const worker = new Worker(workerJsPath, { + workerData: { + start, + end, + batchSize, + }, + }); + + worker.on("message", (message) => { + console.log(`Worker ${i + 1}: ${message}`); + }); + + worker.on("error", (err) => { + console.error(`Worker ${i + 1} error: ${err}`); + reject(err); + }); + + worker.on("exit", (code) => { + if (code !== 0) { + reject(new Error(`Worker ${i + 1} stopped with exit code ${code}`)); + } else { + resolve(); + } + }); + }) + ); + } + + try { + await Promise.all(promises); + } catch (error) { + console.error("Error in worker threads: ", error); + throw error; + } +}; + +// Function to execute ramp-ups +const executeRampUps = async () => { + let activeThreads = 1; + let startDoc = 0; + + for (let i = 0; i < rampUps; i++) { + await spawnWorkers(activeThreads, startDoc, docsPerRampUp); + startDoc += docsPerRampUp; + + if (activeThreads < maxThreads) { + activeThreads++; // Increase the number of threads for next ramp-up + } + + if (i < rampUps - 1) { + console.log( + `Ramping up to ${activeThreads} worker(s) in ${ + rampUpDelay / 1000 + } seconds...` + ); + await new Promise((resolve) => setTimeout(resolve, rampUpDelay)); + } + } +}; + +// Run the ramp-ups +executeRampUps() + .then(() => { + const totalEndTime = performance.now(); + const totalDuration = (totalEndTime - totalStartTime) / 1000; // Convert to seconds + console.log( + `Successfully written ${totalDocs} documents to the collection in ${totalDuration.toFixed( + 2 + )} seconds.` + ); + }) + .catch((error) => { + console.error("Error in worker threads: ", error); + }); diff --git a/firestore-bigquery-export/functions/stress_test/worker.js b/firestore-bigquery-export/functions/stress_test/worker.js new file mode 100644 index 000000000..baea4c3e1 --- /dev/null +++ b/firestore-bigquery-export/functions/stress_test/worker.js @@ -0,0 +1,59 @@ +const { parentPort, workerData } = require("worker_threads"); +const admin = require("firebase-admin"); +const { v4: uuidv4 } = require("uuid"); +const { performance } = require("perf_hooks"); + +// Initialize Firebase Admin SDK +admin.initializeApp({ + projectId: "vertex-testing-1efc3", +}); + +// Get a reference to the Firestore service +const db = admin.firestore(); +const collectionName = "posts_2"; + +// Generate a random document +const generateRandomDocument = () => { + return { + id: uuidv4(), + name: `Name_${Math.random().toString(36).substring(7)}`, + age: Math.floor(Math.random() * 60) + 18, // Random age between 18 and 78 + email: `user_${Math.random().toString(36).substring(7)}@example.com`, + isActive: Math.random() > 0.5, // Random boolean value + createdAt: admin.firestore.Timestamp.now(), + }; +}; + +// Write a batch of documents to Firestore +const writeBatch = async (start, end, batchSize) => { + let count = start; + while (count < end) { + const batchStartTime = performance.now(); + + let batch = db.batch(); + for (let i = 0; i < batchSize && count < end; i++) { + let docRef = db.collection(collectionName).doc(); + batch.set(docRef, generateRandomDocument()); + count++; + } + + await batch.commit(); + + const batchEndTime = performance.now(); + const batchDuration = (batchEndTime - batchStartTime) / 1000; // Convert to seconds + parentPort.postMessage( + `Batch of ${batchSize} documents written in ${batchDuration.toFixed( + 2 + )} seconds.` + ); + } +}; + +// Start writing in batches +writeBatch(workerData.start, workerData.end, workerData.batchSize) + .then(() => { + parentPort.postMessage("Completed writing documents."); + }) + .catch((error) => { + parentPort.postMessage(`Error writing documents: ${error}`); + }); diff --git a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json index c789a1f18..3de5b8a8f 100644 --- a/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json +++ b/firestore-bigquery-export/scripts/gen-schema-view/package-lock.json @@ -41,7 +41,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.36", + "version": "1.1.37", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0", diff --git a/firestore-bigquery-export/scripts/import/package-lock.json b/firestore-bigquery-export/scripts/import/package-lock.json index e0866447d..678639cbb 100644 --- a/firestore-bigquery-export/scripts/import/package-lock.json +++ b/firestore-bigquery-export/scripts/import/package-lock.json @@ -39,7 +39,7 @@ }, "../../firestore-bigquery-change-tracker": { "name": "@firebaseextensions/firestore-bigquery-change-tracker", - "version": "1.1.36", + "version": "1.1.37", "license": "Apache-2.0", "dependencies": { "@google-cloud/bigquery": "^7.6.0",