From 6d39778d54deadcd6be6ab6ab93535b5fc605992 Mon Sep 17 00:00:00 2001 From: cameronpettit Date: Wed, 3 Jan 2024 13:11:18 -0800 Subject: [PATCH] PDR-236 migration to fix malformed date on some records --- migrations-data/20240103113100_data.xlsx | Bin 0 -> 13044 bytes .../20240103113100-fixMalformedDates.js | 77 ++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 migrations-data/20240103113100_data.xlsx create mode 100644 migrations/20240103113100-fixMalformedDates.js diff --git a/migrations-data/20240103113100_data.xlsx b/migrations-data/20240103113100_data.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..8e0b9af0bab04770ab5843bfb29b48bb69581899 GIT binary patch literal 13044 zcmeIZgL@@g+CIEvbdrv3+vu1b+v?c1(Xnl_<8*A>wr#89Id<+=c+7U)+fMnZ-9(iV=Eq?JbOcNATp6@#u%LB>?8~sf%CF(`&gDYK| z(#q=yUxtfk;CH5wX3b0~+_i3YXv9Fn2;EmvowR<)5~wOr=*c(PJtaxB+mkmk&FSa^ zuhR!cx52HAZ-BS7v*AiHV^$Z3-j=BMN1Ic1cxZaKb&yAf-O4^Sv!a-2(`aP!`4IR{ zF9KxE!)j5>8LcKfdwEgvBu6g0@zGfY>wWY;*94)BSXEPs8v0D)Gph z@eLdRcz*{0$o`vI)+x~w-G1nr#h%(+@<%yUpumcx8<%;&_nY z*EVxm7$Pz!QN43{VA7MVBLpR>U81mU*;X%t)7;(MuM`n+S8}JeD9X~tq8y2#4I<&0 zE1@d*30f6QD5M|QfvDUmKI;8aYU_FrKY`{16wb;6D;wCdkK-oNycZG+j$nC%*(Fb> zQ&0x&^^BK(dJI|++&y9{DVVUER_bNiaS*y{8(4JS2&Hu(y?W3{rw_^#G9x}PPKXSU zWZ!wK*D_y?WVrUSLG_dlUJm(&;T5faM5zBXlKEhnMgI>2Ndp4_K7aI#iv^vtwVk=1 zwYB*laVuYO$ZCZiz76e=7oMvQ;%hEiA1Wb4$Z9tDa<@r+n9uhDlQ77yCB!J4kk=PD z;>xVr)56ichDB^QMw$DEE*jzv0}&|dKa#P9&~5u1XlFV4lF*F1Ac;}aVo_>|X zPE;gcB_!VGMt7@243}XkmH4UoNv|;XB_o&ez?&DERk>|UkLZaG%gD;E4mDJv{)0eWsl((5iHae1w%NMLhD+v(fch@5 zW0~I3QT-Ft{JQQe){f0}etQ!?yY`fN^QXyg5T6AicWkW>Wdv|!@^^&g2AImR2aQx( zqr~&O#-U+-g*qOy)Y0MIfkc5DyVI)uY0uJ|f#-w^zDS@Zp&rsP;#<|KJ5bh#9X)O{ zx1{)?pKS-!9<~R*^J~VtJ`r46>suUH+XLmK<|9q6Vu0Q(-9jktrmypR@*O{(Tc_~@ z3A269$BVJGexf;y8sFA|k9h7nYUG8L9^bb35}b#GXdKnq-3v8S>(uviAD5{Qg#0q| zh-}RX6_KQFFzc09sDTy zZ!z0%cS+Bv3AJO>KG^o$|j zLsItgyN-B_6dBaZj14&5v%qbGAl-FQ6@?;#x*=IyaDrKn-Hd@i>YbJ2`ig)dA)X$el%9QpOSMO8 z;)xxk&P4vQ|3FTH5tLUzD0JI29ms4pPVdNQW8!dhCiHLcx=~)v-^k!Aiv;z&+9D6tdNML8Oq4#d%o*gHq;Q# zRKbhg68{ieZ?9cN1`E|~TZ|uif2No&J;`G^#qgLs#MCliH{L|9U$(@0yLEA|+N2gb zQa7FC?~#FjxO+NDnge>Jy#>S|bOi+7Gr-ls^WW(h4%z@jQsCMWVj{h$x;Odj#S9|I zZ(ih8i*d3b>)v5NTBNBO`Rf`sw?VIT4D#OuD}rna)b23S6TBMmXpxtF;IS+nxJby2 zmFJu_g0!q))}B>Ie{j^yY=Nws?J?+}F?EL&PenOvh5o>wcAGP6?)u`ipJV>ekn5G2 zpjoD3%Gbey-QAM>waFC#LMb5OzHO2I4sOeax;|By9!-D}E?S)H6?mZ|+2fs9+ubqb zFTebp4c~ZtRpYVEcOgGd6!JtR^p{7UdzwF*B#cTsj-toncMS@?>n^+ z?<%b?tJceW7o?D%q5E@K|2)N~omj21BKvETT!PF>m{$1cXP-#4svBl3UV}9L^m!8x zTNGVspre3lNl)pKe@VJL3?IKH`f4Vm-THd<^yKyW^5)@swKLJd`|{whKhD$odiU}= z_I{$j_kI(V#@pfX^15-d(2%rnagjA{)S(zwGZL0$b5-#Mym7n#{(fWUWczk=z5QYs zZn=}(v#5k!O7Da-U>3gCTgBdQwX+DUpS81ysPAlR5m5CFTSLDrL?0{sY^y}8Whn={_#>v6m~k-*Y>F*={IwEUfGYw2LbJm_VkjHCTK>l8Qoa_6y^J(!b7`Ci7D=G3|KxU^t;x$2Br(I2)OR>@ocHPEbqrNw`4P z9)FKO(_JRXL77fX)UI0$28b+|`X=ti_2rCM0|6gA`nIulf_igJo?CI)`}_KOx21w!L#TnSN|uqoxkXH8AB* zP;&OzXC=Y3g@!NMI+%|zNY_a8Yh}SkjW5CW{H2}sXd{x+irHDKI`QC_UcO$Ol#4>L znNKG=gLH@d?orS$jm*f2w*WWbfIZW}uGIaPXO9qTG0MEt}Qqpd))N?)*CUy_QA z$R{Z3suy`p%mfu`*=!v-us&dx{K|MWa$2t#?P(+(tlXv_Fvte(7kTq<>sISaW3Z@0 zbQ?&a79s(l8iydP3NWsQSX9*h<-c;P12E++MzB%PIu@u_pfaJ!$u(A#uEfBvGuBph zV!EAtYZxhs7X|Pp^|#>VO+BbqRQZ=meE)%hK}GA>r!!4i3^Z|C_w`j>gGZ@x_Z4AO zPVPh;sW5qEN8cM^MdbM2fUE5}i|&^}GOB@yEPW<6jE#)eF+f!gg$qpyNi&J2jwBf^ z$IDNZs9nX6A@6#&?@1V6Q7frmSmPfsj~vvSphmoPy(6sOcWy`*sF8qDo~DwWJu%j0 znJwpi5I1^Iu=G*Y>s7@a->^SgXm<#rhNPsGl~(1;!xjPjScl`Sx81gu;7YZ1{jOVT zB+!Q!!NP*!aN&OaMkDKq0t9%=p2Kja!THY64~fygV23jKZa z-^Vq2G&Ze8v$HLvYWBv-Qf}t!jVeWT)99*kP!KJ0J}*|m6)(NQp>tIyDS;V&V#!}j zbY-S>f}Xd#0xhN6~jZ{@+feu?-DU1oyuvY*$F?opYB zceX$Nl<-mvKO*WuRW^*BX7hNJLT%w*70v>Hwl@lU8g=?pb1dXCrI-b^+BbZCEAQmO z%GX{@*qoIyuBCYQWq&LtF7a&fO@+|3*b=02=K~!G@5To*y@JtMaL1kPgNNmhje7~i zoE1f*<4;1oz?ltJn7VD~`*cF9r1c11%H~jMSIq?Su0Rv)1R~dl?cx3X=HkP3@l}v=OnVOuy4#%Ib-#7)g4%QoY??yLkTr^z)IC)@ zvhbDFzozsXbx!9ccy+?iMmCPQB)p*KdLbrn=h%ZRr6Vvm6W&u^FE1UMgj!#%3meg{ z46&DQXrbuHOD3fgC5(lc(%_b0w5){-NLOM9Kj%8W5A5Azp6=ieTVowT*e=#a*&CLK zka1)*3-ZivWuraCs(?)+=FED6^bAWIMxk}AOK^))=|+Ce`Q%RUr^s(Mg4*He9|os~ z>shVrN?>cWhlqX27-KT7&QwF8Sd%C}L4w(cy_ z&*g`gJqo@uSnkT_99Yj7HX^}DRijn1*kTj;dI>*1&h*8&`yiv$_o~`SpKT*@>;=ld zWY<0$sl-Ag1k~jejAzmOs3rh}PMC%kFg@JuPduGxG zg7maDd;%HGW}w&eSAVS9JRx70H-Og}(`_G*58^`p2szAyXtiReD_?fD{FUIjrsrClWgQT2Ml|Lg^YrH}-09(q$;2~>-sWG(nLT7g;k zSPnrBEP^}=S>z<*iLtLxqV_=s#x!;seeu~K4IdEgCdxt`z{c@gPe!HsAzzM86#Jm% zt0(L3$w)wS3Y#GA?Oxm}v&9f202@xep;A&sl2f(QQ!Nl2a43I{L$G|Uo5{e5yX`)G zPmYM_W%5Y!dVtnIhdQN9Dzpc#CTNxY&e$ZWC2c;O^vA zy`z4*5XgktH=&?@k>e!6Dq0s-_GvDsMdd9^C|WaC_e#KK5u= zH`P%}SV52|D4fqlX}`a|qOEIpyj>n&u^r8lU3K?%ygZI1b-Z73+*NL`qR_Q_USA)@ zp>25H9mquCEU#eWBwrGQR9*hu(!U@L)AfOSz5?RS=%p9vjpai)bSgR@83fkSz(4MG ztUT*p!B*`DYvt-NUuwp?Y9fwIsSvILTh$MDLU-oaEJ+=?ytK)bT#LJD3TX+Mj4?l_ z6)ywaA24OBY|Af2R2uHXh9ZU(eCe82=%^A5$0=bqZ!2phq6%~IpZf}ae>ajIIwQ`+#* z9DFOo441j;7wg7Cm^e9ZiW8Pw&=nlQ`_bJScpXs5Ic#0R5MO;Lp}HSW<%v=T{EDgz0Q5s4vOJnhJpX9;nx_voVhQTZXhHy z4BDD+Fv^>zP)4b`#d&<%VjAmb5M*KHHl(=Sh`x!CYO)vSVO(b*dvLHQjjQK)jx=(^ z10kOpsCztWb2pRB)lQq(GW&T~C3~&8hxg|aqZuup3d3>plpFQ_yI*(wW+|;6KEtFJ z*N2%htyOzW1o7u1V0|qD&YqyHQ{fQwMJsa6WKCv37G$RC)#0w79 zxi8ksXHUEi6`j?Rz{!_p&~+t^u;d)k<+R}hkhP5cwU$y;ryNwLWRH__)P-&Lpy@49 zP+azYZGLUr3*ukq4>ZD#KhQ@htW9YR{|P;wBJ38&RGb1Nt4GI#ALXF1)NJc&=4e{d z_BEe|DBPhYf_|Fb7j280b1d;$t-m=XXqHi?Nv{dZj_Ek!GlKb7Lb665Qc{JNyos30OkzK8kQ9gi{iq*bEC>^(Yf>kOMLw1!Tz$43%zgvmPP zCL3gSvxNK{osyx#Lzh`f?aDc!w~4E z@E7i@pJcp)8KdfseAFl684pjxtF3cJ5Y9eaZZ_Uu3gr)F1+0^Z$e*UM2IES`*84ExN3zO6yMXTF-+HTie^02)?z|VND8_rgm62jFS^Nh ze8|JRyp?+3m9M8lXTU2CK2lTd!_}viQ(NM<^Hx*u>jaL`;dC5wlAtZ-6#Hxo0~mJ1 z=Z6E9sU{oSxk9e2^W|6y3)|GgXyh6ZcN>|$>#_^t*hXRWLp3gC7u)jhw%4!Dd59OW z4=486`mp%bc4fT!t?ew~jBb%Lm90Yyw%{FV`(H8j8mX0qn`pOfk{Hf(>$w>J=)gAz~TIHb67qT{##BS;O;A5+8l=X-t2`$X zghSA8&DY_bn^rd7EY41Jr^$|vgzfV%_((rB=m>du&u;ldoY=mj!D?9J?F0w^4Ry7|vXiuw{=T)#9s{ z898*x$~>#R6C`uXHl-U?t78Mf`YvsP@ZlA_mAOk9w{Nf%;dIIjWIN+N9<6#nDjD;X zla;L`#L{F2BB-IIv4L#d2Pu1h!G8Lw-$d(p-D<=ENNc!=~BfC4P}imwc{&KV<5h-_r#BjkM44Ps|_Sx|%-Y3?I)uxjklCWF=`oXOV>S zpE|s36JcPt11q@YQ0~RY>jv??5-^+%z78Z?+UXTr7p{>Ew=raSV9+YAsH6tg3*ph| zP_IrV!mRwdV)vMPH6D#UitF3Sttm0qG{>1njw=u+$V}=7p{N(<4>^VpvTG zI8`Pw)|+w@Cjf}9N#>*TL%nuvaIUkl8vS3du1s0Xh!9p?&CSa<0hG@cdH1`>DlLgke6IV zo=Zt6JROzQ2M~eIxTkoYZ;XgeHWA?PpeUG~R(Vr^!(sstRSY&v0DlU2_BfPst1+%B zan)Q1)R#^n-U)vDQR2Vd8;1OeI?=ynv1#x3}~Hw z#prxdC4(;56rz;!<-kRG>R?~@7PcGtsuRrE z$V)o8@B$D$IK&*9;J20F#jjD6Hiz(MsaUMwj{`7KW)DO(l)QEa0Z}zd#^4%CzPnDM z$2WQ9l)TplvRU|+N>H&xeorqZrNdEN7_(%eBCo*Ha#?s8pU0!XH|Wck3pEdJEh7{x zTew#E_b^c^c2>jZ-5%*UADix(N$4iegwPaIsN|uO+GgIsJr9K|Q`)HWaAA z6_wSf1Z>}Y0V9Wf`c+?Ip+|Ln zhLZF?N}%KaEIm#<;`(cb_HBy}Qw4!Jv>=y8~|x{o~ftdfe^nNjN3rdZK|xK(`*59nShr z(zSCycP?r5YK(zvK=(Gu^-99-#~nfvBNP?JQblrVfq(zaMy!I`~C#^x=&@T`3XvbiAC3QPm#t?R%( zIlZB~wfusw@LEoHs1Zrgt7@Eev}K&!cgYBupEg0#+6?|~mIfSg=Z#kY^8Q9p=`Lu` zaoQw(dgo#7wsEt%bo<70DYcURI*{S#JwV~&W`w6DTD7{u=u>n3)zYH;Ma9Xw%pNV_ z>2{4!Hf>?8;dwv9IHj$Vp*(Ztb5NV^#+93o@zTAg^V236Zeh!Pzx{oIuG(1BbNSwy zUVJ~)aH=W`!uEwg^yalcf-n-4-)j(L+<#9-B7rg#(LOfF0oVWl@?Xh_iLRZYfr5jb zsg?1cIZ3P9H~TzRWX}u9caR=?qE8{8^K4t=42)MYOs6H|ZgZ@ml`Z z3*|eRWnq%OtlZmB+*-@vWMQ7x?`^pxRkw|AGGR=ZIkC&7F+pm_lZM<;f|64{DcZlG z*JOU~v9G8lHZ%(~9lX|0s3#_%(BH6qJKy5c6uaGUn`-UM-Qcw!6G)YtCi!@sxo@Y^ zc8xOzBB7tjrkh^7DJL1nk7{^&9bnWmljO3b`Av2x)x7FKDI?6(TC^xyMol+|d)qa? zi8}A`DblKT!8qJlY~)y}`N6Mzn_PEWJxaw;ZRI9;M%6yD-uzvd=12X63Z+AxPls~U3-mHi3{5X9D)#>+0S29FIE>o9^V#sTV8dLS{}+?pW-}ij&xsn zmaG^PHP~LTYOG9g3rFQc8fbr^61dhlV|S<`o%Do4iB|cwt3;YX(J_pC3gZj_0#Ouj zasR4FicA1u!_ts)xl~ahWlfH@^(mM(`Wcd-i-#W$66d9GUjnS7Y2L|UOu6YizMQ}c zoue?O8IB)a^4r39=ED@c0&-3thT)D$gR%VDpGW5g(N!0on$h=*F#JEeP9m@3H{o3( zyX>JoXHM??&`iq@zGnB;8_ggHNoqpUGxDbd`g;NMQjEb`&Mg_sn zY=aWnZ>V0zl@B!AnEG=>_<33-O!|RYx(d@VIfM{CsFlg4?i|f?FH^s+_$B>CSY_>3lzn|T-Yt+DldQo;brlMKp zv%?n{ev?h>BDlgabg8IhP4D5NJgh*_W^py=P^bp-QB#xX2p3G4Pp`GuEOnC=?Aq3B zQ)XD@<(#PpFV<7?vAR3gTQD_dP^OB|=$p=kz9P}3`zU@Yx=Bffcts;DItFqOyD*#mV!Mt)tcNLkiy>4|_at<=0t{aY@*~HG$G!!JT(; zI&misB{ZU3#4*6Vk|>>LxfviORSFs>;RUaMc~t9qY%8nw6QAqI<+u!=jBAR9a+VW~ zvC6$~j=g`l%l}+dH4O#4^MB0uj6TR42p{AP18aR*J8K(zI(=(9!@qd4|Kn_YM40qg z3G)x0sQ*R48$@0U4X2oPzf(2^I}5ytfP}TcGI8!ah^YqIdwX?8Bh*>3$l0jZC^cn) zyON0ZW-D_uv07{_Br}`9v8C92sjG(!VU*-2Wh__{_U~jf$4@1jtOxyF(-@!NNFKT^I*dZ=qIf?LC`@Nh==Fomh_szp^1a{QTz6+jPb&~hwM8S^oj6?C ze*962DY3R53Q`Q+Wqf}Zw;Edze z0A5Zs;T}hN_{8N`ZEBoZd?cxDYTuDd0{3oQ3s;+~HHhYE1I4osNB;xc6=+fOY7%eW z(|@IcLf@@T27c%rCsMeLm9sUPZg*~5&RJ8id+op-OOaQEPT?(gIi z2$<$0(fse<7yNUq{(1c`UmM6u{JVgEXF~rI_{TNr!yx{J7yUc%cXsIC(AEz-`;9C5 zJNVz3dVfO!0Lu?D*8e|m?{_)B)3^SXl>O=dtHghhxqg@O`+nZvQsz+p>+as~0)AhS z{9Aw`&YuGQUZ4CO`uqIhZ|D}uKcK(QCVrRj?}77gJODsR1pxd<82ugo@BZ&!;kh*b c0{_b!%1VHJj4l8G{qg($5vjH4|M>U+0F3{Sa{vGU literal 0 HcmV?d00001 diff --git a/migrations/20240103113100-fixMalformedDates.js b/migrations/20240103113100-fixMalformedDates.js new file mode 100644 index 0000000..fa427d6 --- /dev/null +++ b/migrations/20240103113100-fixMalformedDates.js @@ -0,0 +1,77 @@ +const AWS = require('aws-sdk'); +const { dynamodb, TABLE_NAME, getOne } = require('../lambda/dynamoUtil'); +const readXlsxFile = require('read-excel-file/node'); + +const file = '../migrations-data/20240103113100_data.xlsx'; + +/* +This migration is to fix a handful of activity records that have incorrectly formated dates (and therefore sks). When trying to find an exhaustive list of these records, I did a database scan for all items where: + + begins_with(pk: ':pk') AND begins_with(sk: 020) + +Obviously this is a very expensive query so I only scanned instances where ':pk'=0,1. I got a total of 31 results. Every single result has a 'lastUpdated' param of approximately the same time which indicates the malformed dates are a result of a bad migration and not something that is inherently wrong with the lambda that creates the activity records. + +It is difficult to tell exactly how the sk and date are mangled on these records. The dates all consist of 6 digits, where in other records, the first 4 digits are the year and the last two are the 2-digit month. I am going to assume that dates that begin with '0202' are for 2022 and dates that begin with '0203' are for 2023. This makes sense considering the last two digits of '0202' dates are 11 and 12, and the last two digits of '0203' dates is always 01, pertaining to the contiguous 3 month period between Nov 2022 and Jan 2023 before the 'lastUpdated' date of Feb 24 2023. + +This migration is idempotent. +*/ + +const schema = { + 'pk': { + prop: 'pk', + type: String + }, + 'sk': { + prop: 'sk', + type: String + } +} + +async function updateRecords() { + const records = await readXlsxFile(file, { schema }); + let successes = 0; + try { + for (const record of records.rows) { + let old = await getOne(record.pk, record.sk); + if (Object.keys(old)?.length) { + const oldDate = old.date; + // fix old sk and date + const month = old.date.slice(4, 6); + let year = ''; + if (old.date.slice(0, 4) === '0202') { + // its a 2022 record + year = '2022' + } else if (old.date.slice(0, 4) === '0203') { + // its a 2023 record + year = '2023'; + } else { + throw 'Year is neither 0202 (2022) nor 0203 (2023)'; + } + const newDate = `${year}${month}`; + old.sk = newDate; + old.date = newDate; + // create new record + const newItem = { + TableName: TABLE_NAME, + Item: AWS.DynamoDB.Converter.marshall(old) + } + await dynamodb.putItem(newItem).promise(); + // delete old record + const deleteItem = { + TableName: TABLE_NAME, + Key: { + pk: { S: old.pk }, + sk: { S: oldDate } + } + } + await dynamodb.deleteItem(deleteItem).promise(); + } + successes++; + } + console.log('Successes:', successes); + } catch (err) { + console.log('Error:', err); + } +} + +updateRecords(); \ No newline at end of file