From 26336731c33ba8f3ef2a86c191918c989cb4a048 Mon Sep 17 00:00:00 2001 From: Alex Kerney Date: Thu, 23 May 2024 10:18:22 -0400 Subject: [PATCH] Encoding dataarrays Make sure that dataarrays are encoded as well to hopefully solve IndexError issues when datetimes are not coordinates. Works on #59 --- .github/workflows/tests.yml | 2 +- noxfile.py | 2 ++ requirements-dev.txt | 1 + tests/conftest.py | 2 +- tests/min_coordinates_encoding.nc | Bin 0 -> 51830 bytes tests/server.py | 13 ++++++++++++- tests/test_server.py | 24 +++++++++++++++++++++++- xpublish_opendap/dap_xarray.py | 6 ++++-- 8 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tests/min_coordinates_encoding.nc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d4b4c64..ae92516 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] os: [windows-latest, ubuntu-latest, macos-latest] pydantic-version: ["<2", ">=2"] diff --git a/noxfile.py b/noxfile.py index adabe51..f797df1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -9,6 +9,8 @@ python_versions = workflow["jobs"]["run"]["strategy"]["matrix"]["python-version"] pydantic_versions = workflow["jobs"]["run"]["strategy"]["matrix"]["pydantic-version"] +nox.options.default_venv_backend = "uv" + @nox.session(python=python_versions) @nox.parametrize("pydantic", pydantic_versions) diff --git a/requirements-dev.txt b/requirements-dev.txt index f678616..4e246f7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ black +cf_xarray check-manifest doctr h5netcdf diff --git a/tests/conftest.py b/tests/conftest.py index e18aab4..ef4bfdb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ server_path = Path(__file__).parent / "server.py" -@pytest.fixture +@pytest.fixture(scope="session") def xpublish_server(xprocess): """Launch an Xpublish server in the background. diff --git a/tests/min_coordinates_encoding.nc b/tests/min_coordinates_encoding.nc new file mode 100644 index 0000000000000000000000000000000000000000..c294a23f43c126a98788c79680be8c08e0b07424 GIT binary patch literal 51830 zcmeG_34CKk`IEMLwA;ND3alJ&y9;~J-07)HlWmf1*rq8>+vSqyYx3I0O_PwL+XYlW zL196~i&MY_@BAYOSy1wg7CC># zzLU&9Y@hGvAF_|v^BbmrIsf2k-|eqTOifeHrx zYccS(qpfEK`@8!HY86#vS+OBPxma^A)eA_w5dR?=b=^E|AM>p|xIYiH3sf+uW02Q^ z=JCpQ7Bin*c-)D(E1fg~(VlyTPsRi%7 zORru)zB~_}GK3=W=j*RnL?-4##2P{l2k)v6{B$|F#12m`LXH9N)StZN2-1MM)pso9 zAn-2T{P`ovb8m&GBWW&j+oyM~B=a7Bnex`s(3U(jdXp=NMh? z|MH1#EZtf#1v9*IpGD4R8 zzBNi34uz*+6@npm{rHI?ax!MwLda6^HXr)~h3v*Y1@%C}-LdQ`g*0F{gi^46>8~#z zA-fP`&~~KEZO5!1B}bql23ZhK|2*icV^^&E*-7yDpLCS|wFaD#J6A&ufHJg4@4>)M z98ciRI~sm&v5;yEAodU?6w`W2l2IhLUaw_J$-og~=228w>&=G3rbmS3y^A19& zZu%es01DHdttSp?@WPv?YhjdtTZc4&zOV!XdV(0buYs5bs?@Id5Y+O2+HD*N^#V2Y zXuafTThcoyehE2b2lh>H2M50J)o-%L%I}*91DyZsM_=D5w-(4P$WFtAybZkjf0y1) zu0=*C69Vg9h^6X=e8kpT*_w4wYffc^k_>X4{eD3mZFME+R;6w-&!qg3O9kSu7S5*U$;S`XCepmrI>YXd_J`y34r4DZ+0ke^>pIpiuTC4L5|GbVKt`m{#_6 zzy&()X+Dd6tybnapS{5u7M<`{4xRDENLTk}fv1%mPZCW9Itt95^FRV$0@@15%yO@q zkHtqk{=|49PL=^JP>?2sRT7DK@?Y(CN#|0_5IW(Mx0_dfa*!=b0jY<^ffeed&O+10RYQjbd~ zVmx=H)7f|;rR3F|aa03~F2!Q%{j5(e0w4$`m`Emvlw?6I$-0rzmx2O9_WSBV`NrPI zhr>uhO~z^WMi~KzLsw4DHc7ddl2o$wo3ZZIFzV@0NTySxa!MIjX$D-h!30`%j3!<1&DM)fDW6ZeidVb zykMR#k)r941!6j@)8-I-^uaSA0-Ts{aMUk=f@Z`=X-3LkBAjgl3t$e{AeejiMI9$^ z>WcJG#PVVUlEoak1Jx98T)*aXDhS)S@CA-8Wq>czZ@~p8aH7^CUZlqbDs&=Vr03wY zcu_6~r^Tpn`!41}Ye+ zV4#A53I-||m;nZ;{m}wQSknIJSq`E0V*Y@O6zGJ{Ld@=gKkv(fbq*~jTWOGBVRGHa zMV=Qo-+pc|8VzU$PgRiW)kXOFRl3V(fvWx=-R85vio`2ad$+)buty`;BQLZGn`N`v;cr;Y!dj8jvDKa~jZ8%_hNVF$qqqNpRXsg41pe z9H&Ni?12?m!9WEA6%14`@cLrFIRE2rz?HTyxV8B|kxCTjfBQ7^eeFlEkFDHgrM1`2T= ztdMfh&8nd{v;I7hlC$arwVOawYAj?hHKAtFUP+CE?NU~a<1B9v0IRB${!p&=T|Fs&-$tg&f5U>b2Jt8ZajGBt8aoY48E#(S6 z#Ddv`6p7^}sa?j2yStq3_07uekTWYcnNfTDGkWm5@cw8M$rv?O&2Dq|DB&#A7?UI_tmE2a^ zzIanpsaJ)jX+r-{A zNkgpLOU*X7BsEC`1vM8}wo6tis19cfN_IPty9pZxsFz&W&gK@WFI=ZtOXU-JzKSKr z62rI@RTatUkhYFXSeRxznBSgZi(a@0heNMF5{}vj;73wm3@W2wABBBd6K=p>P)?*q z(sW=1c%~V3c*to-YeUE^539L6OvEGD-{q9L%@$YkXbMFKqukromrZBXY(4?TXZQh7 zBQJ+@g$=GIFl+>?(+wV5yUo=!79VNe;IuhCHa8?J>5StaKZ&^dGlPR#t56^uR&op# zYx!_5Qv`6Lm^r7}RJVxeF#|}m89-Xh0Mcp(5Gucyi`aG(Y68+3lDK4tt3x96RJnDF*@JNP6uT;H^cq`a>E%hHppzxrpI$&Wtvs3N`5?@vkt<<9k)8$ zTOByXl@G>g4j7j?VBF?_@t6aq8NrkwrPCZz4w*wr4NQ5Y)WDQSN)1eTq}0HaM@kJ0 zC*>k8m8(iz&B~wwP)b^1Ku$X-=>TMVb=I7!*g+!Cp_s+Fka7GkgV^jjjSFAfX&h~| zs29o||G8~hC2mcmVp$MzR1-Ks34rBwq!Iwj>ry2Eme;vT04%SYl>j&m?{Ku!ZU8&X z)Au;m*>Gc+DRiZ#fdM6vEq@jeP|72?fKncn1(fngE1;A|PXUD!QMaH_$R;TP(F_1t z1~HC*{E}pNbipAd#(#@Bgg?O)!V4(n5nezkkMIIYd4v~G$|JmhGTrEHHHU~(} zDhnv(kybz{kDdYwCn9bsfJI`Otr_BXkvR4DBvOURDdXP@@t+u6wMknB)olO1UCVNn zz4oU}jN%rvgVM}2Xq0xQL8CM@4H~7TY0xN5O@n5_Mn4#N3YwY$t@!{!YsJNoGVxZd zTbrrS+RcRKq$_J?wG?(h%g0-bpsC5w%oz2>x|uQRi_pv%^+jlAjQS!pGsbDFMpJX4 znK4d_bu(j}7NMCjPBmyQGsbBVni=D?P0-Y3#>gr{Gh<{Gp_wtViqOm$Sw(1OjD>cM z3MN2|^P1 z)jnQGs#bL}uVzz9(yqjkiE;Zzr(?5qv&$iOMfw9>!Kl4Qoluj`*TiGp9Jo+{A`0EK zmycJIqOur|lbf(!aWLzZwvJ{rkwEJezmiv^d{!AzvspmR2&|<`S!J8VOvvc|e@>c6 zC=xER%Sl=Z+qeeswdOF)0>vj;C#0m3r?`ZZS@!%6K*=9=Ht;b`k{@3Va9i-y`mlX&f&jt&nDNHKk9bk!l29?yWCR91$EaV5o0tis(#Bc$9m*e%MdNh@^sR)^azvgKsR zBPSEXS+I~&bNE?dtIcI|wtDPf>{ytz19X3WKCqi z=*Fl_YrzwOf89PnR>Xu?j zC6`OYxMdhOOxA2D9Bktq&Q+h%_vX1~x`vx+<|W>!Ov3%oc36kV=n? z*tWvg8Yy+VO$ED3kVy*}Fga7>faf*`%DW}XbJOxXvn?-|%h_=H)&m`5JK5SYlfZdO z%{1G_poEG$uoy6oe=O^ik1pG#jenFK&A0ypsO2uj&_$o*lhYacZ31Qb=W$ID42Zzq+UNJn z!DwV-7kdo5*ki9x-WZDXL^hoS@fhd}_6(lLpQF0xf#P#tG&0!V6<~>?x(owdJ)y7< z;-k^QaKJA|yM29_Y$(w0+Z2cvpJd-=V`xD44hzD>8wYzf%OcR#6%7{ypi0WYNPkzT zcT)v9fgzqMyqTp4 zNBjYRfbYT)IQ@2DFw8>HaHPivCE=Ta?m&My(j@hS{h%WE0(|xy4xLC_hw;6U-ku1Y zn#gTzmoBN;wJ(1o`TL9y7WDqJE_C+^SFpx#w6o|+JfB<}s%7a7j z)E8=!5PGOLfE8koav%b!dL#W;h8tM+f96)C?0xeNcXA0D52)772hXm4*>uQn) zhp>N9AJ9PkLAO3DB#7!VLa+yWE={S8Nr>fifv2HR6aWta8sTtQig$?V;)J1(Zhu!K z0MywM0D|a{O2aywvTzZlJgmbPjVU6O1-17@E|rGwqEbaJC8Wr%ARaWU3HH5R5z53a z04|4uJwF6G0$)@p@RbuRC|mZ7XZtUhVpvGK%YEUk`U?vlV$ zh4`pQFnjQsT)L2rsnbHwWkuCX7M?4kWqH&ulxa&=66xXCv~S1u{Wh1PnKfDsC(=cs zMktrGMZM`1=%%1n!MAPna~|86_VM6w2EG&0WT7tJ6X&v9r+qvK6`|h^E}NqI=CNxH3V$VMT%4;6VEV^KKNO%4Z#=kPo-oBFH{vh{H;67i6c_qZd_^}hXSI{U4{nNSM z&V9HD3Y|FJ{gso9VG(bZ|DCOW*|K+Wx16BeQ(pikKY^dMrt(|CKm`L83{)^s!N8jh1MIet-&9%r*s`jc*WR9l@;|zn`r@{w61Uvc z;h~Dv>v8YozPnyJOWbSm6u;NPi^YHwH!#(btvguElJl9r;e*U49#I<}F5j#7iX$J( zfmIJ{{d21t-hRR^aVe*K%v--&+@Q2SNSG1@-u=6J5X4YdEYsDWg*TSGo^H#!2Yxp7 zPNx^X!D>`XFAv7&#>01=HuZj}iTkFeI%i3*jhyps|2wB9UH8eUN#FJ4lQ+|2On*p^ z8-=b0mu+pPJ>{P*ztA+f57AlP&ADUY07j9#zifE{6y&?l~joXf2OB2t- zzKv(E#@C*5F-yEiJL%Ql@f5o$dOoJYvsoe4N2aD)v8(S<3X9(qjj7gM%2E{z-t)ch zUrB4e?PKhf<#r1R(VEv;FaH$zSK`9680WHDEXYpipc9;DX7{`a?sFKwOboG#PH>-U zO%I!@Z{bkI!D_-qMW7MlHC!d{9y$gJ!6qB-9s;Ep?j`~opC$=Tampv)3L`15X7Xe8 z5~kKp@hGGcd7~9O&GpjB_ng8I_p-$TqaX4>Pojv$XN&+h)t?<=@e(t*s+!*h$8}*1 zt@EcxD^-tYb&b^2ieeaVo{b)SV)q#EdJZuo+0PWKfLl%d7DP*NYlIw71>G1`s*G11S4?4j?v$Pq~CO`*7YZwE2-mdulHS>UH}G5vPKS zbix-{t08@4l1LC_Z&KH?H+(XrS2APFnz4hfr}w$e`$1vILh2y(Aw(ITs3yn5+ZE*2 zZT96P4v&YC#G^M_=}q;AlIQ=_v5C9_v4@an-??XySmE&q^7nkMK<=U0eBb}c1>`+6 zLC<$jyPA9g9vjHhzlhyRo}vkoA71qg!AbI9vS)KEpV}Z;asPP?Q2Ngo+>H}nlZ+Yh zYecmeQ8gKs5mpma8AUaLlXV6$&vVjijDhu%*g^ROjz^H0JAb1Dg`84I z^ULAc``d}f7`GL(G~=a$r$Q3$7`I&uA(kS>wPrq{X!rDf-PLx@R9SrDeg>o{i%(?t zwz0Se&t<+Cn|FSQg~ji`xVA3B{cr!>JE_&gZik2hf14Dr)QDj=kDAGgs}wCYVl>tj zZQjNGh?W{r?gDu7w%xu zZ}MNt%@>jE!dKRku9uN#r`qvupZlJw^%!Y1a=n+O*>*;Kuv7wLrEAhh{4oI6SKxd8t zrYjAab{n-=wNd!04P9%1>WEdFdUD~FtaiKjboZi|_8vxNP;qY}3>+F(AKUlx04yg^ zC)O9HTOJVd$o`N6@&x=b$3WwP#s$}kxTQOfZeT*1C|Ptm;fs`0g|y9FcoMx;_}P)h zfAdK+fG~V)dZUx*H!)%jn-TB0{_hgI?cUGoNN3e8SRBlmnDG8zf9f5i0iWPPESxjx zta>gykfWJAEAKexLPB{aWWlMLPP1r+OyF2tDhc5<`wNV?h^+)S{P_>xV%*&ak>GGv z{p(-;nP8#HuNfH7Pof;nL{C|wC!&FXh8|%cKGF9kVxY0UTN#MG;_Rx+%xF&q#_TXa zmq=^*vXliD+g9QT3|@P~m1k1@N=$ha>Y+Use|A=j=m-m)-q%*GyL!N!oLa`=Op6kD zabqe9M$rM_6Z4okU9T@3peYyeltmp55-ow-CxCUzz4X{^k3TVi53EF~Y_sLbW8vm?|h^|uP1K1C-tCSv)uQl%| zwIz*a9@^k24#vX(EP{tq5t7?S5;-uhp3#T|IYk@9R&7|qA`XKh!BC5Vc{>=X!vN42 z6AoL-m{7{YVvCGPt6Op4iXk{YDs@TYr&-~OFaCLIES=8A6Dj!UlkKcZ9Gp0qfUgiq zbWKar7c%o*y;8&wtAIy9dB`C}7?-p3;iIO6B?!-g@j~i~`)|VlQ#N+mJNRqc@C(*Z>J4z7)7WN|s{;Y#$hPM2YcEsw3`%1gKVOI--L00F@984X8?Z97e2SI^uC> z-TLrrbVPDLmLf6b_1u-q|91HYmwp3cDy|d;W~Kp3j6_6F9z%~Xkk&wWqVHM;+7Nw* zA^HwS^h755F0l6gPaIdO8I`y>Vt|Ty-kWr}w?6JUZ&qR+4mWhd$7fCanTwi-+hsV) zskk#!L4%cA{=TkO^P*6Rd|IoI5Gb`Pq3|%0LQ;{zbZ(r9Z?Fo+(syu)t*b-AZ9=@F ztV6;*I%ZkdA&s#IIR=k#%ty(++HcajSp_c1Lbjo$)s3g{4Zp*s=^(2 zs1U~G6qx!^!4-`WFcFg}b8te8kVNHC&?I?$b}v(%NoM-gDr9aItKZ1vp1I{N=9-0>e&fu{^xrf_`Va0H9!4`DvDqgK zu!Y4%JxJXPV8$_zWud5xPWXaFArnQXS$(h|(QG|m zI?zUv@LoR~;|)H@=r+@l2pKd6Bw^8bqmg4Aca;>ZrB7$Pu@fWqWxR36@^}9el2qJ* z!hl}u2=hW>=zF;sc^w7>FV%OG62=>X)_OV52qal0pz&f^hA@Zp;d;v6@7a3K}z)}Zg39AEMtE5dp^N@YvG1&A z8UQt>rk|>)R-Di*&>{i7mcYy5D5;=z9yeL2p6g< z?m%HcFOPG)6Fu#Tp3FqwO^Cin5Cg6CbS8Ql6MY|Pykr!4P{qBT7@+gN5QAw&J~;2B zr)M?)3$ar8IHnKa+jBMi&c`(#Y%={Ie%cvd m6hGCtV{T?00^Pz#w2Qgj2G9C1?n~&y4hUD~c0j`ay!(I5EOku) literal 0 HcmV?d00001 diff --git a/tests/server.py b/tests/server.py index 5e8a31d..45fad45 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1,5 +1,7 @@ """Test OpenDAP server with air temperature dataset.""" +from pathlib import Path + import numpy as np import xarray.tutorial import xpublish @@ -14,8 +16,17 @@ ds_attrs_cast.attrs["npint"] = np.int16(16) ds_attrs_cast.attrs["npintthirtytwo"] = np.int32(32) +ds_coordinates_encoding = xarray.open_dataset( + Path(__file__).parent / "min_coordinates_encoding.nc", +) + rest = xpublish.Rest( - {"air": ds, "attrs_quote": ds_attrs_quote, "attrs_cast": ds_attrs_cast}, + { + "air": ds, + "attrs_quote": ds_attrs_quote, + "attrs_cast": ds_attrs_cast, + "coords_encoding": ds_coordinates_encoding, + }, plugins={"opendap": OpenDapPlugin()}, ) diff --git a/tests/test_server.py b/tests/test_server.py index 70030a2..c75705a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -7,6 +7,7 @@ """ import sys +from pathlib import Path import netCDF4 import pytest @@ -31,7 +32,9 @@ def test_default_xarray_engine(xpublish_server, dataset): """Test opening OpenDAP air dataset with default Xarray engine.""" url = f"{xpublish_server}/datasets/air/opendap" ds = xr.open_dataset(url) - assert ds == dataset + # assert ds == dataset + xr.testing.assert_equal(ds, dataset) + # xr.testing.assert_identical(ds, dataset) @pytest.mark.skipif( @@ -76,3 +79,22 @@ def test_attrs_types(xpublish_server): assert ds.attrs["npint"] == 16 assert ds.attrs["npintthirtytwo"] == 32 + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="NetCDF4 is failing on Windows Github Actions workers", +) +def test_coordinates_persist_correctly(xpublish_server): + """Test that encoded coordinate data makes it through OpenDAP correctly. + + xref: https://github.com/xpublish-community/xpublish/discussions/246 + """ + dataset = xr.open_dataset(Path(__file__).parent / "min_coordinates_encoding.nc") + + url = f"{xpublish_server}/datasets/coords_encoding/opendap" + ds = xr.open_dataset(url) + assert ds.dims == dataset.dims + assert ds.coords == dataset.coords + assert ds.variables == dataset.variables + assert ds == dataset diff --git a/xpublish_opendap/dap_xarray.py b/xpublish_opendap/dap_xarray.py index 3dba87a..7402256 100644 --- a/xpublish_opendap/dap_xarray.py +++ b/xpublish_opendap/dap_xarray.py @@ -90,10 +90,12 @@ def dap_dimension(da: xr.DataArray) -> dap.Array: def dap_grid(da: xr.DataArray, dims: dict[str, dap.Array]) -> dap.Grid: """Transform an xarray DataArray into a DAP Grid.""" + encoded_da = xr.conventions.encode_cf_variable(da.variable) + data_grid = dap.Grid( name=da.name, - data=da.astype(da.dtype).data, - dtype=dap_dtype(da), + data=encoded_da.astype(encoded_da.dtype).data, + dtype=dap_dtype(encoded_da), dimensions=[dims[dim] for dim in da.dims], )