diff --git a/.gitignore b/.gitignore index 2f41357..435dc68 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -examples/preload_cache/ \ No newline at end of file +examples/zenodo_cache/ \ No newline at end of file diff --git a/environment.yml b/environment.yml index d170b8e..f0202df 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: dependencies: # Required - python>=3.10 - - IPython + - fsspec - numpy - requests - tabulate diff --git a/examples/zenodo_data_store.ipynb b/examples/zenodo_data_store.ipynb index 43af87c..76d0a1c 100644 --- a/examples/zenodo_data_store.ipynb +++ b/examples/zenodo_data_store.ipynb @@ -24,15 +24,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.04 s, sys: 234 ms, total: 3.27 s\n", - "Wall time: 1.41 s\n" + "CPU times: user 7 μs, sys: 1 μs, total: 8 μs\n", + "Wall time: 10 μs\n" ] } ], @@ -52,15 +52,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 31.6 ms, sys: 13.8 ms, total: 45.5 ms\n", - "Wall time: 44.6 ms\n" + "CPU times: user 33.4 ms, sys: 8.93 ms, total: 42.3 ms\n", + "Wall time: 41.6 ms\n" ] }, { @@ -71,6 +71,11 @@ "access_token": { "title": "Zenodo access token.", "type": "string" + }, + "preload_cache_folder": { + "description": "Datasets which are accessed using prelaod_data will be stored in this folder in a prepared way.", + "title": "Preload cache folder.", + "type": "string" } }, "required": [ @@ -79,10 +84,10 @@ "type": "object" }, "text/plain": [ - "" + "" ] }, - "execution_count": 2, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -102,21 +107,21 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 3.44 ms, sys: 1.04 ms, total: 4.48 ms\n", - "Wall time: 4.43 ms\n" + "CPU times: user 5.17 ms, sys: 15 μs, total: 5.18 ms\n", + "Wall time: 5.1 ms\n" ] } ], "source": [ "%%time\n", - "access_token = \"fill in you Zenodo access token here\"\n", + "access_token = \"XVYlYi840itHkdbVXN2PtQk9cBeIY2WQ9OlVQeIssaP9YgMDAAyjlnIc6H6l\"\n", "store = new_data_store(\"zenodo\", access_token=access_token)" ] }, @@ -129,33 +134,33 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 365 ms, sys: 30.8 ms, total: 396 ms\n", - "Wall time: 2min 18s\n" + "CPU times: user 43.9 ms, sys: 5.57 ms, total: 49.4 ms\n", + "Wall time: 25.5 s\n" ] }, { "data": { "text/plain": [ - "['14054126/AVIRIS-NG-methane-emissions_v2024-11-04.nc',\n", - " '14052963/CESM.b.T31_g37.HighPrec.SO.pop.DIC.nc',\n", - " '14052963/CESM.b.T31_g37.Equilibration.500-1000.pop.nc',\n", - " '14052963/CESM.b.T31_g37.NoPrec.SH.cam.nc',\n", - " '14052963/CESM.b.T31_g37.HighPrec.SH.cam.nc',\n", - " '14052963/CESM.b.T31_g37.Equilibration.0-500.pop.nc',\n", - " '14052963/CESM.b.T31_g37.NoPrec.SO.pop.nc',\n", - " '14052963/CESM.b.T31_g37.NoPrec.SO.pop.DIC.nc',\n", - " '14052963/CESM.b.T31_g37.HighPrec.SO.pop.nc',\n", - " '14051280/Mitoflash_CM-8bit.tif']" + "['14446470/pretrained.zip',\n", + " '14446770/mastodon-sc/mastodon-deep-lineage-mastodon-deep-lineage-0.4.3.zip',\n", + " '14446677/MeTech/haptic-oriring-public-v1.1.zip',\n", + " '14446628/magicrjk/treeofrobots-ToR-V1.0.zip',\n", + " '14446620/Ahmet-Agaoglu/The-Corridor-Method-0.1.0.zip',\n", + " '14446612/FAIRmat-NFDI/pynxtools-xps-v0.4.9.zip',\n", + " '14446605/PowerGridModel/power-grid-model-v1.10.22.zip',\n", + " '14446598/FerdinandKlingenberg/tree-cover-density-comparison-V1.0.0.zip',\n", + " '14446582/KatieWillis/DriveSelectionBalance-DriveSelectionBalance.zip',\n", + " '14441477/simulation_data.zip']" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -182,7 +187,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -271,10 +276,10 @@ ] }, "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -292,15 +297,15 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 192 μs, sys: 19 μs, total: 211 μs\n", - "Wall time: 214 μs\n" + "CPU times: user 217 μs, sys: 20 μs, total: 237 μs\n", + "Wall time: 239 μs\n" ] }, { @@ -348,10 +353,10 @@ "type": "object" }, "text/plain": [ - "" + "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -364,15 +369,15 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 203 ms, sys: 3.83 ms, total: 207 ms\n", - "Wall time: 206 ms\n" + "CPU times: user 21.2 ms, sys: 4.01 ms, total: 25.2 ms\n", + "Wall time: 24.1 ms\n" ] }, { @@ -750,9 +755,9 @@ "Data variables:\n", " band_1 (y, x) uint8 25GB dask.array<chunksize=(1024, 1024), meta=np.ndarray>\n", "Attributes:\n", - " source: https://zenodo.org/records/8154445/files/planet_canopy_cover_30..." + " dtype='float64', name='y', length=149363))
  • source :
    https://zenodo.org/records/8154445/files/planet_canopy_cover_30m_v0.1.tif
    • " ], "text/plain": [ " Size: 25GB\n", @@ -876,7 +881,7 @@ " source: https://zenodo.org/records/8154445/files/planet_canopy_cover_30..." ] }, - "execution_count": 14, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -899,24 +904,24 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 321 ms, sys: 68.4 ms, total: 389 ms\n", - "Wall time: 3.06 s\n" + "CPU times: user 661 ms, sys: 122 ms, total: 783 ms\n", + "Wall time: 4 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 15, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" }, @@ -945,15 +950,15 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 5.15 ms, sys: 0 ns, total: 5.15 ms\n", - "Wall time: 4.9 ms\n" + "CPU times: user 5.73 ms, sys: 1.12 ms, total: 6.85 ms\n", + "Wall time: 161 ms\n" ] }, { @@ -962,7 +967,7 @@ "1" ] }, - "execution_count": 16, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -979,7 +984,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -1357,9 +1362,9 @@ "Data variables:\n", " band_1 (y, x) uint8 25GB dask.array<chunksize=(1024, 1024), meta=np.ndarray>\n", "Attributes:\n", - " source: https://zenodo.org/records/8154445/files/planet_canopy_cover_30..." + " dtype='float64', name='y', length=149363))
  • source :
    https://zenodo.org/records/8154445/files/planet_canopy_cover_30m_v0.1.tif
    • " ], "text/plain": [ " Size: 25GB\n", @@ -1483,7 +1488,7 @@ " source: https://zenodo.org/records/8154445/files/planet_canopy_cover_30..." ] }, - "execution_count": 17, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -1509,15 +1514,15 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 83.2 ms, sys: 15.3 ms, total: 98.4 ms\n", - "Wall time: 9.03 s\n" + "CPU times: user 65.9 ms, sys: 21.8 ms, total: 87.7 ms\n", + "Wall time: 8.05 s\n" ] }, { @@ -1910,7 +1915,7 @@ " Modelrun: ERA5weather\n", " Modelconfig: newfriction\n", " TimeStamp: 11-Jun-2024 13:01:44\n", - " Notes: Statistics based on ERA5 hindcast run from 1980 to 2022. \\n...