fix: updated ftp directory for FES2022 ocean tide model (#376)

* fix: updated ftp directory for FES2022 ocean tide model fix: update JSON databases for new FES2022 directory docs: update FES2022 directory in getting started * test: retrieve spice kernel file from AWS
tsutterley · Jan 8, 2025 · b5e5457 · b5e5457
1 parent b674de7
commit b5e5457
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 80 deletions.
diff --git a/doc/source/getting_started/Getting-Started.rst b/doc/source/getting_started/Getting-Started.rst
@@ -84,7 +84,7 @@ Presently, the following models and their directories are parameterized within `
 
     * `FES2014 <https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes/description-fes2014.html>`_: ``<path_to_tide_models>/fes2014/ocean_tide/``
     * `FES2014_load <https://www.aviso.altimetry.fr/en/data/products/auxiliary-products/global-tide-fes/description-fes2014.html>`_: ``<path_to_tide_models>/fes2014/load_tide/``
-    * `FES2022 <https://doi.org/10.24400/527896/A01-2024.004>`_: ``<path_to_tide_models>/fes2022b/ocean_tide/``
+    * `FES2022 <https://doi.org/10.24400/527896/A01-2024.004>`_: ``<path_to_tide_models>/fes2022b/ocean_tide_20241025/``
     * `FES2022_load <https://doi.org/10.24400/527896/A01-2024.004>`_: ``<path_to_tide_models>/fes2022b/load_tide/``
 
 - Empirical Ocean Tide models :cite:p:`HartDavis:2021dx`

diff --git a/providers/AVISO.json b/providers/AVISO.json
@@ -268,40 +268,40 @@
         "FES2022": {
             "format": "FES-netcdf",
             "model_file": [
-                "fes2022b/ocean_tide/2n2_fes2022.nc",
-                "fes2022b/ocean_tide/eps2_fes2022.nc",
-                "fes2022b/ocean_tide/j1_fes2022.nc",
-                "fes2022b/ocean_tide/k1_fes2022.nc",
-                "fes2022b/ocean_tide/k2_fes2022.nc",
-                "fes2022b/ocean_tide/l2_fes2022.nc",
-                "fes2022b/ocean_tide/lambda2_fes2022.nc",
-                "fes2022b/ocean_tide/m2_fes2022.nc",
-                "fes2022b/ocean_tide/m3_fes2022.nc",
-                "fes2022b/ocean_tide/m4_fes2022.nc",
-                "fes2022b/ocean_tide/m6_fes2022.nc",
-                "fes2022b/ocean_tide/m8_fes2022.nc",
-                "fes2022b/ocean_tide/mf_fes2022.nc",
-                "fes2022b/ocean_tide/mks2_fes2022.nc",
-                "fes2022b/ocean_tide/mm_fes2022.nc",
-                "fes2022b/ocean_tide/mn4_fes2022.nc",
-                "fes2022b/ocean_tide/ms4_fes2022.nc",
-                "fes2022b/ocean_tide/msf_fes2022.nc",
-                "fes2022b/ocean_tide/msqm_fes2022.nc",
-                "fes2022b/ocean_tide/mtm_fes2022.nc",
-                "fes2022b/ocean_tide/mu2_fes2022.nc",
-                "fes2022b/ocean_tide/n2_fes2022.nc",
-                "fes2022b/ocean_tide/n4_fes2022.nc",
-                "fes2022b/ocean_tide/nu2_fes2022.nc",
-                "fes2022b/ocean_tide/o1_fes2022.nc",
-                "fes2022b/ocean_tide/p1_fes2022.nc",
-                "fes2022b/ocean_tide/q1_fes2022.nc",
-                "fes2022b/ocean_tide/r2_fes2022.nc",
-                "fes2022b/ocean_tide/s1_fes2022.nc",
-                "fes2022b/ocean_tide/s2_fes2022.nc",
-                "fes2022b/ocean_tide/s4_fes2022.nc",
-                "fes2022b/ocean_tide/sa_fes2022.nc",
-                "fes2022b/ocean_tide/ssa_fes2022.nc",
-                "fes2022b/ocean_tide/t2_fes2022.nc"
+                "fes2022b/ocean_tide_20241025/2n2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/eps2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/j1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/k1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/k2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/l2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/lambda2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m3_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m6_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m8_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mf_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mks2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mn4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/ms4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/msf_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/msqm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mtm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mu2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/n2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/n4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/nu2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/o1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/p1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/q1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/r2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/sa_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/ssa_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/t2_fes2022.nc"
             ],
             "name": "FES2022",
             "reference": "https://doi.org/10.24400/527896/A01-2024.004",

diff --git a/pyTMD/data/database.json b/pyTMD/data/database.json
@@ -1222,40 +1222,40 @@
         "FES2022": {
             "format": "FES-netcdf",
             "model_file": [
-                "fes2022b/ocean_tide/2n2_fes2022.nc",
-                "fes2022b/ocean_tide/eps2_fes2022.nc",
-                "fes2022b/ocean_tide/j1_fes2022.nc",
-                "fes2022b/ocean_tide/k1_fes2022.nc",
-                "fes2022b/ocean_tide/k2_fes2022.nc",
-                "fes2022b/ocean_tide/l2_fes2022.nc",
-                "fes2022b/ocean_tide/lambda2_fes2022.nc",
-                "fes2022b/ocean_tide/m2_fes2022.nc",
-                "fes2022b/ocean_tide/m3_fes2022.nc",
-                "fes2022b/ocean_tide/m4_fes2022.nc",
-                "fes2022b/ocean_tide/m6_fes2022.nc",
-                "fes2022b/ocean_tide/m8_fes2022.nc",
-                "fes2022b/ocean_tide/mf_fes2022.nc",
-                "fes2022b/ocean_tide/mks2_fes2022.nc",
-                "fes2022b/ocean_tide/mm_fes2022.nc",
-                "fes2022b/ocean_tide/mn4_fes2022.nc",
-                "fes2022b/ocean_tide/ms4_fes2022.nc",
-                "fes2022b/ocean_tide/msf_fes2022.nc",
-                "fes2022b/ocean_tide/msqm_fes2022.nc",
-                "fes2022b/ocean_tide/mtm_fes2022.nc",
-                "fes2022b/ocean_tide/mu2_fes2022.nc",
-                "fes2022b/ocean_tide/n2_fes2022.nc",
-                "fes2022b/ocean_tide/n4_fes2022.nc",
-                "fes2022b/ocean_tide/nu2_fes2022.nc",
-                "fes2022b/ocean_tide/o1_fes2022.nc",
-                "fes2022b/ocean_tide/p1_fes2022.nc",
-                "fes2022b/ocean_tide/q1_fes2022.nc",
-                "fes2022b/ocean_tide/r2_fes2022.nc",
-                "fes2022b/ocean_tide/s1_fes2022.nc",
-                "fes2022b/ocean_tide/s2_fes2022.nc",
-                "fes2022b/ocean_tide/s4_fes2022.nc",
-                "fes2022b/ocean_tide/sa_fes2022.nc",
-                "fes2022b/ocean_tide/ssa_fes2022.nc",
-                "fes2022b/ocean_tide/t2_fes2022.nc"
+                "fes2022b/ocean_tide_20241025/2n2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/eps2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/j1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/k1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/k2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/l2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/lambda2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m3_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m6_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/m8_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mf_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mks2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mn4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/ms4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/msf_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/msqm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mtm_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/mu2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/n2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/n4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/nu2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/o1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/p1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/q1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/r2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s1_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s2_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/s4_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/sa_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/ssa_fes2022.nc",
+                "fes2022b/ocean_tide_20241025/t2_fes2022.nc"
             ],
             "name": "FES2022",
             "reference": "https://doi.org/10.24400/527896/A01-2024.004",

diff --git a/scripts/aviso_fes_tides.py b/scripts/aviso_fes_tides.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 aviso_fes_tides.py
-Written by Tyler Sutterley (07/2024)
+Written by Tyler Sutterley (01/2025)
 Downloads the FES (Finite Element Solution) global tide model from AVISO
 Decompresses the model tar files into the constituent files and auxiliary files
     https://www.aviso.altimetry.fr/data/products/auxiliary-products/
@@ -40,6 +40,8 @@
     utilities.py: download and management utilities for syncing files
 
 UPDATE HISTORY:
+    Updated 01/2025: new ocean tide directory for latest FES2022 version
+        scrubbed use of pathlib.os to just use os directly
     Updated 07/2024: added list and download for FES2022 tide model
         compare modification times with remote to not overwrite files
     Updated 05/2023: added option to change connection timeout
@@ -234,7 +236,9 @@ def aviso_fes_list(MODEL, f, logger,
     FES = {}
     # 2022 model
     FES['FES2022'] = []
-    FES['FES2022'].append(['fes2022b','ocean_tide'])
+    # updated directory for ocean tide model
+    # latest version fixes the valid_max attribute for longitudes
+    FES['FES2022'].append(['fes2022b','ocean_tide_20241025'])
     if LOAD:
         FES['FES2022'].append(['fes2022b','load_tide'])
     if EXTRAPOLATED:
@@ -320,7 +324,7 @@ def ftp_download(logger, ftp, remote_path, local_dir,
                 shutil.copyfileobj(fi, fo)
             # get last modified date of remote file within tar file
             # keep remote modification time of file and local access time
-            pathlib.os.utime(local_file, (local_file.stat().st_atime, m.mtime))
+            os.utime(local_file, (local_file.stat().st_atime, m.mtime))
             local_file.chmod(mode=MODE)
     elif LZMA:
         # get last modified date of remote file and convert into unix time
@@ -350,7 +354,7 @@ def ftp_download(logger, ftp, remote_path, local_dir,
             shutil.copyfileobj(fi, fo)
         # get last modified date of remote file within tar file
         # keep remote modification time of file and local access time
-        pathlib.os.utime(local_file, (local_file.stat().st_atime, mtime))
+        os.utime(local_file, (local_file.stat().st_atime, mtime))
         local_file.chmod(mode=MODE)
     else:
         # copy readme and uncompressed files directly
@@ -375,7 +379,7 @@ def ftp_download(logger, ftp, remote_path, local_dir,
         with opener(local_file, 'wb') as f:
             ftp.retrbinary(f'RETR {remote_file}', f.write, blocksize=CHUNK)
         # keep remote modification time of file and local access time
-        pathlib.os.utime(local_file, (local_file.stat().st_atime, mtime))
+        os.utime(local_file, (local_file.stat().st_atime, mtime))
         local_file.chmod(mode=MODE)
 
 # PURPOSE: compare the modification time of two files

diff --git a/scripts/gsfc_got_tides.py b/scripts/gsfc_got_tides.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 gsfc_got_tides.py
-Written by Tyler Sutterley (09/2024)
+Written by Tyler Sutterley (01/2025)
 Download GSFC Global Ocean Tide (GOT) models
 
 CALLING SEQUENCE:
@@ -32,12 +32,14 @@
     utilities.py: download and management utilities for syncing files
 
 UPDATE HISTORY:
+    Updated 01/2025: scrubbed use of pathlib.os to just use os directly
     Updated 09/2024: added Ray and Erofeeva (2014) long-period tide model
     Updated 08/2024: keep prime nomenclature for 3rd degree tides
     Written 07/2024
 """
 from __future__ import print_function, annotations
 
+import os
 import re
 import gzip
 import shutil
@@ -120,7 +122,7 @@ def gsfc_got_tides(MODEL: str,
             shutil.copyfileobj(fi, fo)
         # get last modified date of remote file within tar file
         # keep remote modification time of file and local access time
-        pathlib.os.utime(local_file, (local_file.stat().st_atime, m.mtime))
+        os.utime(local_file, (local_file.stat().st_atime, m.mtime))
         local_file.chmod(mode=MODE)
 
 # PURPOSE: compare the modification time of two files

diff --git a/scripts/verify_box_tpxo.py b/scripts/verify_box_tpxo.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 verify_box_tpxo.py
-Written by Tyler Sutterley (09/2024)
+Written by Tyler Sutterley (01/2025)
 Verifies downloaded TPXO9-atlas global tide models from the box file
     sharing service
 
@@ -26,6 +26,7 @@
     https://developer.box.com/guides/
 
 UPDATE HISTORY:
+    Updated 01/2025: scrubbed use of pathlib.os to just use os directly
     Updated 09/2024: use model class to define output directory
     Updated 04/2023: using pathlib to define and expand paths
     Updated 01/2023: use default context from utilities module
@@ -38,6 +39,7 @@
 """
 from __future__ import print_function
 
+import os
 import re
 import json
 import logging
@@ -132,7 +134,7 @@ def verify_box_tpxo(tide_dir, folder_id, TIDE_MODEL=None,
             logger.critical(f'Local checksum: {sha1}')
             raise Exception('Checksum verification failed')
         # keep remote modification time of file and local access time
-        pathlib.os.utime(local, (local.stat().st_atime, remote_mtime))
+        os.utime(local, (local.stat().st_atime, remote_mtime))
         # change the permissions mode of the local file
         local.chmod(mode=MODE)
 

diff --git a/test/test_solid_earth.py b/test/test_solid_earth.py
@@ -1,5 +1,5 @@
 """
-test_solid_earth.py (11/2024)
+test_solid_earth.py (01/2025)
 Tests the steps for calculating the solid earth tides
 
 PYTHON DEPENDENCIES:
@@ -10,6 +10,7 @@
         https://pypi.org/project/timescale/
 
 UPDATE HISTORY:
+    Updated 01/2025: added function to get JPL ephemerides file from AWS
     Updated 11/2024: moved normalize_angle and polynomial_sum to math.py
     Updated 07/2024: use normalize_angle from pyTMD astro module
     Updated 04/2024: use timescale for temporal operations
@@ -18,7 +19,10 @@
     Updated 04/2023: added test for using JPL ephemerides for positions
     Written 04/2023
 """
+import boto3
 import pytest
+import shutil
+import posixpath
 import numpy as np
 import pyTMD.astro
 import pyTMD.compute
@@ -291,7 +295,7 @@ def test_solid_earth_radial(EPHEMERIDES):
     assert np.isclose(tide_mean-tide_free, predicted, atol=5e-4).all()
 
 # PURPOSE: Download JPL ephemerides from Solar System Dynamics server
-@pytest.fixture(scope="module", autouse=True)
+@pytest.fixture(scope="module", autouse=False)
 def download_jpl_ephemerides():
     """Download JPL ephemerides from Solar System Dynamics server
     """
@@ -308,6 +312,38 @@ def download_jpl_ephemerides():
         # run tests
         yield
 
+# PURPOSE: Retrieve JPL ephemerides from AWS S3 bucket
+@pytest.fixture(scope="module", autouse=True)
+def aws_ephemerides(aws_access_key_id, aws_secret_access_key, aws_region_name):
+    """Retrieve JPL ephemerides from AWS S3 bucket
+    """
+    # get aws session object
+    session = boto3.Session(
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        region_name=aws_region_name)
+    # get s3 object and bucket object for pytmd data
+    s3 = session.resource('s3')
+    bucket = s3.Bucket('pytmd')
+    # get path to default ephemerides
+    de440s = pyTMD.astro._default_kernel
+    # get JPL ephemerides from AWS if not existing
+    if not de440s.exists():
+        # retrieve spice kernel file
+        obj = bucket.Object(key=posixpath.join('spice',de440s.name))
+        response = obj.get()
+        # save kernel to destination
+        with de440s.open('wb') as destination:
+            shutil.copyfileobj(response['Body'], destination)
+        assert de440s.exists()
+        # run tests
+        yield
+        # clean up
+        de440s.unlink(missing_ok=True)
+    else:
+        # run tests
+        yield  
+
 def test_solar_ecef():
     """Test solar ECEF coordinates with ephemeride predictions
     """