Use temp .netrc file for integration tests and support NETRC environment variable

.github/actions/install-pkg/action.yml

@@ -21,4 +21,4 @@ runs:
 
     - name: Install package and test dependencies
       shell: bash
-      run: pip install .[test]
+      run: pip install --root-user-action ignore ".[test]"

.github/workflows/integration-test.yml

@@ -47,14 +47,19 @@ jobs:
 
     steps:
       - name: Fetch user permission
+        if: github.event_name == 'pull_request_target'
         id: permission
         uses: actions-cool/check-user-permission@v2
         with:
           require: write
           username: ${{ github.triggering_actor }}
 
       - name: Check user permission
-        if: ${{ steps.permission.outputs.require-result == 'false' }}
+        # The name of the output require-result is a bit confusing, but when its value
+        # is 'false', it means that the triggering actor does NOT have the required
+        # permission.
+        if: github.event_name == 'pull_request_target' && steps.permission.outputs.require-result == 'false'
+
         # If the triggering actor does not have write permission (i.e., this is a
         # PR from a fork), then we exit, otherwise most of the integration tests will
         # fail because they require access to secrets.  In this case, a maintainer
@@ -78,8 +83,6 @@ jobs:
         env:
           EARTHDATA_USERNAME: ${{ secrets.EDL_USERNAME }}
           EARTHDATA_PASSWORD: ${{ secrets.EDL_PASSWORD }}
-          EARTHACCESS_TEST_USERNAME: ${{ secrets.EDL_USERNAME }}
-          EARTHACCESS_TEST_PASSWORD: ${{ secrets.EDL_PASSWORD }}
         run: ./scripts/integration-test.sh
 
       - name: Upload coverage report

.github/workflows/test-mindeps.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version-file: pyproject.toml
+          python-version: 3.9
 
       - name: Install minimum-compatible dependencies
         run: uv sync --resolution lowest-direct --extra test

.gitignore

@@ -9,6 +9,7 @@ htmlcov
 dist
 site
 .coverage
+.coverage.*
 coverage.xml
 .netlify
 test.db

CHANGELOG.md

@@ -13,6 +13,9 @@
   instead ([#766](https://github.com/nsidc/earthaccess/issues/766))
   ([**@Sherwin-14**](https://github.com/Sherwin-14),
   [**@chuckwondo**](https://github.com/chuckwondo))
+- Use built-in `assert` statement in integration tests
+  ([#743](https://github.com/nsidc/earthaccess/issues/743))
+  ([**@chuckwondo**](https://github.com/chuckwondo))
 
 ### Added
 
@@ -25,6 +28,9 @@
   [**@chuckwondo**](https://github.com/chuckwondo),
   [**@mfisher87**](https://github.com/mfisher87),
   [**@betolink**](https://github.com/betolink))
+- Support use of `NETRC` environment variable to override default `.netrc` file
+  location ([#480](https://github.com/nsidc/earthaccess/issues/480))
+  ([**@chuckwondo**](https://github.com/chuckwondo))
 
 - Added example PR links to pull request template
   ([#756](https://github.com/nsidc/earthaccess/issues/756))
@@ -38,6 +44,9 @@
 - Removed Broken Link "Introduction to NASA earthaccess"
   ([#779](https://github.com/nsidc/earthaccess/issues/779))
   ([**@Sherwin-14**](https://github.com/Sherwin-14))
+- Integration tests no longer clobber existing `.netrc` file
+  ([#806](https://github.com/nsidc/earthaccess/issues/806))
+  ([**@chuckwondo**](https://github.com/chuckwondo))
 
 ## [0.10.0] 2024-07-19
 

docs/contributing/development.md

@@ -17,20 +17,29 @@ If you don't have pipx (pip for applications), then you can install with
 pip is reasonable). If you use macOS, then pipx and nox are both in brew, use
 `brew install pipx nox`.
 
-To use, run `nox`. This will typecheck and test using every installed version of
-Python on your system, skipping ones that are not installed. You can also run
-specific jobs:
+To use, run `nox` without any arguments.  This will run type checks and unit
+tests using the installed version of Python on your system.
+
+You can also run individual tasks (_sessions_ in `nox` parlance, hence the `-s`
+option below), like so:
 
 ```console
-$ nox -s typecheck              # Typecheck only
-$ nox -s tests                  # Python tests
-$ nox -s build_docs -- --serve  # Build and serve the docs
-$ nox -s build_pkg              # Make an SDist and wheel
+nox -s typecheck              # Run typechecks
+nox -s tests                  # Run unit tests
+nox -s integration-tests      # Run integration tests (see note below)
+nox -s build_docs -- --serve  # Build and serve the docs
+nox -s build_pkg              # Build an SDist and wheel
 ```
 
 Nox handles everything for you, including setting up a temporary virtual
 environment for each run.
 
+**NOTE:** In order to run integration tests locally, you must set the
+environment variables `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` to your
+username and password, respectively, of your
+[NASA Earthdata](https://urs.earthdata.nasa.gov/) account (registration is
+free).
+
 ## Manual development environment setup
 
 While `nox` is the fastest way to get started, you will likely need a full

docs/howto/authenticate.md

@@ -1,74 +1,90 @@
-## Authenticate with Earthdata Login
+# Authenticate with Earthdata Login
 
-The first step to use NASA Earthdata is to create an account with Earthdata Login, please follow the instructions at [NASA EDL](https://urs.earthdata.nasa.gov/)
+The first step to use NASA Earthdata is to create an account with Earthdata
+Login, please follow the instructions at
+[NASA EDL](https://urs.earthdata.nasa.gov/)
 
-Once registered, earthaccess can use environment variables, a `.netrc` file or interactive input from a user to login with NASA EDL.
+Once registered, earthaccess can use environment variables, a `.netrc` file or
+interactive input from a user to login with NASA EDL.
 
-If a strategy is not especified, env vars will be used first, then netrc and finally user's input.
+If a strategy is not specified, environment variables will be used first, then
+a `.netrc` (if found, see below), and finally a user's input.
 
 ```py
 import earthaccess
 
 auth = earthaccess.login()
 ```
 
-If you have a .netrc file with your Earthdata Login credentials
+If you have a `.netrc` file (see below) with your Earthdata Login credentials,
+you can explicitly specify its use:
 
 ```py
 auth = earthaccess.login(strategy="netrc")
 ```
 
-If your Earthdata Login credentials are set as environment variables: EARTHDATA_USERNAME, EARTHDATA_PASSWORD
+If your Earthdata Login credentials are set as the environment variables
+`EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD`, you can explicitly specify their
+use:
 
 ```py
 auth = earthaccess.login(strategy="environment")
 ```
 
-If you wish to enter your Earthdata Login credentials when prompted with optional persistence to .netrc
+If you wish to enter your Earthdata Login credentials when prompted, with
+optional persistence to your `.netrc` file (see below), specify the interactive
+strategy:
 
 ```py
 auth = earthaccess.login(strategy="interactive", persist=True)
 ```
 
+## Authentication
 
+By default, `earthaccess` with automatically look for your EDL account
+credentials in two locations:
 
-### **Authentication**
+1. A `.netrc` file: By default, this is either `~/_netrc` (on a Windows system)
+   or `~/.netrc` (on a non-Windows system).  On *any* system, you may override
+   the default location by setting the `NETRC` environment variable to the path
+   of your desired `.netrc` file.
 
-By default, `earthaccess` with automatically look for your EDL account credentials in two locations:
-
-1. A `~/.netrc` file
+   **NOTE**: When setting the `NETRC` environment variable, there is no
+   requirement to use a specific filename.  The name `.netrc` is common, but
+   used throughout documentation primarily for convenience.  The only
+   requirement is that the *contents* of the file adhere to the
+   [`.netrc` file format](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html).
 2. `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables
 
-If neither of these options are configured, you can authenticate by calling the `earthaccess.login()` method
-and manually entering your EDL account credentials.
+If neither of these options are configured, you can authenticate by calling the
+`earthaccess.login()` method and manually entering your EDL account credentials.
 
 ```python
 import earthaccess
 
 earthaccess.login()
 ```
 
-Note you can pass `persist=True` to `earthaccess.login()` to have the EDL account credentials you enter
-automatically saved to a `~/.netrc` file for future use.
-
+Note you can pass `persist=True` to `earthaccess.login()` to have the EDL
+account credentials you enter automatically saved to your `.netrc` file (see
+above) for future use.
 
 Once you are authenticated with NASA EDL you can:
 
 * Get a file from a DAAC using a `fsspec` session.
-* Request temporary S3 credentials from a particular DAAC (needed to download or stream data from an S3 bucket in the cloud).
+* Request temporary S3 credentials from a particular DAAC (needed to download or
+  stream data from an S3 bucket in the cloud).
 * Use the library to download or stream data directly from S3.
 * Regenerate CMR tokens (used for restricted datasets).
 
+## Earthdata User Acceptance Testing (UAT) environment
 
-### Earthdata User Acceptance Testing (UAT) environment
-
-If your EDL account is authorized to access the User Acceptance Testing (UAT) system,
-you can set earthaccess to work with its EDL and CMR endpoints
-by setting the `system` argument at login, as follows:
+If your EDL account is authorized to access the User Acceptance Testing (UAT)
+system, you can set earthaccess to work with its EDL and CMR endpoints by
+setting the `system` argument at login, as follows:
 
 ```python
 import earthaccess
 
 earthaccess.login(system=earthaccess.UAT)
-
 ```

earthaccess/__init__.py

@@ -1,6 +1,7 @@
 import logging
 import threading
 from importlib.metadata import version
+from typing import Optional
 
 from .api import (
     auth_environ,
@@ -21,7 +22,7 @@
 )
 from .auth import Auth
 from .kerchunk import consolidate_metadata
-from .search import DataCollections, DataGranules
+from .search import DataCollection, DataCollections, DataGranule, DataGranules
 from .services import DataServices
 from .store import Store
 from .system import PROD, UAT
@@ -46,7 +47,9 @@
     "download",
     "auth_environ",
     # search.py
+    "DataGranule",
     "DataGranules",
+    "DataCollection",
     "DataCollections",
     "DataServices",
     # auth.py
@@ -62,7 +65,7 @@
 __version__ = version("earthaccess")
 
 _auth = Auth()
-_store = None
+_store: Optional[Store] = None
 _lock = threading.Lock()
 
 

earthaccess/auth.py

@@ -25,6 +25,24 @@
 logger = logging.getLogger(__name__)
 
 
+def netrc_path() -> Path:
+    """Return the path of the `.netrc` file.
+
+    The path may or may not exist.
+
+    See [the `.netrc` file](https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html).
+
+    Returns:
+        `Path` of the `NETRC` environment variable, if the value is non-empty;
+        otherwise, the path of the platform-specific default location:
+        `~/_netrc` on Windows systems, `~/.netrc` on non-Windows systems.
+    """
+    sys_netrc_name = "_netrc" if platform.system() == "Windows" else ".netrc"
+    env_netrc = os.environ.get("NETRC")
+
+    return Path(env_netrc) if env_netrc else Path.home() / sys_netrc_name
+
+
 class SessionWithHeaderRedirection(requests.Session):
     """Requests removes auth headers if the redirect happens outside the
     original req domain.
@@ -104,11 +122,12 @@ def login(
         if self.authenticated and (system == self.system):
             logger.debug("We are already authenticated with NASA EDL")
             return self
+
         if strategy == "interactive":
             self._interactive(persist)
-        if strategy == "netrc":
+        elif strategy == "netrc":
             self._netrc()
-        if strategy == "environment":
+        elif strategy == "environment":
             self._environment()
 
         return self
@@ -222,25 +241,29 @@ def _interactive(self, persist_credentials: bool = False) -> bool:
         if authenticated:
             logger.debug("Using user provided credentials for EDL")
             if persist_credentials:
-                logger.info("Persisting credentials to .netrc")
                 self._persist_user_credentials(username, password)
         return authenticated
 
     def _netrc(self) -> bool:
+        netrc_loc = netrc_path()
+
         try:
-            my_netrc = Netrc()
+            my_netrc = Netrc(str(netrc_loc))
         except FileNotFoundError as err:
-            raise FileNotFoundError(f"No .netrc found in {Path.home()}") from err
+            raise FileNotFoundError(f"No .netrc found at {netrc_loc}") from err
         except NetrcParseError as err:
-            raise NetrcParseError("Unable to parse .netrc") from err
+            raise NetrcParseError(f"Unable to parse .netrc file {netrc_loc}") from err
+
         if (creds := my_netrc[self.system.edl_hostname]) is None:
             return False
 
         username = creds["login"]
         password = creds["password"]
         authenticated = self._get_credentials(username, password)
+
         if authenticated:
             logger.debug("Using .netrc file for EDL")
+
         return authenticated
 
     def _environment(self) -> bool:
@@ -293,33 +316,41 @@ def _find_or_create_token(self, username: str, password: str) -> Any:
 
     def _persist_user_credentials(self, username: str, password: str) -> bool:
         # See: https://github.com/sloria/tinynetrc/issues/34
+
+        netrc_loc = netrc_path()
+        logger.info(f"Persisting credentials to {netrc_loc}")
+
         try:
-            netrc_path = Path().home().joinpath(".netrc")
-            netrc_path.touch(exist_ok=True)
-            netrc_path.chmod(0o600)
+            netrc_loc.touch(exist_ok=True)
+            netrc_loc.chmod(0o600)
         except Exception as e:
             logger.error(e)
             return False
-        my_netrc = Netrc(str(netrc_path))
+
+        my_netrc = Netrc(str(netrc_loc))
         my_netrc[self.system.edl_hostname] = {
             "login": username,
             "password": password,
         }
         my_netrc.save()
+
         urs_cookies_path = Path.home() / ".urs_cookies"
+
         if not urs_cookies_path.exists():
             urs_cookies_path.write_text("")
 
         # Create and write to .dodsrc file
         dodsrc_path = Path.home() / ".dodsrc"
+
         if not dodsrc_path.exists():
             dodsrc_contents = (
-                f"HTTP.COOKIEJAR={urs_cookies_path}\nHTTP.NETRC={netrc_path}"
+                f"HTTP.COOKIEJAR={urs_cookies_path}\nHTTP.NETRC={netrc_loc}"
             )
             dodsrc_path.write_text(dodsrc_contents)
 
         if platform.system() == "Windows":
             local_dodsrc_path = Path.cwd() / dodsrc_path.name
+
             if not local_dodsrc_path.exists():
                 shutil.copy2(dodsrc_path, local_dodsrc_path)