diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index d34e83c01..6ff4437e4 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -7,6 +7,8 @@ on: - '!documentation' schedule: # once a day at midnight UTC - cron: '0 0 * * *' + pull_request: # requires approval for first-time contributors + types: [synchronize, opened, reopened, labeled] workflow_dispatch: # Manually trigger with 'Run workflow' button concurrency: # Replace Cancel Workflow Action @@ -22,8 +24,6 @@ jobs: env: OS: ubuntu-latest PYTHON: '3.9' - UCSF_BOX_TOKEN: ${{ secrets.UCSF_BOX_TOKEN }} # for download and testing - UCSF_BOX_USER: ${{ secrets.UCSF_BOX_USER }} services: mysql: image: datajoint/mysql:8.0 @@ -57,23 +57,23 @@ jobs: pip install --quiet .[test] - name: Download data env: - BASEURL: ftps://ftp.box.com/trodes_to_nwb_test_data/ - NWBFILE: minirec20230622.nwb # Relative to Base URL - VID_ONE: 20230622_sample_01_a1/20230622_sample_01_a1.1.h264 - VID_TWO: 20230622_sample_02_a1/20230622_sample_02_a1.1.h264 + BASEURL: https://ucsf.box.com/shared/static/ + NWB_URL: k3sgql6z475oia848q1rgms4zdh4rkjn.nwb + VID1URL: ykep8ek4ogad20wz4p0vuyuqfo60cv3w.h264 + VID2URL: d2jjk0y565ru75xqojio3hymmehzr5he.h264 + NWBFILE: minirec20230622.nwb + VID_ONE: 20230622_minirec_01_s1.1.h264 + VID_TWO: 20230622_minirec_02_s2.1.h264 RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ run: | mkdir -p $RAW_DIR $VID_DIR - wget_opts() { # Declare func with download options - wget \ - --recursive --no-verbose --no-host-directories --no-directories \ - --user "$UCSF_BOX_USER" --password "$UCSF_BOX_TOKEN" \ - -P "$1" "$BASEURL""$2" + curl_opts() { # Declare func with download options + curl -L --output "$1""$2" "$BASEURL""$3" } - wget_opts $RAW_DIR $NWBFILE - wget_opts $VID_DIR $VID_ONE - wget_opts $VID_DIR $VID_TWO + curl_opts $RAW_DIR $NWBFILE $NWB_URL + curl_opts $VID_DIR $VID_ONE $VID1URL + curl_opts $VID_DIR $VID_TWO $VID2URL - name: Run tests run: | - pytest --no-docker --no-dlc + pytest --no-docker --no-dlc tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e91b59ed..dd3bbeeb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove debug statement #1164 - Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 - Initialize tables in pytests #1181 + - Download test data without credentials, trigger on approved PRs #1180 - Allow python \< 3.13 #1169 - Remove numpy version restriction #1169 - Merge table delete removes orphaned master entries #1164 diff --git a/environment.yml b/environment.yml index a5229b8a8..26155a7cc 100644 --- a/environment.yml +++ b/environment.yml @@ -9,7 +9,7 @@ name: spyglass channels: - conda-forge - - defaults + # - defaults # deprecated - franklab - edeno # - pytorch # dlc-only diff --git a/environment_dlc.yml b/environment_dlc.yml index 156bed793..df6026c18 100644 --- a/environment_dlc.yml +++ b/environment_dlc.yml @@ -9,7 +9,7 @@ name: spyglass-dlc channels: - conda-forge - - defaults + # - defaults # deprecated - franklab - edeno - pytorch # dlc-only diff --git a/src/spyglass/common/common_behav.py b/src/spyglass/common/common_behav.py index bab4ba075..53f8b7dcd 100644 --- a/src/spyglass/common/common_behav.py +++ b/src/spyglass/common/common_behav.py @@ -197,7 +197,7 @@ def fetch1_dataframe(self): id_rp = [(n["id"], n["raw_position"]) for n in self.fetch_nwb()] if len(set(rp.interval for _, rp in id_rp)) > 1: - logger.warn("Loading DataFrame with multiple intervals.") + logger.warning("Loading DataFrame with multiple intervals.") df_list = [ pd.DataFrame( @@ -395,7 +395,9 @@ def _no_transaction_make(self, key, verbose=True, skip_duplicates=False): ) if videos is None: - logger.warn(f"No video data interface found in {nwb_file_name}\n") + logger.warning( + f"No video data interface found in {nwb_file_name}\n" + ) return else: videos = videos.time_series diff --git a/src/spyglass/common/common_device.py b/src/spyglass/common/common_device.py index 1d20d8fb6..3fe14ec55 100644 --- a/src/spyglass/common/common_device.py +++ b/src/spyglass/common/common_device.py @@ -88,7 +88,9 @@ def insert_from_nwbfile(cls, nwbf, config=None): + f"{ndx_devices.keys()}" ) else: - logger.warn("No conforming data acquisition device metadata found.") + logger.warning( + "No conforming data acquisition device metadata found." + ) @classmethod def get_all_device_names(cls, nwbf, config) -> tuple: @@ -305,7 +307,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): if device_name_list: logger.info(f"Inserted camera devices {device_name_list}") else: - logger.warn("No conforming camera device metadata found.") + logger.warning("No conforming camera device metadata found.") return device_name_list @@ -462,7 +464,7 @@ def insert_from_nwbfile(cls, nwbf, config=None): if all_probes_types: logger.info(f"Inserted probes {all_probes_types}") else: - logger.warn("No conforming probe metadata found.") + logger.warning("No conforming probe metadata found.") return all_probes_types @@ -709,7 +711,7 @@ def create_from_nwbfile( query = ProbeType & {"probe_type": probe_type} if len(query) == 0: - logger.warn( + logger.warning( f"No ProbeType found with probe_type '{probe_type}'. Aborting." ) return @@ -769,7 +771,7 @@ def create_from_nwbfile( ] if not device_found: - logger.warn( + logger.warning( "No electrodes in the NWB file were associated with a device " + f"named '{nwb_device_name}'." ) diff --git a/src/spyglass/common/common_dio.py b/src/spyglass/common/common_dio.py index 228e9caf9..6b8f29c17 100644 --- a/src/spyglass/common/common_dio.py +++ b/src/spyglass/common/common_dio.py @@ -38,7 +38,7 @@ def make(self, key): nwbf, "behavioral_events", pynwb.behavior.BehavioralEvents ) if behav_events is None: - logger.warn( + logger.warning( "No conforming behavioral events data interface found in " + f"{nwb_file_name}\n" ) diff --git a/src/spyglass/common/common_session.py b/src/spyglass/common/common_session.py index 5095ed6ed..3328857e5 100644 --- a/src/spyglass/common/common_session.py +++ b/src/spyglass/common/common_session.py @@ -152,7 +152,7 @@ def _add_data_acquisition_device_part(self, nwb_file_name, nwbf, config={}): "data_acquisition_device_name": device_name } if len(query) == 0: - logger.warn( + logger.warning( "Cannot link Session with DataAcquisitionDevice.\n" + f"DataAcquisitionDevice does not exist: {device_name}" ) @@ -180,7 +180,7 @@ def _add_experimenter_part( # ensure that the foreign key exists and do nothing if not query = LabMember & {"lab_member_name": name} if len(query) == 0: - logger.warn( + logger.warning( "Cannot link Session with LabMember. " + f"LabMember does not exist: {name}" ) diff --git a/src/spyglass/common/common_subject.py b/src/spyglass/common/common_subject.py index 2b8dc071a..c7757e087 100644 --- a/src/spyglass/common/common_subject.py +++ b/src/spyglass/common/common_subject.py @@ -37,7 +37,7 @@ def insert_from_nwbfile(cls, nwbf: NWBFile, config: dict = None): """ config = config or dict() if "Subject" not in config and nwbf.subject is None: - logger.warn("No subject metadata found.\n") + logger.warning("No subject metadata found.\n") return None conf = config["Subject"][0] if "Subject" in config else dict() diff --git a/src/spyglass/common/common_task.py b/src/spyglass/common/common_task.py index 60fba08d3..aabf381ac 100644 --- a/src/spyglass/common/common_task.py +++ b/src/spyglass/common/common_task.py @@ -33,7 +33,7 @@ def insert_from_nwbfile(cls, nwbf: pynwb.NWBFile): """ tasks_mod = nwbf.processing.get("tasks") if tasks_mod is None: - logger.warn(f"No tasks processing module found in {nwbf}\n") + logger.warning(f"No tasks processing module found in {nwbf}\n") return for task in tasks_mod.data_interfaces.values(): if cls.check_task_table(task): @@ -136,7 +136,7 @@ def make(self, key): tasks_mod = nwbf.processing.get("tasks") config_tasks = config.get("Tasks", []) if tasks_mod is None and (not config_tasks): - logger.warn( + logger.warning( f"No tasks processing module found in {nwbf} or config\n" ) return @@ -163,7 +163,7 @@ def make(self, key): for camera_id in valid_camera_ids ] else: - logger.warn( + logger.warning( f"No camera device found with ID {camera_ids} in NWB " + f"file {nwbf}\n" ) @@ -186,7 +186,7 @@ def make(self, key): epoch, session_intervals ) if target_interval is None: - logger.warn("Skipping epoch.") + logger.warning("Skipping epoch.") continue key["interval_list_name"] = target_interval task_inserts.append(key.copy()) @@ -219,7 +219,7 @@ def make(self, key): epoch, session_intervals ) if target_interval is None: - logger.warn("Skipping epoch.") + logger.warning("Skipping epoch.") continue new_key["interval_list_name"] = target_interval task_inserts.append(key.copy()) diff --git a/src/spyglass/common/prepopulate/prepopulate.py b/src/spyglass/common/prepopulate/prepopulate.py index ecad63a25..9ba703cd5 100644 --- a/src/spyglass/common/prepopulate/prepopulate.py +++ b/src/spyglass/common/prepopulate/prepopulate.py @@ -57,7 +57,7 @@ def populate_from_yaml(yaml_path: str): if k in table_cls.primary_key } if not primary_key_values: - logger.warn( + logger.warning( f"Populate: No primary key provided in data {entry_dict} " + f"for table {table_cls.__name__}" ) diff --git a/src/spyglass/position/v1/dlc_utils_makevid.py b/src/spyglass/position/v1/dlc_utils_makevid.py index 2763a7898..6f0df6289 100644 --- a/src/spyglass/position/v1/dlc_utils_makevid.py +++ b/src/spyglass/position/v1/dlc_utils_makevid.py @@ -5,7 +5,6 @@ import subprocess from concurrent.futures import ProcessPoolExecutor, TimeoutError, as_completed from pathlib import Path -from typing import Tuple import matplotlib import matplotlib.pyplot as plt @@ -190,10 +189,6 @@ def _set_frame_info(self): self.pad_len = len(str(self.n_frames)) - def _set_input_stats(self, video_filename=None) -> Tuple[int, int]: - """Get the width and height of the video.""" - logger.debug("Getting video stats with ffprobe") - def _set_plot_bases(self): """Create the figure and axes for the video.""" logger.debug("Setting plot bases") @@ -309,15 +304,20 @@ def centroid_to_px(*idx): ) ) - def _set_orient_line(self, frame, pos_ind): - def orient_list(c): - return [c, c + 30 * np.cos(self.orientation_mean[pos_ind])] + def _get_orient_line(self, pos_ind): + orient = self.orientation_mean[pos_ind] + if isinstance(orient, np.ndarray): + orient = orient[0] # Trodes passes orientation as a 1D array - if np.all(np.isnan(self.orientation_mean[pos_ind])): - self.orientation_line.set_data((np.NaN, np.NaN)) + def orient_list(c, axis="x"): + func = np.cos if axis == "x" else np.sin + return [c, c + 30 * func(orient)] + + if np.all(np.isnan(orient)): + return ([np.NaN], [np.NaN]) else: - c0, c1 = self._get_centroid_data(pos_ind) - self.orientation_line.set_data(orient_list(c0), orient_list(c1)) + x, y = self._get_centroid_data(pos_ind) + return (orient_list(x), orient_list(y, axis="y")) def _generate_single_frame(self, frame_ind): """Generate a single frame and save it as an image.""" @@ -364,7 +364,7 @@ def _generate_single_frame(self, frame_ind): ) ) self.centroid_position_dot.set_offsets(dlc_centroid_data) - _ = self._set_orient_line(frame, pos_ind) + self.orientation_line.set_data(self._get_orient_line(pos_ind)) time_delta = pd.Timedelta( pd.to_datetime(self.position_time[pos_ind] * 1e9, unit="ns") diff --git a/src/spyglass/position/v1/position_trodes_position.py b/src/spyglass/position/v1/position_trodes_position.py index 031c0e6bb..f7684450e 100644 --- a/src/spyglass/position/v1/position_trodes_position.py +++ b/src/spyglass/position/v1/position_trodes_position.py @@ -344,9 +344,14 @@ def make(self, key): adj_df = _fix_col_names(raw_df) # adjust 'xloc1' to 'xloc' limit = params.get("limit", None) - if limit or test_mode: + + if limit and not test_mode: params["debug"] = True output_video_filename = Path(".") / f"TEST_VID_{limit}.mp4" + elif test_mode: + limit = 10 + + if limit: # pytest video data has mismatched shapes in some cases min_len = limit or min(len(adj_df), len(pos_df), len(video_time)) adj_df = adj_df.head(min_len) @@ -401,7 +406,7 @@ def make(self, key): **params, ) - if limit: + if limit and not test_mode: return vid_maker self.insert1(dict(**key, has_video=True)) diff --git a/src/spyglass/spikesorting/utils.py b/src/spyglass/spikesorting/utils.py index d99a71ff2..f69359838 100644 --- a/src/spyglass/spikesorting/utils.py +++ b/src/spyglass/spikesorting/utils.py @@ -108,7 +108,7 @@ def get_group_by_shank( if omit_ref_electrode_group and ( str(e_group) == str(ref_elec_group) ): - logger.warn( + logger.warning( f"Omitting electrode group {e_group} from sort groups " + "because contains reference." ) @@ -117,7 +117,7 @@ def get_group_by_shank( # omit unitrodes if indicated if omit_unitrode and len(shank_elect) == 1: - logger.warn( + logger.warning( f"Omitting electrode group {e_group}, shank {shank} " + "from sort groups because unitrode." ) diff --git a/src/spyglass/spikesorting/v0/sortingview.py b/src/spyglass/spikesorting/v0/sortingview.py index ff62f09b8..baa99131c 100644 --- a/src/spyglass/spikesorting/v0/sortingview.py +++ b/src/spyglass/spikesorting/v0/sortingview.py @@ -72,7 +72,7 @@ def make(self, key: dict): LabMember.LabMemberInfo & {"lab_member_name": team_member} ).fetch("google_user_name") if len(google_user_id) != 1: - logger.warn( + logger.warning( f"Google user ID for {team_member} does not exist or more than one ID detected;\ permission not given to {team_member}, skipping..." ) diff --git a/src/spyglass/spikesorting/v0/sortingview_helper_fn.py b/src/spyglass/spikesorting/v0/sortingview_helper_fn.py index 431d99e88..3ecdd5eae 100644 --- a/src/spyglass/spikesorting/v0/sortingview_helper_fn.py +++ b/src/spyglass/spikesorting/v0/sortingview_helper_fn.py @@ -136,7 +136,7 @@ def _generate_url( ) if initial_curation is not None: - logger.warn("found initial curation") + logger.warning("found initial curation") sorting_curation_uri = kcl.store_json(initial_curation) else: sorting_curation_uri = None diff --git a/src/spyglass/spikesorting/v0/spikesorting_artifact.py b/src/spyglass/spikesorting/v0/spikesorting_artifact.py index b665b2314..71edc5de1 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_artifact.py +++ b/src/spyglass/spikesorting/v0/spikesorting_artifact.py @@ -263,7 +263,7 @@ def _get_artifact_times( [[valid_timestamps[0], valid_timestamps[-1]]] ) artifact_times_empty = np.asarray([]) - logger.warn("No artifacts detected.") + logger.warning("No artifacts detected.") return recording_interval, artifact_times_empty # convert indices to intervals diff --git a/src/spyglass/spikesorting/v0/spikesorting_curation.py b/src/spyglass/spikesorting/v0/spikesorting_curation.py index e8c2f149e..27aa074d3 100644 --- a/src/spyglass/spikesorting/v0/spikesorting_curation.py +++ b/src/spyglass/spikesorting/v0/spikesorting_curation.py @@ -266,7 +266,7 @@ def save_sorting_nwb( AnalysisNwbfile().add(key["nwb_file_name"], analysis_file_name) if object_ids == "": - logger.warn( + logger.warning( "Sorting contains no units." "Created an empty analysis nwb file anyway." ) diff --git a/src/spyglass/spikesorting/v1/artifact.py b/src/spyglass/spikesorting/v1/artifact.py index 9a4fbeaef..14c82dba6 100644 --- a/src/spyglass/spikesorting/v1/artifact.py +++ b/src/spyglass/spikesorting/v1/artifact.py @@ -98,7 +98,7 @@ def insert_selection(cls, key: dict): """ query = cls & key if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["artifact_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) @@ -290,7 +290,7 @@ def _get_artifact_times( [[valid_timestamps[0], valid_timestamps[-1]]] ) artifact_times_empty = np.asarray([]) - logger.warn("No artifacts detected.") + logger.warning("No artifacts detected.") return recording_interval, artifact_times_empty # convert indices to intervals diff --git a/src/spyglass/spikesorting/v1/figurl_curation.py b/src/spyglass/spikesorting/v1/figurl_curation.py index 5be842d15..a79ee6246 100644 --- a/src/spyglass/spikesorting/v1/figurl_curation.py +++ b/src/spyglass/spikesorting/v1/figurl_curation.py @@ -51,7 +51,7 @@ def insert_selection(cls, key: dict): if "figurl_curation_id" in key: query = cls & {"figurl_curation_id": key["figurl_curation_id"]} if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["figurl_curation_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/spikesorting/v1/metric_curation.py b/src/spyglass/spikesorting/v1/metric_curation.py index 8346f8ddd..43e09a8de 100644 --- a/src/spyglass/spikesorting/v1/metric_curation.py +++ b/src/spyglass/spikesorting/v1/metric_curation.py @@ -190,7 +190,7 @@ def insert_selection(cls, key: dict): key for the inserted row """ if cls & key: - logger.warn("This row has already been inserted.") + logger.warning("This row has already been inserted.") return (cls & key).fetch1() key["metric_curation_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/spikesorting/v1/recording.py b/src/spyglass/spikesorting/v1/recording.py index f4e150837..d1257d4f3 100644 --- a/src/spyglass/spikesorting/v1/recording.py +++ b/src/spyglass/spikesorting/v1/recording.py @@ -154,7 +154,7 @@ def insert_selection(cls, key: dict): """ query = cls & key if query: - logger.warn("Similar row(s) already inserted.") + logger.warning("Similar row(s) already inserted.") return query.fetch(as_dict=True) key["recording_id"] = uuid.uuid4() cls.insert1(key, skip_duplicates=True) diff --git a/src/spyglass/utils/database_settings.py b/src/spyglass/utils/database_settings.py index 56e2aa28f..0f08fbfb4 100755 --- a/src/spyglass/utils/database_settings.py +++ b/src/spyglass/utils/database_settings.py @@ -20,6 +20,7 @@ "lfp", "waveform", "mua", + "sharing", ] GRANT_ALL = "GRANT ALL PRIVILEGES ON " GRANT_SEL = "GRANT SELECT ON " diff --git a/src/spyglass/utils/dj_helper_fn.py b/src/spyglass/utils/dj_helper_fn.py index d77416487..42cf67ba0 100644 --- a/src/spyglass/utils/dj_helper_fn.py +++ b/src/spyglass/utils/dj_helper_fn.py @@ -124,7 +124,7 @@ def _subclass_factory( # Define the __call__ method for the new class def init_override(self, *args, **kwargs): - logger.warn( + logger.warning( "Deprecation: this class has been moved out of " + f"{old_module}\n" + f"\t{old_name} -> {new_module}.{new_class.__name__}" diff --git a/src/spyglass/utils/dj_merge_tables.py b/src/spyglass/utils/dj_merge_tables.py index 7d6bf46ba..146b9873a 100644 --- a/src/spyglass/utils/dj_merge_tables.py +++ b/src/spyglass/utils/dj_merge_tables.py @@ -66,14 +66,14 @@ def __init__(self): self._reserved_sk = RESERVED_SECONDARY_KEY if not self.is_declared: if not is_merge_table(self): # Check definition - logger.warn( + logger.warning( "Merge table with non-default definition\n" + f"Expected:\n{MERGE_DEFINITION.strip()}\n" + f"Actual :\n{self.definition.strip()}" ) for part in self.parts(as_objects=True): if part.primary_key != self.primary_key: - logger.warn( # PK is only 'merge_id' in parts, no others + logger.warning( # PK is only 'merge_id' in parts, no others f"Unexpected primary key in {part.table_name}" + f"\n\tExpected: {self.primary_key}" + f"\n\tActual : {part.primary_key}" @@ -721,7 +721,7 @@ def _normalize_source( raise ValueError(f"Unable to find source for {source}") source = fetched_source[0] if len(fetched_source) > 1: - logger.warn(f"Multiple sources. Selecting first: {source}.") + logger.warning(f"Multiple sources. Selecting first: {source}.") if isinstance(source, dj.Table): source = self._part_name(source) if isinstance(source, dict): @@ -814,7 +814,7 @@ def merge_fetch( try: results.extend(part.fetch(*attrs, **kwargs)) except DataJointError as e: - logger.warn( + logger.warning( f"{e.args[0]} Skipping " + to_camel_case(part.table_name.split("__")[-1]) ) diff --git a/src/spyglass/utils/dj_mixin.py b/src/spyglass/utils/dj_mixin.py index 5c0b7dcf3..72e34c04f 100644 --- a/src/spyglass/utils/dj_mixin.py +++ b/src/spyglass/utils/dj_mixin.py @@ -368,7 +368,7 @@ def _check_delete_permission(self) -> None: not self._session_connection # Table has no session or self._member_pk in self.heading.names # Table has experimenter ): - logger.warn( # Permit delete if no session connection + logger.warning( # Permit delete if no session connection "Could not find lab team associated with " + f"{self.__class__.__name__}." + "\nBe careful not to delete others' data." @@ -376,7 +376,7 @@ def _check_delete_permission(self) -> None: return if not (sess_summary := self._get_exp_summary()): - logger.warn( + logger.warning( f"Could not find a connection from {self.camel_name} " + "to Session.\n Be careful not to delete others' data." ) diff --git a/src/spyglass/utils/nwb_helper_fn.py b/src/spyglass/utils/nwb_helper_fn.py index 5d5fdaca4..82af8a626 100644 --- a/src/spyglass/utils/nwb_helper_fn.py +++ b/src/spyglass/utils/nwb_helper_fn.py @@ -364,7 +364,7 @@ def get_valid_intervals( if total_time < min_valid_len: half_total_time = total_time / 2 - logger.warn(f"Setting minimum valid interval to {half_total_time}") + logger.warning(f"Setting minimum valid interval to {half_total_time}") min_valid_len = half_total_time # get rid of NaN elements diff --git a/tests/README.md b/tests/README.md index 36b6ab71f..d1873505c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,12 +2,6 @@ ## Environment -To allow pytest helpers to automatically dowlnoad requisite data, you'll need to -set credentials for Box. Consider adding these to a private `.env` file. - -- `UCSF_BOX_USER`: UCSF email address -- `UCSF_BOX_TOKEN`: Token generated from UCSF Box account - To facilitate headless testing of various Qt-based tools as well as Tensorflow, `pyproject.toml` includes some environment variables associated with the display. These are... diff --git a/tests/conftest.py b/tests/conftest.py index de513d3a9..a7354a383 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,7 @@ warnings.filterwarnings("ignore", category=FutureWarning, module="sklearn") warnings.filterwarnings("ignore", category=PerformanceWarning, module="pandas") warnings.filterwarnings("ignore", category=NumbaWarning, module="numba") +warnings.filterwarnings("ignore", category=ResourceWarning, module="datajoint") # ------------------------------- TESTS CONFIG ------------------------------- @@ -108,7 +109,6 @@ def pytest_configure(config): ) DOWNLOADS = DataDownloader( - nwb_file_name=TEST_FILE, base_dir=BASE_DIR, verbose=VERBOSE, download_dlc=not NO_DLC, @@ -420,7 +420,6 @@ def video_keys(common, base_dir): for file in DOWNLOADS.file_downloads: if file.endswith(".h264"): DOWNLOADS.wait_for(file) - DOWNLOADS.rename_files() return common.VideoFile().fetch(as_dict=True) diff --git a/tests/data_downloader.py b/tests/data_downloader.py index cb58e1c71..40af1ea88 100644 --- a/tests/data_downloader.py +++ b/tests/data_downloader.py @@ -1,5 +1,4 @@ from functools import cached_property -from os import environ as os_environ from pathlib import Path from shutil import copy as shutil_copy from subprocess import DEVNULL, Popen @@ -9,46 +8,44 @@ from datajoint import logger as dj_logger -UCSF_BOX_USER = os_environ.get("UCSF_BOX_USER") -UCSF_BOX_TOKEN = os_environ.get("UCSF_BOX_TOKEN") -BASE_URL = "ftps://ftp.box.com/trodes_to_nwb_test_data/" +BASE_URL = "https://ucsf.box.com/shared/static/" NON_DLC = 3 # First N items below are not for DeepLabCut FILE_PATHS = [ { "relative_dir": "raw", "target_name": "minirec20230622.nwb", - "url": BASE_URL + "minirec20230622.nwb", + "url": BASE_URL + "k3sgql6z475oia848q1rgms4zdh4rkjn.nwb", }, { "relative_dir": "video", "target_name": "20230622_minirec_01_s1.1.h264", - "url": BASE_URL + "20230622_sample_01_a1/20230622_sample_01_a1.1.h264", + "url": BASE_URL + "ykep8ek4ogad20wz4p0vuyuqfo60cv3w.h264", }, { "relative_dir": "video", "target_name": "20230622_minirec_02_s2.1.h264", - "url": BASE_URL + "20230622_sample_02_a1/20230622_sample_02_a1.1.h264", + "url": BASE_URL + "d2jjk0y565ru75xqojio3hymmehzr5he.h264", }, { "relative_dir": "deeplabcut", "target_name": "CollectedData_sc_eb.csv", - "url": BASE_URL + "minirec_dlc_items/CollectedData_sc_eb.csv", + "url": BASE_URL + "3nzqdfty51vrga7470rn2vayrtoor3ot.csv", }, { "relative_dir": "deeplabcut", "target_name": "CollectedData_sc_eb.h5", - "url": BASE_URL + "minirec_dlc_items/CollectedData_sc_eb.h5", + "url": BASE_URL + "sx30rqljppeisi4jdyu53y51na0q9rff.h5", }, { "relative_dir": "deeplabcut", "target_name": "img000.png", - "url": BASE_URL + "minirec_dlc_items/img000.png", + "url": BASE_URL + "wrvgncfbpjuzfhopkfaizzs069tb1ruu.png", }, { "relative_dir": "deeplabcut", "target_name": "img001.png", - "url": BASE_URL + "minirec_dlc_items/img001.png", + "url": BASE_URL + "czbkxeinemat7jj7j0877pcosfqo9psh.png", }, ] @@ -56,41 +53,18 @@ class DataDownloader: def __init__( self, - nwb_file_name, file_paths=FILE_PATHS, base_dir=".", download_dlc=True, verbose=True, ): - if not all([UCSF_BOX_USER, UCSF_BOX_TOKEN]): - raise ValueError( - "Missing os.environ credentials: UCSF_BOX_USER, UCSF_BOX_TOKEN." - ) - if nwb_file_name != file_paths[0]["target_name"]: - raise ValueError( - f"Please adjust data_downloader.py to match: {nwb_file_name}" - ) - - self.cmd = [ - "wget", - "--recursive", - "--no-host-directories", - "--no-directories", - "--user", - UCSF_BOX_USER, - "--password", - UCSF_BOX_TOKEN, - "-P", # Then need relative path, then url - ] - - self.verbose = verbose - if not verbose: - self.cmd.insert(self.cmd.index("--recursive") + 1, "--no-verbose") - self.cmd_kwargs = dict(stdout=DEVNULL, stderr=DEVNULL) - else: + if verbose: self.cmd_kwargs = dict(stdout=stdout, stderr=stderr) + else: + self.cmd_kwargs = dict(stdout=DEVNULL, stderr=DEVNULL) - self.base_dir = Path(base_dir).resolve() + self.verbose = verbose + self.base_dir = Path(base_dir).expanduser().resolve() self.download_dlc = download_dlc self.file_paths = file_paths if download_dlc else file_paths[:NON_DLC] self.base_dir.mkdir(exist_ok=True) @@ -98,46 +72,41 @@ def __init__( # Start downloads _ = self.file_downloads - def rename_files(self): - """Redundant, but allows rerun later in startup process of conftest.""" - for path in self.file_paths: - target, url = path["target_name"], path["url"] - target_dir = self.base_dir / path["relative_dir"] - orig = target_dir / url.split("/")[-1] - dest = target_dir / target - - if orig.exists(): - orig.rename(dest) - @cached_property # Only make list of processes once def file_downloads(self) -> Dict[str, Union[Popen, None]]: """{File: POpen/None} for each file. If exists/finished, None.""" ret = dict() - self.rename_files() for path in self.file_paths: - target, url = path["target_name"], path["url"] target_dir = self.base_dir / path["relative_dir"] target_dir.mkdir(exist_ok=True, parents=True) + + target = path["target_name"] dest = target_dir / target - cmd = ( - ["echo", f"Already have {target}"] - if dest.exists() - else self.cmd + [target_dir, url] - ) + + if dest.exists(): + cmd = ["echo", f"Already have {target}"] + else: + cmd = ["curl", "-L", "--output", str(dest), f"{path['url']}"] + + print(f"cmd: {cmd}") + ret[target] = Popen(cmd, **self.cmd_kwargs) + return ret def wait_for(self, target: str): """Wait for target to finish downloading.""" status = self.file_downloads.get(target).poll() + limit = 10 while status is None and limit > 0: - time_sleep(5) # Some + time_sleep(5) limit -= 1 status = self.file_downloads.get(target).poll() - if status != 0: + + if status != 0: # Error downloading raise ValueError(f"Error downloading: {target}") - if limit < 1: + if limit < 1: # Reached attempt limit raise TimeoutError(f"Timeout downloading: {target}") def move_dlc_items(self, dest_dir: Path):