Skip to content
This repository has been archived by the owner on Oct 30, 2024. It is now read-only.

Commit

Permalink
Saving analysis outputs in folders according to SWC-Blueprint (#60)
Browse files Browse the repository at this point in the history
Co-authored-by: Niko Sirmpilatze <[email protected]>
  • Loading branch information
lauraporta and niksirbi authored Jul 4, 2023
1 parent 5728e8f commit 32ea410
Show file tree
Hide file tree
Showing 12 changed files with 1,120 additions and 63 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ Second, set up the environmental variables and the config file by executing:
```bash
python3 setup_for_demo.py
```
Then edit the config file with the correct paths to the data by overwriting `/path/to/`. The only path that matters at this stage is the `allen_dff` path, which should point to the folder where you stored the pre-processed data.
Then edit the config file with the correct paths to the data by overwriting `/path/to/`.
Please edit the `allen_dff` path to point to the folder where you stored the pre-processed data and the `output` path to point to the folder where you want to store the analysis output.

Finally, run `python3 demo_cli.py` to run the analysis. The script will create a file containing the analysis output which will then be used by the dashboard.

Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ dependencies = [
"scipy",
"numba",
"tqdm",
"GitPython",
]

[project.urls]
Expand Down
40 changes: 27 additions & 13 deletions rsp_vision/console_application/app.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,23 @@
import logging
import pickle
import sys
from pathlib import Path

import rich
from decouple import config
from fancylog import fancylog
from rich.prompt import Prompt

from rsp_vision.analysis.spatial_freq_temporal_freq import (
FrequencyResponsiveness,
)
from rsp_vision.load.load_data import load_data
from rsp_vision.load.load_data import load_data, read_config_file
from rsp_vision.objects.enums import PhotonType
from rsp_vision.objects.photon_data import PhotonData
from rsp_vision.objects.SWC_Blueprint import SWC_Blueprint_Spec
from rsp_vision.save.save_data import save_data

CONFIG_PATH = config("CONFIG_PATH")
config_path = Path(__file__).parents[1] / CONFIG_PATH


def exception_handler(func: object) -> object:
Expand Down Expand Up @@ -45,8 +50,16 @@ def analysis_pipeline() -> None:
CLI or GUI functionality is added here.
"""
# pipeline draft
start_logging()
config = read_config_file(config_path)
swc_blueprint_spec = SWC_Blueprint_Spec(
project_name="rsp_vision",
raw_data=False,
derivatives=True,
local_path=Path(config["paths"]["output"]),
)
start_logging(swc_blueprint_spec)
logging.debug(f"Config file read from {config_path}")
logging.debug(f"Config file content: {config}")

folder_name = Prompt.ask(
" \
Expand All @@ -61,7 +74,7 @@ def analysis_pipeline() -> None:
)

# load data
data, config = load_data(folder_name)
data, folder_naming = load_data(folder_name, config)

# preprocess and make PhotonData object
photon_data = PhotonData(data, PhotonType.TWO_PHOTON, config)
Expand All @@ -75,15 +88,11 @@ def analysis_pipeline() -> None:
logging.info("Analysis finished")
logging.info(f"Updated photon_data object: {photon_data}")

saving_path = (
Path(config["paths"]["output"]) / f"{folder_name}_data.pickle"
)
with open(saving_path, "wb") as f:
pickle.dump(photon_data, f)
logging.info("Analysis saved")
# save results
save_data(swc_blueprint_spec, folder_naming, photon_data, config)


def start_logging(module=None):
def start_logging(swc_blueprint_spec: SWC_Blueprint_Spec, module=None):
"""Start logging to file and console.
The log level to file is set to DEBUG, to console to INFO. The log file is
Expand All @@ -93,8 +102,13 @@ def start_logging(module=None):
if module is None:
module = get_module_for_logging()

Path(swc_blueprint_spec.logs_path).mkdir(parents=True, exist_ok=True)

fancylog.start_logging(
output_dir="./", package=module, filename="rsp_vision", verbose=False
output_dir=str(swc_blueprint_spec.logs_path),
package=module,
filename="rsp_vision",
verbose=False,
)


Expand Down
13 changes: 4 additions & 9 deletions rsp_vision/load/load_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,15 @@

import h5py
import yaml
from decouple import config

from ..objects.data_raw import DataRaw
from ..objects.enums import AnalysisType, DataType
from ..objects.folder_naming_specs import FolderNamingSpecs

CONFIG_PATH = config("CONFIG_PATH")
config_path = Path(__file__).parents[1] / CONFIG_PATH


def load_data(folder_name: str) -> Tuple[DataRaw, dict]:
def load_data(
folder_name: str, config: dict
) -> Tuple[DataRaw, FolderNamingSpecs]:
"""Creates the configuration object and loads the data.
Parameters
Expand All @@ -28,12 +26,11 @@ def load_data(folder_name: str) -> Tuple[DataRaw, dict]:
specs: specs object
data_raw: list containing all raw data
"""
config = read_config_file(config_path)
folder_naming = FolderNamingSpecs(folder_name, config)
folder_naming.extract_all_file_names()
data_raw = load_data_from_filename(folder_naming, config)

return data_raw, config
return data_raw, folder_naming


def load_data_from_filename(
Expand Down Expand Up @@ -82,6 +79,4 @@ def read_config_file(config_path: Path) -> dict:
"""
with open(config_path, "r") as f:
config = yaml.load(f, Loader=yaml.FullLoader)
logging.debug(f"Config file read from {config_path}")
logging.debug(f"Config file content: {config}")
return config
240 changes: 240 additions & 0 deletions rsp_vision/objects/SWC_Blueprint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
from pathlib import Path
from typing import Union

from rsp_vision.objects.folder_naming_specs import FolderNamingSpecs


class SWC_Blueprint_Spec:
def __init__(
self,
project_name: str,
raw_data: bool = False,
derivatives: bool = True,
local_path: Path = Path().absolute(),
) -> None:
"""
Create a `SWC_Blueprint_Spec` object to specify the location of the
raw data and/or derivatives for a project.
Please refer to https://swc-blueprint.neuroinformatics.dev/ for more
information about SWC Blueprint.
Parameters
----------
project_name : str
The name of the project folder.
raw_data : bool, optional
Whether you want to create a subfolder for raw data or not,
by default False.
derivatives : bool, optional
Whether you want to create a subfolder for derivatives or not,
by default True.
local_path : Path, optional
The path to the local folder where you want to create the project,
by default Path().absolute().
"""
self.project_name = project_name
self.raw_data = raw_data
self.derivatives = derivatives
self.path = (
local_path / self.project_name / "derivatives"
if derivatives
else local_path / self.project_name / "raw_data"
)
self.logs_path = local_path / self.project_name / "logs"


class SubjectFolder:
"""
A class to represent a subject folder in SWC Blueprint.
This specific implementation is for the rsp_vision project.
The subject folder will have the following format:
sub-###_line-###_id-### (e.g. sub-001_line-C57_id-123).
Line and id are specified in the `FolderNamingSpecs` object or
in the table row. The table is a pandas DataFrame that contains
the information about the subject that were already analysed.
Attributes
----------
swc_blueprint_spec : SWC_Blueprint_Spec
The SWC Blueprint specification for the project.
sub_num : int
The subject number.
sub_folder_name : str
The name of the subject folder. It contains the subject number,
the mouse line and the mouse id.
sub_folder_path : Path
The path to the subject folder.
Methods
-------
make_from_folder_naming_specs(folder_naming_specs, sub_num)
Create a `SubjectFolder` object from a `FolderNamingSpecs` object
and a subject number.
make_from_table_row(table_row)
Create a `SubjectFolder` object from a row of a table (a dict which
corresponds to a row of the "analysis_log.csv" file).
Raises
------
ValueError
If the argument `folder_or_table` is neither 'folder' nor 'table'.
"""

def __init__(
self,
swc_blueprint_spec: SWC_Blueprint_Spec,
folder_or_table: Union[FolderNamingSpecs, dict],
sub_num: int,
):
self.swc_blueprint_spec = swc_blueprint_spec
if isinstance(folder_or_table, FolderNamingSpecs):
self.make_from_folder_naming_specs(folder_or_table, sub_num)
elif isinstance(folder_or_table, dict):
self.make_from_table_row(folder_or_table)
else:
raise ValueError(
"The argument `folder_or_table` must be an instance of "
+ "`FolderNamingSpecs` or a dictionary."
)
self.sub_folder_path = Path(
self.swc_blueprint_spec.path / self.sub_folder_name
)

def make_from_folder_naming_specs(
self,
folder_naming_specs: FolderNamingSpecs,
sub_num: int,
):
self.sub_num = sub_num
self.sub_folder_name = (
f"sub-{sub_num:03d}"
+ "_line-"
+ folder_naming_specs.mouse_line
+ "_id-"
+ folder_naming_specs.mouse_id
)

def make_from_table_row(self, table_row: dict):
self.sub_num = int(table_row["sub"])
self.sub_folder_name = (
f"sub-{self.sub_num:03d}"
+ "_line-"
+ table_row["mouse_line"]
+ "_id-"
+ str(table_row["mouse_id"])
)


class SessionFolder:
"""
A class to represent a session folder in SWC Blueprint.
This specific implementation is for the rsp_vision project.
The session folder will have the following format:
ses-###_hemisphere-###_region-###_monitor-###_fov-###_cre-### (e.g.
ses-000_hemisphere-hL_region-RSPd_monitor-front);
`fov` and `cre` are optional.
Attributes
----------
subject_folder : SubjectFolder
The subject folder.
ses_num : int
The session number.
monitor : str
The monitor position.
ses_folder_name : str
The name of the session folder. It contains the session number,
the hemisphere, the brain region, the monitor position, the field
of view and the cre line (if applicable).
ses_folder_path : Path
The path to the session folder.
Methods
-------
make_from_folder_naming_specs(folder_naming_specs, ses_num)
Create a `SessionFolder` object from a `FolderNamingSpecs` object
and a session number.
make_from_table_row(table_row)
Create a `SessionFolder` object from a table row.
Raises
------
ValueError
If the argument `folder_or_table` is neither 'folder' nor 'table'.
"""

def __init__(
self,
subject_folder: SubjectFolder,
folder_or_table: Union[FolderNamingSpecs, dict],
ses_num: int,
):
self.subject_folder = subject_folder
if isinstance(folder_or_table, FolderNamingSpecs):
self.make_from_folder_naming_specs(folder_or_table, ses_num)
elif isinstance(folder_or_table, dict):
self.make_from_table_row(folder_or_table)
else:
raise ValueError(
"The argument `folder_or_table` must be an instance of "
+ "`FolderNamingSpecs` or a dictionary."
)
self.ses_folder_path = Path(
self.subject_folder.sub_folder_path / self.ses_folder_name
)

def make_from_folder_naming_specs(
self,
folder_naming_specs: FolderNamingSpecs,
ses_num: int,
):
self.ses_num = ses_num
self.monitor = (
"_".join(folder_naming_specs.monitor_position.split("_")[1:])
.replace("_", "-")
.replace("-", "")
)
self.ses_folder_name = (
f"ses-{self.ses_num:03d}"
+ "_hemisphere-"
+ folder_naming_specs.hemisphere
+ "_region-"
+ folder_naming_specs.brain_region
+ "_monitor-"
+ self.monitor
+ (
"_fov-" + folder_naming_specs.fov
if folder_naming_specs.fov
else ""
)
+ (
"_cre-" + folder_naming_specs.cre
if folder_naming_specs.cre
else ""
)
)

def make_from_table_row(self, table_row: dict):
self.ses_num = int(table_row["ses"])
self.monitor = table_row["monitor_position"]
self.ses_folder_name = (
f"ses-{self.ses_num:03d}"
+ "_hemisphere-"
+ table_row["hemisphere"]
+ "_region-"
+ table_row["brain_region"]
+ "_monitor-"
+ self.monitor
+ (
"_fov-" + str(table_row["fov"])
if (str(table_row["fov"]) != "nan")
else ""
)
+ (
"_cre-" + str(table_row["cre"])
if (str(table_row["cre"]) != "nan")
else ""
)
)
Loading

0 comments on commit 32ea410

Please sign in to comment.