From 51be7dc75f56883effcfd0d4acb97aea396034a2 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Tue, 24 May 2022 10:29:28 +0100 Subject: [PATCH 1/6] Added metadata parser class and example --- rfi_file_monitor/metadataparser.py | 23 +++++++++++ .../metadataparsers/csv_metadata_parser.py | 40 +++++++++++++++++++ setup.cfg | 2 + 3 files changed, 65 insertions(+) create mode 100644 rfi_file_monitor/metadataparser.py create mode 100644 rfi_file_monitor/metadataparsers/csv_metadata_parser.py diff --git a/rfi_file_monitor/metadataparser.py b/rfi_file_monitor/metadataparser.py new file mode 100644 index 0000000..4c73a40 --- /dev/null +++ b/rfi_file_monitor/metadataparser.py @@ -0,0 +1,23 @@ +from abc import ABC, abstractmethod + + +class MetadataParser(ABC): + """Each file will need it's own metadata parser, as every import library works in different ways. + This class aims to standardise the function names across each file loader.""" + + NAME = "Metadataparser" + + def __init__(self): + self.metadata = [] + + @classmethod + @abstractmethod + def supports_file(cls, file: str): + """This method allows you to add the file types that the method supports""" + raise NotImplementedError + + @classmethod + @abstractmethod + def extract_metadata(cls, capture_metadata, file): + """extracts metadata, for a given set of vars""" + raise NotImplementedError diff --git a/rfi_file_monitor/metadataparsers/csv_metadata_parser.py b/rfi_file_monitor/metadataparsers/csv_metadata_parser.py new file mode 100644 index 0000000..e5f23df --- /dev/null +++ b/rfi_file_monitor/metadataparsers/csv_metadata_parser.py @@ -0,0 +1,40 @@ +from rfi_file_monitor.metadataparser import MetadataParser +from itertools import takewhile +import re +import logging + +logger = logging.getLogger(__name__) + + +class CsvMetadataParser(MetadataParser): + NAME = "CSV Parser" + + @classmethod + def supports_file(cls, file: str): + if file.endswith("csv") or file.endswith("tsv"): + return True + return False + + @classmethod + def extract_metadata(cls, capture_metadata, file): + # check for metadata stored as comments + + with open(file, "r") as fobj: + # takewhile returns an iterator over all the lines + # that start with the comment string + headiter = takewhile(lambda s: s.startswith("#"), fobj) + # you may want to process the headers differently, + # but here we just convert it to a list + metadata = list(headiter) + + scicat_metadata = {} + for line in metadata: + line = line.strip("#") + line = line.strip("\n") + for i in capture_metadata: + if i in line: + splitline = re.split(r'[ ,|;"=]+', line) + splitline = [part for part in splitline if part != ""] + scicat_metadata[splitline[0]] = splitline[1] + + return scicat_metadata diff --git a/setup.cfg b/setup.cfg index fb5298a..f202b9f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,6 +67,8 @@ rfi_file_monitor.files = WeightedRegularFile = rfi_file_monitor.files.regular_file:WeightedRegularFile S3Object = rfi_file_monitor.files.s3_object:S3Object Directory = rfi_file_monitor.files.directory:Directory +rfi_file_monitor.metadataparsers = + CsvMetadataParser=rfi_file_monitor.metadataparsers.csv_metadata_parser:CsvMetadataParser gui_scripts = rfi-file-monitor = rfi_file_monitor:main From 37bd58cbbb2ed7c5950173f8ff7fe1821561bee3 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Tue, 24 May 2022 10:30:01 +0100 Subject: [PATCH 2/6] Added manual metadata and parsers --- rfi_file_monitor/operations/scicataloguer.py | 194 +++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index 023cc4f..54a1d75 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -14,6 +14,8 @@ from pyscicat.model import Dataset, RawDataset, DerivedDataset import logging from urllib.parse import urlparse +import importlib.metadata +import importlib from typing import Dict, Optional, List from ..version import __version__ as core_version @@ -396,6 +398,114 @@ def __init__(self, *args, **kwargs): ) self._grid.attach(self._derived_checkbox, 0, 6, 1, 1) + self.tempgrid = Gtk.Grid( + row_spacing=5, + column_spacing=5, + halign=Gtk.Align.FILL, + valign=Gtk.Align.CENTER, + hexpand=True, + vexpand=False, + ) + self._grid.attach(self.tempgrid, 0, 8, 1, 1) + + self.counter = 0 # counter for number of rows added + b = Gtk.Button.new_with_label("Manually add metadata") + b.connect("clicked", self.on_add_clicked) + self.tempgrid.attach(b, 0, 0, 1, 1) + + self.extra_widgets = {} + + self.parser_list = [] + for e in importlib.metadata.entry_points()[ + "rfi_file_monitor.metadataparsers" + ]: + self.parser_list.append(e.load()) + + # Add in textboxes to provide metadata manually + def on_add_clicked(self, button): + i = self.counter + + self.tempgrid.attach( + Gtk.Label( + label="Name", + halign=Gtk.Align.CENTER, + valign=Gtk.Align.CENTER, + hexpand=False, + vexpand=False, + ), + 0, + 1 + i, + 1, + 1, + ) + widget = Gtk.Entry( + placeholder_text="Required", + halign=Gtk.Align.FILL, + valign=Gtk.Align.CENTER, + hexpand=True, + vexpand=False, + ) + self.tempgrid.attach(widget, 1, 1 + i, 1, 1) + self.extra_widgets["name_" + str(i)] = widget + + self.tempgrid.attach( + Gtk.Label( + label="Value", + halign=Gtk.Align.CENTER, + valign=Gtk.Align.CENTER, + hexpand=False, + vexpand=False, + ), + 2, + 1 + i, + 1, + 1, + ) + widget = Gtk.Entry( + placeholder_text="Required", + halign=Gtk.Align.FILL, + valign=Gtk.Align.CENTER, + hexpand=True, + vexpand=False, + ) + self.tempgrid.attach(widget, 3, 1 + i, 1, 1) + self.extra_widgets["value_" + str(i)] = widget + + self.tempgrid.attach( + Gtk.Label( + label="Unit", + halign=Gtk.Align.CENTER, + valign=Gtk.Align.CENTER, + hexpand=False, + vexpand=False, + ), + 4, + 1 + i, + 1, + 1, + ) + widget = Gtk.Entry( + halign=Gtk.Align.FILL, + valign=Gtk.Align.CENTER, + hexpand=True, + vexpand=False, + ) + self.tempgrid.attach(widget, 5, 1 + i, 1, 1) + self.extra_widgets["unit_" + str(i)] = widget + + # b = Gtk.Button() + # b.set_image( + # Gtk.Image(icon_name="user-trash", icon_size=Gtk.IconSize.BUTTON) + # ) + # b.connect("clicked", self.on_delete_clicked) + # self.tempgrid.attach(b, 6, 1, 1, 1) + + self.tempgrid.show_all() + self.counter += 1 + + # def on_delete_clicked(self, button): + # return + @staticmethod def _check_required_fields(params): if not params.hostname: @@ -424,7 +534,16 @@ def checkbox_toggled(self, checkbox): self._used_software_entry.set_sensitive(False) return False + def _fetch_additional_metadata(self, n, v, u): + # here create a dict in correct form? add to self.additional_metadata + self.additional_metadata[n] = { + "type": "string", + "value": v, + "unit": u, + } + def preflight_check(self): + try: ScicatClient( base_url=self.params.hostname, @@ -458,10 +577,26 @@ def preflight_check(self): "Please name a technique for this instrument." ) + self.additional_metadata = {} + _len = int(len(self.extra_widgets) / 3) # there are 3 widgets on each row + for i in range(0, _len): + _name = self.extra_widgets["name_" + str(i)].get_text() + _value = self.extra_widgets["value_" + str(i)].get_text() + _unit = self.extra_widgets["unit_" + str(i)].get_text() + if _name and _value: + self._fetch_additional_metadata(_name, _value, _unit) + elif not _name and not _value: + break # we don't want to do anything here? + else: + raise RequiredInfoNotFound( + "Type and value are required metadata fields." + ) + def run(self, file: File): # Create the payload payload = self.create_payload(file) + print(payload.scientificMetadata) try: try: scicat_session = ScicatClient( @@ -550,8 +685,32 @@ def create_payload(self, file): payload.size = file._total_size payload.numberOfFiles = len(file._filelist) + parser_dict = {} + for f in file: + try: + parser = self.find_parser(f[0]) + except ParserNotFound: + parser = None + if parser: + parser_dict[f[0]] = parser + if not parser_dict: + logger.info( + " Parsers not found. Creating payload without metadata" + ) + # Scientific metadata scientificMetadata: Dict[str, Dict[str, str]] = {} + if parser_dict: + for k, v in parser_dict.items(): + metadata = PayloadHelpers.implement_parser( + self.instr_dict, self.params.technique, k, v + ) + for k, v in metadata.items(): + if k in scientificMetadata.keys(): + if scientificMetadata[k] == v: + continue + else: + scientificMetadata[k] = v payload.scientificMetadata = ( PayloadHelpers.scientific_metadata_concatenation( scientificMetadata, payload.scientificMetadataDefaults @@ -569,8 +728,23 @@ def create_payload(self, file): fstats = Path(file.filename).stat() payload.size = fstats.st_size + try: + parser = self.find_parser(file.filename) + except Exception as e: + logger.info( + " Parser not found. Creating payload without metadata" + ) + parser = None + # Creation of scientific metadata scientificMetadata = {} + if parser: + scientificMetadata = PayloadHelpers.implement_parser( + self.instr_dict, + self.params.technique, + file.filename, + parser, + ) payload.scientificMetadata = ( PayloadHelpers.scientific_metadata_concatenation( scientificMetadata, payload.scientificMetadataDefaults @@ -580,6 +754,16 @@ def create_payload(self, file): del payload.scientificMetadataDefaults return payload + def find_parser(self, filename): + for parser in self.parser_list: + if parser.supports_file(filename): + break + else: + parser = None + if not parser: + raise ParserNotFound("parser not found") + return parser + # Inserts a dataset into Scicat def insert_payload(self, payload, scicat_session): try: @@ -674,6 +858,16 @@ def get_host_location(cls, file: File, operations_list, operation): return source_folders + @classmethod + def implement_parser(cls, instr_dict, technique, filename, parser): + scientific_metadata = {} + instr_vars = instr_dict["techniques"][technique] + extracted = parser.extract_metadata(instr_vars, filename) + if extracted: + for k, v in extracted.items(): + scientific_metadata[k] = {"type": "string", "value": str(v), "unit": ""} + return scientific_metadata + @classmethod def scientific_metadata_concatenation(cls, scientific_metadata, defaults): scientific_metadata |= defaults From e89ae5a0daa3e77b4d4f2e2cc9b2373cdaf49b7f Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Tue, 24 May 2022 10:49:01 +0100 Subject: [PATCH 3/6] black formatting fixes --- rfi_file_monitor/operations/scicataloguer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index 54a1d75..c6c6b35 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -578,7 +578,9 @@ def preflight_check(self): ) self.additional_metadata = {} - _len = int(len(self.extra_widgets) / 3) # there are 3 widgets on each row + _len = int( + len(self.extra_widgets) / 3 + ) # there are 3 widgets on each row for i in range(0, _len): _name = self.extra_widgets["name_" + str(i)].get_text() _value = self.extra_widgets["value_" + str(i)].get_text() @@ -865,7 +867,11 @@ def implement_parser(cls, instr_dict, technique, filename, parser): extracted = parser.extract_metadata(instr_vars, filename) if extracted: for k, v in extracted.items(): - scientific_metadata[k] = {"type": "string", "value": str(v), "unit": ""} + scientific_metadata[k] = { + "type": "string", + "value": str(v), + "unit": "", + } return scientific_metadata @classmethod From 205bce85a2b536b74666b657718e1f873d18f5db Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 25 May 2022 10:07:07 +0100 Subject: [PATCH 4/6] Reformatted grid appearance --- rfi_file_monitor/operations/scicataloguer.py | 88 ++++++++++---------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index c6c6b35..d5584b9 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -39,8 +39,18 @@ def __init__(self, *args, **kwargs): ) self.add(self._grid) + tempgrid = Gtk.Grid( + row_spacing=5, + column_spacing=5, + halign=Gtk.Align.FILL, + valign=Gtk.Align.CENTER, + hexpand=True, + vexpand=False, + ) + self._grid.attach(tempgrid, 0, 0, 1, 1) + # Hostname - self._grid.attach( + tempgrid.attach( Gtk.Label( label="SciCat Hostname", halign=Gtk.Align.START, @@ -62,10 +72,10 @@ def __init__(self, *args, **kwargs): ), "hostname", ) - self._grid.attach(self._hostname_entry, 1, 0, 1, 1) + tempgrid.attach(self._hostname_entry, 1, 0, 1, 1) # Operation upload - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Upload location ", halign=Gtk.Align.START, @@ -84,10 +94,10 @@ def __init__(self, *args, **kwargs): op_combo.append_text(k) op_widget = self.register_widget(op_combo, "operation") - self._grid.attach(op_widget, 3, 0, 1, 1) + tempgrid.attach(op_widget, 3, 0, 1, 1) # Username - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Username", halign=Gtk.Align.START, @@ -109,10 +119,10 @@ def __init__(self, *args, **kwargs): ), "username", ) - self._grid.attach(self._username_entry, 1, 1, 1, 1) + tempgrid.attach(self._username_entry, 1, 1, 1, 1) # Password - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Password", halign=Gtk.Align.START, @@ -135,10 +145,10 @@ def __init__(self, *args, **kwargs): ), "password", ) - self._grid.attach(self._password_entry, 3, 1, 1, 1) + tempgrid.attach(self._password_entry, 3, 1, 1, 1) # Owner - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Owner", halign=Gtk.Align.START, @@ -160,10 +170,10 @@ def __init__(self, *args, **kwargs): ), "owner", ) - self._grid.attach(self._owner_entry, 1, 2, 1, 1) + tempgrid.attach(self._owner_entry, 1, 2, 1, 1) # Owner group - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Owner Group", halign=Gtk.Align.START, @@ -185,10 +195,10 @@ def __init__(self, *args, **kwargs): ), "owner_group", ) - self._grid.attach(self._owner_grp_entry, 3, 2, 1, 1) + tempgrid.attach(self._owner_grp_entry, 3, 2, 1, 1) # Comtact email - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Email", halign=Gtk.Align.START, @@ -210,10 +220,10 @@ def __init__(self, *args, **kwargs): ), "email", ) - self._grid.attach(self._email_entry, 1, 3, 1, 1) + tempgrid.attach(self._email_entry, 1, 3, 1, 1) # Orcid - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Orcid", halign=Gtk.Align.START, @@ -235,10 +245,10 @@ def __init__(self, *args, **kwargs): ), "orcid", ) - self._grid.attach(self._orcid_entry, 3, 3, 1, 1) + tempgrid.attach(self._orcid_entry, 3, 3, 1, 1) # PI - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Principal Investigator", halign=Gtk.Align.START, @@ -260,10 +270,10 @@ def __init__(self, *args, **kwargs): ), "investigator", ) - self._grid.attach(self._pi_entry, 1, 4, 1, 1) + tempgrid.attach(self._pi_entry, 1, 4, 1, 1) # Dataset name - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Experiment name", halign=Gtk.Align.START, @@ -285,11 +295,11 @@ def __init__(self, *args, **kwargs): ), "experiment_name", ) - self._grid.attach(self._exp_name_entry, 3, 4, 1, 1) + tempgrid.attach(self._exp_name_entry, 3, 4, 1, 1) # Instrument # TO DO - this is temporary until instrument preferences configured - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Instrument", halign=Gtk.Align.START, @@ -311,10 +321,10 @@ def __init__(self, *args, **kwargs): ), "instrument_choice", ) - self._grid.attach(self._instrument_entry, 1, 5, 1, 1) + tempgrid.attach(self._instrument_entry, 1, 5, 1, 1) # Technique - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Technique", halign=Gtk.Align.START, @@ -336,10 +346,10 @@ def __init__(self, *args, **kwargs): ), "technique", ) - self._grid.attach(self._technique_entry, 3, 5, 1, 1) + tempgrid.attach(self._technique_entry, 3, 5, 1, 1) # Input boxes for derived dataset specific fields - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Input Datasets", halign=Gtk.Align.START, @@ -363,9 +373,9 @@ def __init__(self, *args, **kwargs): ), "input_datasets", ) - self._grid.attach(self._input_datasets_entry, 1, 7, 1, 1) + tempgrid.attach(self._input_datasets_entry, 1, 7, 1, 1) - self._grid.attach( + tempgrid.attach( Gtk.Label( label="Used Software", halign=Gtk.Align.START, @@ -389,29 +399,29 @@ def __init__(self, *args, **kwargs): ), "used_software", ) - self._grid.attach(self._used_software_entry, 3, 7, 1, 1) + tempgrid.attach(self._used_software_entry, 3, 7, 1, 1) self._derived_checkbox = Gtk.CheckButton(label="Derived Dataset") self._derived_checkbox.connect("toggled", self.checkbox_toggled) self.params.derived_dataset = self.checkbox_toggled( self._derived_checkbox ) - self._grid.attach(self._derived_checkbox, 0, 6, 1, 1) + tempgrid.attach(self._derived_checkbox, 0, 6, 1, 1) self.tempgrid = Gtk.Grid( row_spacing=5, column_spacing=5, - halign=Gtk.Align.FILL, + halign=Gtk.Align.CENTER, valign=Gtk.Align.CENTER, hexpand=True, vexpand=False, ) - self._grid.attach(self.tempgrid, 0, 8, 1, 1) + self._grid.attach(self.tempgrid, 0, 1, 1, 1) self.counter = 0 # counter for number of rows added b = Gtk.Button.new_with_label("Manually add metadata") b.connect("clicked", self.on_add_clicked) - self.tempgrid.attach(b, 0, 0, 1, 1) + self.tempgrid.attach(b, 3, 0, 1, 1) self.extra_widgets = {} @@ -493,19 +503,9 @@ def on_add_clicked(self, button): self.tempgrid.attach(widget, 5, 1 + i, 1, 1) self.extra_widgets["unit_" + str(i)] = widget - # b = Gtk.Button() - # b.set_image( - # Gtk.Image(icon_name="user-trash", icon_size=Gtk.IconSize.BUTTON) - # ) - # b.connect("clicked", self.on_delete_clicked) - # self.tempgrid.attach(b, 6, 1, 1, 1) - self.tempgrid.show_all() self.counter += 1 - # def on_delete_clicked(self, button): - # return - @staticmethod def _check_required_fields(params): if not params.hostname: @@ -588,7 +588,9 @@ def preflight_check(self): if _name and _value: self._fetch_additional_metadata(_name, _value, _unit) elif not _name and not _value: - break # we don't want to do anything here? + logger.info( + "name and value not provided for additional metadata row, skipping" + ) else: raise RequiredInfoNotFound( "Type and value are required metadata fields." From 1c4e8cf997b92cfc8d22ea78024d11ba076441e1 Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Wed, 25 May 2022 14:12:37 +0100 Subject: [PATCH 5/6] modified fetch_additional_metadata function --- rfi_file_monitor/operations/scicataloguer.py | 66 +++++++++----------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index c19c0a7..ddb0211 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -455,7 +455,7 @@ def on_add_clicked(self, button): vexpand=False, ) self.tempgrid.attach(widget, 1, 1 + i, 1, 1) - self.extra_widgets["name_" + str(i)] = widget + self.extra_widgets["name" + str(i)] = widget self.tempgrid.attach( Gtk.Label( @@ -478,7 +478,7 @@ def on_add_clicked(self, button): vexpand=False, ) self.tempgrid.attach(widget, 3, 1 + i, 1, 1) - self.extra_widgets["value_" + str(i)] = widget + self.extra_widgets["value" + str(i)] = widget self.tempgrid.attach( Gtk.Label( @@ -500,7 +500,7 @@ def on_add_clicked(self, button): vexpand=False, ) self.tempgrid.attach(widget, 5, 1 + i, 1, 1) - self.extra_widgets["unit_" + str(i)] = widget + self.extra_widgets["unit" + str(i)] = widget self.tempgrid.show_all() self.counter += 1 @@ -534,13 +534,29 @@ def _check_required_fields(params): if not params.owner_group: raise RequiredInfoNotFound("Owner group required") - def _fetch_additional_metadata(self, n, v, u): - # here create a dict in correct form? add to self.additional_metadata - self.additional_metadata[n] = { - "type": "string", - "value": v, - "unit": u, - } + # Iterates over every metadata row and extracts data + def _fetch_additional_metadata(self): + self.params.additional_metadata = {} + rows = int(len(self.extra_widgets) / 3) + for i in range(0, rows): + _name = self.extra_widgets["name" + str(i)].get_text() + _value = self.extra_widgets["value" + str(i)].get_text() + _unit = self.extra_widgets["unit" + str(i)].get_text() + if _name and _value: + self.additional_metadata[_name] = { + "type": "string", + "value": _value, + "unit": _unit, + } + # don't throw exception if row left empty, just ignore + elif not _name and not _value: + logger.info( + "name and value not provided for additional metadata row, skipping" + ) + else: + raise RequiredInfoNotFound( + "Both type and value are required metadata fields." + ) def preflight_check(self): self._preflight_check(self.params) @@ -574,29 +590,11 @@ def _preflight_check(self, params): "Please name a technique for this instrument." ) - self.additional_metadata = {} - _len = int( - len(self.extra_widgets) / 3 - ) # there are 3 widgets on each row - for i in range(0, _len): - _name = self.extra_widgets["name_" + str(i)].get_text() - _value = self.extra_widgets["value_" + str(i)].get_text() - _unit = self.extra_widgets["unit_" + str(i)].get_text() - if _name and _value: - self._fetch_additional_metadata(_name, _value, _unit) - elif not _name and not _value: - logger.info( - "name and value not provided for additional metadata row, skipping" - ) - else: - raise RequiredInfoNotFound( - "Type and value are required metadata fields." - ) + # Fetch additional metadata + self._fetch_additional_metadata def run(self, file: File): self.params.keywords = [] - # This can be added manually in future? TO DO - self.params.additional_metadata = {} return self._run(file, self.params) def _run(self, file: File, params): @@ -674,9 +672,7 @@ def is_file_payload(self, _payload, file): try: parser = self.find_parser(file.filename) except Exception as e: - logger.info( - " Parser not found. Creating payload without metadata" - ) + logger.info(" Parser not found. Creating payload without metadata") parser = None # Scientific metadata @@ -724,9 +720,7 @@ def is_dir_payload(self, _payload, file): if parser: parser_dict[f[0]] = parser if not parser_dict: - logger.info( - " Parsers not found. Creating payload without metadata" - ) + logger.info(" Parsers not found. Creating payload without metadata") # Scientific metadata scientificMetadata: Dict[str, Dict[str, str]] = {} From 4d5abb1445c95d2773bea1b311230874f9eb24ac Mon Sep 17 00:00:00 2001 From: Abigail Alexander Date: Thu, 26 May 2022 14:23:43 +0100 Subject: [PATCH 6/6] renamed duplicate nested variables --- rfi_file_monitor/operations/scicataloguer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index ddb0211..a8e6e1b 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -729,12 +729,12 @@ def is_dir_payload(self, _payload, file): metadata = PayloadHelpers.implement_parser( self.instr_dict, self.params.technique, k, v ) - for k, v in metadata.items(): - if k in scientificMetadata.keys(): - if scientificMetadata[k] == v: + for key, value in metadata.items(): + if key in scientificMetadata.keys(): + if scientificMetadata[key] == value: continue else: - scientificMetadata[k] = v + scientificMetadata[key] = value _payload.scientificMetadata = ( PayloadHelpers.scientific_metadata_concatenation( scientificMetadata,