diff --git a/README.md b/README.md index 93d0c7a..5bb47e0 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,16 @@ $ pytr login --phone_no +49123456789 --pin 1234 ``` If no arguments are supplied pytr will look for them in the file `~/.pytr/credentials` (the first line must contain the phone number, the second line the pin). If the file doesn't exist pytr will ask for for the phone number and pin. + +## Location and File names of the downloaded Documents +During the first run of the 'dl_docs' command a config file is created in the user home directory `/.pytr/file_destination_config.yaml`. +The file contains destination patterns which describe where the file should be located and how the file name should look like. If a event/document matches the defined pattern it will be located in that specific `path` with the specified `filename`. + +There are three mandatory patterns defined at the top: +* `default` - Defines only `filename` and is used for all other patterns if no "filename" is provided +* `unknown` - Defines `filename` and `path`, this is used when no match can be found for the event and the given document. +* `multiple_match` - If there are multiple matching patterns and the destination would be ambiguous, the document will be stored in the given `path` with the given `filename` + +The other pattern can be as you like but keep in mind that patterns `path` and `filenames` should result in unique document names. If you see something like this ` (some strange string)` the document path + name was not unique. + +> Its also possible to copy the configuration file from `~/pytr/config/file_destination_config__template.yaml` to `/.pytr/file_destination_config.yaml` and modify it before the first download to avoid that the download need to be performed a second time. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 1c2e5c0..b15cab3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ dependencies = [ "shtab", "websockets>=10.1", "babel", + "PyYAML", + "importlib_resources" ] [project.scripts] diff --git a/pytr/api.py b/pytr/api.py index 20ae661..4bfd84c 100644 --- a/pytr/api.py +++ b/pytr/api.py @@ -32,18 +32,15 @@ import ssl import requests import websockets + from ecdsa import NIST256p, SigningKey from ecdsa.util import sigencode_der from http.cookiejar import MozillaCookieJar from pytr.utils import get_logger +from pytr.app_path import CREDENTIALS_FILE, COOKIES_FILE, KEY_FILE -home = pathlib.Path.home() -BASE_DIR = home / '.pytr' -CREDENTIALS_FILE = BASE_DIR / 'credentials' -KEY_FILE = BASE_DIR / 'keyfile.pem' -COOKIES_FILE = BASE_DIR / 'cookies.txt' class TradeRepublicApi: diff --git a/pytr/app_path.py b/pytr/app_path.py new file mode 100644 index 0000000..01b34f0 --- /dev/null +++ b/pytr/app_path.py @@ -0,0 +1,10 @@ +import pathlib + +home = pathlib.Path.home() +BASE_DIR = home / '.pytr' + +CREDENTIALS_FILE = BASE_DIR / 'credentials' +KEY_FILE = BASE_DIR / 'keyfile.pem' +COOKIES_FILE = BASE_DIR / 'cookies.txt' + +DESTINATION_CONFIG_FILE = BASE_DIR / 'file_destination_config.yaml' \ No newline at end of file diff --git a/pytr/config/__init__.py b/pytr/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pytr/config/file_destination_config__template.yaml b/pytr/config/file_destination_config__template.yaml new file mode 100644 index 0000000..29551f0 --- /dev/null +++ b/pytr/config/file_destination_config__template.yaml @@ -0,0 +1,236 @@ +destination: + ################################################################################################ + ## Default and fallback patterns used for the destination of a downloaded document. + ################################################################################################ + # valid for all blocks without explicit filename + default: + filename: "{iso_date}.{iso_time} {event_title}" # {event_title} = Wertpapier-/ETF-/Produkt-Name + + # if pattern not found, use this block + unknown: + path: "Unknown/{section_title}/" + filename: "{iso_date}.{iso_time} {event_type} - {event_subtitle} - {document_title} - {event_title}" + + # if pattern found multiple times, use this block + multiple_match: + path: "MultipleMatch/{section_title}/" + filename: "{iso_date}.{iso_time} {event_type} - {event_subtitle} - {document_title} - {event_title}" + ################################################################################################ + + ################################################################################################ + ## Specific patterns for the destination of a downloaded document. + ################################################################################################ + # stocks + stock_order_settlement: + pattern: [ + {event_type: "ORDER_EXECUTED", document_title: "Abrechnung(.\\d+)?"}, # mit limit verkauft + {event_type: "TRADE_INVOICE", document_title: "Abrechnung(.\\d+)?"}, # mit limit gekauft + {event_type: "STOCK_PERK_REFUNDED", document_title: "Abrechnung(.\\d+)?"}, # Aktiengeschenk + {event_type: "SHAREBOOKING", document_title: "Abrechnung(.\\d+)?"}, # Kapitalmassnahme + ] + path: "Stocks/Settlement/{iso_date_year}/" + + stock_order_cost_report: + pattern: [ + {event_type: "ORDER_CREATED", document_title: "Kosteninformation(.\\d+)?"}, # limit erstellt + {event_type: "ORDER_EXECUTED", document_title: "Kosteninformation(.\\d+)?"}, + {event_type: "TRADE_INVOICE", document_title: "Kosteninformation(.\\d+)?"}, + {event_type: "STOCK_PERK_REFUNDED", document_title: "Kosteninformation(.\\d+)?"}, # Aktiengeschenk + {event_type: "EX_POST_COST_REPORT"}, + ] + path: "Stocks/Cost report/{iso_date_year}/" + + stock_order_created: + pattern: [{event_type: "ORDER_CREATED", document_title: "Auftragsbestätigung(.\\d+)?"}, {event_type: "ORDER_EXECUTED", document_title: "Auftragsbestätigung(.\\d+)?"}, {event_type: "TRADE_INVOICE", document_title: "Auftragsbestätigung(.\\d+)?"}] + path: "Stocks/Order created/{iso_date_year}/" + + stock_order_canceled: + pattern: [{event_type: "TRADE_CANCELED"}, {event_type: "ORDER_CANCELED"}] + path: "Stocks/Order canceled/{iso_date_year}/" + + order_expired: + pattern: [{event_type: "ORDER_EXPIRED"}] + path: "Stocks/Order canceled/{iso_date_year}/" + filename: "{iso_date} {document_title}" + + # Kapitalmaßnahmen + stock_notice_1: + pattern: [{event_type: "EXERCISE"}, {event_type: "SHAREBOOKING", document_title: "Ausführungsanzeige(.\\d+)?"}] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_2: + pattern: [{event_type: "CORPORATE_ACTION"}, {event_type: "SHAREBOOKING_CANCELED"}] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_3: + pattern: [ + {event_type: "SHAREBOOKING_TRANSACTIONAL"}, + {event_type: "INSTRUCTION_CORPORATE_ACTION", document_title: "Kundenanschreiben(.\\d+)?"}, #Kapitalerhöhung gegen Bar + ] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} {event_title}" + + # split for better readability + stock_notice_4: + pattern: + [ + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Spin-off"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Reverse Split"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Aktiendividende"}, + {event_type: "ssp_corporate_action_invoice_shares", event_subtitle: "Zwischenvertrieb von Wertpapieren"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Wechsel"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Information"}, + {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Aufruf von Zwischenpapieren"}, + ] + path: "Stocks/Notice/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle} - {event_title}" + + stock_report: + pattern: [{event_type: "QUARTERLY_REPORT", document_title: "Kontoauszug(.\\d+)?"}, {event_type: "QUARTERLY_REPORT", document_title: "Depotauszug(.\\d+)?"}, {event_type: "QUARTERLY_REPORT", document_title: "Cryptoauszug(.\\d+)?"}] + path: "Stocks/Report/{iso_date_year}/" + filename: "{iso_date} {document_title} {event_title}" + + # General Meetings + stock_general_meetings: + pattern: [{event_type: "GENERAL_MEETING", document_title: "Hauptversammlung"}, {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Jährliche Hauptversammlung"}] + path: "Stocks/General Meetings/{iso_date_year}/" + + stock_general_meetings_multiple_files: + pattern: [{event_type: "GENERAL_MEETING", document_title: "Hauptversammlung \\d+"}] + path: "Stocks/General Meetings/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_title} - {document_title}" + + stock_special_meetings: + pattern: [{event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Außerordentliche oder spezielle Hauptversammlung"}] + path: "Stocks/General Meetings/{iso_date_year}/" + filename: "{iso_date}.{iso_time} {event_subtitle}" + + # Savings plan + savings_plan: + pattern: [{event_type: "SAVINGS_PLAN_INVOICE_CREATED"}, {event_type: "SAVINGS_PLAN_EXECUTED"}, {event_type: "SAVINGS_PLAN_CANCELED"}] + path: "Stocks/Savings plan/{iso_date_year}/" + + # pre-determined tax base earning + stock_pre_earning_tax: + pattern: [{event_type: "PRE_DETERMINED_TAX_BASE_EARNING"}] + path: "Stocks/PreEarningTax/{iso_date_year}/" + + # Dividends + dividends_received: + pattern: [{event_type: "CREDIT", event_subtitle: "Dividende"}, {event_type: "CREDIT", event_subtitle: "Ausschüttung"}, {event_type: "CREDIT_CANCELED"}, {event_type: "ssp_corporate_action_invoice_cash"}] + path: "Dividends/{iso_date_year}/" + + # Dividends Corporate action election + dividends_election: + pattern: [{event_type: "INSTRUCTION_CORPORATE_ACTION", document_title: "Dividende Wahlweise(.\\d+)?"}, {event_type: "ssp_dividend_option_customer_instruction", event_subtitle: "Cash oder Aktie"}, {event_type: "ssp_corporate_action_informative_notification", event_subtitle: "Dividende Wahlweise"}] + path: "Dividendelection/{iso_date_year}/" + + # bonds + bond_repayment: + pattern: [{event_type: "REPAYMENT"}] + path: "Bonds/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Repayment {event_title}" + + bond_interest: + pattern: [{event_type: "COUPON_PAYMENT"}] + path: "Bonds/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Interest {event_title}" + + # Saveback + saveback_enabled: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Enabled(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Enabled {event_title}" + + saveback_executed: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Abrechnung Ausführung(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Report {event_title}" + + saveback_cost_report: + pattern: [{event_type: "benefits_saveback_execution", document_title: "Kosteninformation(.\\d+)?"}] + path: "Saveback/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Cost report {event_title}" + + # Round up + roundup_enabled: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Enabled(.\\d+)?"}] # same files - multiple times at once + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Enabled {event_title}" + + roundup_executed: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Abrechnung Ausführung(.\\d+)?"}] + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Report {event_title}" + + roundup_cost_report: + pattern: [{event_type: "benefits_spare_change_execution", document_title: "Kosteninformation(.\\d+)?"}] + path: "Roundup/{iso_date_year}/" + filename: "{iso_date}.{iso_time} Cost report {event_title}" + + # account + cash_interest: + pattern: [{event_type: "INTEREST_PAYOUT"}, {event_type: "INTEREST_PAYOUT_CREATED"}] + path: "Cash Interest/" + filename: "{iso_date} Report" + + cash_transfer_report: + pattern: [{event_type: "INCOMING_TRANSFER"}, {event_type: "PAYMENT_INBOUND_GOOGLE_PAY"}, {event_type: "PAYMENT_INBOUND_CREDIT_CARD"}] + path: "Cash Report/" + filename: "{iso_date}.{iso_time} {document_title}" # {event_title} = Personal name + + # annual tax report for account + account_tax_report: + pattern: [{event_type: "TAX_REFUND"}, {event_type: "TAX_ENGINE_ANNUAL_REPORT"}, {event_type: "YEAR_END_TAX_REPORT"}] + path: "Tax/" + filename: "{iso_date} {document_title}" + + account_tax_adjustment: + pattern: [{event_type: "ssp_tax_correction_invoice"}, {event_type: "TAX_CORRECTION"}] + path: "Tax/" + filename: "{iso_date} {event_title}" + + # common informations + notice_stocks: + pattern: [{event_type: "ORDER_CREATED", document_title: "Basisinformationsblatt(.\\d+)?"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title} - {event_title}" + + notice_stocks2: + pattern: [{event_type: "TRADE_INVOICE", document_title: "Basisinformationsblatt(.\\d+)?"}, {event_type: "GESH_CORPORATE_ACTION", event_subtitle: "Unternehmensmeldung"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {event_subtitle} - {event_title}" + + notice_stocks3: + pattern: [{event_type: "CUSTOMER_CREATED"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title}" + + notice_option_contract: + pattern: [{event_type: "ORDER_EXECUTED", document_title: "Basisinformationsblatt(.\\d+)?"}] + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {document_title} Option" + + notice_multiple_documents: + pattern: [{event_type: "GESH_CORPORATE_ACTION_MULTIPLE_POSITIONS"}] #event_subtitle: Gesellschaftshinweis + path: "Notice/{iso_date_year}/" + filename: "{iso_date} {event_subtitle} - {event_title} - {document_title}" + + contract_documents: + pattern: [ + {event_type: "card_order_billed"}, # Bestellung Trade Republic Karte + {event_type: "DOCUMENTS_CREATED"}, # Basisinformationen über Wertpapiere + {event_type: "DOCUMENTS_ACCEPTED"}, # Rechtliche Dokumente: Kundenvereinbarung / Vorvertragliche Informationen / Datenschutzinformationen* / Widerrufsbelehrung* / *Crypto* / Risikohinweise + {event_type: "DOCUMENTS_CHANGED", section_title: "Dokumente"}, # Rechtliche Dokumente: Kundenvereinbarung + ] + path: "Contract/" + filename: "{iso_date} {event_title} - {document_title}" + + contract_documents_updated: + pattern: [{event_type: "DOCUMENTS_CHANGED", section_title: "Aktualisierte Dokumente"}] # aktualisierte Rechtliche Dokumente: Kundenvereinbarung + path: "Contract/" + filename: "{iso_date} {event_title} - {document_title} updated" diff --git a/pytr/dl.py b/pytr/dl.py index 6f85714..8e23d48 100644 --- a/pytr/dl.py +++ b/pytr/dl.py @@ -1,21 +1,22 @@ -import re +import os from concurrent.futures import as_completed from pathlib import Path from requests_futures.sessions import FuturesSession +from datetime import datetime from pathvalidate import sanitize_filepath from pytr.utils import preview, get_logger from pytr.api import TradeRepublicError from pytr.timeline import Timeline +from pytr.file_destination_provider import FileDestinationProvider class DL: def __init__( self, tr, output_path, - filename_fmt, since_timestamp=0, history_file='pytr_history', max_workers=8, @@ -25,13 +26,12 @@ def __init__( ''' tr: api object output_path: name of the directory where the downloaded files are saved - filename_fmt: format string to customize the file names since_timestamp: downloaded files since this date (unix timestamp) ''' self.tr = tr self.output_path = Path(output_path) self.history_file = self.output_path / history_file - self.filename_fmt = filename_fmt + self.file_destination_provider = self.__get_file_destination_provider() self.since_timestamp = since_timestamp self.universal_filepath = universal_filepath self.sort_export = sort_export @@ -79,60 +79,34 @@ async def dl_loop(self): else: self.log.warning(f"unmatched subscription of type '{subscription['type']}':\n{preview(response)}") - def dl_doc(self, doc, titleText, subtitleText, subfolder=None): + def dl_doc(self, doc, event_type: str, event_title: str, event_subtitle: str, section_title: str, timestamp: datetime): ''' send asynchronous request, append future with filepath to self.futures ''' doc_url = doc['action']['payload'] - if subtitleText is None: - subtitleText = '' - - try: - date = doc['detail'] - iso_date = '-'.join(date.split('.')[::-1]) - except KeyError: - date = '' - iso_date = '' + document_title = doc.get('title', '') doc_id = doc['id'] - # extract time from subtitleText - try: - time = re.findall('um (\\d+:\\d+) Uhr', subtitleText) - if time == []: - time = '' - else: - time = f' {time[0]}' - except TypeError: - time = '' - - if subfolder is not None: - directory = self.output_path / subfolder - else: - directory = self.output_path + variables = {} + variables['iso_date'] = timestamp.strftime('%Y-%m-%d') + variables['iso_date_year'] = timestamp.strftime('%Y') + variables['iso_date_month'] = timestamp.strftime('%m') + variables['iso_date_day'] = timestamp.strftime('%d') + variables['iso_time'] = timestamp.strftime('%H-%M') - # If doc_type is something like 'Kosteninformation 2', then strip the 2 and save it in doc_type_num - doc_type = doc['title'].rsplit(' ') - if doc_type[-1].isnumeric() is True: - doc_type_num = f' {doc_type.pop()}' - else: - doc_type_num = '' - - doc_type = ' '.join(doc_type) - titleText = titleText.replace('\n', '').replace('/', '-') - subtitleText = subtitleText.replace('\n', '').replace('/', '-') + filepath = self.file_destination_provider.get_file_path( + event_type, event_title, event_subtitle, section_title, document_title, variables) + # Just in case someone defines file names with extension + if filepath.endswith('.pdf') is True: + filepath = filepath[:-4] - filename = self.filename_fmt.format( - iso_date=iso_date, time=time, title=titleText, subtitle=subtitleText, doc_num=doc_type_num, id=doc_id - ) + filepath_with_doc_id = f'{filepath} ({doc_id})' - filename_with_doc_id = filename + f' ({doc_id})' + filepath = f'{filepath}.pdf' + filepath_with_doc_id = f'{filepath_with_doc_id}.pdf' - if doc_type in ['Kontoauszug', 'Depotauszug']: - filepath = directory / 'Abschlüsse' / f'{filename}' / f'{doc_type}.pdf' - filepath_with_doc_id = directory / 'Abschlüsse' / f'{filename_with_doc_id}' / f'{doc_type}.pdf' - else: - filepath = directory / doc_type / f'{filename}.pdf' - filepath_with_doc_id = directory / doc_type / f'{filename_with_doc_id}.pdf' + filepath = Path(os.path.join(self.output_path, filepath)) + filepath_with_doc_id = Path(os.path.join(self.output_path, filepath_with_doc_id)) if self.universal_filepath: filepath = sanitize_filepath(filepath, '_', 'universal') @@ -142,12 +116,15 @@ def dl_doc(self, doc, titleText, subtitleText, subfolder=None): filepath_with_doc_id = sanitize_filepath(filepath_with_doc_id, '_', 'auto') if filepath in self.filepaths: - self.log.debug(f'File {filepath} already in queue. Append document id {doc_id}...') + self.log.debug( + f'File {filepath} already in queue. Append document id {doc_id}...') if filepath_with_doc_id in self.filepaths: - self.log.debug(f'File {filepath_with_doc_id} already in queue. Skipping...') + self.log.debug( + f'File {filepath_with_doc_id} already in queue. Skipping...') return else: filepath = filepath_with_doc_id + doc['local filepath'] = str(filepath) self.filepaths.append(filepath) @@ -200,3 +177,6 @@ def work_responses(self): if self.done == len(self.doc_urls): self.log.info('Done.') exit(0) + + def __get_file_destination_provider(self): + return FileDestinationProvider() diff --git a/pytr/file_destination_provider.py b/pytr/file_destination_provider.py new file mode 100644 index 0000000..1930621 --- /dev/null +++ b/pytr/file_destination_provider.py @@ -0,0 +1,188 @@ +import os +import re +import shutil +import pytr.config + +from importlib_resources import files + +from dataclasses import dataclass, fields +from typing import Optional +from yaml import safe_load +from pathlib import Path +from pytr.app_path import DESTINATION_CONFIG_FILE +from pytr.utils import get_logger + + +DEFAULT_CONFIG = "default" +UNKNOWN_CONFIG = "unknown" +MULTIPLE_MATCH_CONFIG = "multiple_match" + +TEMPLATE_FILE_NAME ="file_destination_config__template.yaml" + +# Invalid characters translation table, for cleaning up the variables before using them. +# This was done to avoid issues with for example 'event_subtitle: “Umtausch/Bezug”' which caused a directory which was unintentional. +INVALID_CHARS_TRANSLATION_TABLE = str.maketrans({ + '"': '', + '?': '', + '<': '', + '>': '', + '*': '', + '|': '-', + '/': '-', + '\\': '-' +}) + + +class DefaultFormateValue(dict): + def __missing__(self, key): + return key.join("{}") + + +@dataclass +class DestinationConfig: + config_name: str + filename: str + path: Optional[str] = None + pattern: Optional[list] = None + + +@dataclass +class Pattern: + event_type: Optional[str] = None + event_title: Optional[str] = None + event_subtitle: Optional[str] = None + section_title: Optional[str] = None + document_title: Optional[str] = None + + +class FileDestinationProvider: + + def __init__(self): + ''' + A provider for file path and file names based on the event type and other parameters. + ''' + self._log = get_logger(__name__) + + config_file_path = Path(DESTINATION_CONFIG_FILE) + if config_file_path.is_file() == False: + self.__create_default_config(config_file_path) + + config_file = open(config_file_path, "r", encoding="utf8") + destination_config = safe_load(config_file) + + self.__validate_config(destination_config) + + destinations = destination_config["destination"] + + self._destination_configs: list[DestinationConfig] = [] + + for config_name in destinations: + if config_name == DEFAULT_CONFIG: + self._default_file_config = DestinationConfig( + DEFAULT_CONFIG, destinations[DEFAULT_CONFIG]["filename"]) + elif config_name == UNKNOWN_CONFIG: + self._unknown_file_config = DestinationConfig( + UNKNOWN_CONFIG, destinations[UNKNOWN_CONFIG]["filename"], destinations[UNKNOWN_CONFIG]["path"]) + elif config_name == MULTIPLE_MATCH_CONFIG: + self._multiple_match_file_config = DestinationConfig( + MULTIPLE_MATCH_CONFIG, destinations[MULTIPLE_MATCH_CONFIG]["filename"], destinations[MULTIPLE_MATCH_CONFIG]["path"]) + else: + patterns = self.__extract_pattern( + destinations[config_name].get("pattern", None)) + for pattern in patterns: + self._destination_configs.append(DestinationConfig( + config_name, destinations[config_name].get("filename", None), destinations[config_name].get("path", None), pattern)) + + def get_file_path(self, event_type: str, event_title: str, event_subtitle: str, section_title: str, document_title: str, variables: dict) -> str: + ''' + Get the file path based on the event type and other parameters. + + Parameters: + event_type (str): The event type + event_title (str): The event title + event_subtitle (str): The event subtitle + section_title (str): The section title + document_title (str): The document title + variables (dict): The variables->value dict to be used in the file path and file name format. + ''' + + doc = Pattern(event_type, event_title, event_subtitle, section_title, document_title) + + matching_configs = self._destination_configs.copy() + # create a dictionary that maps the field names to their values in the pattern instance + pattern_dict = {field.name: getattr(doc, field.name) for field in fields(Pattern)} + + # iterate over the dictionary to filter the matching_configs list and update the variables dictionary + for field_name, search_pattern in pattern_dict.items(): + if search_pattern is not None: + matching_configs = list(filter(lambda config: self.__is_matching_config(config, field_name, search_pattern), matching_configs)) + variables[field_name] = search_pattern.translate(INVALID_CHARS_TRANSLATION_TABLE).strip() + + + if len(matching_configs) == 0: + self._log.debug( + f"No destination config found for the given parameters: event_type:{event_type}, event_title:{event_title},event_subtitle:{event_subtitle},section_title:{section_title},document_title:{document_title}") + return self.__create_file_path(self._unknown_file_config, variables) + + if len(matching_configs) > 1: + self._log.debug(f"Multiple Destination Patterns where found. Using 'multiple_match' config! Parameter: event_type:{event_type}, event_title:{event_title},event_subtitle:{event_subtitle},section_title:{section_title},document_title:{document_title}") + return self.__create_file_path(self._multiple_match_file_config, variables) + + return self.__create_file_path(matching_configs[0], variables) + + @staticmethod + def __is_matching_config(config: DestinationConfig, field_name: str, search_pattern: str) -> bool: + pattern = config.pattern + return ( + getattr(pattern, field_name, None) is None + or re.fullmatch(getattr(pattern, field_name, None), search_pattern) + ) + + def __create_file_path(self, config: DestinationConfig, variables: dict): + formate_variables = DefaultFormateValue(variables) + + path = config.path + filename = config.filename + if filename is None: + filename = self._default_file_config.filename + + return os.path.join(path, filename).format_map(formate_variables) + + def __extract_pattern(self, pattern_config: list) -> list: + patterns = [] + for pattern in pattern_config: + patterns.append(Pattern(pattern.get("event_type", None), + pattern.get("event_title", None), + pattern.get("event_subtitle", None), + pattern.get("section_title", None), + pattern.get("document_title", None))) + + return patterns + + def __validate_config(self, destination_config: dict): + if "destination" not in destination_config: + raise ValueError("'destination' key not found in config file") + + destinations = destination_config["destination"] + + # Check if default config is present + if DEFAULT_CONFIG not in destinations or "filename" not in destinations[DEFAULT_CONFIG]: + raise ValueError( + "'default' config not found or filename is not present in 'default' config") + + if UNKNOWN_CONFIG not in destinations or "filename" not in destinations[UNKNOWN_CONFIG] or "path" not in destinations[UNKNOWN_CONFIG]: + raise ValueError( + "'unknown' config not found or filename/path is not present in 'unknown' config") + + if MULTIPLE_MATCH_CONFIG not in destinations or "filename" not in destinations[MULTIPLE_MATCH_CONFIG] or "path" not in destinations[MULTIPLE_MATCH_CONFIG]: + raise ValueError( + "'multiple_match' config not found or filename/path is not present in 'multiple_match' config") + + for config_name in destinations: + if config_name != DEFAULT_CONFIG and "path" not in destinations[config_name]: + raise ValueError( + f"'{config_name}' has no path defined in destination config") + + def __create_default_config(self, config_file_path: Path): + path = files(pytr.config).joinpath(TEMPLATE_FILE_NAME) + shutil.copyfile(path, config_file_path) diff --git a/pytr/main.py b/pytr/main.py index 1389926..04459f3 100644 --- a/pytr/main.py +++ b/pytr/main.py @@ -76,6 +76,9 @@ def formatter(prog): 'Download all pdf documents from the timeline and sort them into folders.' + ' Also export account transactions (account_transactions.csv)' + ' and JSON files with all events (events_with_documents.json and other_events.json' + + ' The file and folder where the structure is saved is defined in a config file located in your home' + + ' directory "/.pytr/file_destination_config.yaml". This is created during the first dl_docs run.' + + ' Its also possible to provide the config upfront by creating the file manually (copy from git repo).' ) parser_dl_docs = parser_cmd.add_parser( 'dl_docs', @@ -86,12 +89,7 @@ def formatter(prog): ) parser_dl_docs.add_argument('output', help='Output directory', metavar='PATH', type=Path) - parser_dl_docs.add_argument( - '--format', - help='available variables:\tiso_date, time, title, doc_num, subtitle, id', - metavar='FORMAT_STRING', - default='{iso_date}{time} {title}{doc_num}', - ) + parser_dl_docs.add_argument( '--last_days', help='Number of last days to include (use 0 get all days)', metavar='DAYS', default=0, type=int ) @@ -207,10 +205,10 @@ def main(): since_timestamp = 0 else: since_timestamp = (datetime.now().astimezone() - timedelta(days=args.last_days)).timestamp() + dl = DL( login(phone_no=args.phone_no, pin=args.pin, web=not args.applogin), args.output, - args.format, since_timestamp=since_timestamp, max_workers=args.workers, universal_filepath=args.universal, diff --git a/pytr/timeline.py b/pytr/timeline.py index d33eac0..8ad8397 100644 --- a/pytr/timeline.py +++ b/pytr/timeline.py @@ -130,14 +130,7 @@ async def process_timelineDetail(self, response, dl): + f"{event['title']} -- {event['subtitle']} - {event['timestamp'][:19]}" ) - subfolder = { - 'benefits_saveback_execution': 'Saveback', - 'benefits_spare_change_execution': 'RoundUp', - 'ssp_corporate_action_invoice_cash': 'Dividende', - 'CREDIT': 'Dividende', - 'INTEREST_PAYOUT_CREATED': 'Zinsen', - "SAVINGS_PLAN_EXECUTED":'Sparplan' - }.get(event["eventType"]) + event['has_docs'] = False for section in response['sections']: @@ -150,10 +143,7 @@ async def process_timelineDetail(self, response, dl): except (ValueError, KeyError): timestamp = datetime.now().timestamp() if self.max_age_timestamp == 0 or self.max_age_timestamp < timestamp: - title = f"{doc['title']} - {event['title']}" - if event['eventType'] in ["ACCOUNT_TRANSFER_INCOMING", "ACCOUNT_TRANSFER_OUTGOING", "CREDIT"]: - title += f" - {event['subtitle']}" - dl.dl_doc(doc, title, doc.get('detail'), subfolder) + dl.dl_doc(doc, event['eventType'], event['title'], event['subtitle'], section['title'], datetime.fromisoformat(event['timestamp'][:19])) if event['has_docs']: self.events_with_docs.append(event)