diff --git a/tools/extract_function/extract_function.py b/tools/extract_function/extract_function.py index 20cd6fc6..8a82ebbe 100644 --- a/tools/extract_function/extract_function.py +++ b/tools/extract_function/extract_function.py @@ -19,6 +19,7 @@ if function_header.endswith(';'): function_header = function_header[:-1] +# Extract the function name from the function header argument. left_parentheses_index = function_header.find('(') if left_parentheses_index >= 0: function_name = function_header[function_header.rfind(' ', None, left_parentheses_index) + 1 : left_parentheses_index] @@ -56,6 +57,7 @@ def get_line_address(line: str): return line[line.index(ADDRESS_FIND) + len(ADDRESS_FIND) : -1] +# Find the start and end of the function within the ASM file. for i, line in enumerate(original_lines): if first_function_start_line is None and line.startswith(ARM_FUNC_START): first_function_start_line = i @@ -63,7 +65,7 @@ def get_line_address(line: str): function_start_line = i elif line.strip() == f'arm_func_end {function_name}'.strip(): function_end_line = i - + if function_start_line is not None and extract_function_address is None and ADDRESS_FIND in line: extract_function_address = get_line_address(line) if function_end_line is not None and ADDRESS_FIND in line: @@ -93,6 +95,7 @@ def get_line_address(line: str): extract_file_name = f'{file_prefix}{extract_function_address}' +# If needed, add the extracted function's new .o file to main.lsf. merge_prev_file = None merge_next_file = None SRC_LSF_PREFIX = '\tObject src/' @@ -112,12 +115,15 @@ def get_line_address(line: str): if include_new_asm_file: lsf_lines[i] += f'\tObject asm/{file_prefix}{new_file_address}.o\n' break - + BRANCH_LINK_INSTRUCTION = '\tbl ' BRANCH_LINK_EXCHANGE_INSTRUCTION = '\tblx ' BRANCH_INSTRUCTION = '\tb ' WORD_KEY = '.word ' WORD_PLUS_OFFSET = ' + 0x' +""" +Searches through an ASM file's contents for all external symbosl, then populates a .inc file with all the necessary .public definitions. +""" def write_inc_file(lines: List[str], file_path: str): defined_functions = set() used_functions = set() @@ -189,6 +195,9 @@ def write_inc_file(lines: List[str], file_path: str): }}""" +# Add the extracted function to a .h and .c file. +# If there is an existing C file adjacent to the extracted function, add the function to that file. +# Otherwise, make a new set of files. if merge_prev_file: header_file_path = os.path.join(HEADER_FOLDER, f'{merge_prev_file}.h') with open(header_file_path, 'r') as header_file: diff --git a/tools/sync_pmdsky_debug/pmdsky_debug_reader.py b/tools/sync_pmdsky_debug/pmdsky_debug_reader.py index 9ab5f50a..d54ddbd5 100644 --- a/tools/sync_pmdsky_debug/pmdsky_debug_reader.py +++ b/tools/sync_pmdsky_debug/pmdsky_debug_reader.py @@ -21,6 +21,9 @@ 'GAME_STATE_VALUES', ]) +""" +Returns the file path where pmdsky-debug is located locally, defined within pmdsky_debug_location.txt. +""" def get_pmdsky_debug_location() -> str: global pmdsky_debug_path if not pmdsky_debug_path: diff --git a/tools/sync_pmdsky_debug/symbol_details.py b/tools/sync_pmdsky_debug/symbol_details.py index 8bb96a61..b520acd0 100644 --- a/tools/sync_pmdsky_debug/symbol_details.py +++ b/tools/sync_pmdsky_debug/symbol_details.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +# Some symbol names in the decomp do not match pmdsky-debug because of naming convention differences. +# Map these symbol names between the two projects to avoid changes when syncing the projects. MIXED_CASE_SYMBOLS_ARM9 = { '_secure': 'SECURE', '_start_AutoloadDoneCallback': 'StartAutoloadDoneCallback', diff --git a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py index bbb11566..199b0a5e 100644 --- a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py @@ -16,6 +16,9 @@ xmap_symbols = read_xmap_symbols() asm_files = [] +""" +Searches for all files within a directory that have certain extensions. +""" def add_files_with_extensions(folder: str, extensions: List[str]) -> List[str]: found_files = [] for root, _, files in os.walk(folder): @@ -57,6 +60,8 @@ def add_files_with_extensions(folder: str, extensions: List[str]) -> List[str]: print(f'Replacing {old_symbol.name} with {symbol.name}') replaced_symbols.add(old_symbol.name) + + # Replace symbol occurrences in ASM files. if symbol.is_data: asm_search_string_bases = [ f'\n{old_symbol.name}:\n', @@ -100,6 +105,7 @@ def add_files_with_extensions(folder: str, extensions: List[str]) -> List[str]: with open(file_path, 'w') as asm_file: asm_file.write(asm_contents) + # Replace symbol occurrences in C files. src_search_string_data_regex = re.compile(fr'([ &*(]){old_symbol.name}([,); [])') src_search_string_data_regex_replace = fr'\1{symbol.name}\2' diff --git a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py index 38ce2481..7944b6c7 100644 --- a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py @@ -27,6 +27,11 @@ def get_base_symbol_name(symbol_name: str) -> str: return symbol_name[:symbol.name.find('__')] return symbol_name +def read_symbol_array(symbol_path: str, symbol_type_key: str, yaml_manager: YamlManager) -> List[Any]: + symbols_yaml_outer: Dict[str, Any] = yaml_manager.read_yaml(symbol_path) + symbols_yaml: Dict[str, Any] = symbols_yaml_outer[list(symbols_yaml_outer.keys())[0]] + return symbols_yaml[symbol_type_key] + def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_manager: YamlManager): if default_symbol_name.match(symbol.name): return @@ -48,27 +53,6 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma wram_address = address address -= WRAM_OFFSET - if address in pmdsky_debug_section: - # If the address is already defined in pmdsky-debug, replace the old symbol name with the new one in the YAML and header files. - old_symbol = pmdsky_debug_section[address] - base_old_symbol_name = get_base_symbol_name(old_symbol.name) - if base_old_symbol_name != base_symbol_name: - yaml_manager.write_yaml() - print(f'Replacing {base_old_symbol_name} with {base_symbol_name}') - with open(old_symbol.file_path, 'r') as symbols_file: - symbol_contents = symbols_file.read() - symbol_contents = symbol_contents.replace(f'name: {base_old_symbol_name}\n', f'name: {base_symbol_name}\n') - with open(old_symbol.file_path, 'w') as symbol_file: - symbol_file.write(symbol_contents) - - header_path = old_symbol.file_path.replace(SYMBOLS_FOLDER, os.path.join('headers', 'functions')).replace('.yml', '.h') - with open(header_path, 'r') as header_file: - header_contents = header_file.read() - header_contents = header_contents.replace(f' {base_old_symbol_name}(', f' {base_symbol_name}(') - with open(header_path, 'w') as header_file: - header_file.write(header_contents) - return - path_prefix = os.path.join(pmdsky_debug_location, SYMBOLS_FOLDER) if base_symbol_name in symbol_file_paths: symbol_path = symbol_file_paths[base_symbol_name] @@ -85,19 +69,43 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma symbol_path = os.path.join(path_prefix, base_symbol_path) - symbols_yaml_outer: Dict[str, Any] = yaml_manager.read_yaml(symbol_path) - - symbols_yaml: Dict[str, Any] = symbols_yaml_outer[list(symbols_yaml_outer.keys())[0]] - if symbol.is_data: symbol_type_key = 'data' else: symbol_type_key = 'functions' - symbol_array: List[Any] = symbols_yaml[symbol_type_key] + + if address in pmdsky_debug_section: + # If the address is already defined in pmdsky-debug, replace the old symbol name with the new one in the YAML and header files. + old_symbol = pmdsky_debug_section[address] + base_old_symbol_name = get_base_symbol_name(old_symbol.name) + if base_old_symbol_name != base_symbol_name: + print(f'Replacing {base_old_symbol_name} with {base_symbol_name}') + symbol_array = read_symbol_array(symbol_path, symbol_type_key, yaml_manager) + for yaml_symbol in symbol_array: + if yaml_symbol['name'] == base_old_symbol_name: + yaml_symbol['name'] = base_symbol_name + break + + header_path = old_symbol.file_path.replace(SYMBOLS_FOLDER, os.path.join('headers', symbol_type_key)).replace('.yml', '.h') + with open(header_path, 'r') as header_file: + header_contents = header_file.read() + + if symbol.is_data: + # Match data symbols by looking for either the end-of-line semicolon or array start bracket. + header_contents = re.sub(fr' {base_old_symbol_name}([\[;])', fr' {base_symbol_name}\1', header_contents) + else: + # Match function symbols by looking for the open parentheses syntax. + header_contents = header_contents.replace(f' {base_old_symbol_name}(', f' {base_symbol_name}(') + + with open(header_path, 'w') as header_file: + header_file.write(header_contents) + return matching_symbol_entry = None symbol_before = None + symbol_array = read_symbol_array(symbol_path, symbol_type_key, yaml_manager) + # Find the existing symbol and replace its address, or make a new one if it isn't there. symbol_preexisting = False insert_index = None @@ -138,6 +146,7 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma symbol_entry_addresses: int | List[int] = symbol_entry_language_addresses[language_key] + # If needed, reorder language addresses within the YAML for consistency with existing pmdsky-debug entries. hex_address = HexCapsInt(address) reorder_languages = language_key == 'EU' and len(symbol_entry_language_addresses) > 1 and not symbol_entry_language_addresses[language_key] if multiple_symbol_suffix.search(symbol.name): @@ -162,15 +171,18 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma if symbol_preexisting: return + # Add the symbol to the correspond header file. base_symbol_path = base_symbol_path.replace('.yml', '.h') header_path = symbol_path.replace(SYMBOLS_FOLDER, os.path.join('headers', symbol_type_key)).replace('.yml', '.h') with open(header_path, 'r') as header_file: header_contents = header_file.readlines() + # Look for the symbol that was immediately before the new symbol in the YAML. + # The new symbol will be added directly after this anchor symbol. target_line = None if symbol_before is not None: for i, line in enumerate(header_contents): - if symbol.is_data and re.search(fr' {symbol_before}[[;]', line) or not symbol.is_data and f' {symbol_before}(' in line: + if symbol.is_data and re.search(fr' {symbol_before}[\[;]', line) or not symbol.is_data and f' {symbol_before}(' in line: target_line = i break if target_line is None: @@ -222,6 +234,7 @@ def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_ma if f' {base_symbol_name}(' in line: symbol_header = line break + # Match the typedefs used in pmdsky-debug. symbol_header = symbol_header.replace('u32', 'uint32_t') symbol_header = symbol_header.replace('u16', 'uint16_t') symbol_header = symbol_header.replace('u8', 'uint8_t')