From d564ca9d22888568b0d4a0b9776befdb2679d72f Mon Sep 17 00:00:00 2001 From: AnonymousRandomPerson Date: Tue, 21 Nov 2023 19:26:05 -0600 Subject: [PATCH] Supported EU syncing to pmdsky-debug --- asm/include/overlay_01.inc | 2 +- asm/include/overlay_10.inc | 2 +- asm/include/overlay_11.inc | 4 +- asm/include/overlay_29_022E0378.inc | 2 +- asm/include/overlay_29_022EAB50.inc | 2 +- asm/include/overlay_29_022F7388.inc | 2 +- asm/main_0200C4CC.s | 42 +- asm/overlay_01.s | 14 +- asm/overlay_09.s | 1 + asm/overlay_10.s | 12 +- asm/overlay_11.s | 6 +- asm/overlay_29_022E0378.s | 4 +- asm/overlay_29_022EAB50.s | 2 +- asm/overlay_29_022F7388.s | 2 +- tools/sync_pmdsky_debug/requirements.txt | 3 +- tools/sync_pmdsky_debug/symbol_details.py | 1 - .../sync_from_pmdsky_debug.py | 9 +- .../sync_pmdsky_debug/sync_to_pmdsky_debug.py | 429 ++++++++++-------- tools/sync_pmdsky_debug/xmap_reader.py | 4 +- tools/sync_pmdsky_debug/yaml_writer.py | 82 ++++ 20 files changed, 376 insertions(+), 249 deletions(-) create mode 100644 tools/sync_pmdsky_debug/yaml_writer.py diff --git a/asm/include/overlay_01.inc b/asm/include/overlay_01.inc index 69b12b5f..1576da79 100644 --- a/asm/include/overlay_01.inc +++ b/asm/include/overlay_01.inc @@ -63,7 +63,7 @@ .public GetTeamMember .public GetVictoriesOnOneFloor .public InitAnimationControl -.public InitAnimationControlWithSet__0201C14C +.public InitAnimationControlWithSet__0201C0B0 .public InitPreprocessorArgs .public IsAdvancedMenuActive .public IsAdventureLogNotEmpty diff --git a/asm/include/overlay_10.inc b/asm/include/overlay_10.inc index 466a0317..f61b567a 100644 --- a/asm/include/overlay_10.inc +++ b/asm/include/overlay_10.inc @@ -175,7 +175,7 @@ .public HandleSir0Translation .public HidePortraitBox .public IncrementNbFriendRescues -.public InitAnimationControlWithSet__0201C14C +.public InitAnimationControlWithSet__0201C0B0 .public InitEventFlagScriptVars .public InitPortraitBox .public InitPortraitBoxWithMonsterId diff --git a/asm/include/overlay_11.inc b/asm/include/overlay_11.inc index cd586622..e3d47dd6 100644 --- a/asm/include/overlay_11.inc +++ b/asm/include/overlay_11.inc @@ -122,7 +122,7 @@ .public IncrementNbBigTreasureWins .public IncrementNbDungeonsCleared .public IncrementNbSkyGiftsSent -.public InitAnimationControlWithSet__0201C14C +.public InitAnimationControlWithSet__0201C0B0 .public InitBulkItem .public InitMainTeamAfterQuiz .public InitMove @@ -306,7 +306,6 @@ .public SaveScriptVariableValue .public SaveScriptVariableValueAtIndex .public SaveScriptVariableValueBytes -.public SendAudioCommandWrapperVeneer .public SentryRunState .public SetActorTalkMain .public SetActorTalkMainAndActorTalkSub @@ -443,6 +442,7 @@ .public sub_02017AB0 .public sub_02017ACC .public sub_02017B18 +.public sub_02017B64 .public sub_02017B70 .public sub_02017B7C .public sub_02017B88 diff --git a/asm/include/overlay_29_022E0378.inc b/asm/include/overlay_29_022E0378.inc index d1f7bc91..0ad12e79 100644 --- a/asm/include/overlay_29_022E0378.inc +++ b/asm/include/overlay_29_022E0378.inc @@ -22,7 +22,7 @@ .public GetTileSafe .public HandleFaint .public HandleSir0Translation -.public InitAnimationControlWithSet__0201C168 +.public InitAnimationControlWithSet__0201C0CC .public IsCurrentTilesetBackground .public IsExperienceLocked .public IsFloorOver diff --git a/asm/include/overlay_29_022EAB50.inc b/asm/include/overlay_29_022EAB50.inc index 4f986dd8..c4c1cb81 100644 --- a/asm/include/overlay_29_022EAB50.inc +++ b/asm/include/overlay_29_022EAB50.inc @@ -61,7 +61,6 @@ .public RemoveItem .public RoundUpDiv256 .public RunMonsterAi -.public SendAudioCommandWrapperVeneer .public SetItemAcquired .public SetLeaderAction .public ShouldDisplayEntityWrapper @@ -72,6 +71,7 @@ .public sub_0200D208 .public sub_02017ACC .public sub_02017B18 +.public sub_02017B64 .public sub_02017B70 .public sub_02017B7C .public sub_02017C50 diff --git a/asm/include/overlay_29_022F7388.inc b/asm/include/overlay_29_022F7388.inc index 0653a3ed..092a1fd7 100644 --- a/asm/include/overlay_29_022F7388.inc +++ b/asm/include/overlay_29_022F7388.inc @@ -92,7 +92,7 @@ .public GroundToDungeonMoveset .public HasHeldItem .public HasMaxGinsengBoost99 -.public InitAnimationControlWithSet__0201C168 +.public InitAnimationControlWithSet__0201C0CC .public InitMoveCheckId .public InitTeamMember .public IQ_SKILLS diff --git a/asm/main_0200C4CC.s b/asm/main_0200C4CC.s index 16a751bf..e9edf631 100644 --- a/asm/main_0200C4CC.s +++ b/asm/main_0200C4CC.s @@ -13850,7 +13850,7 @@ sub_02016EAC: ; 0x02016EAC ldr r0, _0201712C ; =_020AF760 ldr r0, [r0, #8] add r0, r0, #0x98 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _0201712C ; =_020AF760 ldr r1, [r0, #8] add r0, r1, #0x98 @@ -13889,7 +13889,7 @@ _02016FB0: mov r0, #0x7c mla r8, sb, r0, r6 mov r0, r8 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x94] mov r0, r8 bl SetSpriteIdForAnimationControl @@ -13931,7 +13931,7 @@ _02017054: mov r0, #0x7c mla r8, sb, r0, r6 mov r0, r8 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x96] mov r0, r8 bl SetSpriteIdForAnimationControl @@ -14849,13 +14849,13 @@ PlayBgmByIdVeneer: ; 0x02017B58 _02017B60: .word PlayBgmById arm_func_end PlayBgmByIdVeneer - arm_func_start SendAudioCommandWrapperVeneer -SendAudioCommandWrapperVeneer: ; 0x02017B64 - ldr ip, _02017B6C ; =SendAudioCommandWrapper + arm_func_start sub_02017B64 +sub_02017B64: ; 0x02017B64 + ldr ip, _02017B6C ; =sub_02017E70 bx ip .align 2, 0 -_02017B6C: .word SendAudioCommandWrapper - arm_func_end SendAudioCommandWrapperVeneer +_02017B6C: .word sub_02017E70 + arm_func_end sub_02017B64 arm_func_start sub_02017B70 sub_02017B70: ; 0x02017B70 @@ -15188,8 +15188,8 @@ _02017E68: .word 0x000003E7 _02017E6C: .word _022A4BEC arm_func_end PlayBgmById - arm_func_start SendAudioCommandWrapper -SendAudioCommandWrapper: ; 0x02017E70 + arm_func_start sub_02017E70 +sub_02017E70: ; 0x02017E70 stmdb sp!, {r3, r4, r5, r6, r7, lr} ldr r3, _02017EE0 ; =0x000003E7 mov r6, r0 @@ -15221,7 +15221,7 @@ SendAudioCommandWrapper: ; 0x02017E70 .align 2, 0 _02017EE0: .word 0x000003E7 _02017EE4: .word _022A4BEC - arm_func_end SendAudioCommandWrapper + arm_func_end sub_02017E70 arm_func_start StopBgmCommand StopBgmCommand: ; 0x02017EE8 @@ -20245,8 +20245,8 @@ InitAnimationControl: ; 0x0201C050 ldmia sp!, {r4, pc} arm_func_end InitAnimationControl - arm_func_start InitAnimationControlWithSet__0201C14C -InitAnimationControlWithSet__0201C14C: ; 0x0201C0B0 + arm_func_start InitAnimationControlWithSet__0201C0B0 +InitAnimationControlWithSet__0201C0B0: ; 0x0201C0B0 stmdb sp!, {r4, lr} mov r4, r0 bl InitAnimationControl @@ -20254,10 +20254,10 @@ InitAnimationControlWithSet__0201C14C: ; 0x0201C0B0 orr r0, r0, #1 strh r0, [r4, #2] ldmia sp!, {r4, pc} - arm_func_end InitAnimationControlWithSet__0201C14C + arm_func_end InitAnimationControlWithSet__0201C0B0 - arm_func_start InitAnimationControlWithSet__0201C168 -InitAnimationControlWithSet__0201C168: ; 0x0201C0CC + arm_func_start InitAnimationControlWithSet__0201C0CC +InitAnimationControlWithSet__0201C0CC: ; 0x0201C0CC stmdb sp!, {r4, lr} mov r4, r0 bl InitAnimationControl @@ -20265,7 +20265,7 @@ InitAnimationControlWithSet__0201C168: ; 0x0201C0CC orr r0, r0, #1 strh r0, [r4, #2] ldmia sp!, {r4, pc} - arm_func_end InitAnimationControlWithSet__0201C168 + arm_func_end InitAnimationControlWithSet__0201C0CC arm_func_start SetSpriteIdForAnimationControl SetSpriteIdForAnimationControl: ; 0x0201C0E8 @@ -37230,7 +37230,7 @@ LoadCursors: ; 0x0202950C mov r2, #0xe bl sub_0201D9C8 ldr r0, _02029668 ; =_022AAC80 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _02029668 ; =_022AAC80 ldr r1, _02029660 ; =_022AAC64 ldrsh r1, [r1, #2] @@ -37252,7 +37252,7 @@ LoadCursors: ; 0x0202950C strh r1, [r0, #0x38] bl sub_0201C108 ldr r0, _0202966C ; =_022AACFC - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _0202966C ; =_022AACFC ldr r1, _02029660 ; =_022AAC64 ldrsh r1, [r1] @@ -37922,7 +37922,7 @@ LoadAlert: ; 0x02029EE8 ldr r1, _02029F7C ; =_022AAD78 strh r0, [r1] ldr r0, _02029F80 ; =_022AAD7C - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _02029F7C ; =_022AAD78 ldr r0, _02029F80 ; =_022AAD7C ldrsh r1, [r1] @@ -117777,7 +117777,7 @@ _0206B328: mov r0, #0x7c mla r6, r5, r0, fp mov r0, r6 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r4, #0x18] mov r0, r6 bl SetSpriteIdForAnimationControl diff --git a/asm/overlay_01.s b/asm/overlay_01.s index cf9fde25..2b0da101 100644 --- a/asm/overlay_01.s +++ b/asm/overlay_01.s @@ -3685,7 +3685,7 @@ ov01_0232C148: ; 0x0232C148 ldr r0, _0232C1CC ; =ov01_0233C800 mov r4, r1 mov r6, r2 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _0232C1D0 ; =ov01_0233C780 ldr r0, _0232C1CC ; =ov01_0233C800 ldrsh r1, [r1] @@ -3873,7 +3873,7 @@ ov01_0232C3AC: ; 0x0232C3AC ldr r0, _0232C448 ; =ov01_0233C8FC mov r4, r1 mov r6, r3 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _0232C44C ; =ov01_0233C87C ldr r0, _0232C448 ; =ov01_0233C8FC ldrsh r1, [r1] @@ -8589,7 +8589,7 @@ ov01_02330224: ; 0x02330224 strh r1, [r2, #0x8e] ldr r0, [r0] add r0, r0, #0x94 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _02330584 ; =ov01_0233C164 ldr r2, [r0] ldrh r1, [r2, #0x96] @@ -8628,7 +8628,7 @@ ov01_02330224: ; 0x02330224 strh r2, [r1, #0xcc] ldr r0, [r0] add r0, r0, #0x110 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _02330584 ; =ov01_0233C164 ldr r0, [r1] add r0, r0, #0x100 @@ -8672,7 +8672,7 @@ ov01_02330224: ; 0x02330224 strh r2, [r0, #0x48] ldr r0, [r1] add r0, r0, #0x18c - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _02330584 ; =ov01_0233C164 ldr r0, [r1] add r0, r0, #0x100 @@ -8715,7 +8715,7 @@ ov01_02330224: ; 0x02330224 strh r2, [r0, #0xc4] ldr r0, [r1] add r0, r0, #0x208 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _02330584 ; =ov01_0233C164 ldr r0, [r0] ldrsh r1, [r0, #0x92] @@ -9477,7 +9477,7 @@ _02330F18: moveq r0, #0 beq _02330FDC ldr r0, _02330FF8 ; =ov01_0233C978 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r1, _02330FE4 ; =ov01_0233C174 ldr r0, _02330FF8 ; =ov01_0233C978 ldrsh r1, [r1, #2] diff --git a/asm/overlay_09.s b/asm/overlay_09.s index 1a05f314..f6e7e78a 100644 --- a/asm/overlay_09.s +++ b/asm/overlay_09.s @@ -3096,6 +3096,7 @@ _0233F548: _0233F564: add sp, sp, #0x2c ldmia sp!, {r3, r4, r5, r6, pc} + arm_func_end UpdateInputLockBox ; 0x0233F56C .global ov09_0233F56C diff --git a/asm/overlay_10.s b/asm/overlay_10.s index 1d4f9765..b28c0a61 100644 --- a/asm/overlay_10.s +++ b/asm/overlay_10.s @@ -1807,7 +1807,7 @@ _022BE200: ; jump table b _022BE418 ; case 6 _022BE21C: add r0, r7, #0x68 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x64] add r0, r7, #0x68 bl SetSpriteIdForAnimationControl @@ -1833,7 +1833,7 @@ _022BE21C: b _022BE440 _022BE280: add r0, r7, #0x68 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x64] add r0, r7, #0x68 bl SetSpriteIdForAnimationControl @@ -1866,7 +1866,7 @@ _022BE280: b _022BE440 _022BE300: add r0, r7, #0x68 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x64] add r0, r7, #0x68 bl SetSpriteIdForAnimationControl @@ -1894,7 +1894,7 @@ _022BE300: b _022BE440 _022BE36C: add r0, r7, #0x68 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldrsh r1, [r7, #0x64] add r0, r7, #0x68 bl SetSpriteIdForAnimationControl @@ -4813,7 +4813,7 @@ _022C08D4: ldr r0, [r4] add r0, r0, #0x88 mla r0, r6, sb, r0 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 add r0, r6, #1 ldr r1, [r4] mov r0, r0, lsl #0x10 @@ -4918,7 +4918,7 @@ ov10_022C09E8: ; 0x022C09E8 strh r0, [r2, #4] ldr r0, [r1] add r0, r0, #8 - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 ldr r0, _022C0CC0 ; =ov10_022DC1D0 ldr r0, [r0] ldrsh r1, [r0, #4] diff --git a/asm/overlay_11.s b/asm/overlay_11.s index dab2b3ad..dba5eb78 100644 --- a/asm/overlay_11.s +++ b/asm/overlay_11.s @@ -6761,7 +6761,7 @@ _022E20D0: mov r2, r3, lsl #0x10 mov r1, r1, lsr #0x10 mov r2, r2, lsr #0x10 - bl SendAudioCommandWrapperVeneer + bl sub_02017B64 b _022E2464 _022E2120: bl sub_02017B70 @@ -29166,7 +29166,7 @@ _022F4A38: mov r1, #0xff add r0, r4, #0xc strb r1, [r4, #0xa4] - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 mov r0, #0 strb r0, [r4, #0x8a] strh r0, [r4, #0x8c] @@ -29211,7 +29211,7 @@ ov11_022F4AB8: ; 0x022F4AB8 strh r1, [r6, #0xb2] add r0, r6, #0xc strh r1, [r6, #0xb4] - bl InitAnimationControlWithSet__0201C14C + bl InitAnimationControlWithSet__0201C0B0 mov r1, #0 strh r1, [r6, #0x8c] strh r1, [r6, #0x8e] diff --git a/asm/overlay_29_022E0378.s b/asm/overlay_29_022E0378.s index 78a1f909..5615df30 100644 --- a/asm/overlay_29_022E0378.s +++ b/asm/overlay_29_022E0378.s @@ -1522,7 +1522,7 @@ _022E1658: add r0, r0, #0x12000 ldr r0, [r0, #0xb28] add r0, r0, #0x2c - bl InitAnimationControlWithSet__0201C168 + bl InitAnimationControlWithSet__0201C0CC ldr r0, [r4] add r0, r0, r6, lsl #2 add r0, r0, #0x12000 @@ -1560,7 +1560,7 @@ _022E16E8: add r0, r0, #0x12000 ldr r0, [r0, #0xb38] add r0, r0, #0x2c - bl InitAnimationControlWithSet__0201C168 + bl InitAnimationControlWithSet__0201C0CC ldr r0, [r4] add r0, r0, r7, lsl #2 add r0, r0, #0x12000 diff --git a/asm/overlay_29_022EAB50.s b/asm/overlay_29_022EAB50.s index 03fa1380..87801ada 100644 --- a/asm/overlay_29_022EAB50.s +++ b/asm/overlay_29_022EAB50.s @@ -474,7 +474,7 @@ _022EB0D4: ldrh r1, [r4, #8] mov r0, r0, lsr #0x10 mov r2, #0x100 - bl SendAudioCommandWrapperVeneer + bl sub_02017B64 _022EB0EC: ldrh r1, [r4, #0xa] mov r0, #4 diff --git a/asm/overlay_29_022F7388.s b/asm/overlay_29_022F7388.s index 6ca9ba8f..319e6b16 100644 --- a/asm/overlay_29_022F7388.s +++ b/asm/overlay_29_022F7388.s @@ -5537,7 +5537,7 @@ _022FBB38: add r1, r2, r1, lsl #1 ldrsh r1, [r1, sl] strh r1, [r8, #0xa8] - bl InitAnimationControlWithSet__0201C168 + bl InitAnimationControlWithSet__0201C0CC ldrsh r1, [r8, #0xa8] add r0, r8, #0x2c bl SetSpriteIdForAnimationControl diff --git a/tools/sync_pmdsky_debug/requirements.txt b/tools/sync_pmdsky_debug/requirements.txt index b369901a..6d150f40 100644 --- a/tools/sync_pmdsky_debug/requirements.txt +++ b/tools/sync_pmdsky_debug/requirements.txt @@ -1,2 +1,3 @@ -PyYAML==6.0.1 requests==2.31.0 +ruamel.yaml==0.18.5 +ruamel.yaml.clib==0.2.8 diff --git a/tools/sync_pmdsky_debug/symbol_details.py b/tools/sync_pmdsky_debug/symbol_details.py index cb26d675..c591ba76 100644 --- a/tools/sync_pmdsky_debug/symbol_details.py +++ b/tools/sync_pmdsky_debug/symbol_details.py @@ -2,7 +2,6 @@ MIXED_CASE_SYMBOLS = { '_secure': 'SECURE', - '_start': 'Start', '_start_AutoloadDoneCallback': 'StartAutoloadDoneCallback', '_start_ModuleParams': 'START_MODULE_PARAMS', 'MIi_UncompressBackward': 'MIiUncompressBackward', diff --git a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py index 975723c3..1eb2eb1e 100644 --- a/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_from_pmdsky_debug.py @@ -1,8 +1,9 @@ +import re +from typing import List + from pmdsky_debug_reader import * from symbol_details import * from xmap_reader import * -from typing import List -import re # Syncs symbols from pmdsky-debug (https://github.com/UsernameFodder/pmdsky-debug) to the decomp. # To use this script, you will need: @@ -40,8 +41,8 @@ def add_files_with_extensions(folder: str, extensions: List[str]) -> List[str]: xmap_section = {} for address, symbol in pmdsky_debug_section.items(): - if section_name == 'arm7' and address > 0x2380000: - # TODO Figure out exact boundary of WRAM. + if section_name == 'arm7' and address < 0x27E0000: + # Shift ARM 7 WRAM to its RAM location. address += 0x1477E18 if address in xmap_section and xmap_section[address].name != symbol.name and xmap_section[address].name not in MIXED_CASE_SYMBOLS and xmap_section[address].name not in replaced_symbols: old_symbol = xmap_section[address] diff --git a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py index e5bac696..84edefe4 100644 --- a/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py +++ b/tools/sync_pmdsky_debug/sync_to_pmdsky_debug.py @@ -1,7 +1,11 @@ +import re +from typing import List +from ruamel.yaml.scalarint import HexCapsInt + from pmdsky_debug_reader import * from symbol_details import * from xmap_reader import * -import re +from yaml_writer import * # Syncs symbols from the decomp to a local clone of pmdsky-debug (https://github.com/UsernameFodder/pmdsky-debug). # To use this script, you will need: @@ -13,202 +17,241 @@ xmap_symbols = read_xmap_symbols() pmdsky_debug_location = get_pmdsky_debug_location() -default_symbol_name = re.compile(r'^(?:ov\d{2}|sub)?_[\dA-F]{8}$') -multiple_symbol_suffix = re.compile(r'__[\dA-F]{8}$') +default_symbol_name = re.compile(r'^(?:ov\d{2}|sub)?_[\dA-F]{8}(?:_[\w]{2})?$') +multiple_symbol_suffix = re.compile(r'__[\dA-F]{8}(?:_[\w]{2})?$') + +LANGUAGE_KEYS = { + 'us': 'NA', + 'eu': 'EU', +} + +INDENT_BLANK = '' + +def get_base_symbol_name(symbol_name: str) -> str: + if multiple_symbol_suffix.search(symbol_name): + return symbol_name[:symbol.name.find('__')] + return symbol_name + +def sync_xmap_symbol(address: int, symbol: SymbolDetails, language: str, yaml_manager: YamlManager): + if default_symbol_name.match(symbol.name): + return -for section_name, xmap_section in xmap_symbols.items(): - if section_name in pmdsky_debug_symbols: - pmdsky_debug_section = pmdsky_debug_symbols[section_name] + language_key = LANGUAGE_KEYS[language] + + if symbol.name in MIXED_CASE_SYMBOLS: + symbol.name = MIXED_CASE_SYMBOLS[symbol.name] + + base_symbol_name = get_base_symbol_name(symbol.name) + + wram_address = None + if section_name == 'arm7' and address >= 0x37F8000: + # Shift ARM 7 WRAM to its ROM location. + wram_address = address + address -= 0x1477E18 + + if address in pmdsky_debug_section: + # If the address is already defined in pmdsky-debug, replace the old symbol name with the new one in the YAML and header files. + old_symbol = pmdsky_debug_section[address] + base_old_symbol_name = get_base_symbol_name(old_symbol.name) + if base_old_symbol_name != base_symbol_name: + yaml_manager.write_yaml() + print(f'Replacing {base_old_symbol_name} with {base_symbol_name}') + with open(old_symbol.file_path, 'r') as symbols_file: + symbol_contents = symbols_file.read() + symbol_contents = symbol_contents.replace(f'name: {base_old_symbol_name}\n', f'name: {base_symbol_name}\n') + with open(old_symbol.file_path, 'w') as symbol_file: + symbol_file.write(symbol_contents) + + header_path = old_symbol.file_path.replace(SYMBOLS_FOLDER, os.path.join('headers', 'functions')).replace('.yml', '.h') + with open(header_path, 'r') as header_file: + header_contents = header_file.read() + header_contents = header_contents.replace(f' {base_old_symbol_name}(', f' {base_symbol_name}(') + with open(header_path, 'w') as header_file: + header_file.write(header_contents) + return + + path_prefix = os.path.join(pmdsky_debug_location, SYMBOLS_FOLDER) + if base_symbol_name in symbol_file_paths: + symbol_path = symbol_file_paths[base_symbol_name] + base_symbol_path = symbol_path[len(path_prefix) + 1:] else: - pmdsky_debug_section = {} - - for address, symbol in xmap_section.items(): - if default_symbol_name.match(symbol.name): - continue - - if symbol.name in MIXED_CASE_SYMBOLS: - symbol.name = MIXED_CASE_SYMBOLS[symbol.name] - - if address in pmdsky_debug_section: - old_symbol = pmdsky_debug_section[address] - if pmdsky_debug_section[address].name != symbol.name: - print(f'Replacing {old_symbol.name} with {symbol.name}') - with open(old_symbol.file_path, 'r') as symbol_file: - symbol_contents = symbol_file.read() - symbol_contents = symbol_contents.replace(f'name: {old_symbol.name}\n', f'name: {symbol.name}\n') - with open(old_symbol.file_path, 'w') as symbol_file: - symbol_file.write(symbol_contents) - - header_path = old_symbol.file_path.replace(SYMBOLS_FOLDER, os.path.join('headers', 'functions')).replace('.yml', '.h') - with open(header_path, 'r') as header_file: - header_contents = header_file.read() - header_contents = header_contents.replace(f' {old_symbol.name}(', f' {symbol.name}(') - with open(header_path, 'w') as header_file: - header_file.write(header_contents) + if section_name == 'main': + base_symbol_path = 'arm9.yml' + elif section_name == 'arm7': + base_symbol_path = 'arm7.yml' + elif section_name == 'ITCM': + base_symbol_path = os.path.join('arm9', 'itcm.yml') else: - if section_name == 'main': - symbol_path = 'arm9.yml' - elif section_name == 'ITCM': - symbol_path = os.path.join('arm9', 'itcm.yml') - else: - symbol_path = f'overlay{int(section_name):02d}.yml' - print(f'Adding {symbol.name} to {symbol_path}') - - symbol_path = os.path.join(pmdsky_debug_location, SYMBOLS_FOLDER, symbol_path) - with open(symbol_path, 'r') as symbol_file: - symbol_contents = symbol_file.readlines() - - if multiple_symbol_suffix.search(symbol.name): - base_symbol_name = symbol.name[:symbol.name.find('__')] - found_base_symbol = False - found_base_symbol_na = False - write_multisymbol_address = False - for i, line in enumerate(symbol_contents): - if not found_base_symbol: - found_base_symbol = line == f' - name: {base_symbol_name}\n' - elif found_base_symbol and not found_base_symbol_na: - found_base_symbol_na = line == ' NA:\n' - elif found_base_symbol_na: - if line.startswith(' - '): - if int(line[-11:-1], 16) > address: - write_multisymbol_address = True - else: - write_multisymbol_address = True - if write_multisymbol_address: - symbol_contents[i - 1] += f' - 0x{address:X}\n' - break - - if write_multisymbol_address: - with open(symbol_path, 'w') as symbol_file: - symbol_file.writelines(symbol_contents) - continue - else: - symbol.name = base_symbol_name - - symbol_length = 0 - string_length = None - if symbol.is_data: - symbols_start = ' data:\n' - - asm_path = os.path.join('asm', symbol.file_path.replace('.o', '.s')) - if os.path.exists(asm_path): - with open(asm_path) as asm_file: - asm_contents = asm_file.readlines() - for i, line in enumerate(asm_contents): - if line.startswith(f'\t.global {symbol.name}'): - target_line = asm_contents[i + 2] - string_index = target_line.find('.string "') - if string_index >= 0: - target_string = target_line[string_index + len('.string "'):-2].replace('\\n', 'n') - string_length = len(target_string) - symbol_length = string_length + 1 - if symbol_length % 4 > 0: - symbol_length += 4 - symbol_length % 4 - break + base_symbol_path = f'overlay{int(section_name):02d}.yml' - else: - symbols_start = ' functions:\n' - - found_symbols = False - symbol_before = None - write_end_list = False - for i, line in enumerate(symbol_contents): - if found_symbols: - write_new_symbol = False - if line.startswith(' - name:'): - current_symbol_index = i - elif line.startswith(' NA:'): - if line.endswith('NA:\n'): - address_line = symbol_contents[i + 1] - else: - address_line = line - address_line_string = address_line[-10 : -1] - if ':' not in address_line_string: - current_symbol_address = int(address_line_string, 16) - write_new_symbol = address < current_symbol_address - elif symbol.is_data and i >= len(symbol_contents) - 2 or not symbol.is_data and line == ' data:\n': - write_new_symbol = True - write_end_list = True - current_symbol_index = i - - if write_new_symbol: - if write_end_list: - symbol_before = None - else: - symbol_before = symbol_contents[current_symbol_index][len(' - name: ') : -1] - if symbol.is_data: - symbol_contents[current_symbol_index - 1] += f""" - name: {symbol.name} - address: - NA: 0x{address:X} - length: - NA: 0x{symbol_length:X} -""" - else: - symbol_contents[current_symbol_index - 1] += f""" - name: {symbol.name} - address: - NA: 0x{address:X} -""" - break - - elif line == symbols_start: - found_symbols = True - current_symbol_index = i - - with open(symbol_path, 'w') as symbol_file: - symbol_file.writelines(symbol_contents) - - - if symbol.is_data: - header_file_name = 'data' - else: - header_file_name = 'functions' + symbol_path = os.path.join(path_prefix, base_symbol_path) - header_path = symbol_path.replace(SYMBOLS_FOLDER, os.path.join('headers', header_file_name)).replace('.yml', '.h') - with open(header_path, 'r') as header_file: - header_contents = header_file.readlines() - - target_line = None - if symbol_before is not None: - for i, line in enumerate(header_contents): - if symbol.is_data and re.search(fr' {symbol_before}[[;]', line) or symbol.is_data and f' {symbol_before}(' in line: - target_line = i - break - if target_line is None: - print(f'Could not find preceding symbol {symbol_before} to {symbol.name} in {header_path}') - continue - - if target_line is None: - if 'arm9' in header_path: - for i, line in enumerate(header_contents): - if line.startswith('// If declaring'): - target_line = i - break - else: - target_line = len(header_contents) - 2 - - symbol_header_path = os.path.join(HEADER_FOLDER, symbol.file_path.replace('.o', '.h')) - if symbol.is_data: - if string_length is not None: - symbol_header = f'extern char {symbol.name}[{string_length}];\n' - else: - symbol_header = f'extern undefined {symbol.name};\n' - elif os.path.exists(symbol_header_path): - with open(symbol_header_path, 'r') as symbol_header_file: - symbol_header_contents = symbol_header_file.readlines() - for line in symbol_header_contents: - if f' {symbol.name}(' in line: - symbol_header = line - break - symbol_header = symbol_header.replace('u32', 'uint32_t') - symbol_header = symbol_header.replace('u16', 'uint16_t') - symbol_header = symbol_header.replace('u8', 'uint8_t') - symbol_header = symbol_header.replace('s32', 'int32_t') - symbol_header = symbol_header.replace('s16', 'int16_t') - symbol_header = symbol_header.replace('s8', 'int8_t') - else: - symbol_header = f'void {symbol.name}(void);\n' + symbols_yaml_outer: Dict[str, Any] = yaml_manager.read_yaml(symbol_path) - header_contents[target_line - 1] += symbol_header + symbols_yaml: Dict[str, Any] = symbols_yaml_outer[list(symbols_yaml_outer.keys())[0]] - with open(header_path, 'w') as header_file: - header_file.writelines(header_contents) + if symbol.is_data: + symbol_type_key = 'data' + else: + symbol_type_key = 'functions' + symbol_array: List[Any] = symbols_yaml[symbol_type_key] + + matching_symbol_entry = None + symbol_before = None + + # Find the existing symbol and replace its address, or make a new one if it isn't there. + symbol_preexisting = False + insert_index = None + for i, symbol_entry in enumerate(symbol_array): + if base_symbol_name == symbol_entry['name']: + matching_symbol_entry = symbol_entry + symbol_preexisting = True + break + else: + # Keep track of the symbol directly before the target symbol. + # This will be used later as an anchor when appending to the header file. + symbol_before: str = symbol_entry['name'] + if language_key in symbol_entry['address']: + current_symbol_address: int | List[int] = symbol_entry['address'][language_key] + if isinstance(current_symbol_address, list): + current_symbol_address = current_symbol_address[0] + if current_symbol_address > address: + insert_index = i + break + if not matching_symbol_entry: + matching_symbol_entry = { + 'name': base_symbol_name, + 'address': {} + } + if insert_index is None: + symbol_array.append(matching_symbol_entry) + else: + symbol_array.insert(insert_index, matching_symbol_entry) + + if symbol_preexisting: + print(f'Updating address of {base_symbol_name} in {base_symbol_path}') + else: + print(f'Adding {base_symbol_name} to {base_symbol_path}') + + symbol_entry_language_addresses: Dict[str, Any] = matching_symbol_entry['address'] + if language_key not in symbol_entry_language_addresses: + symbol_entry_language_addresses[language_key] = None + symbol_entry_addresses: int | List[int] = symbol_entry_language_addresses[language_key] + + + hex_address = HexCapsInt(address) + if multiple_symbol_suffix.search(symbol.name): + if symbol_entry_addresses is None: + symbol_entry_language_addresses[language_key] = [hex_address] + else: + if address not in symbol_entry_addresses: + symbol_entry_addresses.append(hex_address) + return + else: + reorder_languages = language_key == 'EU' and len(symbol_entry_language_addresses) > 1 and not symbol_entry_language_addresses[language_key] + symbol_entry_language_addresses[language_key] = HexCapsInt(hex_address) + if reorder_languages: + symbol_entry_language_addresses.move_to_end(language_key, last=False) + if 'length' in matching_symbol_entry and 'NA' in matching_symbol_entry['length']: + matching_symbol_entry['length'][language_key] = matching_symbol_entry['length']['NA'] + matching_symbol_entry['length'].move_to_end(language_key, last=False) + + if wram_address is not None: + symbol_entry_language_addresses[language_key + '-WRAM'] = HexCapsInt(wram_address) + if reorder_languages: + symbol_entry_language_addresses.move_to_end('NA-WRAM') + + if symbol_preexisting: + return + + base_symbol_path = base_symbol_path.replace('.yml', '.h') + header_path = symbol_path.replace(SYMBOLS_FOLDER, os.path.join('headers', symbol_type_key)).replace('.yml', '.h') + with open(header_path, 'r') as header_file: + header_contents = header_file.readlines() + + target_line = None + if symbol_before is not None: + for i, line in enumerate(header_contents): + if symbol.is_data and re.search(fr' {symbol_before}[[;]', line) or symbol.is_data and f' {symbol_before}(' in line: + target_line = i + break + if target_line is None: + print(f'Could not find preceding symbol {symbol_before} to {base_symbol_name} in {base_symbol_path}') + return + + if target_line is None: + if 'arm9' in header_path: + for i, line in enumerate(header_contents): + if line.startswith('// If declaring'): + target_line = i + break + else: + target_line = len(header_contents) - 2 + + # If the symbol is a data symbol, look through the ASM to find how much space the symbol takes. + symbol_length = 0 + string_length = None + if symbol.is_data: + asm_path = os.path.join('asm', symbol.file_path.replace('.o', '.s')) + if os.path.exists(asm_path): + with open(asm_path) as asm_file: + asm_contents = asm_file.readlines() + for i, line in enumerate(asm_contents): + if line.startswith(f'\t.global {base_symbol_name}'): + target_asm_line = asm_contents[i + 2] + string_index = target_asm_line.find('.string "') + if string_index >= 0: + target_string = target_asm_line[string_index + len('.string "'):-2].replace('\\n', 'n') + string_length = len(target_string) + symbol_length = string_length + 1 + if symbol_length % 4 > 0: + symbol_length += 4 - symbol_length % 4 + break + + # Write the new symbol within the header file. + symbol_header_path = os.path.join(HEADER_FOLDER, symbol.file_path.replace('.o', '.h')) + if symbol.is_data: + if string_length is not None: + symbol_header = f'extern char {base_symbol_name}[{string_length}];\n' + else: + symbol_header = f'extern undefined {base_symbol_name};\n' + elif os.path.exists(symbol_header_path): + with open(symbol_header_path, 'r') as symbol_header_file: + symbol_header_contents = symbol_header_file.readlines() + for line in symbol_header_contents: + if f' {base_symbol_name}(' in line: + symbol_header = line + break + symbol_header = symbol_header.replace('u32', 'uint32_t') + symbol_header = symbol_header.replace('u16', 'uint16_t') + symbol_header = symbol_header.replace('u8', 'uint8_t') + symbol_header = symbol_header.replace('s32', 'int32_t') + symbol_header = symbol_header.replace('s16', 'int16_t') + symbol_header = symbol_header.replace('s8', 'int8_t') + else: + symbol_header = f'void {base_symbol_name}(void);\n' + + header_contents[target_line - 1] += symbol_header + + with open(header_path, 'w') as header_file: + header_file.writelines(header_contents) + +# Extract all pmdsky-debug file paths for symbols between all languages. +symbol_file_paths: Dict[str, str] = {} +for language, pmdsky_debug_language_symbols in pmdsky_debug_symbols.items(): + for section_name, pmdsky_debug_section in pmdsky_debug_language_symbols.items(): + for address, symbol in pmdsky_debug_section.items(): + base_symbol_name = get_base_symbol_name(symbol.name) + symbol_file_paths[base_symbol_name] = symbol.file_path + +with YamlManager() as yaml_manager: + for language, xmap_language_symbols in xmap_symbols.items(): + pmdsky_debug_language_symbols = pmdsky_debug_symbols[language] + for section_name, xmap_section in xmap_language_symbols.items(): + if section_name in pmdsky_debug_language_symbols: + pmdsky_debug_section = pmdsky_debug_language_symbols[section_name] + else: + pmdsky_debug_section = {} + + for address, symbol in xmap_section.items(): + sync_xmap_symbol(address, symbol, language, yaml_manager) diff --git a/tools/sync_pmdsky_debug/xmap_reader.py b/tools/sync_pmdsky_debug/xmap_reader.py index bf2dfd88..7a477054 100644 --- a/tools/sync_pmdsky_debug/xmap_reader.py +++ b/tools/sync_pmdsky_debug/xmap_reader.py @@ -39,11 +39,11 @@ def read_xmap_symbols_for_language(language: str) -> Dict[str, Dict[int, SymbolD SECTION_START = '# .' if os.path.exists(xmap_path): - print('Using local xMAP file.') + print(f'Using local xMAP file for {language}.') with open(xmap_path, 'r') as xmap_file: xmap_lines = xmap_file.readlines() else: - print('No local xMAP file found. Using remote xMAP file from upstream repo.') + print(f'No local xMAP file found for {language}. Using remote xMAP file from upstream repo.') remote_xmap_response = requests.get(remote_xmap_url) if remote_xmap_response.status_code == 200: xmap_lines = remote_xmap_response.text.split('\n') diff --git a/tools/sync_pmdsky_debug/yaml_writer.py b/tools/sync_pmdsky_debug/yaml_writer.py new file mode 100644 index 00000000..e7c30888 --- /dev/null +++ b/tools/sync_pmdsky_debug/yaml_writer.py @@ -0,0 +1,82 @@ +from typing import Any +from ruamel.yaml import YAML +from ruamel.yaml.emitter import RoundTripEmitter + +# Used to avoid reloading the same YAML file multiple times if it is already loaded. +class YamlManager(object): + + def __init__(self): + self.yaml_contents = None + self.yaml_path = None + + def read_yaml(self, yaml_path: str): + if yaml_path == self.yaml_path: + return self.yaml_contents + + self.write_yaml() + self.yaml_path = yaml_path + + with open(yaml_path, 'r') as yaml_file: + self.yaml_contents = yaml.load(yaml_file) + return self.yaml_contents + + def write_yaml(self): + if self.yaml_path: + with open(self.yaml_path, 'w') as yaml_file: + yaml.dump(self.yaml_contents, yaml_file) + self.yaml_path = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.write_yaml() + +class PreserveWhitespaceEmitter(RoundTripEmitter): + + # Removes the check for consecutive newline characters when indenting, so that blank lines within strings are indented properly. + def write_literal(self, text: Any, comment: Any = None) -> None: + hints, _indent, _indicator = self.determine_block_hints(text) + if not isinstance(comment, str): + comment = '' + self.write_indicator('|' + hints + comment, True) + if _indicator == '+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + if self.root_context: + idnx = self.indent if self.indent is not None else 0 + self.stream.write(' ' * (_indent + idnx)) + else: + self.write_indent() + start = end + else: + if ch is None or ch in '\n\x85\u2028\u2029': + data = text[start:end] + if bool(self.encoding): + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = ch in '\n\x85\u2028\u2029' + end += 1 + +yaml = YAML() +yaml.preserve_quotes = True +yaml.width = float('inf') +yaml.indent(sequence=4, offset=2) +yaml.Emitter = PreserveWhitespaceEmitter