Skip to content

Commit

Permalink
v80.1 (#77)
Browse files Browse the repository at this point in the history
* fix 4.27 error

* update submodule

* optimize impl

* support 5.1

* fix 5.1 WP error

* optimize impl

* optimize impl

* optimize impl

* optimize impl

* optimize impl

* update version to v78.1

* fix compile error

* update mods

* fix shipping error & update version to v79.0

* split ByGameFeature as GameFeaturePacker

* add ShaderPatcher Mod

* optimize mod

* delete GameFeature Mod

* add GameFeaturePacker Mod

* add HDiffPatchUE Mod

* update version to v80

* optimize impl

* fix Property Category issue

* update Mod

* update Mods

* support UE5.1

* update Mods commit

* Update UpdaterWidget

* optimize Mod impl

* update Updater

* fix Localization key dumplicate

* remove GameFeaturePacker

* add GameFeaturePacker Mod

* update Mods

* optimize mod impl

* rm ShaderPacther Mod

* add ShaderPatcher Mod

* delete mod

* add ShaderPatcherUE Mod

* fix -ddcurl= issue

* update Mods

* optimize CmdHandler/MultiCooker

* update MultiCooker Mod

* optimize DDC Backend Hierarchy

* update

* update mods

* add PakAnalyzerUE Mod

* UpdateMods

* optimize impl

* add submodule

* update

* fix multi-thread assetparser issue

* optimize impl

* fix ushaderbytecode ignore issue

* 1. 修复TargetPlatform注册时机晚于CDO的问题
2. 支持获取参与打包的资源列表
3. 修复PackageTracker的资源没有进包的问题(PackageTracker的资源会进行ForceSkip检测)

* update submodule

* optimize/fix forceskip directory/asset issue

* fix issue

* update submodule

* support override property by cmdlet

* optimize impl

* update version to v80.1

---------

Co-authored-by: lipengzha <[email protected]>
  • Loading branch information
hxhb and lipengzha authored Mar 1, 2023
1 parent 78203c7 commit 6071309
Show file tree
Hide file tree
Showing 27 changed files with 250 additions and 69 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,6 @@
[submodule "Mods/ShaderPatcherUE"]
path = Mods/ShaderPatcherUE
url = [email protected]:hxhb/ShaderPatcherUE.git
[submodule "Mods/PakAnalyzerUE"]
path = Mods/PakAnalyzerUE
url = [email protected]:hxhb/PakAnalyzerUE.git
Original file line number Diff line number Diff line change
Expand Up @@ -181,4 +181,4 @@ TArray<FString> CommandletHelper::GetCookCommandletTargetPlatformName()
}

return result;
}
}
2 changes: 1 addition & 1 deletion HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public HotPatcherCore(ReadOnlyTargetRules Target) : base(Target)
{
"TOOL_NAME=\"HotPatcher\"",
"CURRENT_VERSION_ID=80",
"CURRENT_PATCH_ID=0",
"CURRENT_PATCH_ID=1",
"REMOTE_VERSION_FILE=\"https://imzlp.com/opensource/version.json\""
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -603,10 +603,15 @@ FCookCluster USingleCookerProxy::GetPackageTrackerAsCluster()
PackageTrackerCluster.AssetDetails.Empty();
for(FName LongPackageName:PackageTracker->GetPendingPackageSet())
{
if(!FPackageName::DoesPackageExist(LongPackageName.ToString()))
{
continue;
}

// make asset data to asset registry
FSoftObjectPath ObjectPath(
UFlibAssetManageHelper::LongPackageNameToPackagePath(LongPackageName.ToString())
);
FString PackagePath = UFlibAssetManageHelper::LongPackageNameToPackagePath(LongPackageName.ToString());

FSoftObjectPath ObjectPath(PackagePath);
UFlibAssetManageHelper::UpdateAssetRegistryData(ObjectPath.GetLongPackageName());

FAssetData AssetData;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ namespace PatchWorker
{
for(auto& Chunk:Context.PakChunks)
{
FString SavePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),Context.CurrentVersion.VersionId,TEXT("Metadatas"),Chunk.ChunkName,PlatformName);
FString SavePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),Context.CurrentVersion.VersionId,PlatformName,TEXT("Metadatas"),PlatformName,TEXT("Metadata/ShaderLibrarySource"));
TArray<FString> FoundShaderLibs = UFlibShaderCodeLibraryHelper::FindCookedShaderLibByPlatform(PlatformName,SavePath);

if(Context.PakChunks.Num())
Expand Down Expand Up @@ -563,7 +563,8 @@ namespace PatchWorker
);

const TArray<FAssetDetail>& ChunkAssets = ChunkAssetsDescrible.Assets.GetAssetDetails();

Context.PatchProxy->GetPatcherResult().PatcherAssetDetails.Append(ChunkAssets);

if(Context.GetSettingObject()->IsCookPatchAssets())
{
FTrackPackageAction TrackChunkPackageAction(Context,Chunk,TArray<ETargetPlatform>{Platform});
Expand Down Expand Up @@ -605,6 +606,22 @@ namespace PatchWorker
SingleCookerProxy->AddToRoot();
SingleCookerProxy->Init(&EmptySetting);
bool bExportStatus = SingleCookerProxy->DoExport();
const FCookCluster& AdditionalCluster = SingleCookerProxy->GetPackageTrackerAsCluster();
for(const auto& AssetDetail:AdditionalCluster.AssetDetails)
{
FSoftObjectPath ObjectPath{AssetDetail.PackagePath};
FString ReceiveReason;
if(!Context.GetSettingObject()->GetAssetScanConfig().IsMatchForceSkip(ObjectPath,ReceiveReason))
{
Context.PatchProxy->GetPatcherResult().PatcherAssetDetails.Add(AssetDetail);
Context.VersionDiff.AssetDiffInfo.AddAssetDependInfo.AddAssetsDetail(AssetDetail);
}
else
{
UE_LOG(LogHotPatcher,Display,TEXT("[PackageTracker] %s Match ForceSkipRule,Reason %s"),*ObjectPath.GetLongPackageName(),*ReceiveReason);
}
}

SingleCookerProxy->Shutdown();
SingleCookerProxy->RemoveFromRoot();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "Serialization/ArrayWriter.h"
#include "Settings/ProjectPackagingSettings.h"
#include "ShaderCompiler.h"
#include "Async/ParallelFor.h"
#include "CreatePatch/PatcherProxy.h"
#include "Materials/MaterialInstance.h"
#include "Materials/MaterialInstanceConstant.h"
Expand Down Expand Up @@ -1199,10 +1200,10 @@ FPatchVersionDiff UFlibHotPatcherCoreHelper::DiffPatchVersionWithPatchSetting(co

if(PatchSetting.IsForceSkipContent())
{
TArray<FString> AllSkipContents;
AllSkipContents.Append(UFlibAssetManageHelper::DirectoriesToStrings(PatchSetting.GetForceSkipContentRules()));
AllSkipContents.Append(UFlibAssetManageHelper::SoftObjectPathsToStrings(PatchSetting.GetForceSkipAssets()));
UFlibPatchParserHelper::ExcludeContentForVersionDiff(VersionDiffInfo,AllSkipContents);
TArray<FString> AllSkipDirContents = UFlibAssetManageHelper::DirectoriesToStrings(PatchSetting.GetForceSkipContentRules());
UFlibPatchParserHelper::ExcludeContentForVersionDiff(VersionDiffInfo,AllSkipDirContents,EHotPatcherMatchModEx::StartWith);
TArray<FString> AllSkipAssets = UFlibAssetManageHelper::SoftObjectPathsToStrings(PatchSetting.GetForceSkipAssets());
UFlibPatchParserHelper::ExcludeContentForVersionDiff(VersionDiffInfo,AllSkipAssets,EHotPatcherMatchModEx::Equal);
}
// clean deleted asset info in patch
if(PatchSetting.IsIgnoreDeletedAssetsInfo())
Expand Down Expand Up @@ -1620,8 +1621,9 @@ void UFlibHotPatcherCoreHelper::AppendPakCommandOptions(TArray<FString>& OriginC
const TArray<FString>& Options, bool bAppendAllMatch, const TArray<FString>& AppendFileExtersions,
const TArray<FString>& IgnoreFormats, const TArray<FString>& InIgnoreOptions)
{
for(auto& Command:OriginCommands)
ParallelFor(OriginCommands.Num(),[&](int32 index)
{
FString& Command = OriginCommands[index];
FString PakOptionsStr;
for (const auto& Param : Options)
{
Expand All @@ -1640,7 +1642,7 @@ void UFlibHotPatcherCoreHelper::AppendPakCommandOptions(TArray<FString>& OriginC
PakOptionsStr += AppendOptionStr;
}
Command = FString::Printf(TEXT("%s%s"),*Command,*PakOptionsStr);
}
});
}

FProjectPackageAssetCollection UFlibHotPatcherCoreHelper::ImportProjectSettingsPackages()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,4 @@ namespace CommandletHelper
HOTPATCHERCORE_API bool IsCookCommandlet();
HOTPATCHERCORE_API TArray<ETargetPlatform> GetCookCommandletTargetPlatforms();
HOTPATCHERCORE_API TArray<FString> GetCookCommandletTargetPlatformName();

}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@ using FPatchWorkers = TMap<FString,FPatchWorkerType>;
DECLARE_MULTICAST_DELEGATE_FourParams(FOnPakListGenerated,FHotPatcherPatchContext&,FChunkInfo&,ETargetPlatform,TArray<FPakCommand>&);
DECLARE_MULTICAST_DELEGATE_FourParams(FAddPatchWorkerEvent,FHotPatcherPatchContext&,FChunkInfo&,ETargetPlatform,TArray<FPakCommand>&);


USTRUCT()
struct HOTPATCHERCORE_API FPatherResult
{
GENERATED_BODY()
UPROPERTY()
TArray<FAssetDetail> PatcherAssetDetails;
};

UCLASS()
class HOTPATCHERCORE_API UPatcherProxy:public UHotPatcherProxyBase
{
Expand All @@ -40,11 +49,14 @@ class HOTPATCHERCORE_API UPatcherProxy:public UHotPatcherProxyBase
PatchWorkers.Add(WorkerName,Worker);
}
FORCEINLINE const FPatchWorkers& GetPatchWorkers()const{ return PatchWorkers; }
FORCEINLINE FPatherResult& GetPatcherResult(){ return PatcherResult; }

public:
FOnPakListGenerated OnPakListGenerated;

protected:
FPatchWorkers PatchWorkers;
private:
TSharedPtr<FHotPatcherPatchContext> PatchContext;
FPatherResult PatcherResult;
};
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,11 @@ void FHotPatcherEditorModule::OpenDockTab()

void FHotPatcherEditorModule::PluginButtonClicked()
{
if(!DockTab.IsValid())
if (!DockTab.IsValid())
{
FGlobalTabmanager::Get()->RegisterNomadTabSpawner(HotPatcherTabName, FOnSpawnTab::CreateRaw(this, &FHotPatcherEditorModule::OnSpawnPluginTab))
.SetDisplayName(LOCTEXT("FHotPatcherTabTitle", "HotPatcher"))
.SetMenuType(ETabSpawnerMenuType::Hidden);
.SetDisplayName(LOCTEXT("FHotPatcherTabTitle", "HotPatcher"))
.SetMenuType(ETabSpawnerMenuType::Hidden);
}
FGlobalTabmanager::Get()->InvokeTab(HotPatcherTabName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,8 @@ bool FAssetDependenciesInfo::HasAsset(const FString& InAssetPackageName)const
TArray<FAssetDetail> FAssetDependenciesInfo::GetAssetDetails()const
{
SCOPED_NAMED_EVENT_TEXT("FAssetDependenciesInfo::GetAssetDetails",FColor::Red);
TArray<FAssetDetail> OutAssetDetails;

OutAssetDetails.Empty();
TArray<FAssetDetail> AssetDetails;
TArray<FString> Keys;
AssetsDependenciesMap.GetKeys(Keys);

Expand All @@ -59,11 +58,11 @@ TArray<FAssetDetail> FAssetDependenciesInfo::GetAssetDetails()const
for (const auto& ModuleAssetKey : ModuleAssetKeys)
{
FScopeLock Lock(&SynchronizationObject);
OutAssetDetails.Add(*ModuleAssetDetails.Find(ModuleAssetKey));
AssetDetails.Add(*ModuleAssetDetails.Find(ModuleAssetKey));
}
},GForceSingleThread);

return OutAssetDetails;
return AssetDetails;
}

bool FAssetDependenciesInfo::GetAssetDetailByPackageName(const FString& InAssetPackageName,FAssetDetail& OutDetail) const
Expand Down Expand Up @@ -92,3 +91,28 @@ TArray<FString> FAssetDependenciesInfo::GetAssetLongPackageNames()const
}
return OutAssetLongPackageName;
}

void FAssetDependenciesInfo::RemoveAssetDetail(const FAssetDetail& AssetDetail)
{
SCOPED_NAMED_EVENT_TEXT("FAssetDependenciesInfo::RemoveAssetDetail",FColor::Red);
FString LongPackageName = UFlibAssetManageHelper::PackagePathToLongPackageName(AssetDetail.PackagePath.ToString());
RemoveAssetDetail(LongPackageName);
}

void FAssetDependenciesInfo::RemoveAssetDetail(const FString& LongPackageName)
{
FString BelongModuleName = UFlibAssetManageHelper::GetAssetBelongModuleName(LongPackageName);
if (AssetsDependenciesMap.Contains(BelongModuleName))
{
TMap<FString,FAssetDetail>& AssetDependencyDetails = AssetsDependenciesMap.Find(BelongModuleName)->AssetDependencyDetails;
bool bHas = AssetDependencyDetails.Contains(LongPackageName);
if(bHas)
{
AssetDependencyDetails.Remove(LongPackageName);
if(!AssetDependencyDetails.Num())
{
AssetsDependenciesMap.Remove(BelongModuleName);
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1 +1,38 @@
#include "BaseTypes/FAssetScanConfig.h"

bool FAssetScanConfig::IsMatchForceSkip(const FSoftObjectPath& ObjectPath,FString& OutReason)
{
SCOPED_NAMED_EVENT_TEXT("IsMatchForceSkip",FColor::Red);
bool bSkip = false;
if(bForceSkipContent)
{
bool bSkipAsset = ForceSkipAssets.Contains(ObjectPath);
if(bSkipAsset)
{
OutReason = FString::Printf(TEXT("IsForceSkipAsset"));
}
bool bSkipDir = false;
for(const auto& ForceSkipDir:ForceSkipContentRules)
{
if(ObjectPath.GetLongPackageName().StartsWith(ForceSkipDir.Path))
{
bSkipDir = true;
OutReason = FString::Printf(TEXT("ForceSkipDir %s"),*ForceSkipDir.Path);
break;
}
}
bool bSkipClasses = false;
FName AssetClassesName = UFlibAssetManageHelper::GetAssetType(ObjectPath);
for(const auto& Classes:ForceSkipClasses)
{
if(Classes->GetFName().IsEqual(AssetClassesName))
{
OutReason = FString::Printf(TEXT("ForceSkipClasses %s"),*Classes->GetName());
bSkipClasses = true;
break;
}
}
bSkip = bSkipAsset || bSkipDir || bSkipClasses;
}
return bSkip;
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ FString FChunkInfo::GetShaderLibraryName() const

TArray<FSoftObjectPath> FChunkInfo::GetManagedAssets() const
{
SCOPED_NAMED_EVENT_TEXT("FChunkInfo::GetManagedAssets",FColor::Red);
FScopedNamedEventStatic GetManagedAssetsTag(FColor::Red,*FString::Printf(TEXT("GetManagedAssets_%s"),*ChunkName));
TArray<FSoftObjectPath> NewPaths;

UAssetManager& Manager = UAssetManager::Get();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,8 @@ TSet<FName> FAssetDependenciesParser::GatherAssetDependicesInfoRecursively(
const TSet<FName>& IgnoreAssetTypes,
FScanedCachesType& InScanedCaches)
{
static bool bVerboseLog = FParse::Param(FCommandLine::Get(), TEXT("VerboseLog"));

TSet<FName> AssetDependencies;
SCOPED_NAMED_EVENT_TEXT("GatherAssetDependicesInfoRecursively",FColor::Red);
TArray<FString> TempForceSkipPackageNames = ForceSkipPackageNames;
Expand Down Expand Up @@ -284,18 +286,28 @@ TSet<FName> FAssetDependenciesParser::GatherAssetDependicesInfoRecursively(
// check is ignore directories or ingore types
{
SCOPED_NAMED_EVENT_TEXT("check ignore directories",FColor::Red);
if(!IsForceSkipAsset(LongPackageNameStr,IgnoreAssetTypes,IgnoreDirectories,ForceSkipDirectories,TempForceSkipPackageNames,false))
if(!IsForceSkipAsset(LongPackageNameStr,IgnoreAssetTypes,IgnoreDirectories,ForceSkipDirectories,TempForceSkipPackageNames,true))
{
FScopeLock Lock(&SynchronizationObject);
AssetDependencies.Add(LongPackageName);
}
}
},GForceSingleThread);
},true);
}

if(bRecursively)
{
TSet<FName> Dependencies;
#if ASSET_DEPENDENCIES_DEBUG_LOG
if(bVerboseLog)
{
UE_LOG(LogHotPatcher,Display,TEXT("AssetParser %s Dependencies: (%d)"),*InLongPackageName.ToString(),AssetDependencies.Num());
for(const auto& AssetPackageName:AssetDependencies)
{
UE_LOG(LogHotPatcher,Display,TEXT("\t%s"),*AssetPackageName.ToString());
}
}
#endif
for(const auto& AssetPackageName:AssetDependencies)
{
if(AssetPackageName.IsNone())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -863,7 +863,7 @@ bool UFlibAssetManageHelper::MakePakCommandFromAssetDependencies(
OutCookCommand.Append(FinalCookedCommand);
}
}
},GForceSingleThread);
},true);
return true;
}

Expand Down Expand Up @@ -1151,14 +1151,14 @@ FString UFlibAssetManageHelper::PackagePathToLongPackageName(const FString& Pack
return ObjectPath.GetLongPackageName();
}

void UFlibAssetManageHelper::ExcludeContentForAssetDependenciesDetail(FAssetDependenciesInfo& AssetDependencies,const TArray<FString>& ExcludeRules)
void UFlibAssetManageHelper::ExcludeContentForAssetDependenciesDetail(FAssetDependenciesInfo& AssetDependencies,const TArray<FString>& ExcludeRules,EHotPatcherMatchModEx matchMod)
{
SCOPED_NAMED_EVENT_TEXT("ExcludeContentForAssetDependenciesDetail",FColor::Red);
auto ExcludeEditorContent = [&ExcludeRules](TMap<FString,FAssetDependenciesDetail>& AssetCategorys)
auto ExcludeEditorContent = [&ExcludeRules,matchMod](TMap<FString,FAssetDependenciesDetail>& AssetCategorys)
{
TArray<FString> Keys;
AssetCategorys.GetKeys(Keys);

for(const auto& Key:Keys)
{
FAssetDependenciesDetail& ModuleAssetList = *AssetCategorys.Find(Key);
Expand All @@ -1175,7 +1175,9 @@ void UFlibAssetManageHelper::ExcludeContentForAssetDependenciesDetail(FAssetDepe
FString MatchRule;
for(const auto& Rule:ExcludeRules)
{
if(AssetKeys[index].StartsWith(Rule))
if(matchMod == EHotPatcherMatchModEx::StartWith && AssetKeys[index].StartsWith(Rule)||
matchMod == EHotPatcherMatchModEx::Equal && AssetKeys[index].Equals(Rule)
)
{
MatchRule = Rule;
customStartWith = true;
Expand Down Expand Up @@ -1410,8 +1412,9 @@ bool UFlibAssetManageHelper::MatchIgnoreFilters(const FString& LongPackageName,
{
for(const auto& IgnoreFilter:IgnoreDirs)
{
if(LongPackageName.StartsWith(IgnoreFilter) ||
IgnoreFilter.Contains(TEXT("*")) ? LongPackageName.MatchesWildcard(IgnoreFilter,ESearchCase::CaseSensitive):false)
bool bWithInSkipDir = LongPackageName.StartsWith(IgnoreFilter);
bool bMatchSkipWildcard = IgnoreFilter.Contains(TEXT("*")) ? LongPackageName.MatchesWildcard(IgnoreFilter,ESearchCase::CaseSensitive) : false;
if( bWithInSkipDir || bMatchSkipWildcard)
{
MatchDir = IgnoreFilter;
return true;
Expand Down Expand Up @@ -1612,4 +1615,19 @@ void UFlibAssetManageHelper::UpdateAssetRegistryData(const FString& PackageName)
AssetRegistry.ScanModifiedAssetFiles(TArray<FString>{PackageFilename});
}
}
TArray<FString> UFlibAssetManageHelper::GetPackgeFiles(const FString& LongPackageName,const FString& Extension)
{
SCOPED_NAMED_EVENT_TEXT("GetPackgeFiles",FColor::Red);
TArray<FString> Files;

FString FilePath = FPackageName::LongPackageNameToFilename(LongPackageName) + Extension;

if(FPaths::FileExists(FilePath))
{
FPaths::MakeStandardFilename(FilePath);
Files.Add(FilePath);
}
return Files;
};

// PRAGMA_ENABLE_DEPRECATION_WARNINGS
Loading

0 comments on commit 6071309

Please sign in to comment.