From 4339657e85f70d8ccb2095c100a350456d4a676e Mon Sep 17 00:00:00 2001 From: CptMoore <39010654+CptMoore@users.noreply.github.com> Date: Mon, 9 Dec 2024 19:44:00 +0100 Subject: [PATCH] Another iteration --- ModTek/Features/Logging/LogLevelExtension.cs | 34 +- ...JSONSerializationUtility_FromJSON_Patch.cs | 70 +++- ModTek/Util/HBSJsonUtils.cs | 324 +++++++++++------- 3 files changed, 282 insertions(+), 146 deletions(-) diff --git a/ModTek/Features/Logging/LogLevelExtension.cs b/ModTek/Features/Logging/LogLevelExtension.cs index 88ab6ec..1ef582f 100644 --- a/ModTek/Features/Logging/LogLevelExtension.cs +++ b/ModTek/Features/Logging/LogLevelExtension.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using HBS.Logging; namespace ModTek.Features.Logging; @@ -7,7 +8,7 @@ internal static class LogLevelExtension { internal static string LogToString(LogLevel level) { - var eLogLevel = Convert(level); + var eLogLevel = ConvertHbsLogLevelToELogLevel(level); return eLogLevel switch // fast switch with static string, in order of most occuring { ELogLevels.Trace => "TRACE", @@ -23,10 +24,10 @@ internal static string LogToString(LogLevel level) internal static bool IsLogLevelGreaterThan(LogLevel loggerLevel, LogLevel messageLevel) { - return Convert(messageLevel) >= Convert(loggerLevel); + return ConvertHbsLogLevelToELogLevel(messageLevel) >= ConvertHbsLogLevelToELogLevel(loggerLevel); } - private static ELogLevels Convert(LogLevel level) + private static ELogLevels ConvertHbsLogLevelToELogLevel(LogLevel level) { var intLevel = (int)level; if (intLevel is >= (int)LogLevel.Debug and <= (int)LogLevel.Error) @@ -36,6 +37,33 @@ private static ELogLevels Convert(LogLevel level) return (ELogLevels)intLevel; } + internal static TraceLevel GetLevelFilter(ILog log) + { + if (log is not Logger.LogImpl logImpl) + { + return TraceLevel.Verbose; // either off or verbose, better too much + } + var effectiveLevel = logImpl.EffectiveLevel; + var eLogLevel = ConvertHbsLogLevelToELogLevel(effectiveLevel); + return ConvertELogLevelToTraceLevel(eLogLevel); + } + + private static TraceLevel ConvertELogLevelToTraceLevel(ELogLevels level) + { + level = (ELogLevels)((int)level / 10 * 10); + return level switch + { + ELogLevels.OFF => TraceLevel.Off, + ELogLevels.Fatal => TraceLevel.Error, + ELogLevels.Error => TraceLevel.Error, + ELogLevels.Warning => TraceLevel.Warning, + ELogLevels.Log => TraceLevel.Info, + ELogLevels.Debug => TraceLevel.Verbose, + ELogLevels.Trace => TraceLevel.Verbose, + _ => throw new ArgumentOutOfRangeException(nameof(level), level, null) + }; + } + // log levels private enum ELogLevels { diff --git a/ModTek/Features/Profiler/Patches/JSONSerializationUtility_FromJSON_Patch.cs b/ModTek/Features/Profiler/Patches/JSONSerializationUtility_FromJSON_Patch.cs index ad87d1a..a8b26f9 100644 --- a/ModTek/Features/Profiler/Patches/JSONSerializationUtility_FromJSON_Patch.cs +++ b/ModTek/Features/Profiler/Patches/JSONSerializationUtility_FromJSON_Patch.cs @@ -5,6 +5,7 @@ using System.Reflection; using System.Threading; using BattleTech; +using fastJSON; using HBS.Util; using ModTek.Features.Logging; using ModTek.Misc; @@ -86,6 +87,7 @@ MethodBase GetMethod(BindingFlags bindingAttr = BindingFlags.Default) Log.Main.Trace?.Log("IJsonTemplated.FromJSON " + fromJsonMethod); yield return genericMethod.MakeGenericMethod(jsonTemplated); + // break; } } @@ -109,6 +111,33 @@ internal class MyState { internal MTStopwatch.Tracker Tracker; internal long counter = Interlocked.Increment(ref s_counter); + + internal string nn; + internal string nh; + internal string hn; + internal string hh; + + internal void Save() + { + var differentPopulate = true; + var differentSerializers = false; // tag sets and dictionaries work differently + if ((differentSerializers && nn != nh) || (differentPopulate && hn != nn)) + { + File.WriteAllText(Path.Combine(basePath, $"{counter}_NN.json"), nn); + } + if ((differentSerializers && nn != nh) || (differentPopulate && nh != hh)) + { + File.WriteAllText(Path.Combine(basePath, $"{counter}_NH.json"), nn); + } + if ((differentSerializers && hn != hh) || (differentPopulate && hn != nn)) + { + File.WriteAllText(Path.Combine(basePath, $"{counter}_HN.json"), hn); + } + if ((differentSerializers && hn != hh) || (differentPopulate && nh != hh)) + { + File.WriteAllText(Path.Combine(basePath, $"{counter}_HH.json"), hh); + } + } } private static long s_counter; @@ -116,6 +145,15 @@ internal class MyState [HarmonyPriority(Priority.First)] public static void Prefix(object target, string json, ref MyState __state) { + if (target == null) + { + return; + } + if (typeof(MechDef).Assembly != target.GetType().Assembly) + { + return; + } + __state = new MyState(); if (testNewton) // && target.GetType() == typeof(MechDef) @@ -123,15 +161,14 @@ public static void Prefix(object target, string json, ref MyState __state) s_newton.Start(); try { - var mechDef = new MechDef(); - HBSJsonUtils.PopulateObject(mechDef, json); - var output = HBSJsonUtils.SerializeObject(mechDef); - var path = Path.Combine(basePath, $"{__state.counter}_N.json"); - File.WriteAllText(path, output); + var objectCopy = Activator.CreateInstance(target.GetType()); + HBSJsonUtils.PopulateObject(objectCopy, json); + __state.nn = HBSJsonUtils.SerializeObject(objectCopy); + __state.nh = JSON.ToJSON(objectCopy); } catch (Exception ex) { - Log.Main.Error?.Log("Error N PopulateObject SerializeObject" + target.GetType(), ex); + Log.Main.Error?.Log("Error Populating and Serializing " + target.GetType(), ex); } finally { @@ -145,19 +182,32 @@ public static void Prefix(object target, string json, ref MyState __state) [HarmonyPriority(Priority.Last)] public static void Postfix(object target, string json, ref MyState __state) { + if (__state == null) + { + return; + } + s_stopwatch.AddMeasurement(__state.Tracker.End()); if (testNewton) // && target.GetType() == typeof(MechDef) { try { - var output = HBSJsonUtils.SerializeObject(target); - var path = Path.Combine(basePath, $"{__state.counter}_H.json"); - File.WriteAllText(path, output); + __state.hn = HBSJsonUtils.SerializeObject(target); + __state.hh = JSON.ToJSON(target); + } + catch (Exception ex) + { + Log.Main.Error?.Log("Error Serializing " + target.GetType(), ex); + } + + try + { + __state.Save(); } catch (Exception ex) { - Log.Main.Error?.Log("Error H SerializeObject " + target.GetType(), ex); + Log.Main.Error?.Log("Error Saving JSONs " + target.GetType(), ex); } } } diff --git a/ModTek/Util/HBSJsonUtils.cs b/ModTek/Util/HBSJsonUtils.cs index 9c79a07..3cdf74d 100644 --- a/ModTek/Util/HBSJsonUtils.cs +++ b/ModTek/Util/HBSJsonUtils.cs @@ -1,17 +1,22 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; using System.Text.RegularExpressions; using fastJSON; using HBS.Collections; +using HBS.Logging; using HBS.Util; +using ModTek.Features.Logging; using Newtonsoft.Json; -using Newtonsoft.Json.Converters; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; +using Newtonsoft.Json.Utilities; namespace ModTek.Util; @@ -55,16 +60,6 @@ private static string FixHBSJsonCommas(string json) return s_fixMissingCommasInJson.Replace(json, "$1,\n$2"); } - private static readonly JsonSerializerSettings s_jsonSerializerSettings = new() - { - ContractResolver = new FastJsonContractResolver(), - NullValueHandling = NullValueHandling.Ignore, - Converters = [ - new TagSetConverter(), - new StringEnumConverter(), - ], - }; - // might work, only slightly tested internal static string SerializeObject(object target) { @@ -79,136 +74,51 @@ internal static void PopulateObject(object target, string json) { var commentsStripped = JSONSerializationUtility.StripHBSCommentsFromJSON(json); var commasAdded = FixHBSJsonCommas(commentsStripped); - var dictionariesFixed = FixStructures(commasAdded); - JsonConvert.PopulateObject(dictionariesFixed, target, s_jsonSerializerSettings); - } - - private static string FixStructures(string json) - { - var newton = JObject.Parse(json); - FindAndFixStructures(newton); - return newton.ToString(); + JsonConvert.PopulateObject(commasAdded, target, s_jsonSerializerSettings); } - private static void FindAndFixStructures(JToken newton) + private static readonly JsonSerializerSettings s_jsonSerializerSettings = new() { - switch (newton) - { - case JArray a: - var dict = ConvertHbsDictArrayToDictionary(a); - if (dict != null) - { - foreach (var v in dict.Values()) - { - FindAndFixStructures(v); - } - } - - return; - case JObject o: - if (ConvertHbsTagsToArray(o)) - { - return; - } - foreach (var v in o.Values()) - { - FindAndFixStructures(v); - } - break; - } - } + ContractResolver = new FastJsonContractResolver(), + NullValueHandling = NullValueHandling.Ignore, + Converters = [ + new TagSetConverter(), + new FastJsonStringEnumConverter(), + new FastJsonDictionaryConverter(), + ], + // TraceWriter = new JsonTraceWriter(), + // SerializationBinder = new SerializationBinderAdapter(new DefaultSerializationBinder()) + }; - /* - * [ { "k" : "A" , "v" : B } , { "k" : "C" , "v": D } ] - * { "A" : B , "C" : D } - */ - // similar to TagSetConverter - private static JObject ConvertHbsDictArrayToDictionary(JArray a) + private class JsonTraceWriter : ITraceWriter { - var newDict = new JObject(); - if (a.Count <= 1) - { - return null; - } - - foreach (var i in a) + public void Trace(TraceLevel level, string message, Exception ex) { - if (i is not JObject o) - { - return null; - } - - if (!ExtractKeyValue(o, out var key, out var value)) + if (level == TraceLevel.Off) { - return null; - } - - if (newDict[key] != null) - { - return null; + return; } - newDict.Add(key, value); + Log.Main.Log.LogAtLevel(ConvertLevel(level), message, ex); } - a.Replace(newDict); - - return newDict; - } - - /* - * { "items": [ I ], "tagSetSourceFile": ... } - * to - * [ I ] - */ - private static bool ConvertHbsTagsToArray(JObject o) - { - if (o.Count is >= 1 and <= 2) + private static LogLevel ConvertLevel(TraceLevel level) { - if (o.Count == 2) - { - if (!o.TryGetValue("tagSetSourceFile", out _)) - { - return false; - } - } - if (o.TryGetValue("items", out var items) && items.Type == JTokenType.Array) + return level switch { - o.Replace(items); - } + TraceLevel.Error => LogLevel.Error, + TraceLevel.Warning => LogLevel.Warning, + TraceLevel.Info => LogLevel.Log, + TraceLevel.Verbose => LogLevel.Debug, + _ => throw new ArgumentOutOfRangeException(nameof(level), level, null) + }; } - return false; - } - private static bool ExtractKeyValue( - JObject o, - [NotNullWhen(true)] out string keyAsString, - [NotNullWhen(true)] out JToken valueAsToken - ) { - if (o.Count == 2) - { - if (o.TryGetValue("k", out var keyAsToken) && keyAsToken.Type == JTokenType.String) - { - if (keyAsToken is JValue keyAsValue) - { - keyAsString = keyAsValue._value as string; - if (o.TryGetValue("v", out valueAsToken)) - { - return true; - } - } - } - } - keyAsString = null; - valueAsToken = null; - return false; + public TraceLevel LevelFilter => LogLevelExtension.GetLevelFilter(Log.Main.Log); } - private class FastJsonContractResolver : DefaultContractResolver { - - private readonly Dictionary> _serializableMembersCache = new(); protected override List GetSerializableMembers(Type objectType) { @@ -222,29 +132,33 @@ protected override List GetSerializableMembers(Type objectType) } members.AddRange( - objectType.GetFields(Reflection.defaultFlags & ~BindingFlags.Static) + objectType.GetProperties( BindingFlags.Instance | BindingFlags.Public) + .Where(member => member.CanWrite && member.CanRead) .Where(member => !member.IsDefined(typeof(JsonIgnore), false)) ); members.AddRange( - objectType.GetFields(Reflection.privateFlags) - .Where(member => !member.IsInitOnly) + objectType.GetProperties(BindingFlags.Instance | BindingFlags.NonPublic) .Where(member => member.IsDefined(typeof(JsonSerialized), false)) ); members.AddRange( - objectType.GetProperties(Reflection.defaultFlags & ~BindingFlags.Static) - .Where(member => member.CanWrite && member.CanRead) + objectType.GetFields( BindingFlags.Instance | BindingFlags.Public) .Where(member => !member.IsDefined(typeof(JsonIgnore), false)) ); members.AddRange( - objectType.GetProperties(Reflection.privateFlags) + objectType.GetFields(BindingFlags.Instance | BindingFlags.NonPublic) + .Where(member => !member.IsInitOnly) .Where(member => member.IsDefined(typeof(JsonSerialized), false)) ); - // if (objectType == typeof(MechDef)) - // { - // Log.Main.Debug?.Log($"Found {members.Count} members for type {objectType.FullName}: {string.Join(",", members.Select(m => m.Name))}"); - // } + // Order of properties & fields processing is different between fastJson+HBS compared to Newtonsoft.Json + // -> some setters are called in different order + // -> sometimes different things are null or not at a different time + // solutions: + // - emulate fastJson+HBS ordering? -> uses some kind of 1. properties 2. fields but not always + // - fix vanilla code to avoid needing setters at all? -> can break existing code + // - introduce explicit converters? -> similar to CustomPrewam; require mod support APIs to let mods fix themselves + Log.Main.Debug?.Log($"Found {members.Count} members for type {objectType.FullName}: {string.Join(",", members.Select(m => m.Name))}"); lock (_serializableMembersCache) { @@ -271,7 +185,73 @@ protected override JsonProperty CreateProperty(MemberInfo member, MemberSerializ } } - // similar to ConvertHbsDictArrayToDictionary + private class FastJsonStringEnumConverter : JsonConverter + { + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + if (value == null) + { + writer.WriteNull(); + } + else + { + writer.WriteValue(value.ToString()); + } + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + var isNullable = ReflectionUtils.IsNullableType(objectType); + if (reader.TokenType == JsonToken.Null) + { + if (!isNullable) + throw JsonSerializationException.Create(reader, "Cannot convert null value to {0}.".FormatWith(CultureInfo.InvariantCulture, objectType)); + return null; + } + + var enumType = isNullable ? Nullable.GetUnderlyingType(objectType) : objectType; + if (enumType == null) + { + throw new InvalidCastException(); + } + + if (reader.TokenType == JsonToken.String) + { + try + { + var value = reader.Value.ToString(); + if (value.Length == 0 && isNullable) + { + return null; + } + return Enum.Parse(enumType, value, true); + } + catch (Exception ex) + { + throw JsonSerializationException.Create(reader, "Error converting value {0} to type '{1}'.".FormatWith(CultureInfo.InvariantCulture, MiscellaneousUtils.FormatValueForPrint(reader.Value), objectType), ex); + } + } + if (reader.TokenType == JsonToken.Integer) + { + var numericValue = (int)reader.Value; + return Enum.ToObject(enumType, numericValue); + } + + throw JsonSerializationException.Create(reader, "Unexpected token {0} when parsing enum.".FormatWith(CultureInfo.InvariantCulture, reader.TokenType)); + } + + public override bool CanConvert(Type objectType) + { + // copied from StringEnumConverter.CanConvert + return (ReflectionUtils.IsNullableType(objectType) ? Nullable.GetUnderlyingType(objectType) : objectType).IsEnum(); + } + } + + /* + * { "items": [ I ], "tagSetSourceFile": ... } + * to + * [ I ] + */ private class TagSetConverter : JsonConverter { public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) @@ -329,4 +309,82 @@ public override bool CanConvert(Type objectType) return objectType == typeof(TagSet); } } + + /* + * [ { "k" : "A" , "v" : B } , { "k" : "C" , "v": D } ] + * to + * { "A" : B , "C" : D } + */ + private class FastJsonDictionaryConverter : JsonConverter + { + public override bool CanWrite => false; + + public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + { + throw new NotSupportedException(); + } + + public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + { + if (reader.TokenType == JsonToken.Null) + { + return null; + } + + var valueType = objectType.GetGenericArguments()[1]; + var intermediateDictionaryType = typeof(Dictionary<,>).MakeGenericType(typeof(string), valueType); + var intermediateDictionary = (IDictionary)Activator.CreateInstance(intermediateDictionaryType); + + if (reader.TokenType == JsonToken.StartArray) + { + var intermediateListItemType = typeof(FastJsonDictionaryArrayItem<>).MakeGenericType(valueType); + var intermediateListType = typeof(List<>).MakeGenericType(intermediateListItemType); + var intermediateList = (IList)Activator.CreateInstance(intermediateListType); + serializer.Populate(reader, intermediateList); + foreach (var item in intermediateList) + { + var traverse = Traverse.Create(item); + var key = traverse.Field("k").GetValue(); + var value = traverse.Field("v").GetValue(); + intermediateDictionary.Add(key, value); + } + } + else if (reader.TokenType == JsonToken.StartObject) + { + serializer.Populate(reader, intermediateDictionary); + } + + var keyType = objectType.GetGenericArguments()[0]; + if (keyType == typeof(string)) + { + return intermediateDictionary; + } + + var finalDictionary = (IDictionary)Activator.CreateInstance(objectType); + if (keyType.IsEnum) + { + foreach (DictionaryEntry pair in intermediateDictionary) + { + var key = Enum.Parse(keyType, (string)pair.Key, true); + finalDictionary.Add(key, pair.Value); + } + } + else + { + throw JsonSerializationException.Create(reader, $"Dictionary key type {keyType} is not supported."); + + } + return finalDictionary; + } + + public override bool CanConvert(Type objectType) + { + return typeof(IDictionary).IsAssignableFrom(objectType); + } + } + private class FastJsonDictionaryArrayItem + { + public string k; + public T v; + } } \ No newline at end of file