fixed #17012: Loading assets synchronously causes rendering to freeze. (

#17464)

cocos/native-binding/index.ts

@@ -815,6 +815,31 @@ export declare namespace native {
          *
          */
         export function getStringFromFile(filename: string): string;
+
+        /**
+         * @en Read utf-8 text file asynchronously.
+         * @zh 异步读取 utf-8 编码的文本文件
+         * @param filepath @en The file path. @zh 文件路径
+         * @param onComplete @en The complete callback. @zh 读取完成回调
+         */
+        export function readTextFile(filepath: string, onComplete: (err: string | null, content: string) => void): void;
+
+        /**
+         * @en Read utf-8 json file asynchronously.
+         * @zh 异步读取 utf-8 编码的 json 文件
+         * @param filepath @en The file path. @zh 文件路径
+         * @param onComplete @en The complete callback. @zh 读取完成回调
+         */
+        export function readJsonFile(filepath: string, onComplete: (err: string | null, content: object) => void): void;
+
+        /**
+         * @en Read binary file asynchronously.
+         * @zh 异步读取二进制文件
+         * @param filepath @en The file path. @zh 文件路径
+         * @param onComplete @en The complete callback. @zh 读取完成回调
+         */
+        export function readDataFile(filepath: string, onComplete: (err: string | null, content: ArrayBuffer) => void): void;
+
         /**
          *  @en
          *  Removes a file.

native/cocos/base/Scheduler.cpp

@@ -332,6 +332,40 @@ void Scheduler::removeAllFunctionsToBePerformedInCocosThread() {
     _functionsToPerform.clear();
 }
 
+void Scheduler::runFunctionsToBePerformedInCocosThread() {
+    //
+    // Functions allocated from another thread
+    //
+    auto beginTime = std::chrono::steady_clock::now();
+    auto nowTime = beginTime;
+
+    // Testing size is faster than locking / unlocking.
+    // And almost never there will be functions scheduled to be called.
+    if (!_functionsToPerform.empty()) {
+        _performMutex.lock();
+        // fixed #4123: Save the callback functions, they must be invoked after '_performMutex.unlock()', otherwise if new functions are added in callback, it will cause thread deadlock.
+        auto temp = std::move(_functionsToPerform);
+        _performMutex.unlock();
+
+        auto iter = temp.begin();
+        for (; iter != temp.end(); ++iter) {
+            nowTime = std::chrono::steady_clock::now();
+            auto passedMS = std::chrono::duration_cast<std::chrono::milliseconds>(nowTime - beginTime).count();
+            // If the callbacks takes more than 16ms, delay the remaining jobs to next frame.
+            if (passedMS > 16) {
+                break;
+            }
+
+            (*iter)();
+        }
+
+        std::lock_guard<std::mutex> lk(_performMutex);
+        for (; iter != temp.end(); ++iter) {
+            _functionsToPerform.emplace_back(std::move(*iter));
+        }
+    }
+}
+
 // main loop
 void Scheduler::update(float dt) {
     _updateHashLocked = true;
@@ -377,22 +411,7 @@ void Scheduler::update(float dt) {
     _updateHashLocked = false;
     _currentTarget = nullptr;
 
-    //
-    // Functions allocated from another thread
-    //
-
-    // Testing size is faster than locking / unlocking.
-    // And almost never there will be functions scheduled to be called.
-    if (!_functionsToPerform.empty()) {
-        _performMutex.lock();
-        // fixed #4123: Save the callback functions, they must be invoked after '_performMutex.unlock()', otherwise if new functions are added in callback, it will cause thread deadlock.
-        auto temp = _functionsToPerform;
-        _functionsToPerform.clear();
-        _performMutex.unlock();
-        for (const auto &function : temp) {
-            function();
-        }
-    }
+    runFunctionsToBePerformedInCocosThread();
 }
 
 } // namespace cc

native/cocos/base/Scheduler.h

@@ -268,6 +268,8 @@ class CC_DLL Scheduler final {
      * @js NA
      */
     void removeAllFunctionsToBePerformedInCocosThread();
+
+    void runFunctionsToBePerformedInCocosThread();
 
     bool isCurrentTargetSalvaged() const { return _currentTargetSalvaged; };
 

native/cocos/base/UTF8.cpp

@@ -24,12 +24,20 @@
  THE SOFTWARE.
 ****************************************************************************/
 
+#define CC_USE_SIMD_UTF 1
+
 #include "base/UTF8.h"
 
 #include <cstdarg>
 #include <cstdlib>
 
 #include "ConvertUTF/ConvertUTF.h"
+
+#if CC_USE_SIMD_UTF
+#include "simdutf/simdutf.cpp" //NOLINT
+#include "simdutf/simdutf.h"
+#endif
+
 #include "base/Log.h"
 
 namespace cc {
@@ -211,7 +219,29 @@ CC_DLL void UTF8LooseFix(const ccstd::string &in, ccstd::string &out) { //NOLINT
 }
 
 bool UTF8ToUTF16(const ccstd::string &utf8, std::u16string &outUtf16) { //NOLINT
+#if CC_USE_SIMD_UTF
+    bool validutf8 = simdutf::validate_utf8(utf8.c_str(), utf8.length());
+    if (!validutf8) {
+        outUtf16.clear();
+        return false;
+    }
+
+    // We need a buffer of size where to write the UTF-16LE words.
+    size_t expectedUtf16words = simdutf::utf16_length_from_utf8(utf8.c_str(), utf8.length());
+    outUtf16.resize(expectedUtf16words);
+
+    // convert to UTF-16LE
+    size_t utf16words = simdutf::convert_utf8_to_utf16le(utf8.c_str(), utf8.length(), outUtf16.data());
+    bool validutf16 = simdutf::validate_utf16le(outUtf16.c_str(), utf16words);
+    if (!validutf16) {
+        outUtf16.clear();
+        return false;
+    }
+
+    return true;
+#else
     return utfConvert(utf8, outUtf16, ConvertUTF8toUTF16);
+#endif
 }
 
 bool UTF8ToUTF32(const ccstd::string &utf8, std::u32string &outUtf32) { //NOLINT
@@ -237,7 +267,7 @@ bool UTF32ToUTF16(const std::u32string &utf32, std::u16string &outUtf16) { //NOL
 #if (CC_PLATFORM == CC_PLATFORM_ANDROID || CC_PLATFORM == CC_PLATFORM_OHOS)
 ccstd::string getStringUTFCharsJNI(JNIEnv *env, jstring srcjStr, bool *ret) {
     ccstd::string utf8Str;
-    auto *unicodeChar = static_cast<const uint16_t *>(env->GetStringChars(srcjStr, nullptr));
+    const auto *unicodeChar = static_cast<const uint16_t *>(env->GetStringChars(srcjStr, nullptr));
     size_t unicodeCharLength = env->GetStringLength(srcjStr);
     const std::u16string unicodeStr(reinterpret_cast<const char16_t *>(unicodeChar), unicodeCharLength);
     bool flag = UTF16ToUTF8(unicodeStr, utf8Str);
@@ -312,7 +342,7 @@ void StringUTF8::replace(const ccstd::string &newStr) {
 ccstd::string StringUTF8::getAsCharSequence() const {
     ccstd::string charSequence;
 
-    for (auto &charUtf8 : _str) {
+    for (const auto &charUtf8 : _str) {
         charSequence.append(charUtf8._char);
     }
 

native/cocos/bindings/jswrapper/v8/Object.cpp

@@ -400,6 +400,22 @@ Object *Object::createJSONObject(const ccstd::string &jsonStr) {
     return Object::_createJSObject(nullptr, jsobj);
 }
 
+Object *Object::createJSONObject(std::u16string &jsonStr) {
+    auto v8Str = v8::String::NewExternalTwoByte(__isolate, ccnew internal::ExternalStringResource(jsonStr));
+    if (v8Str.IsEmpty()) {
+        return nullptr;
+    }
+
+    v8::Local<v8::Context> context = __isolate->GetCurrentContext();
+    v8::MaybeLocal<v8::Value> ret = v8::JSON::Parse(context, v8Str.ToLocalChecked());
+    if (ret.IsEmpty()) {
+        return nullptr;
+    }
+
+    v8::Local<v8::Object> jsobj = v8::Local<v8::Object>::Cast(ret.ToLocalChecked());
+    return Object::_createJSObject(nullptr, jsobj);
+}
+
 bool Object::init(Class *cls, v8::Local<v8::Object> obj) {
     _cls = cls;
 

native/cocos/bindings/jswrapper/v8/Object.h

@@ -152,6 +152,14 @@ class Object final : public RefCounter {
      *  @note The return value (non-null) has to be released manually.
      */
     static Object *createJSONObject(const ccstd::string &jsonStr);
+
+    /**
+     *  @brief Creates a JavaScript Object from a JSON formatted string.
+     *  @param[in] jsonStr The utf-16 string containing the JSON string to be parsed.
+     *  @return A JavaScript Object containing the parsed value, or nullptr if the input is invalid.
+     *  @note The return value (non-null) has to be released manually. In order to avoid memory copy, use std::u16string reference directly without const, after this method is invoked, jsonStr will be empty since it was moved.
+     */
+    static Object *createJSONObject(std::u16string &jsonStr);
 
     /**
      *  @brief Creates a JavaScript Native Binding Object from an existing se::Class instance.

native/cocos/bindings/jswrapper/v8/Utils.h

@@ -50,6 +50,30 @@ void setPrivate(v8::Isolate *isolate, ObjectWrap &wrap, Object *obj);
 Object *getPrivate(v8::Isolate *isolate, v8::Local<v8::Value> value);
 void clearPrivate(v8::Isolate *isolate, ObjectWrap &wrap);
 
+class ExternalStringResource : public v8::String::ExternalStringResource {
+public:
+    explicit ExternalStringResource(std::u16string &s)
+    : _s(std::move(s)) {
+
+    }
+
+    ~ExternalStringResource() override = default;
+
+    const uint16_t* data() const override {
+        return reinterpret_cast<const uint16_t*>(_s.data());
+    }
+
+    size_t length() const override {
+        return _s.length();
+    }
+
+    void Dispose() override {
+        delete this;
+    }
+private:
+    std::u16string _s;
+};
+
 } // namespace internal
 } // namespace se
 

native/cocos/bindings/manual/jsb_cocos_manual.cpp

@@ -24,11 +24,14 @@
 
 #include "jsb_cocos_manual.h"
 
+#include "base/ThreadPool.h"
+#include "base/UTF8.h"
+
 #include "bindings/manual/jsb_global.h"
-#include "cocos/bindings/auto/jsb_cocos_auto.h"
-#include "cocos/bindings/jswrapper/SeApi.h"
-#include "cocos/bindings/manual/jsb_conversions.h"
-#include "cocos/bindings/manual/jsb_global_init.h"
+#include "bindings/auto/jsb_cocos_auto.h"
+#include "bindings/jswrapper/SeApi.h"
+#include "bindings/manual/jsb_conversions.h"
+#include "bindings/manual/jsb_global_init.h"
 
 #include "application/ApplicationManager.h"
 #include "platform/interfaces/modules/ISystemWindowManager.h"
@@ -672,8 +675,127 @@ static bool js_se_setExceptionCallback(se::State &s) { // NOLINT(readability-ide
 }
 SE_BIND_FUNC(js_se_setExceptionCallback) // NOLINT(readability-identifier-naming)
 
+template<typename T, bool isJson = false>
+static bool js_readFile(se::State &s) { // NOLINT
+    const auto &args = s.args();
+    size_t argc = args.size();
+    CC_UNUSED bool ok = true;
+    if (argc == 2) {
+        ccstd::string path;
+        ok &= sevalue_to_native(args[0], &path);
+        SE_PRECONDITION2(ok, false, "Error processing arguments");
+
+        const auto &callbackVal = args[1];
+        CC_ASSERT(callbackVal.isObject());
+        CC_ASSERT(callbackVal.toObject()->isFunction());
+
+
+        if (path.empty()) {
+            se::ValueArray seArgs;
+            seArgs.reserve(2);
+            seArgs.emplace_back(se::Value("Path is empty"));
+            seArgs.emplace_back(se::Value::Null);
+            callbackVal.toObject()->call(seArgs, nullptr);
+            return true;
+        }
+
+        std::shared_ptr<se::Value> callbackPtr = std::make_shared<se::Value>(callbackVal);
+
+        // fullPathForFilename is not threadsafe, so don't invoke it in thread pool.
+        ccstd::string fullPath = cc::FileUtils::getInstance()->fullPathForFilename(path);
+
+        gIOThreadPool->pushTask([fullPath, callbackPtr](int/* tid */) {
+            auto *fs = cc::FileUtils::getInstance();
+            if (fs == nullptr) {
+                return;
+            }
+
+            auto app = CC_CURRENT_APPLICATION();
+            if (!app) {
+                return;
+            }
+
+            auto content = std::make_shared<T>();
+            fs->getContents(fullPath, content.get());
+
+            auto engine = app->getEngine();
+            if (!engine) {
+                return;
+            }
+
+            std::shared_ptr<std::u16string> u16str;
+            // TODO(cjh): OpenHarmony NAPI support
+#if SCRIPT_ENGINE_TYPE != SCRIPT_ENGINE_NAPI
+            if constexpr (std::is_same_v<T, ccstd::string> && isJson) {
+                u16str = std::make_shared<std::u16string>();
+                if (!cc::StringUtils::UTF8ToUTF16(*content, *u16str)) {
+                    CC_LOG_ERROR("UTF8ToUTF16 failed, file: %s", fullPath.c_str());
+                }
+            }
+#endif
+
+            engine->getScheduler()->performFunctionInCocosThread([callbackPtr, content, u16str](){
+                se::AutoHandleScope hs;
+                se::ValueArray seArgs;
+                seArgs.reserve(2);
+
+                if constexpr (std::is_same_v<T, ccstd::string>) {
+                    if constexpr (isJson) {
+#if SCRIPT_ENGINE_TYPE == SCRIPT_ENGINE_NAPI
+                        se::HandleObject jsonObj(se::Object::createJSONObject(*content));
+#else
+                        se::HandleObject jsonObj(se::Object::createJSONObject(*u16str));
+#endif
+                        if (!jsonObj.get()) {
+                            seArgs.emplace_back(se::Value("Parse json failed!"));
+                            seArgs.emplace_back(se::Value::Null);
+                        } else {
+                            seArgs.emplace_back(se::Value::Null);
+                            seArgs.emplace_back(se::Value(jsonObj));
+                        }
+                        callbackPtr->toObject()->call(seArgs, nullptr);
+                    } else {
+                        seArgs.emplace_back(se::Value::Null);
+                        seArgs.emplace_back(se::Value(*content));
+                        callbackPtr->toObject()->call(seArgs, nullptr);
+                    }
+                } else if constexpr (std::is_same_v<T, cc::Data>) {
+                    se::HandleObject dataObj(se::Object::createArrayBufferObject(content->getBytes(), content->getSize()));
+                    seArgs.emplace_back(se::Value::Null);
+                    seArgs.emplace_back(se::Value(dataObj));
+                    callbackPtr->toObject()->call(seArgs, nullptr);
+                }
+            });
+        });
+
+        return true;
+    }
+    SE_REPORT_ERROR("wrong number of arguments: %d, was expecting %d", (int)argc, 2);
+    return false;
+}
+
+static bool js_readTextFile(se::State &s) { // NOLINT
+    return js_readFile<ccstd::string>(s);
+}
+SE_BIND_FUNC(js_readTextFile)
+
+static bool js_readDataFile(se::State &s) { // NOLINT
+    return js_readFile<cc::Data>(s);
+}
+SE_BIND_FUNC(js_readDataFile)
+
+static bool js_readJsonFile(se::State &s) { // NOLINT
+    return js_readFile<ccstd::string, true>(s);
+}
+SE_BIND_FUNC(js_readJsonFile)
+
+
 static bool register_filetuils_ext(se::Object * /*obj*/) { // NOLINT(readability-identifier-naming)
     __jsb_cc_FileUtils_proto->defineFunction("listFilesRecursively", _SE(js_engine_FileUtils_listFilesRecursively));
+    __jsb_cc_FileUtils_proto->defineFunction("readTextFile", _SE(js_readTextFile));
+    __jsb_cc_FileUtils_proto->defineFunction("readDataFile", _SE(js_readDataFile));
+    __jsb_cc_FileUtils_proto->defineFunction("readJsonFile", _SE(js_readJsonFile));
+
     return true;
 }