VisionOS iOS support

[yosun]

Describe the feature

Hi! Is this supported on iOS/VisionOS?

[amakropoulos]

hi! not yet, but I get more and more requests about it.
I have to do an exploration to see how much effort is needed before prioritising it.

[amakropoulos]

For iOS the following adaptations are needed:

iOS build (for @amakropoulos )

• New DLL build of LlamaLib
• adapt the download function for setup of LLMUnity:
  

  LLMUnity/Runtime/LLMUnitySetup.cs

  Lines 393 to 429 in 5f94629

  	static async Task DownloadLibrary()
  	{
  	if (libraryProgress < 1) return;
  	libraryProgress = 0;
  	try
  	{
  	string setupDir = Path.Combine(libraryPath, "setup");
  	Directory.CreateDirectory(setupDir);
  	// setup LlamaLib in StreamingAssets
  	await DownloadAndExtractInsideDirectory(LlamaLibURL, libraryPath, setupDir);
  	// setup LlamaLib in Plugins for Android
  	AssetDatabase.StartAssetEditing();
  	string androidDir = Path.Combine(libraryPath, "android");
  	if (Directory.Exists(androidDir))
  	{
  	string androidPluginsDir = Path.Combine(Application.dataPath, "Plugins", "Android");
  	Directory.CreateDirectory(androidPluginsDir);
  	string pluginDir = Path.Combine(androidPluginsDir, Path.GetFileName(libraryPath));
  	if (Directory.Exists(pluginDir)) Directory.Delete(pluginDir, true);
  	Directory.Move(androidDir, pluginDir);
  	if (File.Exists(androidDir + ".meta")) File.Delete(androidDir + ".meta");
  	}
  	AssetDatabase.StopAssetEditing();
  	// setup LlamaLib extras in StreamingAssets
  	if (FullLlamaLib) await DownloadAndExtractInsideDirectory(LlamaLibExtensionURL, libraryPath, setupDir);
  	}
  	catch (Exception e)
  	{
  	LogError(e.Message);
  	}
  	libraryProgress = 1;
  	}

DLL loader

• create a DLL loader similar to Android:

  LLMUnity/Runtime/LLMLib.cs

  Lines 241 to 274 in 5f94629

  	private static class Android
  	{
  	public static IntPtr dlopen(string path) => dlopen(path, 1);
  	#if UNITY_ANDROID
  	// LoadLibrary for Android
  	[DllImport("__Internal")]
  	public static extern IntPtr dlopen(string filename, int flags);
  	// GetSymbol for Android
  	[DllImport("__Internal")]
  	public static extern IntPtr dlsym(IntPtr handle, string symbol);
  	// FreeLibrary for Android
  	[DllImport("__Internal")]
  	public static extern int dlclose(IntPtr handle);
  	#else
  	public static IntPtr dlopen(string filename, int flags)
  	{
  	return default;
  	}
  	public static IntPtr dlsym(IntPtr handle, string symbol)
  	{
  	return default;
  	}
  	public static int dlclose(IntPtr handle)
  	{
  	return default;
  	}
  	#endif
  	}

• and add it in the LibraryLoader functions:
  

  LLMUnity/Runtime/LLMLib.cs

  Lines 107 to 108 in 5f94629

  	else if (Application.platform == RuntimePlatform.Android)
  	handle = Android.dlopen(libraryName);

  
  

  LLMUnity/Runtime/LLMLib.cs

  Lines 127 to 128 in 5f94629

  	else if (Application.platform == RuntimePlatform.Android)
  	handle = Android.dlsym(library, symbolName);

  
  

  LLMUnity/Runtime/LLMLib.cs

  Lines 146 to 147 in 5f94629

  	else if (Application.platform == RuntimePlatform.Android)
  	Android.dlclose(library);

• add the DLL in the architectures and the DLL path
  

  LLMUnity/Runtime/LLMLib.cs

  Lines 146 to 147 in 5f94629

  	else if (Application.platform == RuntimePlatform.Android)
  	Android.dlclose(library);

  (just "iOS" here)
  

  LLMUnity/Runtime/LLMLib.cs

  Lines 445 to 448 in 5f94629

  	else if (Application.platform == RuntimePlatform.Android)
  	{
  	return "libundreamai_android.so";
  	}

  (the name of the DLL file here)

Model extraction in iOS

• similar extraction functions to Android - if files in the iOS build can't be accessed directly:
  

  LLMUnity/Runtime/LLMUnitySetup.cs

  Lines 271 to 326 in 5f94629

  	public static async Task AndroidExtractFile(string assetName, bool overwrite = false, bool log = true, int chunkSize = 1024*1024)
  	{
  	Task extractionTask;
  	lock (lockObject)
  	{
  	if (!androidExtractTasks.TryGetValue(assetName, out extractionTask))
  	{
  	extractionTask = AndroidExtractFileOnce(assetName, overwrite, log, chunkSize);
  	androidExtractTasks[assetName] = extractionTask;
  	}
  	}
  	await extractionTask;
  	}
  	public static async Task AndroidExtractFileOnce(string assetName, bool overwrite = false, bool log = true, int chunkSize = 1024*1024)
  	{
  	string source = "jar:file://" + Application.dataPath + "!/assets/" + assetName;
  	string target = GetAssetPath(assetName);
  	if (!overwrite && File.Exists(target))
  	{
  	if (log) Log($"File {target} already exists");
  	return;
  	}
  	Log($"Extracting {source} to {target}");
  	// UnityWebRequest to read the file from StreamingAssets
  	UnityWebRequest www = UnityWebRequest.Get(source);
  	// Send the request and await its completion
  	var operation = www.SendWebRequest();
  	while (!operation.isDone) await Task.Delay(1);
  	if (www.result != UnityWebRequest.Result.Success)
  	{
  	LogError("Failed to load file from StreamingAssets: " + www.error);
  	}
  	else
  	{
  	byte[] buffer = new byte[chunkSize];
  	using (Stream responseStream = new MemoryStream(www.downloadHandler.data))
  	using (FileStream fileStream = new FileStream(target, FileMode.Create, FileAccess.Write))
  	{
  	int bytesRead;
  	while ((bytesRead = await responseStream.ReadAsync(buffer, 0, buffer.Length)) > 0)
  	{
  	await fileStream.WriteAsync(buffer, 0, bytesRead);
  	}
  	}
  	}
  	}
  	public static async Task AndroidExtractAsset(string path, bool overwrite = false)
  	{
  	if (Application.platform != RuntimePlatform.Android) return;
  	await AndroidExtractFile(Path.GetFileName(path), overwrite);
  	}

• and use them here:
  

  LLMUnity/Runtime/LLMManager.cs

  Lines 98 to 153 in 5f94629

  	public static async Task<bool> SetupOnce()
  	{
  	await LLMUnitySetup.AndroidExtractAsset(LLMUnitySetup.LLMManagerPath, true);
  	LoadFromDisk();
  	List<StringPair> downloads = new List<StringPair>();
  	foreach (ModelEntry modelEntry in modelEntries)
  	{
  	string target = LLMUnitySetup.GetAssetPath(modelEntry.filename);
  	if (File.Exists(target)) continue;
  	if (!downloadOnStart || string.IsNullOrEmpty(modelEntry.url))
  	{
  	await LLMUnitySetup.AndroidExtractFile(modelEntry.filename);
  	if (!File.Exists(target)) LLMUnitySetup.LogError($"Model {modelEntry.filename} could not be found!");
  	}
  	else
  	{
  	downloads.Add(new StringPair {source = modelEntry.url, target = target});
  	}
  	}
  	if (downloads.Count == 0) return true;
  	try
  	{
  	downloadProgress = 0;
  	totalSize = 0;
  	completedSize = 0;
  	ResumingWebClient client = new ResumingWebClient();
  	Dictionary<string, long> fileSizes = new Dictionary<string, long>();
  	foreach (StringPair pair in downloads)
  	{
  	long size = client.GetURLFileSize(pair.source);
  	fileSizes[pair.source] = size;
  	totalSize += size;
  	}
  	foreach (StringPair pair in downloads)
  	{
  	currFileSize = fileSizes[pair.source];
  	await LLMUnitySetup.DownloadFile(pair.source, pair.target, false, null, SetDownloadProgress);
  	await LLMUnitySetup.AndroidExtractFile(Path.GetFileName(pair.target));
  	completedSize += currFileSize;
  	}
  	completedSize = totalSize;
  	SetDownloadProgress(0);
  	}
  	catch (Exception ex)
  	{
  	LLMUnitySetup.LogError($"Error downloading the models: {ex.Message}");
  	return false;
  	}
  	return true;
  	}

  
  

  LLMUnity/Runtime/LLMCharacter.cs

  Line 355 in 5f94629

  await LLMUnitySetup.AndroidExtractAsset(path, true);

• use persistentDataPath similarly to Android here:

  LLMUnity/Runtime/LLMUnitySetup.cs

  Lines 205 to 210 in 5f94629

  	public static string GetAssetPath(string relPath = "")
  	{
  	// Path to store llm server binaries and models
  	string assetsDir = Application.platform == RuntimePlatform.Android ? Application.persistentDataPath : Application.streamingAssetsPath;
  	return Path.Combine(assetsDir, relPath).Replace('\\', '/');
  	}

Runtime

• check if iOS can run with all threads. in Android I have to limit it to the big cores only otherwise it takes forever:
  

  LLMUnity/Runtime/LLM.cs

  Line 401 in 5f94629

  if (Application.platform == RuntimePlatform.Android && numThreads <= 0) numThreadsToUse = LLMUnitySetup.AndroidGetNumBigCores();

  
  The big cores calculation is here:
  

  LLMUnity/Runtime/LLMUnitySetup.cs

  Lines 543 to 591 in 5f94629

  	public static int AndroidGetNumBigCores()
  	{
  	int maxFreqKHzMin = int.MaxValue;
  	int maxFreqKHzMax = 0;
  	List<int> cpuMaxFreqKHz = new List<int>();
  	List<bool> cpuIsSmtCpu = new List<bool>();
  	try
  	{
  	string cpuPath = "/sys/devices/system/cpu/";
  	int coreIndex;
  	if (Directory.Exists(cpuPath))
  	{
  	foreach (string cpuDir in Directory.GetDirectories(cpuPath))
  	{
  	string dirName = Path.GetFileName(cpuDir);
  	if (!dirName.StartsWith("cpu")) continue;
  	if (!int.TryParse(dirName.Substring(3), out coreIndex)) continue;
  	int maxFreqKHz = GetMaxFreqKHz(coreIndex);
  	cpuMaxFreqKHz.Add(maxFreqKHz);
  	if (maxFreqKHz > maxFreqKHzMax) maxFreqKHzMax = maxFreqKHz;
  	if (maxFreqKHz < maxFreqKHzMin) maxFreqKHzMin = maxFreqKHz;
  	cpuIsSmtCpu.Add(IsSmtCpu(coreIndex));
  	}
  	}
  	}
  	catch (Exception e)
  	{
  	LogError(e.Message);
  	}
  	int numBigCores = 0;
  	int numCores = SystemInfo.processorCount;
  	int maxFreqKHzMedium = (maxFreqKHzMin + maxFreqKHzMax) / 2;
  	if (maxFreqKHzMedium == maxFreqKHzMax) numBigCores = numCores;
  	else
  	{
  	for (int i = 0; i < cpuMaxFreqKHz.Count; i++)
  	{
  	if (cpuIsSmtCpu[i] || cpuMaxFreqKHz[i] >= maxFreqKHzMedium) numBigCores++;
  	}
  	}
  	if (numBigCores == 0) numBigCores = SystemInfo.processorCount / 2;
  	else numBigCores = Math.Min(numBigCores, SystemInfo.processorCount);
  	return numBigCores;
  	}