From 8e171c785d25f2479c277ae3fa2351c8fbcc1c63 Mon Sep 17 00:00:00 2001 From: Chester Liu <4710575+skyline75489@users.noreply.github.com> Date: Thu, 12 Dec 2024 11:50:03 +0800 Subject: [PATCH] Initialize Objective-C bindings (#709) This PR contains Objective-C wrapper code, podspec generation infras and the accompanying test code. Internal workitem: https://task.ms/aii/35199 --------- Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com> --- .pipelines/macos-ios-cocoapods-publishing.yml | 22 ++ .pipelines/stages/jobs/capi-packaging-job.yml | 1 + .../macos-ios-cocoapods-packaging-job.yml | 40 ++ .../stages/jobs/nuget-packaging-job.yml | 2 + .../jobs/steps/capi-appleframework-step.yml | 9 +- .../macos-ios-cocoapods-packaging-stage.yml | 15 + build.py | 1 + src/objectivec/cxx_api.h | 17 + src/objectivec/error_utils.h | 29 ++ src/objectivec/error_utils.mm | 34 ++ src/objectivec/include/ort_genai_objc.h | 355 ++++++++++++++++++ src/objectivec/oga_generator.mm | 95 +++++ src/objectivec/oga_generator_params.mm | 60 +++ src/objectivec/oga_images.mm | 35 ++ src/objectivec/oga_internal.h | 64 ++++ src/objectivec/oga_model.mm | 28 ++ src/objectivec/oga_multi_modal_processor.mm | 50 +++ src/objectivec/oga_named_tensors.mm | 25 ++ src/objectivec/oga_sequences.mm | 54 +++ src/objectivec/oga_tensor.mm | 55 +++ src/objectivec/oga_tokenizer.mm | 50 +++ src/objectivec/oga_tokenizer_stream.mm | 44 +++ src/objectivec/test/assertion_utils.h | 46 +++ src/objectivec/test/ort_genai_api_test.mm | 134 +++++++ .../apple/build_and_assemble_apple_pods.py | 218 +++++++++++ .../github/apple/c/assemble_c_pod_package.py | 163 ++++++++ .../github/apple/c/c.podspec.template | 29 ++ .../apple/c/onnxruntime-genai-c.config.json | 5 + ...ulator_apple_framework_build_settings.json | 21 ++ .../github/apple/get_simulator_device_info.py | 165 ++++++++ .../objectivec/assemble_objc_pod_package.py | 185 +++++++++ .../apple/objectivec/objc.podspec.template | 53 +++ .../onnxruntime-genai-objc.config.json | 5 + .../github/apple/package_assembly_utils.py | 173 +++++++++ .../github/apple/test_apple_packages.py | 299 +++++++++++++++ 35 files changed, 2579 insertions(+), 2 deletions(-) create mode 100644 .pipelines/macos-ios-cocoapods-publishing.yml create mode 100644 .pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml create mode 100644 .pipelines/stages/macos-ios-cocoapods-packaging-stage.yml create mode 100644 src/objectivec/cxx_api.h create mode 100644 src/objectivec/error_utils.h create mode 100644 src/objectivec/error_utils.mm create mode 100644 src/objectivec/include/ort_genai_objc.h create mode 100644 src/objectivec/oga_generator.mm create mode 100644 src/objectivec/oga_generator_params.mm create mode 100644 src/objectivec/oga_images.mm create mode 100644 src/objectivec/oga_internal.h create mode 100644 src/objectivec/oga_model.mm create mode 100644 src/objectivec/oga_multi_modal_processor.mm create mode 100644 src/objectivec/oga_named_tensors.mm create mode 100644 src/objectivec/oga_sequences.mm create mode 100644 src/objectivec/oga_tensor.mm create mode 100644 src/objectivec/oga_tokenizer.mm create mode 100644 src/objectivec/oga_tokenizer_stream.mm create mode 100644 src/objectivec/test/assertion_utils.h create mode 100644 src/objectivec/test/ort_genai_api_test.mm create mode 100755 tools/ci_build/github/apple/build_and_assemble_apple_pods.py create mode 100644 tools/ci_build/github/apple/c/assemble_c_pod_package.py create mode 100644 tools/ci_build/github/apple/c/c.podspec.template create mode 100644 tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json create mode 100644 tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json create mode 100755 tools/ci_build/github/apple/get_simulator_device_info.py create mode 100755 tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py create mode 100644 tools/ci_build/github/apple/objectivec/objc.podspec.template create mode 100644 tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json create mode 100644 tools/ci_build/github/apple/package_assembly_utils.py create mode 100644 tools/ci_build/github/apple/test_apple_packages.py diff --git a/.pipelines/macos-ios-cocoapods-publishing.yml b/.pipelines/macos-ios-cocoapods-publishing.yml new file mode 100644 index 000000000..b4dc0be3d --- /dev/null +++ b/.pipelines/macos-ios-cocoapods-publishing.yml @@ -0,0 +1,22 @@ +parameters: +- name: ort_version + displayName: 'OnnxRuntime version' + type: string + default: '1.18.0' + +- name: build_config + displayName: 'Build Configuration' + type: string + default: 'Release' + values: + - 'Debug' + - 'RelWithDebInfo' + - 'Release' + - 'MinSizeRel' + +trigger: none +stages: +- template: stages/macos-ios-cocoapods-packaging-stage.yml + parameters: + ort_version: ${{ parameters.ort_version }} + build_config: ${{ parameters.build_config }} diff --git a/.pipelines/stages/jobs/capi-packaging-job.yml b/.pipelines/stages/jobs/capi-packaging-job.yml index e837a070c..885abd574 100644 --- a/.pipelines/stages/jobs/capi-packaging-job.yml +++ b/.pipelines/stages/jobs/capi-packaging-job.yml @@ -111,5 +111,6 @@ jobs: - template: steps/capi-appleframework-step.yml parameters: build_config: ${{ parameters.build_config }} + build_settings_file: "tools/ci_build/github/apple/default_full_ios_framework_build_settings.json" - template: steps/compliant-and-cleanup-step.yml diff --git a/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml new file mode 100644 index 000000000..e6bc458a8 --- /dev/null +++ b/.pipelines/stages/jobs/macos-ios-cocoapods-packaging-job.yml @@ -0,0 +1,40 @@ +parameters: +- name: build_config + type: string +- name: ort_version + type: string + +jobs: +- job: macos_ios_cocoapods_packaging + pool: + vmImage: 'macOS-latest' + variables: + buildSettingsFile: "tools/ci_build/github/apple/default_full_apple_framework_build_settings.json" + + workspace: + clean: all + steps: + - checkout: self + clean: true + submodules: none + + - template: steps/utils/set-genai-version.yml + - template: steps/utils/set-cmake-build-type.yml + parameters: + build_config: ${{parameters.build_config}} + + + - task: CmdLine@2 + displayName: Build MacOS & iOS CocoaPods Packages + inputs: + script: | + set -e -x + python tools/ci_build/github/apple/build_and_assemble_apple_pods.py \ + --build-dir "$(Build.BinariesDirectory)/apple_framework" \ + --staging-dir "$(Build.BinariesDirectory)/staging" \ + --pod-version "$(genai_version)" \ + --test \ + --variant Full \ + --build-settings-file "${{ variables.buildSettingsFile }}" \ + --build-apple-framework-args==--config=$(cmake_build_type) \ + --ort-version ${{parameters.ort_version}} diff --git a/.pipelines/stages/jobs/nuget-packaging-job.yml b/.pipelines/stages/jobs/nuget-packaging-job.yml index 1be6a9cd2..5d2a33b12 100644 --- a/.pipelines/stages/jobs/nuget-packaging-job.yml +++ b/.pipelines/stages/jobs/nuget-packaging-job.yml @@ -179,6 +179,8 @@ jobs: - template: steps/utils/set-genai-version.yml + - task: NuGetAuthenticate@1 + - powershell: | dotnet --info dotnet build Microsoft.ML.OnnxRuntimeGenAI.csproj -p:Configuration="$(buildConfig)" -p:IncludeMobileTargets=true --verbosity normal diff --git a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml index 1970812ed..4dcf32a8f 100644 --- a/.pipelines/stages/jobs/steps/capi-appleframework-step.yml +++ b/.pipelines/stages/jobs/steps/capi-appleframework-step.yml @@ -2,7 +2,8 @@ parameters: - name: build_config type: string default: 'release' - +- name: build_settings_file + type: string steps: - checkout: self @@ -11,6 +12,9 @@ steps: submodules: recursive - template: utils/set-genai-version.yml +- template: utils/set-cmake-build-type.yml + parameters: + build_config: ${{parameters.build_config}} - template: utils/set-nightly-build-option-variable.yml @@ -19,7 +23,8 @@ steps: python3 -m pip install requests python3 tools/ci_build/github/apple/build_apple_framework.py \ --build_dir "$(Build.BinariesDirectory)/apple_framework" \ - tools/ci_build/github/apple/default_full_ios_framework_build_settings.json + --config $(cmake_build_type) \ + "${{ parameters.build_settings_file }}" mkdir $(Build.BinariesDirectory)/artifacts mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-genai.xcframework diff --git a/.pipelines/stages/macos-ios-cocoapods-packaging-stage.yml b/.pipelines/stages/macos-ios-cocoapods-packaging-stage.yml new file mode 100644 index 000000000..0e3b27ac8 --- /dev/null +++ b/.pipelines/stages/macos-ios-cocoapods-packaging-stage.yml @@ -0,0 +1,15 @@ +parameters: +- name: ort_version + type: string + +- name: build_config + type: string + default: 'Release' + +stages: +- stage: Build_MacOS_iOS_CocoaPods_Archive + jobs: + - template: jobs/macos-ios-cocoapods-packaging-job.yml + parameters: + ort_version: ${{ parameters.ort_version }} + build_config: ${{ parameters.build_config }} diff --git a/build.py b/build.py index 8a750d776..34619e839 100644 --- a/build.py +++ b/build.py @@ -505,6 +505,7 @@ def update(args: argparse.Namespace, env: dict[str, str]): "-DENABLE_PYTHON=OFF", "-DENABLE_TESTS=OFF", "-DENABLE_MODEL_BENCHMARK=OFF", + "-DCMAKE_OSX_DEPLOYMENT_TARGET=" + args.apple_deploy_target, f"-DBUILD_APPLE_FRAMEWORK={'ON' if args.build_apple_framework else 'OFF'}", "-DPLATFORM_NAME=" + platform_name, ] diff --git a/src/objectivec/cxx_api.h b/src/objectivec/cxx_api.h new file mode 100644 index 000000000..3e423c17e --- /dev/null +++ b/src/objectivec/cxx_api.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// wrapper for ORT GenAI C/C++ API headers + +#if defined(__clang__) +#pragma clang diagnostic push +// ignore clang documentation-related warnings +// instead, we will rely on Doxygen warnings for the C/C++ API headers +#pragma clang diagnostic ignored "-Wdocumentation" +#endif // defined(__clang__) + +#include "ort_genai.h" + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif // defined(__clang__) diff --git a/src/objectivec/error_utils.h b/src/objectivec/error_utils.h new file mode 100644 index 000000000..8c73faa97 --- /dev/null +++ b/src/objectivec/error_utils.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +#include + +#import "cxx_api.h" + +NS_ASSUME_NONNULL_BEGIN + +extern NSString* const kOgaErrorDomain; + +void OGASaveCodeAndDescriptionToError(int code, const char* description, NSError** error); +void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** error); +void OGASaveExceptionToError(const std::exception& e, NSError** error); + +// helper macros to catch and handle C++ exceptions +#define OGA_OBJC_API_IMPL_CATCH(error, failure_return_value) \ + catch (const std::exception& e) { \ + OGASaveExceptionToError(e, (error)); \ + return (failure_return_value); \ + } + +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) OGA_OBJC_API_IMPL_CATCH(error, NO) + +#define OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) OGA_OBJC_API_IMPL_CATCH(error, nil) + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/error_utils.mm b/src/objectivec/error_utils.mm new file mode 100644 index 000000000..9bed15320 --- /dev/null +++ b/src/objectivec/error_utils.mm @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" + +NS_ASSUME_NONNULL_BEGIN + +NSString* const kOgaErrorDomain = @"onnxruntime-genai"; +const int kOgaErrorCode = 0x0A; + +void OGASaveCodeAndDescriptionToError(int code, const char* descriptionCstr, NSError** error) { + if (!error) return; + + NSString* description = [NSString stringWithCString:descriptionCstr + encoding:NSUTF8StringEncoding]; + + *error = [NSError errorWithDomain:kOgaErrorDomain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; +} + +void OGASaveCodeAndDescriptionToError(int code, NSString* description, NSError** error) { + if (!error) return; + + *error = [NSError errorWithDomain:kOgaErrorDomain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; +} + +void OGASaveExceptionToError(const std::exception& e, NSError** error) { + OGASaveCodeAndDescriptionToError(kOgaErrorCode, e.what(), error); +} + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/include/ort_genai_objc.h b/src/objectivec/include/ort_genai_objc.h new file mode 100644 index 000000000..e17e61c3a --- /dev/null +++ b/src/objectivec/include/ort_genai_objc.h @@ -0,0 +1,355 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// GenAI Objective-C API +// +// This is a wrapper around the C++ API, and provides for a set of Objective-C classes with automatic resource management + +/* A simple end to end example of how to generate an answer from a prompt: + * + * OGAModel* model = [[OGAModel alloc] initWithPath:path error:&error]; + * OGATokenizer* tokenizer = [[OGATokenizer alloc] initWithModel:model error:&error]; + * + * OGASequences* sequences = [tokenizer encode:@"A great recipe for Kung Pao chicken is " error:&error]; + * + * OGAGeneratorParams* params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + * [params setInputSequences:sequences error:&error]; + * [params setSearchOption:@"max_length" doubleValue:200 error:&error]; + * + * OGASequences* output_sequences = [model generate:params error:&error]; + * NSString* out_string = [tokenizer decode:[output_sequences sequenceDataAtIndex:0] length:[output_sequences sequenceCountAtIndex:0] error:&error]; + * + */ + +NS_ASSUME_NONNULL_BEGIN + +@class OGATensor; +@class OGASequences; +@class OGANamedTensors; +@class OGAGeneratorParams; +@class OGATokenizerStream; +@class OGAMultiModalProcessor; + +typedef NS_ENUM(NSInteger, OGAElementType) { + OGAElementTypeUndefined, + OGAElementTypeFloat32, // maps to c type float + OGAElementTypeUint8, // maps to c type uint8_t + OGAElementTypeInt8, // maps to c type int8_t + OGAElementTypeUint16, // maps to c type uint16_t + OGAElementTypeInt16, // maps to c type int16_t + OGAElementTypeInt32, // maps to c type int32_t + OGAElementTypeInt64, // maps to c type int64_t + OGAElementTypeString, // string type (not currently supported by Oga) + OGAElementTypeBool, // maps to c type bool + OGAElementTypeFloat16, // IEEE 752-2008 binary16 format, 1 sign bit, 5 bit exponent, 10 bit fraction + OGAElementTypeFloat64, // maps to c type double + OGAElementTypeUint32, // maps to c type uint32_t + OGAElementTypeUint64, // maps to c type uint64_t +}; + +/** + * An ORT GenAI model. + */ +@interface OGAModel : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a model. + * + * @param path The path to the ONNX GenAI model folder. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithPath:(NSString*)path + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +@end + +/** + * An ORT GenAI tokenizer. + */ +@interface OGATokenizer : NSObject +- (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a tokenizer. + * + * @param model The model to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +/** + * Encode text to sequences + * + * @param str The text to be encoded. + * @param error Optional error information set if an error occurs. + * @return The encoding result, or nil if an error occurs. + */ +- (nullable OGASequences*)encode:(NSString*)str + error:(NSError**)error; + +/** + * Decode sequences to text + * + * @param tokensData The sequences data to be encoded. + * @param tokensLength The length of the sequences data to be encoded. + * @param error Optional error information set if an error occurs. + * @return The decoding result, or nil if an error occurs. + */ +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error; + +@end + +/** + * A tokenizer stream is used to decode token strings incrementally, one token at a time. + */ +@interface OGATokenizerStream : NSObject +- (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a tokenizer stream with an underlying tokenizer. + * + * @param tokenizer The underlying tokenizer to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +/** + * Creates a tokenizer stream with a multi modal processor + * + * @param processor The underlying processor to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor + error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Decode one token. + * + * @param token The token to be decoded. + * @param error Optional error information set if an error occurs. + * @return The decoding result, or nil if an error occurs. + */ +- (nullable NSString*)decode:(int32_t)token + error:(NSError**)error; +@end + +/** + * A series of generated sequences + */ +@interface OGASequences : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +/** + * The count of generated sequences + * @param error Optional error information set if an error occurs. + * @return The count of sequences, or size_t(-1) if an error occurs. + */ +- (size_t)getCountWithError:(NSError**)error NS_SWIFT_NAME(count()); + +/** + * Retrieve the sequence data at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence data at the given index, or nil if an error occurs. + */ +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error; + +/** + * Retrieve the sequence count at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence count at the given index, or size_t(-1) if an error occurs. + */ +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error; + +@end + +/** + * The parameters for generation. + */ +@interface OGAGeneratorParams : NSObject +- (instancetype)init NS_UNAVAILABLE; + +/** + * Creates a GeneratorParams from the given model. + * @param model The model to use for generation. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +/** + * Set input with NamedTensors type. + * @param namedTensors The named tensors. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)setInputs:(OGANamedTensors*)namedTensors + error:(NSError**)error; + +/** + * Set input with name and corresponding tensor. + * @param name The input name. + * @param tensor The tensor. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)setModelInput:(NSString*)name + tensor:(OGATensor*)tensor + error:(NSError**)error; + +/** + * Set double option value. + * @param key The option key. + * @param value The option value. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)setSearchOption:(NSString*)key + doubleValue:(double)value + error:(NSError**)error; +/** + * Set boolean option value. + * @param key The option key. + * @param value The option value. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)setSearchOption:(NSString*)key + boolValue:(BOOL)value + error:(NSError**)error; +@end + +/** + * The main generator interface that can be used for generation loop. + */ +@interface OGAGenerator : NSObject + +- (instancetype)init NS_UNAVAILABLE; +/** + * Creates a generator. + * + * @param model The model to use. + * @param params The generation params to use. + * @param error Optional error information set if an error occurs. + * @return The instance, or nil if an error occurs. + */ +- (nullable instancetype)initWithModel:(OGAModel*)model + params:(OGAGeneratorParams*)params + error:(NSError**)error NS_DESIGNATED_INITIALIZER; +/** + * Whether generation is done. + * @param error Optional error information set if an error occurs. + * @return The result, or false if an error occurs. + */ +- (BOOL)isDoneWithError:(NSError**)error __attribute__((swift_error(nonnull_error))); + +/** + * Appends token sequences to the generator. + * @param sequences The sequences to append. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)appendTokenSequences:(OGASequences*)sequences error:(NSError**)error; + +/** + * Appends token sequences to the generator. + * @param tokens The tokens to append. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error; + +/** + * Rewinds the generator to the given length. + * @param newLength The desired length in tokens after rewinding. + * @param error Optional error information set if an error occurs. + */ +- (BOOL)rewindTo:(size_t)newLength error:(NSError**)error; + +/** + * Generate next token + * @param error Optional error information set if an error occurs. + */ +- (BOOL)generateNextTokenWithError:(NSError**)error; +/** + * Get the output tensor. + * @param name The output name. + * @param error Optional error information set if an error occurs. + * @return The result, or nil if an error occurs. + */ +- (nullable OGATensor*)getOutput:(NSString*)name error:(NSError**)error; + +/** + * Retrieve the sequence data at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence data at the given index, or nil if an error occurs. + */ +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index + error:(NSError**)error; + +/** + * Retrieve the sequence count at the given index. + * @param index The index needed. + * @param error Optional error information set if an error occurs. + * @return The sequence count at the given index, or size_t(-1) if an error occurs. + */ +- (size_t)sequenceCountAtIndex:(size_t)index + error:(NSError**)error; + +/** + * Clean up the resource before process exits. + */ ++ (void)shutdown; + +@end + +@interface OGATensor : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithDataPointer:(void*)data + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error; +- (OGAElementType)getTypeWithError:(NSError**)error NS_SWIFT_NAME(type()); +- (nullable void*)getDataPointerWithError:(NSError**)error NS_SWIFT_NAME(dataPointer()); + +@end + +@interface OGANamedTensors : NSObject + +- (instancetype)init NS_UNAVAILABLE; + +@end + +@interface OGAImages : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithPath:(NSArray*)paths + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +@end + +@interface OGAMultiModalProcessor : NSObject + +- (instancetype)init NS_UNAVAILABLE; +- (nullable instancetype)initWithModel:(OGAModel*)model + error:(NSError**)error NS_DESIGNATED_INITIALIZER; + +- (nullable OGANamedTensors*)processImages:(NSString*)prompt + images:(OGAImages*)images + error:(NSError**)error; + +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error; +@end + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/oga_generator.mm b/src/objectivec/oga_generator.mm new file mode 100644 index 000000000..aefcd7be0 --- /dev/null +++ b/src/objectivec/oga_generator.mm @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGAGenerator { + std::unique_ptr _generator; +} + +- (nullable instancetype)initWithModel:(OGAModel*)model + params:(OGAGeneratorParams*)params + error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generator = OgaGenerator::Create([model CXXAPIOgaModel], [params CXXAPIOgaGeneratorParams]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (BOOL)isDoneWithError:(NSError**)error { + try { + return _generator->IsDone(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)appendTokenSequences:(OGASequences*)sequences error:(NSError**)error { + try { + _generator->AppendTokenSequences([sequences CXXAPIOgaSequences]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)appendTokens:(NSArray*)tokens error:(NSError**)error { + try { + std::vector cxxTokens; + for (NSNumber* object in tokens) { + cxxTokens.push_back([object intValue]); + } + + _generator->AppendTokens(cxxTokens.data(), cxxTokens.size()); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)rewindTo:(size_t)newLength error:(NSError**)error { + try { + _generator->RewindTo(newLength); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)generateNextTokenWithError:(NSError**)error { + try { + _generator->GenerateNextToken(); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (nullable OGATensor*)getOutput:(NSString*)name error:(NSError**)error { + try { + std::unique_ptr output = _generator->GetOutput([name UTF8String]); + return [[OGATensor alloc] initWithCXXPointer:std::move(output)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index error:(NSError**)error { + try { + return _generator->GetSequenceData(index); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error { + try { + return _generator->GetSequenceCount(index); + } + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) +} + ++ (void)shutdown { + OgaShutdown(); +} +@end diff --git a/src/objectivec/oga_generator_params.mm b/src/objectivec/oga_generator_params.mm new file mode 100644 index 000000000..3e3b6050d --- /dev/null +++ b/src/objectivec/oga_generator_params.mm @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGAGeneratorParams { + std::unique_ptr _generatorParams; +} + +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _generatorParams = OgaGeneratorParams::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (BOOL)setInputs:(OGANamedTensors*)namedTensors error:(NSError**)error { + try { + _generatorParams->SetInputs([namedTensors CXXAPIOgaNamedTensors]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)setModelInput:(NSString*)name tensor:(OGATensor*)tensor error:(NSError**)error { + try { + _generatorParams->SetModelInput([name UTF8String], [tensor CXXAPIOgaTensor]); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)setSearchOption:(NSString*)key doubleValue:(double)value error:(NSError**)error { + try { + _generatorParams->SetSearchOption([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (BOOL)setSearchOption:(NSString*)key boolValue:(BOOL)value error:(NSError**)error { + try { + _generatorParams->SetSearchOptionBool([key UTF8String], value); + return YES; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_BOOL(error) +} + +- (OgaGeneratorParams&)CXXAPIOgaGeneratorParams { + return *(_generatorParams.get()); +} + +@end diff --git a/src/objectivec/oga_images.mm b/src/objectivec/oga_images.mm new file mode 100644 index 000000000..8b338f958 --- /dev/null +++ b/src/objectivec/oga_images.mm @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGAImages { + std::unique_ptr _images; +} + +- (nullable instancetype)initWithPath:(NSArray*)paths error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + std::vector cpp_paths; + cpp_paths.reserve([paths count]); + + for (NSString* path in paths) { + cpp_paths.push_back([path UTF8String]); + } + + _images = OgaImages::Load(cpp_paths); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (OgaImages&)CXXAPIOgaImages { + return *(_images.get()); +} + +@end \ No newline at end of file diff --git a/src/objectivec/oga_internal.h b/src/objectivec/oga_internal.h new file mode 100644 index 000000000..808b02324 --- /dev/null +++ b/src/objectivec/oga_internal.h @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "cxx_api.h" +#import "error_utils.h" +#import "ort_genai_objc.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface OGAModel () + +- (const OgaModel&)CXXAPIOgaModel; + +@end + +@interface OGATokenizer () + +- (const OgaTokenizer&)CXXAPIOgaTokenizer; + +@end + +@interface OGASequences () + +- (nullable instancetype)initWithError:(NSError**)error; +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; + +- (OgaSequences&)CXXAPIOgaSequences; + +@end + +@interface OGAGeneratorParams () + +- (OgaGeneratorParams&)CXXAPIOgaGeneratorParams; + +@end + +@interface OGAImages () + +- (OgaImages&)CXXAPIOgaImages; + +@end + +@interface OGATensor () + +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; + +- (OgaTensor&)CXXAPIOgaTensor; + +@end + +@interface OGANamedTensors () + +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr NS_DESIGNATED_INITIALIZER; +- (OgaNamedTensors&)CXXAPIOgaNamedTensors; + +@end + +@interface OGAMultiModalProcessor () + +- (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor; + +@end + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/oga_model.mm b/src/objectivec/oga_model.mm new file mode 100644 index 000000000..a38414059 --- /dev/null +++ b/src/objectivec/oga_model.mm @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGAModel { + std::unique_ptr _model; +} + +- (nullable instancetype)initWithPath:(NSString*)path error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _model = OgaModel::Create([path UTF8String]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaModel&)CXXAPIOgaModel { + return *(_model.get()); +} + +@end diff --git a/src/objectivec/oga_multi_modal_processor.mm b/src/objectivec/oga_multi_modal_processor.mm new file mode 100644 index 000000000..917e625f2 --- /dev/null +++ b/src/objectivec/oga_multi_modal_processor.mm @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGAMultiModalProcessor { + std::unique_ptr _processor; +} + +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _processor = OgaMultiModalProcessor::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable OGANamedTensors*)processImages:(NSString*)prompt + images:(OGAImages*)images + error:(NSError**)error { + try { + OGANamedTensors* result = [[OGANamedTensors alloc] + initWithCXXPointer:_processor->ProcessImages([prompt UTF8String], + &[images CXXAPIOgaImages])]; + return result; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error { + try { + OgaString result = _processor->Decode(tokensData, tokensLength); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaMultiModalProcessor&)CXXAPIOgaMultiModalProcessor { + return *(_processor.get()); +} + +@end diff --git a/src/objectivec/oga_named_tensors.mm b/src/objectivec/oga_named_tensors.mm new file mode 100644 index 000000000..8243f8403 --- /dev/null +++ b/src/objectivec/oga_named_tensors.mm @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGANamedTensors { + std::unique_ptr _tensor; +} + +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + + _tensor = std::move(ptr); + return self; +} + +- (OgaNamedTensors&)CXXAPIOgaNamedTensors { + return *(_tensor.get()); +} + +@end diff --git a/src/objectivec/oga_sequences.mm b/src/objectivec/oga_sequences.mm new file mode 100644 index 000000000..7dd717b36 --- /dev/null +++ b/src/objectivec/oga_sequences.mm @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGASequences { + std::unique_ptr _sequences; +} + +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + + _sequences = std::move(ptr); + return self; +} + +- (nullable instancetype)initWithError:(NSError**)error { + try { + self = [self initWithCXXPointer:OgaSequences::Create()]; + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (size_t)getCountWithError:(NSError**)error { + try { + return _sequences->Count(); + } + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) +} + +- (nullable const int32_t*)sequenceDataAtIndex:(size_t)index error:(NSError**)error { + try { + return _sequences->SequenceData(index); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (size_t)sequenceCountAtIndex:(size_t)index error:(NSError**)error { + try { + return _sequences->SequenceCount(index); + } + OGA_OBJC_API_IMPL_CATCH(error, size_t(-1)) +} + +- (OgaSequences&)CXXAPIOgaSequences { + return *(_sequences.get()); +} + +@end diff --git a/src/objectivec/oga_tensor.mm b/src/objectivec/oga_tensor.mm new file mode 100644 index 000000000..bd936da06 --- /dev/null +++ b/src/objectivec/oga_tensor.mm @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGATensor { + std::unique_ptr _tensor; +} + +- (instancetype)initWithCXXPointer:(std::unique_ptr)ptr { + if ((self = [super init]) == nil) { + return nil; + } + + _tensor = std::move(ptr); + return self; +} + +- (nullable instancetype)initWithDataPointer:(void*)data + shape:(NSArray*)shape + type:(OGAElementType)elementType + error:(NSError**)error { + try { + std::vector cxxShape; + for (NSNumber* object in shape) { + cxxShape.push_back([object longLongValue]); + } + self = [self initWithCXXPointer:OgaTensor::Create(data, cxxShape.data(), cxxShape.size(), + static_cast(elementType))]; + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (OGAElementType)getTypeWithError:(NSError**)error { + try { + return OGAElementType(_tensor->Type()); + } + OGA_OBJC_API_IMPL_CATCH(error, OGAElementTypeUndefined) +} + +- (nullable void*)getDataPointerWithError:(NSError**)error { + try { + return _tensor->Data(); + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (OgaTensor&)CXXAPIOgaTensor { + return *(_tensor.get()); +} + +@end diff --git a/src/objectivec/oga_tokenizer.mm b/src/objectivec/oga_tokenizer.mm new file mode 100644 index 000000000..c961faabc --- /dev/null +++ b/src/objectivec/oga_tokenizer.mm @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGATokenizer { + std::unique_ptr _tokenizer; +} + +- (nullable instancetype)initWithModel:(OGAModel*)model error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _tokenizer = OgaTokenizer::Create([model CXXAPIOgaModel]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable OGASequences*)encode:(NSString*)str error:(NSError**)error { + OGASequences* sequences = [[OGASequences alloc] initWithError:error]; + if (!sequences) { + return nil; + } + try { + _tokenizer->Encode([str UTF8String], [sequences CXXAPIOgaSequences]); + return sequences; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString*)decode:(const int32_t*)tokensData + length:(size_t)tokensLength + error:(NSError**)error { + try { + OgaString result = _tokenizer->Decode(tokensData, tokensLength); + return [NSString stringWithUTF8String:result]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (const OgaTokenizer&)CXXAPIOgaTokenizer { + return *(_tokenizer.get()); +} + +@end diff --git a/src/objectivec/oga_tokenizer_stream.mm b/src/objectivec/oga_tokenizer_stream.mm new file mode 100644 index 000000000..fb10adf99 --- /dev/null +++ b/src/objectivec/oga_tokenizer_stream.mm @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import "error_utils.h" +#import "oga_internal.h" +#import "ort_genai_objc.h" + +@implementation OGATokenizerStream { + std::unique_ptr _stream; +} + +- (nullable instancetype)initWithTokenizer:(OGATokenizer*)tokenizer error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([tokenizer CXXAPIOgaTokenizer]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable instancetype)initWithMultiModalProcessor:(OGAMultiModalProcessor*)processor + error:(NSError**)error { + if ((self = [super init]) == nil) { + return nil; + } + + try { + _stream = OgaTokenizerStream::Create([processor CXXAPIOgaMultiModalProcessor]); + return self; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +- (nullable NSString*)decode:(int32_t)token error:(NSError**)error { + try { + return [NSString stringWithUTF8String:_stream->Decode(token)]; + } + OGA_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error) +} + +@end \ No newline at end of file diff --git a/src/objectivec/test/assertion_utils.h b/src/objectivec/test/assertion_utils.h new file mode 100644 index 000000000..2b72435b9 --- /dev/null +++ b/src/objectivec/test/assertion_utils.h @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +#define ORTAssertNullableResultSuccessful(result, error) \ + do { \ + XCTAssertNotNil(result, @"Expected non-nil result but got nil. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertBoolResultSuccessful(result, error) \ + do { \ + XCTAssertTrue(result, @"Expected true result but got false. Error: %@", error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertNullableResultUnsuccessful(result, error) \ + do { \ + XCTAssertNil(result); \ + XCTAssertNotNil(error); \ + } while (0) + +#define ORTAssertBoolResultUnsuccessful(result, error) \ + do { \ + XCTAssertFalse(result); \ + XCTAssertNotNil(error); \ + } while (0) + +#define ORTAssertEqualFloatAndNoError(expected, result, error) \ + do { \ + XCTAssertEqualWithAccuracy(expected, result, 1e-3f, @"Expected %f but got %f. Error:%@", expected, result, error); \ + XCTAssertNil(error); \ + } while (0) + +#define ORTAssertEqualFloatArrays(expected, result) \ + do { \ + XCTAssertEqual(expected.count, result.count); \ + for (size_t i = 0; i < expected.count; ++i) { \ + XCTAssertEqualWithAccuracy([expected[i] floatValue], [result[i] floatValue], 1e-3f); \ + } \ + } while (0) + +NS_ASSUME_NONNULL_END diff --git a/src/objectivec/test/ort_genai_api_test.mm b/src/objectivec/test/ort_genai_api_test.mm new file mode 100644 index 000000000..37db97e41 --- /dev/null +++ b/src/objectivec/test/ort_genai_api_test.mm @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#import + +#import "ort_genai_objc.h" +#import "assertion_utils.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface ORTGenAIAPITest : XCTestCase + +@end + +@implementation ORTGenAIAPITest + +- (void)setUp { + [super setUp]; + self.continueAfterFailure = NO; +} + ++ (void)tearDown { + [OGAGenerator shutdown]; +} + ++ (NSString*)getModelPath { + NSBundle* bundle = [NSBundle bundleForClass:[ORTGenAIAPITest class]]; + NSString* path = [[bundle resourcePath] stringByAppendingString:@"/tiny-random-gpt2-fp32"]; + return path; +} + +- (void)testTensor_And_AddExtraInput { + // Create a [3 4] shaped tensor + std::array data{0, 1, 2, 3, + 10, 11, 12, 13, + 20, 21, 22, 23}; + NSArray* shape = @[@3, @4]; + + NSError *error = nil; + BOOL ret = NO; + OGAModel* model = [[OGAModel alloc] initWithPath:[ORTGenAIAPITest getModelPath] error:&error]; + ORTAssertNullableResultSuccessful(model, error); + + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); + + OGATensor* tensor = [[OGATensor alloc] initWithDataPointer:data.data() shape:shape type:OGAElementTypeFloat32 error:&error]; + ORTAssertNullableResultSuccessful(tensor, error); + + ret = [params setModelInput:@"test_input" tensor:tensor error:&error]; + ORTAssertBoolResultSuccessful(ret, error); +} + +- (void)testGetOutput { + std::vector input_ids_shape{2, 4}; + NSArray* input_ids = @[@0, @0, @0, @52, @0, @0, @195, @731]; + const auto batch_size = input_ids_shape[0]; + int max_length = 10; + + NSError *error = nil; + BOOL ret = NO; + OGAModel* model = [[OGAModel alloc] initWithPath:[ORTGenAIAPITest getModelPath] error:&error]; + ORTAssertNullableResultSuccessful(model, error); + + OGAGeneratorParams *params = [[OGAGeneratorParams alloc] initWithModel:model error:&error]; + ORTAssertNullableResultSuccessful(params, error); + + [params setSearchOption:@"max_length" doubleValue:max_length error:&error]; + XCTAssertNil(error); + + [params setSearchOption:@"batch_size" doubleValue:batch_size error:&error]; + XCTAssertNil(error); + + OGAGenerator* generator = [[OGAGenerator alloc] initWithModel:model + params:params + error:&error]; + ORTAssertNullableResultSuccessful(generator, error); + [generator appendTokens:input_ids error:&error]; + XCTAssertNil(error); + + // check prompt + // full logits has shape [2, 4, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 4, 5] + std::vector expected_sampled_logits_prompt{0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + 0.30323744f, 0.0545997f, 0.03894716f, 0.11702324f, 0.0410665f, + -0.12675379f, -0.04443946f, 0.14492269f, 0.03021223f, -0.03212897f, + 0.29694548f, 0.00955007f, 0.0430819f, 0.10063869f, 0.0437237f, + 0.27329233f, 0.00841076f, -0.1060291f, 0.11328877f, 0.13369876f, + -0.04699047f, 0.17915794f, 0.20838135f, 0.10888482f, -0.00277808f, + 0.2938929f, -0.10538938f, -0.00226692f, 0.12050669f, -0.10622668f}; + + OGATensor* prompt_logits_ptr = [generator getOutput:@"logits" error:&error]; + ORTAssertNullableResultSuccessful(prompt_logits_ptr, error); + auto prompt_logits = static_cast([prompt_logits_ptr getDataPointerWithError:&error]); + XCTAssertNil(error); + XCTAssertNotEqual(prompt_logits, nullptr); + const int num_prompt_outputs_to_check = 40; + const int sample_size = 200; + const float tolerance = 0.001f; + // Verify outputs match expected outputs + for (int i = 0; i < num_prompt_outputs_to_check; i++) { + XCTAssertEqualWithAccuracy(expected_sampled_logits_prompt[i], prompt_logits[i * sample_size], tolerance); + } + + ret = [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); + ret = [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); + + // check for the 1st token generation + // full logits has shape [2, 1, 1000]. Sample 1 for every 200 tokens and the expected sampled logits has shape [2, 1, 5] + std::vector expected_sampled_logits_token_gen{0.03742531f, -0.05752287f, 0.14159015f, 0.04210977f, -0.1484456f, + 0.3041716f, -0.08701379f, -0.03778192f, 0.07471392f, -0.02049096f}; + + OGATensor* token_gen_logits_ptr = [generator getOutput:@"logits" error:&error]; + ORTAssertNullableResultSuccessful(token_gen_logits_ptr, error); + + auto token_gen_logits = static_cast([token_gen_logits_ptr getDataPointerWithError:&error]); + XCTAssertNil(error); + XCTAssertNotEqual(token_gen_logits, nullptr); + int num_token_gen_outputs_to_check = 10; + + for (int i = 0; i < num_token_gen_outputs_to_check; i++) { + XCTAssertEqualWithAccuracy(expected_sampled_logits_token_gen[i], token_gen_logits[i * sample_size], tolerance); + } + [generator generateNextTokenWithError:&error]; + ORTAssertBoolResultSuccessful(ret, error); +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/tools/ci_build/github/apple/build_and_assemble_apple_pods.py b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py new file mode 100755 index 000000000..35bbb6ba4 --- /dev/null +++ b/tools/ci_build/github/apple/build_and_assemble_apple_pods.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import logging +import pathlib +import shutil +import sys +import tempfile + +from c.assemble_c_pod_package import assemble_c_pod_package +from objectivec.assemble_objc_pod_package import assemble_objc_pod_package +from package_assembly_utils import PackageVariant, get_ort_genai_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve() +SCRIPT_DIR = SCRIPT_PATH.parent +REPO_DIR = SCRIPT_PATH.parents[4] + + +logging.basicConfig(format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG) +log = logging.getLogger(SCRIPT_PATH.stem) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Builds an iOS framework and uses it to assemble iOS pod package files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + parser.add_argument( + "--build-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "apple_framework", + help="The build directory. This will contain the iOS framework build output.", + ) + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=REPO_DIR / "build" / "apple_pod_staging", + help="The staging directory. This will contain the iOS pod package files. " + "The pod package files do not have dependencies on files in the build directory.", + ) + + parser.add_argument( + "--pod-version", + default=f"{get_ort_genai_version()}-local", + help="The version string of the pod. The same version is used for all pods.", + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.release_variant_names(), + default=PackageVariant.Full.name, + help="Pod package variant.", + ) + + parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.") + parser.add_argument( + "--skip-build", + action="store_true", + help="Use build from previous run. Useful to debug test issues or packaging changes.", + ) + + build_framework_group = parser.add_argument_group( + title="iOS framework build arguments", + description="See the corresponding arguments in build_apple_framework.py for details.", + ) + + build_framework_group.add_argument("--include-ops-by-config") + build_framework_group.add_argument( + "--build-settings-file", required=True, help="The positional argument of build_apple_framework.py." + ) + build_framework_group.add_argument( + "-b", + "--build-apple-framework-arg", + action="append", + dest="build_apple_framework_extra_args", + default=[], + help="Pass an argument through to build_apple_framework.py. This may be specified multiple times.", + ) + + parser.add_argument( + "--skip-macos-test", + action="store_true", + help="Skip macos platform tests. Specify this argument when build targets only contain ios archs. ", + ) + + parser.add_argument( + "--ort-version", required=True, help="The ORT version to depend on." + ) + + parser.add_argument( + "--ort-home", required=False, help="The ORT home for building dependency." + ) + + args = parser.parse_args() + + return args + + +def run(arg_list, cwd=None): + import os + import shlex + import subprocess + + log.info("Running subprocess in '%s'\n %s", cwd or os.getcwd(), " ".join([shlex.quote(arg) for arg in arg_list])) + + return subprocess.run(arg_list, check=True, cwd=cwd) + + +def main(): + args = parse_args() + + build_dir = args.build_dir.resolve() + staging_dir = args.staging_dir.resolve() + + # build framework + package_variant = PackageVariant[args.variant] + framework_info_file = build_dir / "xcframework_info.json" + + log.info("Building Apple framework.") + + build_apple_framework_args = [ + sys.executable, + str(SCRIPT_DIR / "build_apple_framework.py"), + *args.build_apple_framework_extra_args, + ] + + if args.ort_home: + build_apple_framework_args.append('--ort_home') + build_apple_framework_args.append(args.ort_home) + + if args.include_ops_by_config is not None: + build_apple_framework_args += ["--include_ops_by_config", args.include_ops_by_config] + + build_apple_framework_args += ["--build_dir", str(build_dir), args.build_settings_file] + + if not args.skip_build: + run(build_apple_framework_args) + + if args.test: + test_apple_packages_args = [ + sys.executable, + str(SCRIPT_DIR / "test_apple_packages.py"), + "--fail_if_cocoapods_missing", + "--framework_info_file", + str(framework_info_file), + "--c_framework_dir", + str(build_dir / "framework_out"), + "--variant", + package_variant.name, + '--ort_version', + args.ort_version + ] + if args.skip_macos_test: + test_apple_packages_args.append("--skip_macos_test") + + run(test_apple_packages_args) + + # assemble pods and then move them to their target locations (staging_dir/) + staging_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name: + pod_assembly_dir = pathlib.Path(pod_assembly_dir_name) + + log.info("Assembling C/C++ pod.") + + c_pod_staging_dir = pod_assembly_dir / "c_pod" + c_pod_name, c_pod_podspec = assemble_c_pod_package( + staging_dir=c_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + framework_dir=build_dir / "framework_out" / "onnxruntime-genai.xcframework", + public_headers_dir=build_dir / "framework_out" / "Headers", + package_variant=package_variant, + ort_version=args.ort_version + ) + + if args.test: + test_c_pod_args = ["pod", "lib", "lint", "--verbose"] + + if args.skip_macos_test: + test_c_pod_args.append("--platforms=ios") + + run(test_c_pod_args, cwd=c_pod_staging_dir) + + log.info("Assembling Objective-C pod.") + + objc_pod_staging_dir = pod_assembly_dir / "objc_pod" + objc_pod_name, objc_pod_podspec = assemble_objc_pod_package( + staging_dir=objc_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + package_variant=package_variant + ) + + if args.test: + test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"] + if args.skip_macos_test: + test_objc_pod_args.append("--platforms=ios") + + run(test_objc_pod_args, cwd=objc_pod_staging_dir) + + def move_dir(src, dst): + if dst.is_dir(): + shutil.rmtree(dst) + shutil.copytree(src, dst, symlinks=True) + shutil.rmtree(src) + + move_dir(c_pod_staging_dir, staging_dir / c_pod_name) + move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name) + + log.info(f"Successfully assembled iOS pods at '{staging_dir}'.") + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py new file mode 100644 index 000000000..878333c38 --- /dev/null +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import shutil +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + +# the license file +license_file = "LICENSE" + +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + gen_file_from_template, + get_podspec_values, + load_json_config, +) + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-genai-c.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_c_pod_package( + staging_dir: pathlib.Path, + pod_version: str, + framework_info_file: pathlib.Path, + public_headers_dir: pathlib.Path, + framework_dir: pathlib.Path, + package_variant: PackageVariant, + ort_version: str +): + """ + Assembles the files for the C/C++ pod package in a staging directory. + + :param staging_dir Path to the staging directory for the C/C++ pod files. + :param pod_version C/C++ pod version. + :param framework_info_file Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. + :param public_headers_dir Path to the public headers directory to include in the pod. + :param framework_dir Path to the onnxruntime framework directory to include in the pod. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + public_headers_dir = public_headers_dir.resolve(strict=True) + framework_dir = framework_dir.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + # copy the necessary files to the staging directory + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True, symlinks=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True, symlinks=True) + copy_repo_relative_to_dir(None, [license_file], staging_dir) + + (ios_deployment_target, macos_deployment_target, weak_framework) = get_podspec_values(framework_info) + + # generate the podspec file from the template + variable_substitutions = { + "DESCRIPTION": pod_config["description"], + # By default, we build both "iphoneos" and "iphonesimulator" architectures, and the deployment target should be the same between these two. + "IOS_DEPLOYMENT_TARGET": ios_deployment_target, + "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, + "LICENSE_FILE": license_file, + "NAME": pod_name, + "ORTGENAI_C_FRAMEWORK": framework_dir.name, + "ORTGENAI_C_HEADERS_DIR": public_headers_dir.name, + "SUMMARY": pod_config["summary"], + "VERSION": pod_version, + "ORT_VERSION": ort_version, + "WEAK_FRAMEWORK": weak_framework, + } + + podspec_template = _script_dir / "c.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the C/C++ pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./c-staging"), + help="Path to the staging directory for the C/C++ pod files.", + ) + parser.add_argument("--pod-version", required=True, help="C/C++ pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--public-headers-dir", + type=pathlib.Path, + required=True, + help="Path to the public headers directory to include in the pod.", + ) + parser.add_argument( + "--framework-dir", + type=pathlib.Path, + required=True, + help="Path to the onnxruntime framework directory to include in the pod.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." + ) + + parser.add_argument( + "--ort-version", required=True, help="The ORT version to depend on." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_c_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + public_headers_dir=args.public_headers_dir, + framework_dir=args.framework_dir, + package_variant=PackageVariant[args.variant], + ort_version=args.ort_version + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/apple/c/c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template new file mode 100644 index 000000000..7927add89 --- /dev/null +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -0,0 +1,29 @@ +Pod::Spec.new do |s| + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } + s.author = {"ONNX Runtime" => "onnxruntime@microsoft.com" } + s.source = { :http => 'file:///http_source_placeholder' } + + s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@" + s.osx.deployment_target = "@MACOSX_DEPLOYMENT_TARGET@" + + s.vendored_frameworks = "@ORTGENAI_C_FRAMEWORK@" + s.static_framework = true + s.framework = ['ImageIO', 'CoreGraphics'] + s.weak_framework = [ @WEAK_FRAMEWORK@ ] + + s.source_files = "@ORTGENAI_C_HEADERS_DIR@/*.h" + + s.preserve_paths = [ "@LICENSE_FILE@" ] + s.description = "@DESCRIPTION@" + s.libraries = "c++" + s.dependency 'onnxruntime-c', '~> @ORT_VERSION@' + + s.pod_target_xcconfig = { + "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", + } +end + diff --git a/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json new file mode 100644 index 000000000..5129808f6 --- /dev/null +++ b/tools/ci_build/github/apple/c/onnxruntime-genai-c.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-genai-c", + "summary": "ONNX Runtime GenAI C/C++ Pod", + "description": "A pod for the ONNX Runtime GenAI C/C++ library." +} diff --git a/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json b/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json new file mode 100644 index 000000000..555f7f6e4 --- /dev/null +++ b/tools/ci_build/github/apple/default_ios_simulator_apple_framework_build_settings.json @@ -0,0 +1,21 @@ +{ + "build_osx_archs": { + "iphonesimulator": [ + "arm64", + "x86_64" + ] + }, + "build_params": { + "base": [ + "--parallel", + "--cmake_generator", + "Xcode", + "--build_apple_framework", + "--skip_tests" + ], + "iphonesimulator": [ + "--ios", + "--apple_deploy_target=13.0" + ] + } +} diff --git a/tools/ci_build/github/apple/get_simulator_device_info.py b/tools/ci_build/github/apple/get_simulator_device_info.py new file mode 100755 index 000000000..aa693038b --- /dev/null +++ b/tools/ci_build/github/apple/get_simulator_device_info.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from __future__ import annotations + +import argparse +import functools +import itertools +import json +import os +import subprocess + + +@functools.total_ordering +class Version: + """ + A simple Version class. + We opt to use this instead of `packaging.version.Version` to avoid depending on the external `packaging` package. + It only supports integer version components. + """ + + def __init__(self, version_string: str): + self._components = tuple(int(component) for component in version_string.split(".")) + + def __eq__(self, other: Version) -> bool: + component_pairs = itertools.zip_longest(self._components, other._components, fillvalue=0) + return all(pair[0] == pair[1] for pair in component_pairs) + + def __lt__(self, other: Version) -> bool: + component_pairs = itertools.zip_longest(self._components, other._components, fillvalue=0) + for self_component, other_component in component_pairs: + if self_component != other_component: + return self_component < other_component + return False + + +def get_simulator_device_info( + requested_runtime_platform: str = "iOS", + requested_device_type_product_family: str = "iPhone", + requested_runtime_version_str: str | None = None, +) -> dict[str, str]: + """ + Retrieves simulator device information from Xcode. + This simulator device should be appropriate for running tests on this machine. + + :param requested_runtime_platform: The runtime platform to select. + :param requested_device_type_product_family: The device type product family to select. + :param requested_runtime_version_str: The runtime version to select. If unspecified, selects the latest one. + + :return: A dictionary containing information about the selected simulator device. + """ + requested_runtime_version = ( + Version(requested_runtime_version_str) if requested_runtime_version_str is not None else None + ) + + simctl_proc = subprocess.run( + ["xcrun", "simctl", "list", "--json", "--no-escape-slashes"], + text=True, + capture_output=True, + check=True, + ) + + simctl_json = json.loads(simctl_proc.stdout) + + # device type id -> device type structure + device_type_map = {device_type["identifier"]: device_type for device_type in simctl_json["devicetypes"]} + + # runtime id -> runtime structure + runtime_map = {runtime["identifier"]: runtime for runtime in simctl_json["runtimes"]} + + def runtime_filter(runtime) -> bool: + if not runtime["isAvailable"]: + return False + + if runtime["platform"] != requested_runtime_platform: + return False + + if requested_runtime_version is not None and Version(runtime["version"]) != requested_runtime_version: + return False + + return True + + def runtime_id_filter(runtime_id: str) -> bool: + runtime = runtime_map.get(runtime_id) + if runtime is None: + return False + return runtime_filter(runtime) + + def device_type_filter(device_type) -> bool: + if device_type["productFamily"] != requested_device_type_product_family: + return False + + return True + + def device_filter(device) -> bool: + if not device["isAvailable"]: + return False + + if not device_type_filter(device_type_map[device["deviceTypeIdentifier"]]): + return False + + return True + + # simctl_json["devices"] is a map of runtime id -> list of device structures + # expand this into a list of (runtime id, device structure) and filter out invalid entries + runtime_id_and_device_pairs = [] + for runtime_id, device_list in filter( + lambda runtime_id_and_device_list: runtime_id_filter(runtime_id_and_device_list[0]), + simctl_json["devices"].items(), + ): + runtime_id_and_device_pairs.extend((runtime_id, device) for device in filter(device_filter, device_list)) + + if len(runtime_id_and_device_pairs) == 0: + raise ValueError("Failed to find requested simulator device info.") + + # sort key - tuple of (runtime version, device type min runtime version) + # the secondary device type min runtime version value is to treat more recent device types as greater + def runtime_id_and_device_pair_key(runtime_id_and_device_pair): + runtime_id, device = runtime_id_and_device_pair + + runtime = runtime_map[runtime_id] + device_type = device_type_map[device["deviceTypeIdentifier"]] + + return (Version(runtime["version"]), device_type["minRuntimeVersion"]) + + selected_runtime_id, selected_device = max(runtime_id_and_device_pairs, key=runtime_id_and_device_pair_key) + selected_runtime = runtime_map[selected_runtime_id] + selected_device_type = device_type_map[selected_device["deviceTypeIdentifier"]] + + result = { + "device_name": selected_device["name"], + "device_udid": selected_device["udid"], + "device_type_identifier": selected_device_type["identifier"], + "device_type_name": selected_device_type["name"], + "device_type_product_family": selected_device_type["productFamily"], + "runtime_identifier": selected_runtime["identifier"], + "runtime_platform": selected_runtime["platform"], + "runtime_version": selected_runtime["version"], + } + + return result + + +def main(): + requested_runtime_version_environment_variable_name = "ORT_GET_SIMULATOR_DEVICE_INFO_REQUESTED_RUNTIME_VERSION" + + parser = argparse.ArgumentParser(description="Gets simulator info from Xcode and prints it in JSON format.") + parser.add_argument( + "--requested-runtime-version", + default=os.environ.get(requested_runtime_version_environment_variable_name, None), + help="The requested runtime version. " + f"This may also be specified with the {requested_runtime_version_environment_variable_name} " + "environment variable. The command line option takes precedence. " + "An unspecified value means the latest available runtime version.", + ) + args = parser.parse_args() + + info = get_simulator_device_info(requested_runtime_version_str=args.requested_runtime_version) + + print(json.dumps(info, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py new file mode 100755 index 000000000..e11774b70 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import sys + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from c.assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402 +from package_assembly_utils import ( # noqa: E402 + PackageVariant, + copy_repo_relative_to_dir, + filter_files, + gen_file_from_template, + get_podspec_values, + load_json_config, +) + +# these variables contain paths or path patterns that are relative to the repo root + +# the license file +license_file = "LICENSE" + +# include directories for compiling the pod itself +include_dirs = [ + "objectivec", +] + +all_objc_files = { + "source_files": [ + "objectivec/include/*.h", + "objectivec/*.h", + "objectivec/*.m", + "objectivec/*.mm", + ], + "public_header_files": [ + "objectivec/include/*.h", + ], + "test_source_files": [ + "objectivec/test/*.h", + "objectivec/test/*.m", + "objectivec/test/*.mm", + ], + "test_resource_files": [ + "test/test_models/hf-internal-testing/tiny-random-gpt2-fp32", + ], +} + +def get_pod_files(package_variant: PackageVariant): + """ + Gets the source and header files for the given package variant. + """ + filtered_pod_files = {} + for key in all_objc_files: + filtered_pod_files[key] = filter_files("src", all_objc_files[key], []) + filtered_pod_files[key].extend(filter_files(None, all_objc_files[key], [])) + return filtered_pod_files + + +def get_pod_config_file(package_variant: PackageVariant): + """ + Gets the pod configuration file path for the given package variant. + """ + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-genai-objc.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_objc_pod_package( + staging_dir: pathlib.Path, pod_version: str, framework_info_file: pathlib.Path, package_variant: PackageVariant +): + """ + Assembles the files for the Objective-C pod package in a staging directory. + + :param staging_dir Path to the staging directory for the Objective-C pod files. + :param pod_version Objective-C pod version. + :param framework_info_file Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + """ + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + c_pod_config = load_json_config(get_c_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + pod_files = get_pod_files(package_variant) + + copy_repo_relative_to_dir(None, [license_file], staging_dir) + + # copy the necessary files to the staging directory + need_copy = [*pod_files["source_files"], *pod_files["test_source_files"]] + copy_repo_relative_to_dir("src", need_copy, staging_dir) + + if "test_resource_files" in pod_files: + need_copy = [*pod_files["test_resource_files"]] + copy_repo_relative_to_dir(None, need_copy, staging_dir) + + # generate the podspec file from the template + + def path_patterns_as_variable_value(patterns: list[str]): + return ", ".join([f'"{pattern}"' for pattern in patterns]) + + (ios_deployment_target, macos_deployment_target, _) = get_podspec_values(framework_info) + + variable_substitutions = { + "C_POD_NAME": c_pod_config["name"], + "DESCRIPTION": pod_config["description"], + "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs), + "IOS_DEPLOYMENT_TARGET": ios_deployment_target, + "MACOSX_DEPLOYMENT_TARGET": macos_deployment_target, + "LICENSE_FILE": license_file, + "NAME": pod_name, + "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(pod_files["public_header_files"]), + "SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["source_files"]), + "SUMMARY": pod_config["summary"], + "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_resource_files"]), + "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(pod_files["test_source_files"]), + "VERSION": pod_version, + } + + podspec_template = _script_dir / "objc.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser( + description=""" + Assembles the files for the Objective-C pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """ + ) + + parser.add_argument( + "--staging-dir", + type=pathlib.Path, + default=pathlib.Path("./objc-staging"), + help="Path to the staging directory for the Objective-C pod files.", + ) + parser.add_argument("--pod-version", required=True, help="Objective-C pod version.") + parser.add_argument( + "--framework-info-file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + parser.add_argument( + "--variant", choices=PackageVariant.release_variant_names(), required=True, help="Pod package variant." + ) + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_objc_pod_package( + staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + package_variant=PackageVariant[args.variant], + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/apple/objectivec/objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template new file mode 100644 index 000000000..a5f543276 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -0,0 +1,53 @@ +Pod::Spec.new do |s| + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.description = "@DESCRIPTION@" + + s.homepage = "https://github.com/microsoft/onnxruntime-genai" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } + s.author = { "ONNX Runtime" => "onnxruntime@microsoft.com" } + s.source = { :http => 'file:///http_source_placeholder' } + s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@" + s.osx.deployment_target = "@MACOSX_DEPLOYMENT_TARGET@" + + s.default_subspec = 'Core' + s.static_framework = true + + s.subspec 'Core' do |core| + core.dependency "@C_POD_NAME@", "#{s.version}" + core.requires_arc = true + core.compiler_flags = "-std=c++17", "-fobjc-arc-exceptions", "-Wall", "-Wextra", "-Werror" + + include_dirs = [ + @INCLUDE_DIR_LIST@ + ].map { |relative_include_dir| + '"${PODS_TARGET_SRCROOT}/' + relative_include_dir + '"' + } + + core.public_header_files = [ + @PUBLIC_HEADER_FILE_LIST@ + ] + + core.source_files = [ + @SOURCE_FILE_LIST@ + ] + + core.test_spec "Tests" do |test| + test.source_files = [ + @TEST_SOURCE_FILE_LIST@ + ] + + test.resources = [ + @TEST_RESOURCE_FILE_LIST@ + ] + end + + core.pod_target_xcconfig = { + "HEADER_SEARCH_PATHS" => include_dirs.join(" "), + "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", + } + + end +end + diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json b/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json new file mode 100644 index 000000000..017564db7 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/onnxruntime-genai-objc.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-genai-objc", + "summary": "ONNX Runtime GenAI Objective-C Pod", + "description": "A pod for the ONNX Runtime GenAI Objective-C API." +} diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py new file mode 100644 index 000000000..cca1c6eb1 --- /dev/null +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -0,0 +1,173 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import enum +import json +import os +import pathlib +import re +import shutil +from typing import Dict, List + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +repo_root = _script_dir.parents[3] + + +class PackageVariant(enum.Enum): + Full = 0 + + @classmethod + def release_variant_names(cls): + return [v.name for v in cls if v.value >= 0] + + +_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" + + +def gen_file_from_template( + template_file: pathlib.Path, output_file: pathlib.Path, variable_substitutions: Dict[str, str], strict: bool = True +): + """ + Generates a file from a template file. + The template file may contain template variables that will be substituted + with the provided values in the generated output file. + In the template file, template variable names are delimited by "@"'s, + e.g., "@var@". + + :param template_file The template file path. + :param output_file The generated output file path. + :param variable_substitutions The mapping from template variable name to value. + :param strict Whether to require the set of template variable names in the file and the keys of + `variable_substitutions` to be equal. + """ + with open(template_file) as template: + content = template.read() + + variables_in_file = set() + + def replace_template_variable(match): + variable_name = match.group(1) + variables_in_file.add(variable_name) + return variable_substitutions.get(variable_name, match.group(0)) + + content = _template_variable_pattern.sub(replace_template_variable, content) + + if strict and variables_in_file != variable_substitutions.keys(): + variables_in_substitutions = set(variable_substitutions.keys()) + raise ValueError( + f"Template file variables and substitution variables do not match. " + f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. " + f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}." + ) + + with open(output_file, mode="w") as output: + output.write(content) + + +def filter_files(subpath: str, all_file_patterns: List[str], excluded_file_patterns: List[str]): + """ + Filters file paths based on inclusion and exclusion patterns + + :param all_file_patterns The list of file paths to filter. + :param excluded_file_patterns The list of exclusion patterns. + + :return The filtered list of file paths + """ + # get all files matching the patterns in all_file_patterns + if subpath: + src_root = repo_root / subpath + else: + src_root = repo_root + + all_files = [str(path.relative_to(src_root)) for pattern in all_file_patterns for path in src_root.glob(pattern)] + + # get all files matching the patterns in excluded_file_patterns + exclude_files = [ + str(path.relative_to(src_root)) for pattern in excluded_file_patterns for path in src_root.glob(pattern) + ] + + # return the difference + return list(set(all_files) - set(exclude_files)) + + +def copy_repo_relative_to_dir(subpath: str, patterns: List[str], dest_dir: pathlib.Path): + """ + Copies file paths relative to the repo root to a directory. + The given paths or path patterns are relative to the repo root, and the + repo root-relative intermediate directory structure is maintained. + + :param patterns The paths or path patterns relative to the repo root. + :param dest_dir The destination directory. + """ + if subpath: + src_root = repo_root / subpath + else: + src_root = repo_root + paths = [path for pattern in patterns for path in src_root.glob(pattern)] + for path in paths: + repo_relative_path = path.relative_to(src_root) + dst_path = dest_dir / repo_relative_path + os.makedirs(dst_path.parent, exist_ok=True) + if os.path.isdir(path): + shutil.copytree(path, dst_path) + else: + shutil.copy(path, dst_path) + + +def load_json_config(json_config_file: pathlib.Path): + """ + Loads configuration info from a JSON file. + + :param json_config_file The JSON configuration file path. + :return The configuration info values. + """ + with open(json_config_file) as config: + return json.load(config) + + +def get_podspec_values(framework_info): + """ + Get the podspec deployement targets and weak framework info from the dictionary that load_json_config returned. + Looks for iphonesimulator, iphoneos and macos settings. + Handles missing platforms and checks consistency. + Returns empty string for deployment target if that platofrm is not enabled. + + :return (ios_deployment_target, macos_deployment_target, weak_framework) + """ + ios_deployment_target = "" + macos_deployment_target = "" + weak_framework = "" # should be the same for all platforms + # get info, allowing for a subset of platforms to be specified + for framework in ("iphonesimulator", "iphoneos", "macosx"): + if framework not in framework_info: + continue + + target = framework_info[framework]["APPLE_DEPLOYMENT_TARGET"] + weak = framework_info[framework]["WEAK_FRAMEWORK"] + + if not weak_framework: + weak_framework = weak + else: + # should be consistent + assert weak == weak_framework + + if framework == "macosx": + macos_deployment_target = target + else: + if not ios_deployment_target: + ios_deployment_target = target + else: + # should be consistent + assert ios_deployment_target == target + + return (ios_deployment_target, macos_deployment_target, weak_framework) + + +def get_ort_genai_version(): + """ + Gets the version string from the repo. + + :return The version string. + """ + with open(repo_root / "VERSION_INFO") as version_file: + return version_file.read().strip() diff --git a/tools/ci_build/github/apple/test_apple_packages.py b/tools/ci_build/github/apple/test_apple_packages.py new file mode 100644 index 000000000..a16ad89b0 --- /dev/null +++ b/tools/ci_build/github/apple/test_apple_packages.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import contextlib +import json +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile + +from huggingface_hub import snapshot_download + +from c.assemble_c_pod_package import assemble_c_pod_package +from package_assembly_utils import PackageVariant, gen_file_from_template, get_ort_genai_version + +SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True) +REPO_DIR = SCRIPT_PATH.parents[4] + + +def _test_apple_packages(args): + # check if CocoaPods is installed + if shutil.which("pod") is None: + if args.fail_if_cocoapods_missing: + raise ValueError("CocoaPods is required for this test") + else: + print("CocoaPods is not installed, ignore this test") + return + + # Now we need to create a zip file contains the framework and the podspec file, both of these 2 files + # should be under the c_framework_dir + c_framework_dir = args.c_framework_dir.resolve() + if not c_framework_dir.is_dir(): + raise FileNotFoundError(f"c_framework_dir {c_framework_dir} is not a folder.") + + has_framework = (c_framework_dir / "onnxruntime-genai.framework").exists() + has_xcframework = (c_framework_dir / "onnxruntime-genai.xcframework").exists() + + if not has_framework and not has_xcframework: + raise FileNotFoundError(f"{c_framework_dir} does not have onnxruntime-genai.framework/xcframework") + + if has_framework and has_xcframework: + raise ValueError("Cannot proceed when both onnxruntime-genai.framework and onnxruntime-genai.xcframework exist") + + framework_name = "onnxruntime-genai.framework" if has_framework else "onnxruntime-genai.xcframework" + + # create a temp folder + + with contextlib.ExitStack() as context_stack: + if args.test_project_stage_dir is None: + stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve() + else: + # If we specify the stage dir, then use it to create test project + stage_dir = args.test_project_stage_dir.resolve() + if os.path.exists(stage_dir): + shutil.rmtree(stage_dir) + os.makedirs(stage_dir) + + # assemble the test project here + target_proj_path = stage_dir / "apple_package_test" + + # copy the test project source files to target_proj_path + test_proj_path = pathlib.Path(REPO_DIR, "test/platform/apple/apple_package_test") + shutil.copytree(test_proj_path, target_proj_path) + + # assemble local pod files here + local_pods_dir = stage_dir / "local_pods" + + # We will only publish xcframework, however, assembly of the xcframework is a post process + # and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752 + # For a single sysroot and arch built by build.py or cmake, we can only generate framework + # We still need a way to test it. framework_dir and public_headers_dir have different values when testing a + # framework and a xcframework. + framework_dir = args.c_framework_dir / framework_name + public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers" + + pod_name, podspec = assemble_c_pod_package( + staging_dir=local_pods_dir, + pod_version=get_ort_genai_version(), + framework_info_file=args.framework_info_file, + public_headers_dir=public_headers_dir, + framework_dir=framework_dir, + package_variant=PackageVariant[args.variant], + ort_version=args.ort_version + ) + + # move podspec out to target_proj_path first + podspec = shutil.move(podspec, target_proj_path / podspec.name) + + # create a zip file contains the framework + zip_file_path = local_pods_dir / f"{pod_name}.zip" + + # shutil.make_archive doesn't preserve symlinks. we know this is running on macOS so use zip + subprocess.run(["zip", "-r", "-y", str(zip_file_path), "."], cwd=local_pods_dir, check=True) + + # update the podspec to point to the local framework zip file + with open(podspec) as file: + file_data = file.read() + + file_data = file_data.replace("file:///http_source_placeholder", f"file:///{zip_file_path}") + + with open(podspec, "w") as file: + file.write(file_data) + + # generate Podfile to point to pod + gen_file_from_template( + target_proj_path / "Podfile.template", + target_proj_path / "Podfile", + {"C_POD_NAME": pod_name, "C_POD_PODSPEC": f"./{podspec.name}"}, + ) + + # clean the Cocoapods cache first, in case the same pod was cached in previous runs + subprocess.run(["pod", "cache", "clean", "--all"], shell=False, check=True, cwd=target_proj_path) + + # install pods + # set env to skip macos test targets accordingly + env = os.environ.copy() + env["SKIP_MACOS_TEST"] = "true" if args.skip_macos_test else "false" + subprocess.run(["pod", "install"], shell=False, check=True, cwd=target_proj_path, env=env) + + # Download phi3 model + model_dir = target_proj_path / "models" / "Phi-3-mini-4k-instruct-onnx" + + print(f"Downloading models:\n{model_dir}") + snapshot_download( + repo_id="microsoft/Phi-3-mini-4k-instruct-onnx", + allow_patterns="cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/*", + local_dir=model_dir + ) + + # run the tests + if not args.prepare_test_project_only: + simulator_device_info = subprocess.check_output( + [ + sys.executable, + str(REPO_DIR / "tools" / "ci_build" / "github" / "apple" / "get_simulator_device_info.py"), + ], + text=True, + ).strip() + print(f"Simulator device info:\n{simulator_device_info}") + + simulator_device_info = json.loads(simulator_device_info) + + # Xcode UI tests seem to be flaky: https://github.com/orgs/community/discussions/68807 + # Add a couple of retries if we get this error: + # ios_package_testUITests-Runner Failed to initialize for UI testing: + # Error Domain=com.apple.dt.XCTest.XCTFuture Code=1000 "Timed out while loading Accessibility." + attempts = 0 + cmd = [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "ios_package_test", + "-destination", + f"platform=iOS Simulator,id={simulator_device_info['device_udid']}", + ] + + while True: + attempts += 1 + completed_process = subprocess.run( + cmd, + shell=False, + capture_output=True, + check=False, + text=True, + cwd=target_proj_path, + ) + + # print so it's in CI output + print(completed_process.stdout) + + if completed_process.returncode != 0: + print(f"Running ios_package_test failed. Return code was {completed_process.returncode}") + print("xcrun xcodebuild test stderr:") + print(completed_process.stderr) + print("---") + + if "Timed out while loading Accessibility" in completed_process.stderr and attempts < 3: + continue + + raise subprocess.CalledProcessError( + completed_process.returncode, " ".join(cmd), completed_process.stdout, completed_process.stderr + ) + + break + + if args.mac_catalyst_enabled: + subprocess.run( + [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "ios_package_test", + "-destination", + "platform=macOS,variant=Mac Catalyst", + "CODE_SIGNING_ALLOWED=NO", + ], + shell=False, + check=True, + cwd=target_proj_path, + ) + + if not args.skip_macos_test: + subprocess.run( + [ + "xcrun", + "xcodebuild", + "test", + "-workspace", + "./apple_package_test.xcworkspace", + "-scheme", + "macos_package_test", + "-destination", + "platform=macOS", + ], + shell=False, + check=True, + cwd=target_proj_path, + ) + + +def parse_args(): + parser = argparse.ArgumentParser( + os.path.basename(__file__), description="Test iOS framework using CocoaPods package." + ) + + parser.add_argument( + "--fail_if_cocoapods_missing", + action="store_true", + help="This script will fail if CocoaPods is not installed, " + "will not throw error unless fail_if_cocoapod_missing is set.", + ) + + parser.add_argument( + "--framework_info_file", + type=pathlib.Path, + required=True, + help="Path to the framework_info.json or xcframework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.", + ) + + parser.add_argument( + "--c_framework_dir", type=pathlib.Path, required=True, help="Provide the parent directory for C/C++ framework" + ) + + parser.add_argument( + "--variant", + choices=PackageVariant.release_variant_names(), + required=True, + help="Pod package variant.", + ) + + parser.add_argument( + "--test_project_stage_dir", + type=pathlib.Path, + help="The stage dir for the test project, if not specified, will use a temporary path", + ) + + parser.add_argument( + "--prepare_test_project_only", + action="store_true", + help="Prepare the test project only, without running the tests", + ) + + parser.add_argument( + "--skip_macos_test", + action="store_true", + help="Skip macos platform tests. Specify this argument when build targets only contain ios archs. ", + ) + + parser.add_argument( + "--mac_catalyst_enabled", + action="store_true", + help="Run tests for mac catalyst variants. Specify this argument when build targets contains catalyst archs. ", + ) + + parser.add_argument( + "--ort_version", required=True, help="The ORT version to depend on." + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + _test_apple_packages(args) + + +if __name__ == "__main__": + main()