Skip to content

Commit

Permalink
Final adjustments
Browse files Browse the repository at this point in the history
  • Loading branch information
@philippzagar
philippzagar committed Mar 27, 2024
1 parent bd8ae82 commit c2187bb
Show file tree
Hide file tree
Showing 12 changed files with 107 additions and 202 deletions.
9 changes: 1 addition & 8 deletions Package.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,9 @@ let package = Package(
.library(name: "SpeziLLMFog", targets: ["SpeziLLMFog"])
],
dependencies: [
.package(url: "https://github.com/StanfordBDHG/OpenAI", .upToNextMinor(from: "0.2.7")),
.package(url: "https://github.com/firebase/firebase-ios-sdk", from: "10.13.0"),
.package(url: "https://github.com/StanfordBDHG/OpenAI", .upToNextMinor(from: "0.2.8")),
.package(url: "https://github.com/StanfordBDHG/llama.cpp", .upToNextMinor(from: "0.2.1")),
.package(url: "https://github.com/StanfordSpezi/Spezi", from: "1.2.1"),
.package(url: "https://github.com/StanfordSpezi/SpeziAccount", from: "1.2.1"),
.package(url: "https://github.com/StanfordSpezi/SpeziFirebase", from: "1.0.1"),
.package(url: "https://github.com/StanfordSpezi/SpeziFoundation", from: "1.0.4"),
.package(url: "https://github.com/StanfordSpezi/SpeziStorage", from: "1.0.2"),
.package(url: "https://github.com/StanfordSpezi/SpeziOnboarding", from: "1.1.1"),
Expand Down Expand Up @@ -84,10 +81,6 @@ let package = Package(
dependencies: [
.target(name: "SpeziLLM"),
.product(name: "Spezi", package: "Spezi"),
// As SpeziAccount, SpeziFirebase and the firebase-ios-sdk currently don't support visionOS and macOS, perform fog node token authentication only on iOS
.product(name: "SpeziAccount", package: "SpeziAccount", condition: .when(platforms: [.iOS])),
.product(name: "SpeziFirebaseAccount", package: "SpeziFirebase", condition: .when(platforms: [.iOS])),
.product(name: "FirebaseAuth", package: "firebase-ios-sdk", condition: .when(platforms: [.iOS])),
.product(name: "OpenAI", package: "OpenAI")
]
),
Expand Down
16 changes: 8 additions & 8 deletions Sources/SpeziLLMFog/Configuration/LLMFogParameters.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,38 +38,38 @@ public struct LLMFogParameters: Sendable {
let modelType: FogModel
/// The to-be-used system prompt(s) of the LLM.
let systemPrompts: [String]
/// Separate user identification token issued by Firebase that overrides the one defined within the ``LLMFogPlatform``.
let overwritingToken: String?
/// Closure that returns an up-to-date auth token for requests to Fog LLMs (e.g., a Firebase ID token).
let authToken: @Sendable () async -> String?


/// Creates the ``LLMFogParameters``.
///
/// - Parameters:
/// - modelType: The to-be-used Fog LLM model such as Google's Gemma models or Meta Llama models.
/// - systemPrompt: The to-be-used system prompt of the LLM enabling fine-tuning of the LLMs behaviour. Defaults to the regular Llama2 system prompt.
/// - overwritingToken: Separate user identification token issued by Firebase that overrides the one defined within the ``LLMFogPlatform``.
/// - authToken: Closure that returns an up-to-date auth token for requests to Fog LLMs (e.g., a Firebase ID token).
public init(
modelType: FogModel,
systemPrompt: String? = nil,
overwritingToken: String? = nil
authToken: @Sendable @escaping () async -> String?
) {
self.init(modelType: modelType, systemPrompts: systemPrompt.map { [$0] } ?? [], overwritingToken: overwritingToken)
self.init(modelType: modelType, systemPrompts: systemPrompt.map { [$0] } ?? [], authToken: authToken)
}

/// Creates the ``LLMFogParameters``.
///
/// - Parameters:
/// - modelType: The to-be-used Fog LLM model such as Google's Gemma models or Meta Llama models.
/// - systemPrompts: The to-be-used system prompt of the LLM enabling fine-tuning of the LLMs behaviour. Defaults to the regular Llama2 system prompt.
/// - overwritingToken: Separate uLser identification token issued by Firebase that overrides the one defined within the ``LLMFogPlatform``.
/// - authToken: Closure that returns an up-to-date auth token for requests to Fog LLMs (e.g., a Firebase ID token).
@_disfavoredOverload
public init(
modelType: FogModel,
systemPrompts: [String] = [],
overwritingToken: String? = nil
authToken: @Sendable @escaping () async -> String?
) {
self.modelType = modelType
self.systemPrompts = systemPrompts
self.overwritingToken = overwritingToken
self.authToken = authToken
}
}
9 changes: 0 additions & 9 deletions Sources/SpeziLLMFog/LLMFogError.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ public enum LLMFogError: LLMError {
case mDnsServicesNotFound
/// Network error during mDNS service discovery.
case mDnsServiceDiscoveryNetworkError
/// User is not logged in via Firebase
case userNotAuthenticated
/// Unknown error
case unknownError(Error)

Expand All @@ -49,8 +47,6 @@ public enum LLMFogError: LLMError {
String(localized: LocalizedStringResource("LLM_NO_MDNS_SERVICE_FOUND_ERROR_DESCRIPTION", bundle: .atURL(from: .module)))
case .mDnsServiceDiscoveryNetworkError:
String(localized: LocalizedStringResource("LLM_SERIVE_DISCOVERY_ERROR_DESCRIPTION", bundle: .atURL(from: .module)))
case .userNotAuthenticated:
String(localized: LocalizedStringResource("LLM_MISSING_AUTHENTICATION_ERROR_DESCRIPTION", bundle: .atURL(from: .module)))
case .unknownError:
String(localized: LocalizedStringResource("LLM_UNKNOWN_ERROR_DESCRIPTION", bundle: .atURL(from: .module)))
}
Expand All @@ -72,8 +68,6 @@ public enum LLMFogError: LLMError {
String(localized: LocalizedStringResource("LLM_NO_MDNS_SERVICE_FOUND_ERROR_RECOVERY_SUGGESTION", bundle: .atURL(from: .module)))
case .mDnsServiceDiscoveryNetworkError:
String(localized: LocalizedStringResource("LLM_SERIVE_DISCOVERY_ERROR_RECOVERY_SUGGESTION", bundle: .atURL(from: .module)))
case .userNotAuthenticated:
String(localized: LocalizedStringResource("LLM_MISSING_AUTHENTICATION_ERROR_RECOVERY_SUGGESTION", bundle: .atURL(from: .module)))
case .unknownError:
String(localized: LocalizedStringResource("LLM_UNKNOWN_ERROR_RECOVERY_SUGGESTION", bundle: .atURL(from: .module)))
}
Expand All @@ -95,8 +89,6 @@ public enum LLMFogError: LLMError {
String(localized: LocalizedStringResource("LLM_NO_MDNS_SERVICE_FOUND_ERROR_FAILURE_REASON", bundle: .atURL(from: .module)))
case .mDnsServiceDiscoveryNetworkError:
String(localized: LocalizedStringResource("LLM_SERIVE_DISCOVERY_ERROR_FAILURE_REASON", bundle: .atURL(from: .module)))
case .userNotAuthenticated:
String(localized: LocalizedStringResource("LLM_MISSING_AUTHENTICATION_ERROR_FAILURE_REASON", bundle: .atURL(from: .module)))
case .unknownError:
String(localized: LocalizedStringResource("LLM_UNKNOWN_ERROR_FAILURE_REASON", bundle: .atURL(from: .module)))
}
Expand All @@ -112,7 +104,6 @@ public enum LLMFogError: LLMError {
case (.missingCaCertificate, .missingCaCertificate): true
case (.mDnsServicesNotFound, .mDnsServicesNotFound): true
case (.mDnsServiceDiscoveryNetworkError, .mDnsServiceDiscoveryNetworkError): true
case (.userNotAuthenticated, .userNotAuthenticated): true
case (.unknownError, .unknownError): true
default: false
}
Expand Down
38 changes: 4 additions & 34 deletions Sources/SpeziLLMFog/LLMFogPlatform.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@

import Foundation
import Spezi
#if os(iOS)
import SpeziFirebaseAccount
#endif
import SpeziFoundation
import SpeziLLM

Expand All @@ -33,14 +30,9 @@ import SpeziLLM
///
/// - Tip: For more information, refer to the documentation of the `LLMPlatform` from SpeziLLM.
///
/// - Important: As the ``LLMFogPlatform`` uses Firebase to verify the identify of users and determine their authorization to use fog LLM resources, one must setup [`SpeziAccount`](https://github.com/StanfordSpezi/SpeziAccount)
/// as well as [`SpeziFirebaseAccount`](https://github.com/StanfordSpezi/SpeziFirebase) in the Spezi `Configuration`.
/// Specifically, one must state the `AccountConfiguration` as well as the `FirebaseAccountConfiguration` in the `Configuration`, otherwise a crash upon startup of Spezi will occur.
///
/// ### Usage
///
/// The example below demonstrates the setup of the ``LLMFogPlatform`` within the Spezi `Configuration`. The initializer requires the passing of a local `URL` to the root CA certificate in the `.crt` format that signed the web service certificate on the fog node. See the `FogNode/README.md` and specifically the `setup.sh` script for more details.
/// It is important to note that the `AccountConfiguration` as well as the `FirebaseAccountConfiguration` have to be stated as well in order to use the ``LLMFogPlatform``.
///
/// ```swift
/// class TestAppDelegate: SpeziAppDelegate {
Expand All @@ -50,32 +42,23 @@ import SpeziLLM
///
/// override var configuration: Configuration {
/// Configuration {
/// // Sets up SpeziAccount and the required account details
/// AccountConfiguration(configuration: [
/// .requires(\.userId),
/// .requires(\.password)
/// ])
///
/// // Sets up SpeziFirebaseAccount which serves as the identity provider for the SpeziAccount setup above
/// FirebaseAccountConfiguration(authenticationMethods: .emailAndPassword)
///
/// LLMRunner {
/// LLMFogPlatform(configuration: .init(caCertificate: Self.caCertificateUrl))
/// }
/// }
/// }
/// }
/// ```
///
/// - Important: For development purposes, one is able to configure the fog node in the development mode, meaning no TLS connection (resulting in no need for custom certificates). See the `FogNode/README.md` for more details regarding server-side (so fog node) instructions.
/// On the client-side within Spezi, one has to pass `nil` for the `caCertificate` parameter of the ``LLMFogPlatform`` as shown above. If used in development mode, no custom CA certificate is required, ensuring a smooth and straightforward development process.

public actor LLMFogPlatform: LLMPlatform {
/// Enforce an arbitrary number of concurrent execution jobs of Fog LLMs.
private let semaphore: AsyncSemaphore
let configuration: LLMFogPlatformConfiguration

@MainActor public var state: LLMPlatformState = .idle
#if os(iOS)
/// Dependency to the FirebaseAccountConfiguration, ensuring that it is present in the Spezi `Configuration`.
@Dependency private var firebaseAuth: FirebaseAccountConfiguration?
#endif


/// Creates an instance of the ``LLMFogPlatform``.
Expand All @@ -88,19 +71,6 @@ public actor LLMFogPlatform: LLMPlatform {
}


public nonisolated func configure() {
#if os(iOS)
Task {
guard await firebaseAuth != nil else {
preconditionFailure("""
SpeziLLMFog: Ensure that the `AccountConfiguration` and `FirebaseAccountConfiguration` are stated within the Spezi `Configuration`
to set up the required Firebase account authentication of the `LLMFogPlatform`.
""")
}
}
#endif
}

public nonisolated func callAsFunction(with llmSchema: LLMFogSchema) -> LLMFogSession {
LLMFogSession(self, schema: llmSchema)
}
Expand Down
19 changes: 19 additions & 0 deletions Sources/SpeziLLMFog/LLMFogSchema.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,26 @@ import SpeziLLM
/// The ``LLMFogSchema`` is used as a configuration for the to-be-used Fog LLM. It contains all information necessary for the creation of an executable ``LLMFogSession``.
/// It is bound to the ``LLMFogPlatform`` that is responsible for turning the ``LLMFogSchema`` to an ``LLMFogSession``.
///
/// - Important: The ``LLMFogSchema`` accepts a closure that returns an authorization token that is passed with every request to the Fog node in the `Bearer` HTTP field via the ``LLMFogParameters/init(modelType:systemPrompt:authToken:)``. The token is created via the closure upon every LLM inference request, as the ``LLMFogSession`` may be long lasting and the token could therefore expire. Ensure that the closure appropriately caches the token in order to prevent unnecessary token refresh roundtrips to external systems.
///
/// - Tip: For more information, refer to the documentation of the `LLMSchema` from SpeziLLM.
///
/// ### Usage
///
/// The code example below showcases a minimal instantiation of an ``LLMFogSchema``.
/// Note the `authToken` closure that is specified in the ``LLMFogSchema/init(parameters:modelParameters:injectIntoContext:)``, as this closure should return a token that is then passed as a `Bearer` HTTP token to the fog node with every LLM inference request.
///
/// ```swift
/// var schema = LLMFogSchema(
/// parameters: .init(
/// modelType: .llama7B,
/// systemPrompt: "You're a helpful assistant that answers questions from users.",
/// authToken: {
/// // Return authorization token as `String` or `nil` if no token is required by the Fog node.
/// }
/// )
/// )
/// ```
public struct LLMFogSchema: LLMSchema, @unchecked Sendable {
public typealias Platform = LLMFogPlatform

Expand Down
31 changes: 2 additions & 29 deletions Sources/SpeziLLMFog/LLMFogSession+Generation.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,14 @@ extension LLMFogSession {
///
/// - Parameters:
/// - continuation: A Swift `AsyncThrowingStream` that streams the generated output.
func _generate( // swiftlint:disable:this identifier_name function_body_length
func _generate( // swiftlint:disable:this identifier_name
continuation: AsyncThrowingStream<String, Error>.Continuation
) async {
Self.logger.debug("SpeziLLMFog: Fog LLM started a new inference")
await MainActor.run {
self.state = .generating
}

// Check if the node is still active by pinging it
guard await ensureFogNodeAvailability(continuation: continuation) else {
return
}

let chatStream: AsyncThrowingStream<ChatStreamResult, Error> = await self.model.chatsStream(query: self.openAIChatQuery)

do {
Expand Down Expand Up @@ -70,7 +65,7 @@ extension LLMFogSession {
Self.logger.error("SpeziLLMFog: LLM model type could not be accessed on fog node - \(error.error.message)")
await finishGenerationWithError(LLMFogError.modelAccessError(error), on: continuation)
} else if error.error.code == "401" || error.error.code == "403" {
Self.logger.error("SpeziLLMFog: LLM model could not be accessed as the Firebase User ID token is invalid.")
Self.logger.error("SpeziLLMFog: LLM model could not be accessed as the passed token is invalid.")
await finishGenerationWithError(LLMFogError.invalidAPIToken, on: continuation)
} else {
Self.logger.error("SpeziLLMFog: Generation error occurred - \(error)")
Expand All @@ -93,26 +88,4 @@ extension LLMFogSession {
self.state = .ready
}
}

private func ensureFogNodeAvailability(continuation: AsyncThrowingStream<String, Error>.Continuation) async -> Bool {
guard let discoveredServiceAddress,
let discoveredServiceAddressUrl = URL(string: discoveredServiceAddress) else {
Self.logger.error("SpeziLLMFog: mDNS service could not be resolved to an IP.")
await finishGenerationWithError(LLMFogError.mDnsServicesNotFound, on: continuation)
return false
}

do {
_ = try await URLSession.shared.data(from: discoveredServiceAddressUrl)
} catch {
// If node not reachable anymore, try to discover another fog node, otherwise fail
guard await setup(continuation: continuation) else {
Self.logger.error("SpeziLLMFog: mDNS service could not be resolved to an IP.")
await finishGenerationWithError(LLMFogError.mDnsServicesNotFound, on: continuation)
return false
}
}

return true
}
}
45 changes: 10 additions & 35 deletions Sources/SpeziLLMFog/LLMFogSession+Setup.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,42 +63,17 @@ extension LLMFogSession {
return false
}

// Overwrite user id token if passed
if let overwritingToken = schema.parameters.overwritingToken {
self.wrappedModel = OpenAI(
configuration: .init(
token: overwritingToken,
host: fogServiceAddress,
port: (caCertificate != nil) ? 443 : 80,
scheme: (caCertificate != nil) ? "https" : "http",
timeoutInterval: platform.configuration.timeout,
caCertificate: caCertificate,
expectedHost: platform.configuration.host
)
self.wrappedModel = OpenAI(
configuration: .init(
token: await schema.parameters.authToken(),
host: fogServiceAddress,
port: (caCertificate != nil) ? 443 : 80,
scheme: (caCertificate != nil) ? "https" : "http",
timeoutInterval: platform.configuration.timeout,
caCertificate: caCertificate,
expectedHost: platform.configuration.host
)
} else {
// Use firebase user id token otherwise
guard let userToken else {
Self.logger.error("""
SpeziLLMFog: Missing user token.
Please ensure that the user is logged in via SpeziAccount and the Firebase identity provider before dispatching the first inference.
""")
await finishGenerationWithError(LLMFogError.userNotAuthenticated, on: continuation)
return false
}

self.wrappedModel = OpenAI(
configuration: .init(
token: userToken,
host: fogServiceAddress,
port: (caCertificate != nil) ? 443 : 80,
scheme: (caCertificate != nil) ? "https" : "http",
timeoutInterval: platform.configuration.timeout,
caCertificate: caCertificate,
expectedHost: platform.configuration.host
)
)
}
)

await MainActor.run {
self.state = .ready
Expand Down
Loading

0 comments on commit c2187bb

Please sign in to comment.