@meridius-labs/apple-on-device-ai
Version:
TypeScript library for accessing Apple's on-device foundation models (Apple Intelligence) with full Vercel AI SDK compatibility
1,352 lines (1,337 loc) • 41 kB
JavaScript
import { createRequire } from "node:module";
import { Readable } from "node:stream";
import { zodToJsonSchema } from "zod-to-json-schema";
import { existsSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { generateId } from "@ai-sdk/provider-utils";
import assert from "node:assert";
import { createApp, createError, defineEventHandler, readBody, toNodeListener } from "h3";
import { createServer } from "node:http";
import { createServer as createServer$1 } from "node:https";
import { generateObject, generateText, streamText, tool } from "ai";
import { z } from "zod";
//#region rolldown:runtime
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __esm = (fn, res) => function() {
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
};
var __export = (target, all) => {
for (var name in all) __defProp(target, name, {
get: all[name],
enumerable: true
});
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
key = keys[i];
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
get: ((k) => from[k]).bind(null, key),
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
});
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
//#endregion
//#region src/native-loader.ts
/**
* Robust native module loader for Apple AI FFI components
* Handles both development and production/bundled scenarios
*/
function loadNativeModule() {
const possiblePaths = [
resolve(__dirname, "../build/apple_ai_napi.node"),
resolve(process.cwd(), "node_modules/@meridius-labs/apple-on-device-ai/build/apple_ai_napi.node"),
resolve(__dirname, "../../build/apple_ai_napi.node"),
resolve(__dirname, "../native/target/release/apple_ai_napi.node"),
resolve(__dirname, "../../native/target/release/apple_ai_napi.node"),
resolve(__dirname, "../native/target/release/libapple_ai_napi.dylib"),
resolve(__dirname, "../../native/target/release/libapple_ai_napi.dylib"),
resolve(process.cwd(), "build/apple_ai_napi.node")
];
let lastError = null;
const existingPath = possiblePaths.find((path) => {
try {
return existsSync(path);
} catch {
return false;
}
});
if (existingPath) try {
return require$1(existingPath);
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
}
for (const path of possiblePaths) try {
return require$1(path);
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
}
const errorDetails = [
`Failed to load apple_ai_napi native module.`,
`Searched paths:`,
...possiblePaths.map((p) => ` - ${p} ${existsSync(p) ? "(exists)" : "(not found)"}`),
``,
`Platform: ${process.platform} ${process.arch}`,
`Node version: ${process.version}`,
`Working directory: ${process.cwd()}`,
`Script location: ${__dirname}`,
``,
`Last error: ${lastError?.message || "Unknown error"}`,
``,
`This package requires:`,
`- macOS 26+ (current: ${process.platform})`,
`- Apple Silicon (ARM64) (current: ${process.arch})`
].join("\n");
throw new Error(errorDetails);
}
/**
* Get the native module instance (singleton)
*/
function getNativeModule() {
if (!nativeModule) nativeModule = loadNativeModule();
return nativeModule;
}
var __dirname, require$1, nativeModule;
var init_native_loader = __esm({ "src/native-loader.ts"() {
try {
__dirname = globalThis.__dirname || dirname(fileURLToPath(import.meta.url));
} catch {
__dirname = dirname(fileURLToPath(import.meta.url));
}
try {
require$1 = globalThis.require || createRequire(import.meta.url);
} catch {
require$1 = createRequire(import.meta.url);
}
nativeModule = null;
} });
//#endregion
//#region src/apple-ai.ts
/**
* Unified structured generation that accepts either Zod schemas or JSON Schema
*/
async function structured(options) {
const { prompt, schema, temperature, maxTokens } = options;
let jsonSchemaString;
if (typeof schema === "object" && schema !== null && "parse" in schema) {
const jsonSchemaObj = zodToJsonSchema(schema, "Root");
jsonSchemaString = JSON.stringify(jsonSchemaObj);
} else jsonSchemaString = JSON.stringify(schema);
return appleAISDK.generateStructured({
prompt,
schemaJson: jsonSchemaString,
temperature,
maxTokens
});
}
/**
* @deprecated Don't use this function directly. It's used internally by the Vercel AI SDK.
* Stream chat that properly integrates with Vercel AI SDK's multi-step tool calling.
* This function emits tool-call events and ends the stream, allowing the SDK to
* orchestrate tool execution and restart generation with updated messages.
*
* Early termination is enabled by default when tools are present, saving compute
* resources by stopping the Swift streaming loop after tool calls complete.
*
*/
function _streamChatForVercelAISDK(options) {
const toolSchemas = options.tools.map((tool$1, idx) => ({
id: idx + 1,
name: tool$1.name,
description: tool$1.description ?? "",
parameters: tool$1.jsonSchema
}));
const collectedToolCalls = [];
const readable = new Readable({
read() {},
objectMode: true
});
toolBindings.setToolCallback(async (err, id, argsJson) => {
if (err) {
toolBindings.toolResult(id, "{}");
return;
}
const tool$1 = options.tools[id - 1];
if (!tool$1) {
toolBindings.toolResult(id, "{}");
return;
}
try {
const args = JSON.parse(argsJson);
collectedToolCalls.push({
id,
toolName: tool$1.name,
args
});
toolBindings.toolResult(id, "{}");
} catch (_error) {
toolBindings.toolResult(id, "{}");
}
});
const messagesJson = JSON.stringify(options.messages);
const schemasJson = JSON.stringify(toolSchemas);
let generationComplete = false;
const finishWithToolCalls = () => {
if (generationComplete) return;
generationComplete = true;
for (const call of collectedToolCalls) readable.push({
type: "tool-call",
toolCallId: `tool-call-${crypto.randomUUID()}`,
toolName: call.toolName,
args: call.args
});
readable.push(null);
toolBindings.clearToolCallback?.();
};
unifiedBindings.generateUnifiedStream(messagesJson, schemasJson, null, options.temperature ?? void 0, void 0, true, (err, chunk) => {
if (err) {
readable.destroy(err);
toolBindings.clearToolCallback?.();
return;
}
if (chunk === null || chunk === "") {
finishWithToolCalls();
return;
}
readable.push({
type: "text",
text: chunk
});
});
return readable[Symbol.asyncIterator]();
}
function chat(options) {
const { messages, tools, schema, temperature, maxTokens, stopAfterToolCalls = true, stream = false } = options;
const normalizedMessages = typeof messages === "string" ? [{
role: "user",
content: messages
}] : messages;
const messagesJson = JSON.stringify(normalizedMessages);
let toolsJson = null;
const toolMap = /* @__PURE__ */ new Map();
if (tools && tools.length > 0) {
const toolSchemas = tools.map((tool$1, idx) => {
const id = idx + 1;
toolMap.set(id, tool$1);
return {
id,
name: tool$1.name,
description: tool$1.description ?? "",
parameters: tool$1.jsonSchema
};
});
toolsJson = JSON.stringify(toolSchemas);
toolBindings.setToolCallback(async (err, id, argsJson) => {
if (err) {
toolBindings.toolResult(id, "{}");
return;
}
const tool$1 = toolMap.get(id);
if (!tool$1) {
toolBindings.toolResult(id, "{}");
return;
}
try {
const result = await tool$1.handler(JSON.parse(argsJson));
toolBindings.toolResult(id, JSON.stringify(result ?? null));
} catch {
toolBindings.toolResult(id, "{}");
}
});
}
let schemaJson = null;
if (!tools && schema) if (typeof schema === "object" && schema !== null && "parse" in schema) {
const jsonSchemaObj = zodToJsonSchema(schema, "Root");
schemaJson = JSON.stringify(jsonSchemaObj);
} else schemaJson = JSON.stringify(schema);
if (stream) {
const readable = new Readable({
read() {},
objectMode: true
});
unifiedBindings.generateUnifiedStream(messagesJson, toolsJson, schemaJson, temperature, maxTokens, stopAfterToolCalls, (err, chunk) => {
if (err) {
readable.destroy(err);
if (toolMap.size > 0) toolBindings.clearToolCallback?.();
return;
}
if (chunk === null || chunk === "") {
readable.push(null);
if (toolMap.size > 0) toolBindings.clearToolCallback?.();
return;
}
readable.push(chunk);
});
return readable[Symbol.asyncIterator]();
} else return (async () => {
try {
const raw = await unifiedBindings.generateUnified(messagesJson, toolsJson, schemaJson, temperature, maxTokens, stopAfterToolCalls);
if (raw?.startsWith("Error: ")) throw new Error(raw.slice(7));
const parsed = JSON.parse(raw);
if (schemaJson && parsed.object) return {
text: parsed.text,
object: parsed.object
};
else if (parsed.toolCalls) return {
text: parsed.text,
toolCalls: parsed.toolCalls
};
else return { text: parsed.text };
} finally {
if (toolMap.size > 0) toolBindings.clearToolCallback?.();
}
})();
}
var native, unifiedBindings, toolBindings, AppleAISDK, appleAISDK;
var init_apple_ai = __esm({ "src/apple-ai.ts"() {
init_native_loader();
native = getNativeModule();
unifiedBindings = {
generateUnified: native.generateUnified,
generateUnifiedStream: native.generateUnifiedStream
};
toolBindings = {
setToolCallback: native.setToolCallback,
clearToolCallback: native.clearToolCallback,
toolResult: native.toolResult
};
AppleAISDK = class {
/** Check availability of Apple Intelligence */
async checkAvailability() {
return native.checkAvailability();
}
/** Get supported languages */
getSupportedLanguages() {
return native.getSupportedLanguages();
}
/** Generate a response for a prompt */
async generateResponse(prompt, options = {}) {
const messages = [{
role: "user",
content: prompt
}];
const messagesJson = JSON.stringify(messages);
const result = await unifiedBindings.generateUnified(messagesJson, null, null, options.temperature ?? void 0, options.maxTokens ?? void 0, true);
const parsed = JSON.parse(result);
return parsed.text || result;
}
/** Generate a response using conversation history */
async generateResponseWithHistory(messages, options = {}) {
const messagesJson = JSON.stringify(messages);
const result = await unifiedBindings.generateUnified(messagesJson, null, null, options.temperature ?? void 0, options.maxTokens ?? void 0, true);
const parsed = JSON.parse(result);
return parsed.text || result;
}
/**
* Stream chat completion as async generator yielding OpenAI-compatible chunks
*/
streamChatCompletion(messages, options = {}) {
const completionId = `chatcmpl-${crypto.randomUUID()}`;
const created = Math.floor(Date.now() / 1e3);
const queue = [];
let done = false;
let isFirstChunk = true;
let pendingResolve = null;
let pendingReject = null;
let error = null;
const handleChunk = (err, chunk) => {
if (err) {
error = err;
done = true;
if (pendingReject) {
pendingReject(err);
pendingResolve = null;
pendingReject = null;
}
return;
}
let chatChunk;
if (chunk == null || chunk === "") {
chatChunk = {
id: completionId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta: {},
finish_reason: "stop"
}]
};
done = true;
} else {
chatChunk = {
id: completionId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta: {
...isFirstChunk ? { role: "assistant" } : {},
content: chunk
},
finish_reason: null
}]
};
isFirstChunk = false;
}
if (pendingResolve) {
pendingResolve({
value: chatChunk,
done: false
});
pendingResolve = null;
pendingReject = null;
} else queue.push(chatChunk);
};
const messagesJson = JSON.stringify(messages);
unifiedBindings.generateUnifiedStream(messagesJson, null, null, options.temperature ?? void 0, options.maxTokens ?? void 0, true, handleChunk);
return {
next() {
if (queue.length > 0) {
const value = queue.shift();
return Promise.resolve({
value,
done: false
});
}
if (done) return Promise.resolve({
value: void 0,
done: true
});
if (error) return Promise.reject(error);
return new Promise((resolve$1, reject) => {
pendingResolve = resolve$1;
pendingReject = reject;
});
},
async return() {
done = true;
return {
value: void 0,
done: true
};
},
async throw(err) {
done = true;
throw err;
},
[Symbol.asyncIterator]() {
return this;
}
};
}
/** Generate a structured object based on a Zod/JSON schema */
async generateStructured(params) {
const { prompt, schemaJson, temperature, maxTokens } = params;
const messages = [{
role: "user",
content: prompt
}];
const messagesJson = JSON.stringify(messages);
const raw = await unifiedBindings.generateUnified(messagesJson, null, schemaJson, temperature ?? void 0, maxTokens ?? void 0);
if (!raw) throw new Error("apple_ai_generate_unified returned null");
if (raw.startsWith("Error: ")) throw new Error(raw.slice(7));
let parsed;
try {
parsed = JSON.parse(raw);
} catch {
throw new Error(`Invalid JSON returned from native: ${raw}`);
}
if (typeof parsed !== "object" || parsed === null) throw new Error(`Unexpected response shape: ${raw}`);
return parsed;
}
};
appleAISDK = new AppleAISDK();
} });
//#endregion
//#region src/apple-ai-chat-model.ts
var AppleAIChatLanguageModel;
var init_apple_ai_chat_model = __esm({ "src/apple-ai-chat-model.ts"() {
init_apple_ai();
AppleAIChatLanguageModel = class {
specificationVersion = "v2";
provider;
modelId;
defaultObjectGenerationMode = "json";
settings;
config;
supportsImageUrls = false;
supportsStructuredOutputs = true;
constructor(modelId, settings, config) {
this.provider = config.provider;
this.modelId = modelId;
this.settings = settings;
this.config = config;
}
supportedUrls = {};
doGenerate(options) {
return this.generateResponse(options);
}
async generateResponse(options) {
const availability = await appleAISDK.checkAvailability();
if (!availability.available) throw new Error(`Apple Intelligence not available: ${availability.reason}`);
const isStructuredGeneration = options.responseFormat && options.responseFormat.type === "json" && options.responseFormat.schema;
if (isStructuredGeneration) return this.handleStructuredGeneration(options);
else return this.handleRegularGeneration(options);
}
async handleStructuredGeneration(options) {
if (options.responseFormat?.type !== "json" || !options.responseFormat?.schema) throw new Error("Structured generation must have a JSON schema. Please use the responseFormat property to specify the schema.");
const schema = options.responseFormat.schema;
const messages = this.convertPromptToMessages(options.prompt);
try {
const result = await chat({
messages,
schema,
temperature: this.settings.temperature,
maxTokens: options.maxOutputTokens ?? this.settings.maxTokens
});
if (result.object) return {
content: [{
type: "text",
text: JSON.stringify(result.object)
}],
finishReason: "stop",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
},
warnings: []
};
else return {
content: [{
type: "text",
text: result.text
}],
finishReason: "stop",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
},
warnings: []
};
} catch (error) {
throw new Error(`Structured generation failed: ${error.message}`);
}
}
async handleRegularGeneration(options) {
if (options.tools && options.tools.length > 0) {
const epTools = options.tools.map((t) => {
if (t.type !== "function") throw new Error(`Unsupported tool type: ${t?.type ?? "unknown"}`);
return {
name: t.name,
description: t.description,
jsonSchema: t.inputSchema,
handler: async (_args) => {
return {};
}
};
});
const messages = this.convertPromptToMessages(options.prompt);
const result = await chat({
messages,
tools: epTools,
temperature: this.settings.temperature,
maxTokens: options.maxOutputTokens ?? this.settings.maxTokens
});
if (result.toolCalls && result.toolCalls.length > 0) {
const toolCallContent = result.toolCalls.map((call) => ({
type: "tool-call",
toolCallType: "function",
toolCallId: call.id,
toolName: call.function.name,
input: call.function.arguments
}));
return {
content: toolCallContent,
finishReason: "tool-calls",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
},
warnings: []
};
}
return {
content: [{
type: "text",
text: result.text || ""
}],
finishReason: "stop",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
},
warnings: []
};
} else {
const messages = this.convertPromptToMessages(options.prompt);
const result = await chat({
messages,
temperature: this.settings.temperature
});
return {
content: [{
type: "text",
text: result.text || ""
}],
finishReason: "stop",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
},
warnings: []
};
}
}
supportsUrl(_url) {
return true;
}
async doStream(options) {
const { prompt, tools } = options;
const availability = await appleAISDK.checkAvailability();
if (!availability.available) throw new Error(`Apple Intelligence not available: ${availability.reason}`);
const messages = this.convertPromptToMessages(prompt);
if (tools && tools.length > 0) return this.createToolEnabledStream(messages, tools);
else return this.createRegularStream(messages);
}
createToolEnabledStream(messages, tools) {
const epTools = tools.map((t) => {
if (t.type !== "function") throw new Error(`Unsupported tool type: ${t?.type ?? "unknown"}`);
return {
name: t.name,
description: t.description,
jsonSchema: t.inputSchema,
handler: async (_args) => {
return {};
}
};
});
const nativeStream = _streamChatForVercelAISDK({
messages,
tools: epTools,
temperature: this.settings.temperature
});
const stream = this.createStreamFromEvents(nativeStream);
return Promise.resolve({ stream });
}
createRegularStream(messages) {
const streamNoTools = appleAISDK.streamChatCompletion(messages, {
temperature: this.settings.temperature,
maxTokens: this.settings.maxTokens
});
const stream = this.createStreamFromChunks(streamNoTools);
return Promise.resolve({ stream });
}
createStreamFromEvents(nativeStream) {
const finishStream = this.finishStream;
return new ReadableStream({ async start(controller) {
try {
for await (const event of nativeStream) if (event.type === "text") controller.enqueue({
type: "text-delta",
delta: event.text,
id: crypto.randomUUID()
});
else if (event.type === "tool-call") controller.enqueue({
type: "tool-call",
toolCallId: event.toolCallId,
toolName: event.toolName,
input: JSON.stringify(event.args)
});
finishStream(controller);
} catch (err) {
controller.error(err);
}
} });
}
createStreamFromChunks(streamNoTools) {
const finishStream = this.finishStream;
return new ReadableStream({ async start(controller) {
try {
for await (const chunk of streamNoTools) {
let text = "";
if (typeof chunk === "string") text = chunk;
else text = chunk.choices?.[0]?.delta?.content ?? "";
if (text) controller.enqueue({
type: "text-delta",
delta: text,
id: crypto.randomUUID()
});
}
finishStream(controller);
} catch (error) {
controller.error(error);
}
} });
}
finishStream(controller) {
controller.enqueue({
type: "finish",
finishReason: "stop",
usage: {
inputTokens: void 0,
outputTokens: void 0,
totalTokens: void 0
}
});
controller.close();
}
convertPromptToMessages(prompt) {
return prompt.map((message) => {
switch (message.role) {
case "system": return this.convertSystemMessage(message);
case "user": return this.convertUserMessage(message);
case "assistant": return this.convertAssistantMessage(message);
case "tool": return this.convertToolMessage(message);
default: return this.convertFallbackMessage(message);
}
});
}
convertSystemMessage(message) {
return {
role: "system",
content: message.content
};
}
convertUserMessage(message) {
return {
role: "user",
content: Array.isArray(message.content) ? message.content.map((part) => part.type === "text" ? part.text : "[unsupported content]").join("\n") : message.content
};
}
convertAssistantMessage(message) {
if (Array.isArray(message.content)) {
const toolCalls = message.content.filter((part) => part.type === "tool-call");
const textParts = message.content.filter((part) => part.type === "text");
if (toolCalls.length > 0) return {
role: "assistant",
content: textParts.map((part) => part.text).join("\n") || "",
tool_calls: toolCalls.map((part) => ({
id: part.toolCallId,
type: "function",
function: {
name: part.toolName,
arguments: JSON.stringify(part.input)
}
}))
};
return {
role: "assistant",
content: message.content.map((part) => {
switch (part.type) {
case "text":
case "reasoning": return part.text;
default: return `[unsupported content - ${part.type}]`;
}
}).join("\n")
};
}
return {
role: "assistant",
content: message.content || ""
};
}
convertToolMessage(message) {
assert(message.content.every((part) => part.type === "tool-result"), "Tool message must contain only tool-result parts");
const toolCalls = message.content.map((part) => {
if (part.type === "tool-result") return {
id: part.toolCallId,
toolName: part.toolName,
segments: [{
type: "text",
text: typeof part.output === "string" ? part.output : JSON.stringify(part.output)
}]
};
return null;
}).filter(Boolean);
const convertedMessage = {
role: "tool",
content: JSON.stringify({ tool_calls: toolCalls })
};
return convertedMessage;
}
convertFallbackMessage(message) {
return {
role: "user",
content: String(message.content || "")
};
}
};
} });
//#endregion
//#region src/apple-ai-provider.ts
function createAppleAI(options = {}) {
const createModel = (modelId, settings = {}) => new AppleAIChatLanguageModel(modelId, settings, {
provider: "localhost",
headers: options.headers ?? {},
generateId: options.generateId ?? generateId
});
const provider = function(modelId, settings) {
if (new.target) throw new Error("The Apple AI provider function cannot be called with the new keyword.");
return createModel(modelId, settings);
};
provider.chat = createModel;
return provider;
}
var appleAI;
var init_apple_ai_provider = __esm({ "src/apple-ai-provider.ts"() {
init_apple_ai_chat_model();
appleAI = createAppleAI();
} });
//#endregion
//#region src/server/routes/models.ts
var models_exports = {};
__export(models_exports, { models: () => models });
var models;
var init_models = __esm({ "src/server/routes/models.ts"() {
models = defineEventHandler(async (_event) => {
return {
object: "list",
data: [{
id: "apple-on-device",
object: "model",
created: 1686935002,
owned_by: "apple",
permission: [{
id: "modelperm-apple-on-device",
object: "model_permission",
created: 1686935002,
allow_create_engine: false,
allow_sampling: true,
allow_logprobs: false,
allow_search_indices: false,
allow_view: true,
allow_fine_tuning: false,
organization: "*",
group: null,
is_blocking: false
}],
root: "apple-on-device",
parent: null
}]
};
});
} });
//#endregion
//#region src/server/routes/chat-completions.ts
var chat_completions_exports = {};
__export(chat_completions_exports, { chatCompletions: () => chatCompletions });
function convertOpenAIMessages(messages) {
return messages.map((msg) => {
const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content);
return {
role: msg.role,
content
};
});
}
function jsonSchemaToZod(schema) {
if (schema.type === "object" && schema.properties) {
const zodFields = {};
for (const [key, value] of Object.entries(schema.properties)) {
let fieldSchema;
if (value.type === "string") fieldSchema = z.string();
else if (value.type === "number") fieldSchema = z.number();
else if (value.type === "integer") fieldSchema = z.number().int();
else if (value.type === "boolean") fieldSchema = z.boolean();
else if (value.type === "array") fieldSchema = z.array(z.any());
else fieldSchema = z.any();
if (value.description) fieldSchema = fieldSchema.describe(value.description);
if (!schema.required?.includes(key)) fieldSchema = fieldSchema.optional();
zodFields[key] = fieldSchema;
}
return z.object(zodFields);
}
return z.any();
}
function convertOpenAITools(openAITools) {
const tools = {};
for (const openAITool of openAITools) {
const toolName = openAITool.function.name;
const inputSchema = jsonSchemaToZod(openAITool.function.parameters || {});
tools[toolName] = tool({
description: openAITool.function.description || "",
inputSchema
});
}
return tools;
}
function createChatCompletionMessage(role, content, tool_calls) {
const message = {
role,
content,
refusal: null
};
if (tool_calls) message.tool_calls = tool_calls;
return message;
}
var chatCompletions;
var init_chat_completions = __esm({ "src/server/routes/chat-completions.ts"() {
init_apple_ai_provider();
chatCompletions = defineEventHandler(async (event) => {
if (event.node.req.method !== "POST") {
event.node.res.statusCode = 405;
return { error: "Method not allowed" };
}
const body = await readBody(event);
const { messages, temperature: rawTemperature, stream = false, tools, response_format } = body;
if (!Array.isArray(messages)) {
event.node.res.statusCode = 400;
return { error: "'messages' field is required and must be an array" };
}
if (messages.length === 0) {
event.node.res.statusCode = 400;
return { error: "messages array cannot be empty" };
}
const temperature = rawTemperature !== null ? rawTemperature : void 0;
const maxOutputTokens = body.max_completion_tokens ?? body.max_tokens ?? void 0;
const appleAIModel = appleAI("apple-on-device", { temperature });
const convertedMessages = convertOpenAIMessages(messages);
const aiSDKTools = tools ? convertOpenAITools(tools) : void 0;
try {
await generateText({
model: appleAIModel,
prompt: "test",
maxOutputTokens: 1
});
} catch (error) {
event.node.res.statusCode = 503;
return { error: {
message: `Apple Intelligence not available: ${error.message}`,
type: "service_unavailable",
code: "apple_intelligence_unavailable"
} };
}
if (stream) {
event.node.res.setHeader("Content-Type", "text/event-stream");
event.node.res.setHeader("Cache-Control", "no-cache");
event.node.res.setHeader("Connection", "keep-alive");
event.node.res.setHeader("X-Accel-Buffering", "no");
const res = event.node.res;
const streamId = `chatcmpl-${crypto.randomUUID()}`;
const created = Math.floor(Date.now() / 1e3);
try {
if (response_format?.type === "json_schema" && response_format.json_schema?.schema) {
const zodSchema = jsonSchemaToZod(response_format.json_schema.schema);
const result = await generateObject({
model: appleAIModel,
messages: convertedMessages,
schema: zodSchema,
temperature,
maxOutputTokens
});
const data = {
id: streamId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta: {
role: "assistant",
content: JSON.stringify(result.object)
},
finish_reason: null
}]
};
res.write(`data: ${JSON.stringify(data)}\n\n`);
} else {
const { fullStream } = streamText({
model: appleAIModel,
messages: convertedMessages,
tools: aiSDKTools,
temperature,
maxOutputTokens
});
let sentRole = false;
for await (const chunk of fullStream) if (chunk.type === "text") {
const delta = {};
if (!sentRole) {
delta.role = "assistant";
sentRole = true;
}
if (chunk.text) delta.content = chunk.text;
const data = {
id: streamId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta,
finish_reason: null
}]
};
res.write(`data: ${JSON.stringify(data)}\n\n`);
} else if (chunk.type === "tool-call") {
const toolCallDelta = {
id: streamId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta: { tool_calls: [{
index: 0,
id: chunk.toolCallId,
type: "function",
function: {
name: chunk.toolName,
arguments: JSON.stringify(chunk.input)
}
}] },
finish_reason: null
}]
};
res.write(`data: ${JSON.stringify(toolCallDelta)}\n\n`);
}
}
const finalData = {
id: streamId,
object: "chat.completion.chunk",
created,
model: "apple-on-device",
choices: [{
index: 0,
delta: {},
finish_reason: "stop"
}]
};
res.write(`data: ${JSON.stringify(finalData)}\n\n`);
res.write("data: [DONE]\n\n");
res.end();
} catch (error) {
const errorData = { error: {
message: error.message,
type: "internal_error",
code: "stream_error"
} };
res.write(`data: ${JSON.stringify(errorData)}\n\n`);
res.end();
}
} else try {
if (response_format?.type === "json_schema" && response_format.json_schema?.schema) {
const zodSchema = jsonSchemaToZod(response_format.json_schema.schema);
const result = await generateObject({
model: appleAIModel,
messages: convertedMessages,
schema: zodSchema,
temperature,
maxOutputTokens
});
const completionId = `chatcmpl-${crypto.randomUUID()}`;
const created = Math.floor(Date.now() / 1e3);
const response = {
id: completionId,
object: "chat.completion",
created,
model: "apple-on-device",
choices: [{
index: 0,
message: createChatCompletionMessage("assistant", JSON.stringify(result.object)),
finish_reason: "stop",
logprobs: null
}],
usage: {
prompt_tokens: result.usage?.inputTokens ?? 0,
completion_tokens: result.usage?.outputTokens ?? 0,
total_tokens: result.usage?.totalTokens ?? 0
}
};
return response;
} else {
const result = await generateText({
model: appleAIModel,
messages: convertedMessages,
tools: aiSDKTools,
temperature,
maxOutputTokens
});
const completionId = `chatcmpl-${crypto.randomUUID()}`;
const created = Math.floor(Date.now() / 1e3);
const toolCalls = result.toolCalls?.map((call) => ({
id: call.toolCallId,
type: "function",
function: {
name: call.toolName,
arguments: JSON.stringify(call.input)
}
}));
const response = {
id: completionId,
object: "chat.completion",
created,
model: "apple-on-device",
choices: [{
index: 0,
message: createChatCompletionMessage("assistant", result.text || null, toolCalls && toolCalls.length > 0 ? toolCalls : void 0),
finish_reason: toolCalls && toolCalls.length > 0 ? "tool_calls" : "stop",
logprobs: null
}],
usage: {
prompt_tokens: result.usage?.inputTokens ?? 0,
completion_tokens: result.usage?.outputTokens ?? 0,
total_tokens: result.usage?.totalTokens ?? 0
}
};
return response;
}
} catch (error) {
event.node.res.statusCode = 500;
return { error: {
message: error.message,
type: "internal_error",
code: "completion_error"
} };
}
});
} });
//#endregion
//#region src/server/routes/health.ts
var health_exports = {};
__export(health_exports, { health: () => health });
var health;
var init_health = __esm({ "src/server/routes/health.ts"() {
init_apple_ai();
health = defineEventHandler(async (_event) => {
try {
const availability = await appleAISDK.checkAvailability();
return {
status: "ok",
apple_intelligence: availability,
timestamp: (/* @__PURE__ */ new Date()).toISOString()
};
} catch (error) {
return {
status: "error",
error: error.message,
timestamp: (/* @__PURE__ */ new Date()).toISOString()
};
}
});
} });
//#endregion
//#region src/server/routes/api-tags.ts
var api_tags_exports = {};
__export(api_tags_exports, { apiTags: () => apiTags });
var apiTags;
var init_api_tags = __esm({ "src/server/routes/api-tags.ts"() {
apiTags = defineEventHandler(async (_event) => {
return { models: [{
model: "apple-on-device",
name: "apple-on-device",
size: 0,
digest: "sha256:apple-on-device",
details: {
parent_model: "",
format: "gguf",
family: "apple",
families: ["apple"],
parameter_size: "unknown",
quantization_level: "unknown"
},
expires_at: "0001-01-01T00:00:00Z",
size_vram: 0
}] };
});
} });
//#endregion
//#region src/server/routes/api-show.ts
var api_show_exports = {};
__export(api_show_exports, { apiShow: () => apiShow });
var apiShow;
var init_api_show = __esm({ "src/server/routes/api-show.ts"() {
apiShow = defineEventHandler(async (event) => {
const body = await readBody(event);
const { model } = body;
if (!model) throw createError({
statusCode: 400,
statusMessage: "Model name is required"
});
if (model !== "apple-on-device") throw createError({
statusCode: 404,
statusMessage: "Model not found"
});
return {
modelfile: `# Modelfile for Apple On-Device AI
FROM apple-on-device
PARAMETER stop "<|im_end|>"
PARAMETER stop "<|im_start|>"
SYSTEM """You are a helpful assistant."""`,
parameters: "stop [\"<|im_end|>\", \"<|im_start|>\"]",
template: `{{ if .System }}<|im_start|>system
{{ .System }}<|im_end|>
{{ end }}{{ if .Prompt }}<|im_start|>user
{{ .Prompt }}<|im_end|>
{{ end }}<|im_start|>assistant
`,
details: {
parent_model: "",
format: "gguf",
family: "apple",
families: ["apple"],
parameter_size: "unknown",
quantization_level: "unknown"
},
model_info: {
"general.architecture": "apple",
"general.basename": "apple-on-device",
"general.description": "Apple On-Device AI Model",
"general.license": "Apple",
"general.name": "Apple On-Device AI",
"general.parameter_count": 3e9,
"general.size_label": "3B",
"apple.context_length": 8192,
"apple.embedding_length": 4096,
"apple.block_count": 32,
"apple.attention.head_count": 32,
"apple.attention.key_length": 128,
"apple.attention.value_length": 128,
"apple.feed_forward_length": 11008
},
capabilities: ["vision", "tools"],
modified_at: (/* @__PURE__ */ new Date()).toISOString()
};
});
} });
//#endregion
//#region src/server/index.ts
/**
* Authentication middleware
*/
function createAuthMiddleware(bearerToken) {
return defineEventHandler(async (event) => {
if (!bearerToken) return;
if (event.node.req.url === "/health") return;
const authHeader = event.node.req.headers.authorization;
if (!authHeader || !authHeader.startsWith("Bearer ")) {
event.node.res.statusCode = 401;
event.node.res.setHeader("Content-Type", "application/json");
event.node.res.end(JSON.stringify({ error: "Unauthorized" }));
return;
}
const token = authHeader.substring(7);
if (token !== bearerToken) {
event.node.res.statusCode = 401;
event.node.res.setHeader("Content-Type", "application/json");
event.node.res.end(JSON.stringify({ error: "Invalid token" }));
return;
}
});
}
/**
* Setup routes for the app
*/
function setupRoutes(app, options) {
const { models: models$1 } = (init_models(), __toCommonJS(models_exports));
const { chatCompletions: chatCompletions$1 } = (init_chat_completions(), __toCommonJS(chat_completions_exports));
const { health: health$1 } = (init_health(), __toCommonJS(health_exports));
const { apiTags: apiTags$1 } = (init_api_tags(), __toCommonJS(api_tags_exports));
const { apiShow: apiShow$1 } = (init_api_show(), __toCommonJS(api_show_exports));
if (options.bearerToken) app.use(createAuthMiddleware(options.bearerToken));
app.use("/v1/models", models$1);
app.use("/v1/chat/completions", chatCompletions$1);
app.use("/health", health$1);
app.use("/v1/api/tags", apiTags$1);
app.use("/v1/api/show", apiShow$1);
app.use(defineEventHandler((event) => {
if (options.onError && event.node.res.statusCode >= 400) options.onError(/* @__PURE__ */ new Error(`${event.node.res.statusCode}: ${event.node.req.url}`));
event.node.res.statusCode = 404;
event.node.res.setHeader("Content-Type", "application/json");
event.node.res.end(JSON.stringify({ error: "Not found" }));
}));
}
/**
* Start the OpenAI-compatible server
*/
async function startServer(opts = {}) {
const host = opts.host ?? "localhost";
let port = opts.port ?? 8080;
if (port === 0) port = await new Promise((resolve$1, reject) => {
const server$1 = createServer();
server$1.once("error", reject);
server$1.listen(0, () => {
const address = server$1.address();
if (address && typeof address === "object") resolve$1(address.port);
else reject(/* @__PURE__ */ new Error("Port is 0, but failed to get random port"));
server$1.close();
});
});
const httpsConf = opts.https;
const app = createApp();
setupRoutes(app, opts);
let server;
let httpsServer;
if (httpsConf) {
const creds = typeof httpsConf === "object" ? {
cert: readFileSync(httpsConf.cert, "utf8"),
key: readFileSync(httpsConf.key, "utf8")
} : void 0;
if (!creds) throw new Error("HTTPS credentials are required, pass them in the https object");
const httpPort = port;
const httpsPort = typeof httpsConf === "object" && httpsConf.port ? httpsConf.port : port + 1;
httpsServer = createServer$1(creds, toNodeListener(app));
await new Promise((resolve$1) => {
httpsServer.listen(httpsPort, host, () => resolve$1());
});
const httpServer = createServer(toNodeListener(app));
await new Promise((resolve$1) => {
httpServer.listen(httpPort, host, () => resolve$1());
});
server = httpsServer;
const urls = {
http: `http://${host}:${httpPort}`,
https: `https://${host}:${httpsPort}`
};
async function stop() {
await Promise.all([new Promise((r) => httpsServer.close(() => r())), new Promise((r) => httpServer.close(() => r()))]);
}
return {
urls,
url: urls.https,
stop,
port: httpPort,
httpsPort
};
} else {
server = createServer(toNodeListener(app));
await new Promise((resolve$1) => {
server.listen(port, host, () => resolve$1());
});
const urls = {
http: `http://${host}:${port}`,
https: void 0
};
async function stop() {
return new Promise((resolve$1) => {
server.close(() => resolve$1());
});
}
return {
urls,
url: urls.http,
stop,
port
};
}
}
//#endregion
//#region src/index.ts
init_apple_ai();
init_apple_ai_provider();
init_apple_ai_chat_model();
init_apple_ai();
init_apple_ai_provider();
//#endregion
export { AppleAIChatLanguageModel, AppleAISDK, _streamChatForVercelAISDK, appleAI, appleAISDK, chat, appleAI as createAppleAI, startServer, structured };