**Refactor:** Restructure server package to streamline imports and improve file organization

- Moved `providers`, `services`, `models`, `lib`, and related files to `src` directory within `server` package.
- Adjusted imports across the codebase to reflect the new paths.
- Renamed several `.ts` files for consistency.
- Introduced an `index.ts` in the `ai/providers` package to export all providers.

This improves maintainability and aligns with the project's updated directory structure.

packages/ai/index.ts

@@ -1,2 +1 @@
-// for future use
-export {};
+export * from './providers';

packages/ai/package.json

@@ -1,4 +1,7 @@
 {
   "name": "@open-gsio/ai",
-  "module": "index.ts"
+  "module": "index.ts",
+  "devDependencies": {
+    "@open-gsio/env": "workspace:*"
+  }
 }

packages/ai/providers/_ProviderRepository.ts

@@ -0,0 +1,86 @@
+export type GenericEnv = Record<string, any>;
+
+export class ProviderRepository {
+  #providers: { name: string; key: string; endpoint: string }[] = [];
+  #env: Record<string, any>;
+
+  constructor(env: GenericEnv) {
+    this.#env = env;
+    this.setProviders(env);
+  }
+
+  static OPENAI_COMPAT_ENDPOINTS = {
+    xai: 'https://api.x.ai/v1',
+    groq: 'https://api.groq.com/openai/v1',
+    google: 'https://generativelanguage.googleapis.com/v1beta/openai',
+    fireworks: 'https://api.fireworks.ai/inference/v1',
+    cohere: 'https://api.cohere.ai/compatibility/v1',
+    cloudflare: 'https://api.cloudflare.com/client/v4/accounts/{CLOUDFLARE_ACCOUNT_ID}/ai/v1',
+    anthropic: 'https://api.anthropic.com/v1',
+    openai: 'https://api.openai.com/v1',
+    cerebras: 'https://api.cerebras.com/v1',
+    ollama: 'http://localhost:11434/v1',
+    mlx: 'http://localhost:10240/v1',
+  };
+
+  static async getModelFamily(model: any, env: Env) {
+    const allModels = await env.KV_STORAGE.get('supportedModels');
+    const models = JSON.parse(allModels);
+    const modelData = models.filter(m => m.id === model);
+    return modelData[0].provider;
+  }
+
+  static async getModelMeta(meta, env) {
+    const allModels = await env.KV_STORAGE.get('supportedModels');
+    const models = JSON.parse(allModels);
+    return models.filter(m => m.id === meta.model).pop();
+  }
+
+  getProviders(): { name: string; key: string; endpoint: string }[] {
+    return this.#providers;
+  }
+
+  setProviders(env: Record<string, any>) {
+    const envKeys = Object.keys(env);
+    for (let i = 0; i < envKeys.length; i++) {
+      if (envKeys[i].endsWith('KEY')) {
+        const detectedProvider = envKeys[i].split('_')[0].toLowerCase();
+        const detectedProviderValue = env[envKeys[i]];
+        if (detectedProviderValue) {
+          switch (detectedProvider) {
+            case 'anthropic':
+              this.#providers.push({
+                name: 'anthropic',
+                key: env.ANTHROPIC_API_KEY,
+                endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['anthropic'],
+              });
+              break;
+            case 'gemini':
+              this.#providers.push({
+                name: 'google',
+                key: env.GEMINI_API_KEY,
+                endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS['google'],
+              });
+              break;
+            case 'cloudflare':
+              this.#providers.push({
+                name: 'cloudflare',
+                key: env.CLOUDFLARE_API_KEY,
+                endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider].replace(
+                  '{CLOUDFLARE_ACCOUNT_ID}',
+                  env.CLOUDFLARE_ACCOUNT_ID,
+                ),
+              });
+              break;
+            default:
+              this.#providers.push({
+                name: detectedProvider,
+                key: env[envKeys[i]],
+                endpoint: ProviderRepository.OPENAI_COMPAT_ENDPOINTS[detectedProvider],
+              });
+          }
+        }
+      }
+    }
+  }
+}

packages/ai/providers/__tests__/chat-stream-provider.test.ts

@@ -0,0 +1,75 @@
+import { OpenAI } from 'openai';
+import { describe, it, expect, vi } from 'vitest';
+
+import {
+  BaseChatProvider,
+  CommonProviderParams,
+  ChatStreamProvider,
+} from '../chat-stream-provider.ts';
+
+// Create a concrete implementation of BaseChatProvider for testing
+class TestChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return param.openai as OpenAI;
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+      max_tokens: param.maxTokens as number,
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+// Mock dependencies
+vi.mock('../../lib/chat-sdk', () => ({
+  default: {
+    buildAssistantPrompt: vi.fn().mockReturnValue('Assistant prompt'),
+    buildMessageChain: vi.fn().mockReturnValue([
+      { role: 'system', content: 'System prompt' },
+      { role: 'user', content: 'User message' },
+    ]),
+  },
+}));
+
+describe('ChatStreamProvider', () => {
+  it('should define the required interface', () => {
+    // Verify the interface has the required method
+    const mockProvider: ChatStreamProvider = {
+      handleStream: vi.fn(),
+    };
+
+    expect(mockProvider.handleStream).toBeDefined();
+  });
+});
+
+describe('BaseChatProvider', () => {
+  it('should implement the ChatStreamProvider interface', () => {
+    // Create a concrete implementation
+    const provider = new TestChatProvider();
+
+    // Verify it implements the interface
+    expect(provider.handleStream).toBeInstanceOf(Function);
+    expect(provider.getOpenAIClient).toBeInstanceOf(Function);
+    expect(provider.getStreamParams).toBeInstanceOf(Function);
+    expect(provider.processChunk).toBeInstanceOf(Function);
+  });
+
+  it('should have abstract methods that need to be implemented', () => {
+    // This test verifies that the abstract methods exist
+    // We can't instantiate BaseChatProvider directly, so we use the concrete implementation
+    const provider = new TestChatProvider();
+
+    // Verify the abstract methods are implemented
+    expect(provider.getOpenAIClient).toBeDefined();
+    expect(provider.getStreamParams).toBeDefined();
+    expect(provider.processChunk).toBeDefined();
+  });
+});

packages/ai/providers/cerebras.ts

@@ -0,0 +1,73 @@
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, type CommonProviderParams } from './chat-stream-provider.ts';
+
+export class CerebrasChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.cerebras,
+      apiKey: param.env.CEREBRAS_API_KEY,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    // models provided by cerebras do not follow standard tune params
+    // they must be individually configured
+    // const tuningParams = {
+    //   temperature: 0.86,
+    //   top_p: 0.98,
+    //   presence_penalty: 0.1,
+    //   frequency_penalty: 0.3,
+    //   max_tokens: param.maxTokens as number,
+    // };
+
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+      // ...tuningParams
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class CerebrasSdk {
+  private static provider = new CerebrasChatProvider();
+
+  static async handleCerebrasStream(
+    param: {
+      openai: OpenAI;
+      systemPrompt: any;
+      disableWebhookGeneration: boolean;
+      preprocessedContext: any;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      model: string;
+      env: Env;
+    },
+    dataCallback: (data: any) => void,
+  ) {
+    return this.provider.handleStream(
+      {
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+        disableWebhookGeneration: param.disableWebhookGeneration,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/chat-stream-provider.ts

@@ -0,0 +1,44 @@
+import ChatSdk from '@open-gsio/server/src/lib/chat-sdk.ts';
+import { OpenAI } from 'openai';
+
+export interface CommonProviderParams {
+  openai?: OpenAI; // Optional for providers that use a custom client.
+  systemPrompt: any;
+  preprocessedContext: any;
+  maxTokens: number | unknown | undefined;
+  messages: any;
+  model: string;
+  env: Env;
+  disableWebhookGeneration?: boolean;
+  // Additional fields can be added as needed
+}
+
+export interface ChatStreamProvider {
+  handleStream(param: CommonProviderParams, dataCallback: (data: any) => void): Promise<any>;
+}
+
+export abstract class BaseChatProvider implements ChatStreamProvider {
+  abstract getOpenAIClient(param: CommonProviderParams): OpenAI;
+  abstract getStreamParams(param: CommonProviderParams, safeMessages: any[]): any;
+  abstract async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean>;
+
+  async handleStream(param: CommonProviderParams, dataCallback: (data: any) => void) {
+    const assistantPrompt = ChatSdk.buildAssistantPrompt({ maxTokens: param.maxTokens });
+    const safeMessages = await ChatSdk.buildMessageChain(param.messages, {
+      systemPrompt: param.systemPrompt,
+      model: param.model,
+      assistantPrompt,
+      toolResults: param.preprocessedContext,
+      env: param.env,
+    });
+
+    const client = this.getOpenAIClient(param);
+    const streamParams = this.getStreamParams(param, safeMessages);
+    const stream = await client.chat.completions.create(streamParams);
+
+    for await (const chunk of stream) {
+      const shouldBreak = await this.processChunk(chunk, dataCallback);
+      if (shouldBreak) break;
+    }
+  }
+}

packages/ai/providers/claude.ts

@@ -0,0 +1,123 @@
+import Anthropic from '@anthropic-ai/sdk';
+import ChatSdk from '@open-gsio/server/src/lib/chat-sdk.ts';
+import type {
+  _NotCustomized,
+  ISimpleType,
+  ModelPropertiesDeclarationToProperties,
+  ModelSnapshotType2,
+  UnionStringArray,
+} from 'mobx-state-tree';
+import { OpenAI } from 'openai';
+
+import { BaseChatProvider, type CommonProviderParams } from './chat-stream-provider.ts';
+
+export class ClaudeChatProvider extends BaseChatProvider {
+  private anthropic: Anthropic | null = null;
+
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    // Claude doesn't use OpenAI client directly, but we need to return something
+    // to satisfy the interface. The actual Anthropic client is created in getStreamParams.
+    return param.openai as OpenAI;
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    this.anthropic = new Anthropic({
+      apiKey: param.env.ANTHROPIC_API_KEY,
+    });
+
+    const claudeTuningParams = {
+      temperature: 0.7,
+      max_tokens: param.maxTokens as number,
+    };
+
+    return {
+      stream: true,
+      model: param.model,
+      messages: safeMessages,
+      ...claudeTuningParams,
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.type === 'message_stop') {
+      dataCallback({
+        type: 'chat',
+        data: {
+          choices: [
+            {
+              delta: { content: '' },
+              logprobs: null,
+              finish_reason: 'stop',
+            },
+          ],
+        },
+      });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+
+  // Override the base handleStream method to use Anthropic client instead of OpenAI
+  async handleStream(param: CommonProviderParams, dataCallback: (data: any) => void) {
+    const assistantPrompt = ChatSdk.buildAssistantPrompt({ maxTokens: param.maxTokens });
+    const safeMessages = await ChatSdk.buildMessageChain(param.messages, {
+      systemPrompt: param.systemPrompt,
+      model: param.model,
+      assistantPrompt,
+      toolResults: param.preprocessedContext,
+      env: param.env,
+    });
+
+    const streamParams = this.getStreamParams(param, safeMessages);
+
+    if (!this.anthropic) {
+      throw new Error('Anthropic client not initialized');
+    }
+
+    const stream = await this.anthropic.messages.create(streamParams);
+
+    for await (const chunk of stream as unknown as AsyncIterable<any>) {
+      const shouldBreak = await this.processChunk(chunk, dataCallback);
+      if (shouldBreak) break;
+    }
+  }
+}
+
+// Legacy class for backward compatibility
+export class ClaudeChatSdk {
+  private static provider = new ClaudeChatProvider();
+
+  static async handleClaudeStream(
+    param: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: ModelSnapshotType2<
+        ModelPropertiesDeclarationToProperties<{
+          role: ISimpleType<UnionStringArray<string[]>>;
+          content: ISimpleType<unknown>;
+        }>,
+        _NotCustomized
+      >;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      model: string;
+      env: Env;
+    },
+    dataCallback: (data) => void,
+  ) {
+    return this.provider.handleStream(
+      {
+        openai: param.openai,
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/cloudflareAi.ts

@@ -0,0 +1,142 @@
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, type CommonProviderParams } from './chat-stream-provider.ts';
+
+export class CloudflareAiChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      apiKey: param.env.CLOUDFLARE_API_KEY,
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.cloudflare.replace(
+        '{CLOUDFLARE_ACCOUNT_ID}',
+        param.env.CLOUDFLARE_ACCOUNT_ID,
+      ),
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    const generationParams: Record<string, any> = {
+      model: this.getModelWithPrefix(param.model),
+      messages: safeMessages,
+      stream: true,
+    };
+
+    // Set max_tokens based on model
+    if (this.getModelPrefix(param.model) === '@cf/meta') {
+      generationParams['max_tokens'] = 4096;
+    }
+
+    if (this.getModelPrefix(param.model) === '@hf/mistral') {
+      generationParams['max_tokens'] = 4096;
+    }
+
+    if (param.model.toLowerCase().includes('hermes-2-pro-mistral-7b')) {
+      generationParams['max_tokens'] = 1000;
+    }
+
+    if (param.model.toLowerCase().includes('openhermes-2.5-mistral-7b-awq')) {
+      generationParams['max_tokens'] = 1000;
+    }
+
+    if (param.model.toLowerCase().includes('deepseek-coder-6.7b-instruct-awq')) {
+      generationParams['max_tokens'] = 590;
+    }
+
+    if (param.model.toLowerCase().includes('deepseek-math-7b-instruct')) {
+      generationParams['max_tokens'] = 512;
+    }
+
+    if (param.model.toLowerCase().includes('neural-chat-7b-v3-1-awq')) {
+      generationParams['max_tokens'] = 590;
+    }
+
+    if (param.model.toLowerCase().includes('openchat-3.5-0106')) {
+      generationParams['max_tokens'] = 2000;
+    }
+
+    return generationParams;
+  }
+
+  private getModelPrefix(model: string): string {
+    let modelPrefix = `@cf/meta`;
+
+    if (model.toLowerCase().includes('llama')) {
+      modelPrefix = `@cf/meta`;
+    }
+
+    if (model.toLowerCase().includes('hermes-2-pro-mistral-7b')) {
+      modelPrefix = `@hf/nousresearch`;
+    }
+
+    if (model.toLowerCase().includes('mistral-7b-instruct')) {
+      modelPrefix = `@hf/mistral`;
+    }
+
+    if (model.toLowerCase().includes('gemma')) {
+      modelPrefix = `@cf/google`;
+    }
+
+    if (model.toLowerCase().includes('deepseek')) {
+      modelPrefix = `@cf/deepseek-ai`;
+    }
+
+    if (model.toLowerCase().includes('openchat-3.5-0106')) {
+      modelPrefix = `@cf/openchat`;
+    }
+
+    const isNueralChat = model.toLowerCase().includes('neural-chat-7b-v3-1-awq');
+    if (
+      isNueralChat ||
+      model.toLowerCase().includes('openhermes-2.5-mistral-7b-awq') ||
+      model.toLowerCase().includes('zephyr-7b-beta-awq') ||
+      model.toLowerCase().includes('deepseek-coder-6.7b-instruct-awq')
+    ) {
+      modelPrefix = `@hf/thebloke`;
+    }
+
+    return modelPrefix;
+  }
+
+  private getModelWithPrefix(model: string): string {
+    return `${this.getModelPrefix(model)}/${model}`;
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class CloudflareAISdk {
+  private static provider = new CloudflareAiChatProvider();
+
+  static async handleCloudflareAIStream(
+    param: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: any;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      model: string;
+      env: Env;
+    },
+    dataCallback: (data: any) => void,
+  ) {
+    return this.provider.handleStream(
+      {
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/fireworks.ts

@@ -0,0 +1,65 @@
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, type CommonProviderParams } from './chat-stream-provider.ts';
+
+export class FireworksAiChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      apiKey: param.env.FIREWORKS_API_KEY,
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.fireworks,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    let modelPrefix = 'accounts/fireworks/models/';
+    if (param.model.toLowerCase().includes('yi-')) {
+      modelPrefix = 'accounts/yi-01-ai/models/';
+    }
+
+    return {
+      model: `${modelPrefix}${param.model}`,
+      messages: safeMessages,
+      stream: true,
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class FireworksAiChatSdk {
+  private static provider = new FireworksAiChatProvider();
+
+  static async handleFireworksStream(
+    param: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: any;
+      maxTokens: number;
+      messages: any;
+      model: any;
+      env: any;
+    },
+    dataCallback: (data) => void,
+  ) {
+    return this.provider.handleStream(
+      {
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/google.ts

@@ -0,0 +1,72 @@
+import { type StreamParams } from '@open-gsio/server/src/services/ChatService';
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, type CommonProviderParams } from './chat-stream-provider.ts';
+
+export class GoogleChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.google,
+      apiKey: param.env.GEMINI_API_KEY,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices?.[0]?.finish_reason === 'stop') {
+      dataCallback({
+        type: 'chat',
+        data: {
+          choices: [
+            {
+              delta: { content: chunk.choices[0].delta.content || '' },
+              finish_reason: 'stop',
+              index: chunk.choices[0].index,
+            },
+          ],
+        },
+      });
+      return true;
+    } else {
+      dataCallback({
+        type: 'chat',
+        data: {
+          choices: [
+            {
+              delta: { content: chunk.choices?.[0]?.delta?.content || '' },
+              finish_reason: null,
+              index: chunk.choices?.[0]?.index || 0,
+            },
+          ],
+        },
+      });
+      return false;
+    }
+  }
+}
+
+export class GoogleChatSdk {
+  private static provider = new GoogleChatProvider();
+
+  static async handleGoogleStream(param: StreamParams, dataCallback: (data: any) => void) {
+    return this.provider.handleStream(
+      {
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/groq.ts

@@ -0,0 +1,82 @@
+import {
+  _NotCustomized,
+  ISimpleType,
+  ModelPropertiesDeclarationToProperties,
+  ModelSnapshotType2,
+  UnionStringArray,
+} from 'mobx-state-tree';
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, CommonProviderParams } from './chat-stream-provider.ts';
+
+export class GroqChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: ProviderRepository.OPENAI_COMPAT_ENDPOINTS.groq,
+      apiKey: param.env.GROQ_API_KEY,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    const tuningParams = {
+      temperature: 0.86,
+      top_p: 0.98,
+      presence_penalty: 0.1,
+      frequency_penalty: 0.3,
+      max_tokens: param.maxTokens as number,
+    };
+
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+      ...tuningParams,
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class GroqChatSdk {
+  private static provider = new GroqChatProvider();
+
+  static async handleGroqStream(
+    param: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: ModelSnapshotType2<
+        ModelPropertiesDeclarationToProperties<{
+          role: ISimpleType<UnionStringArray<string[]>>;
+          content: ISimpleType<unknown>;
+        }>,
+        _NotCustomized
+      >;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      model: string;
+      env: Env;
+    },
+    dataCallback: (data) => void,
+  ) {
+    return this.provider.handleStream(
+      {
+        systemPrompt: param.systemPrompt,
+        preprocessedContext: param.preprocessedContext,
+        maxTokens: param.maxTokens,
+        messages: param.messages,
+        model: param.model,
+        env: param.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/index.ts

@@ -0,0 +1,8 @@
+export * from './claude';
+export * from './cerebras';
+export * from './cloudflareAi';
+export * from './fireworks';
+export * from './groq';
+export * from './mlx-omni';
+export * from './ollama';
+export * from './xai';

packages/ai/providers/mlx-omni.ts

@@ -0,0 +1,96 @@
+import { Utils } from '@open-gsio/server/src/lib/utils.ts';
+import { OpenAI } from 'openai';
+import { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions/completions';
+
+import { BaseChatProvider, CommonProviderParams } from './chat-stream-provider.ts';
+
+export class MlxOmniChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: 'http://localhost:10240',
+      apiKey: param.env.MLX_API_KEY,
+    });
+  }
+
+  getStreamParams(
+    param: CommonProviderParams,
+    safeMessages: any[],
+  ): ChatCompletionCreateParamsStreaming {
+    const baseTuningParams = {
+      temperature: 0.86,
+      top_p: 0.98,
+      presence_penalty: 0.1,
+      frequency_penalty: 0.3,
+      max_tokens: param.maxTokens as number,
+    };
+
+    const getTuningParams = () => {
+      return baseTuningParams;
+    };
+
+    let completionRequest: ChatCompletionCreateParamsStreaming = {
+      model: param.model,
+      stream: true,
+      messages: safeMessages,
+    };
+
+    const client = this.getOpenAIClient(param);
+    const isLocal = client.baseURL.includes('localhost');
+
+    if (isLocal) {
+      completionRequest['messages'] = Utils.normalizeWithBlanks(safeMessages);
+      completionRequest['stream_options'] = {
+        include_usage: true,
+      };
+    } else {
+      completionRequest = { ...completionRequest, ...getTuningParams() };
+    }
+
+    return completionRequest;
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    const isLocal = chunk.usage !== undefined;
+
+    if (isLocal && chunk.usage) {
+      dataCallback({
+        type: 'chat',
+        data: {
+          choices: [
+            {
+              delta: { content: '' },
+              logprobs: null,
+              finish_reason: 'stop',
+            },
+          ],
+        },
+      });
+      return true; // Break the stream
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false; // Continue the stream
+  }
+}
+
+export class MlxOmniChatSdk {
+  private static provider = new MlxOmniChatProvider();
+
+  static async handleMlxOmniStream(ctx: any, dataCallback: (data: any) => any) {
+    if (!ctx.messages?.length) {
+      return new Response('No messages provided', { status: 400 });
+    }
+
+    return this.provider.handleStream(
+      {
+        systemPrompt: ctx.systemPrompt,
+        preprocessedContext: ctx.preprocessedContext,
+        maxTokens: ctx.maxTokens,
+        messages: Utils.normalizeWithBlanks(ctx.messages),
+        model: ctx.model,
+        env: ctx.env,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/ollama.ts

@@ -0,0 +1,75 @@
+import { OpenAI } from 'openai';
+
+import { ProviderRepository } from './_ProviderRepository.ts';
+import { BaseChatProvider, CommonProviderParams } from './chat-stream-provider.ts';
+
+export class OllamaChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: param.env.OLLAMA_API_ENDPOINT ?? ProviderRepository.OPENAI_COMPAT_ENDPOINTS.ollama,
+      apiKey: param.env.OLLAMA_API_KEY,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    const tuningParams = {
+      temperature: 0.75,
+    };
+
+    const getTuningParams = () => {
+      return tuningParams;
+    };
+
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+      ...getTuningParams(),
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class OllamaChatSdk {
+  private static provider = new OllamaChatProvider();
+
+  static async handleOllamaStream(
+    ctx: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: any;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      disableWebhookGeneration: boolean;
+      model: any;
+      env: Env;
+    },
+    dataCallback: (data: any) => any,
+  ) {
+    if (!ctx.messages?.length) {
+      return new Response('No messages provided', { status: 400 });
+    }
+
+    return this.provider.handleStream(
+      {
+        systemPrompt: ctx.systemPrompt,
+        preprocessedContext: ctx.preprocessedContext,
+        maxTokens: ctx.maxTokens,
+        messages: ctx.messages,
+        model: ctx.model,
+        env: ctx.env,
+        disableWebhookGeneration: ctx.disableWebhookGeneration,
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/openai.ts

@@ -0,0 +1,118 @@
+import { Utils } from '@open-gsio/server/src/lib/utils.ts';
+import { OpenAI } from 'openai';
+import { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions/completions';
+
+import { BaseChatProvider, CommonProviderParams } from './chat-stream-provider.ts';
+
+export class OpenAiChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return param.openai as OpenAI;
+  }
+
+  getStreamParams(
+    param: CommonProviderParams,
+    safeMessages: any[],
+  ): ChatCompletionCreateParamsStreaming {
+    const isO1 = () => {
+      if (param.model === 'o1-preview' || param.model === 'o1-mini') {
+        return true;
+      }
+    };
+
+    const tuningParams: Record<string, any> = {};
+
+    const gpt4oTuningParams = {
+      temperature: 0.86,
+      top_p: 0.98,
+      presence_penalty: 0.1,
+      frequency_penalty: 0.3,
+      max_tokens: param.maxTokens as number,
+    };
+
+    const getTuningParams = () => {
+      if (isO1()) {
+        tuningParams['temperature'] = 1;
+        tuningParams['max_completion_tokens'] = (param.maxTokens as number) + 10000;
+        return tuningParams;
+      }
+      return gpt4oTuningParams;
+    };
+
+    let completionRequest: ChatCompletionCreateParamsStreaming = {
+      model: param.model,
+      stream: true,
+      messages: safeMessages,
+    };
+
+    const client = this.getOpenAIClient(param);
+    const isLocal = client.baseURL.includes('localhost');
+
+    if (isLocal) {
+      completionRequest['messages'] = Utils.normalizeWithBlanks(safeMessages);
+      completionRequest['stream_options'] = {
+        include_usage: true,
+      };
+    } else {
+      completionRequest = { ...completionRequest, ...getTuningParams() };
+    }
+
+    return completionRequest;
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    const isLocal = chunk.usage !== undefined;
+
+    if (isLocal && chunk.usage) {
+      dataCallback({
+        type: 'chat',
+        data: {
+          choices: [
+            {
+              delta: { content: '' },
+              logprobs: null,
+              finish_reason: 'stop',
+            },
+          ],
+        },
+      });
+      return true; // Break the stream
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false; // Continue the stream
+  }
+}
+
+// Legacy class for backward compatibility
+export class OpenAiChatSdk {
+  private static provider = new OpenAiChatProvider();
+
+  static async handleOpenAiStream(
+    ctx: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: any;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      model: any;
+    },
+    dataCallback: (data: any) => any,
+  ) {
+    if (!ctx.messages?.length) {
+      return new Response('No messages provided', { status: 400 });
+    }
+
+    return this.provider.handleStream(
+      {
+        openai: ctx.openai,
+        systemPrompt: ctx.systemPrompt,
+        preprocessedContext: ctx.preprocessedContext,
+        maxTokens: ctx.maxTokens,
+        messages: ctx.messages,
+        model: ctx.model,
+        env: {} as Env, // This is not used in OpenAI provider
+      },
+      dataCallback,
+    );
+  }
+}

packages/ai/providers/xai.ts

@@ -0,0 +1,74 @@
+import { OpenAI } from 'openai';
+
+import { BaseChatProvider, CommonProviderParams } from './chat-stream-provider.ts';
+
+export class XaiChatProvider extends BaseChatProvider {
+  getOpenAIClient(param: CommonProviderParams): OpenAI {
+    return new OpenAI({
+      baseURL: 'https://api.x.ai/v1',
+      apiKey: param.env.XAI_API_KEY,
+    });
+  }
+
+  getStreamParams(param: CommonProviderParams, safeMessages: any[]): any {
+    const tuningParams = {
+      temperature: 0.75,
+    };
+
+    const getTuningParams = () => {
+      return tuningParams;
+    };
+
+    return {
+      model: param.model,
+      messages: safeMessages,
+      stream: true,
+      ...getTuningParams(),
+    };
+  }
+
+  async processChunk(chunk: any, dataCallback: (data: any) => void): Promise<boolean> {
+    if (chunk.choices && chunk.choices[0]?.finish_reason === 'stop') {
+      dataCallback({ type: 'chat', data: chunk });
+      return true;
+    }
+
+    dataCallback({ type: 'chat', data: chunk });
+    return false;
+  }
+}
+
+export class XaiChatSdk {
+  private static provider = new XaiChatProvider();
+
+  static async handleXaiStream(
+    ctx: {
+      openai: OpenAI;
+      systemPrompt: any;
+      preprocessedContext: any;
+      maxTokens: unknown | number | undefined;
+      messages: any;
+      disableWebhookGeneration: boolean;
+      model: any;
+      env: Env;
+    },
+    dataCallback: (data: any) => any,
+  ) {
+    if (!ctx.messages?.length) {
+      return new Response('No messages provided', { status: 400 });
+    }
+
+    return this.provider.handleStream(
+      {
+        systemPrompt: ctx.systemPrompt,
+        preprocessedContext: ctx.preprocessedContext,
+        maxTokens: ctx.maxTokens,
+        messages: ctx.messages,
+        model: ctx.model,
+        env: ctx.env,
+        disableWebhookGeneration: ctx.disableWebhookGeneration,
+      },
+      dataCallback,
+    );
+  }
+}