geoffsee/open-gsio
1
0
Fork 0
mirror of https://github.com/geoffsee/open-gsio.git synced 2025-09-08 22:56:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

- Add killport.js script for terminating processes on specific ports

Browse Source 
- Introduce `supportedModels` in `ClientChatStore` and update model validation logic
- Enhance OpenAI inferencing with local setup adaptations and improved streaming options
- Modify ChatService to handle local and remote model fetching
- Update input menu to dynamically fetch and display supported models
- Add start_inference_server.sh for initiating local inference server
- Upgrade OpenAI SDK to v5.0.1 and adjust dependencies accordingly
This commit is contained in:
geoffsee
2025-05-29 19:28:54 -04:00
committed by Geoff Seemueller
parent c9ee7c7690
commit cc0da17b5f
11 changed files with 204 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
workers/site/services/ChatService.ts
View File

@@ -3,7 +3,7 @@ import OpenAI from 'openai';
import ChatSdk from '../lib/chat-sdk';
import Message from "../models/Message";
import O1Message from "../models/O1Message";
import {getModelFamily, ModelFamily} from "../../../src/components/chat/lib/SupportedModels";
import {getModelFamily, ModelFamily, SUPPORTED_MODELS} from "../../../src/components/chat/lib/SupportedModels";
import {OpenAiChatSdk} from "../providers/openai";
import {GroqChatSdk} from "../providers/groq";
import {ClaudeChatSdk} from "../providers/claude";
@@ -73,11 +73,21 @@ const ChatService = types
throw new Error('Unsupported message format');
};
const getSupportedModels = async () => {
if(self.env.OPENAI_API_ENDPOINT.includes("localhost")) {
const openaiClient = new OpenAI({baseURL: self.env.OPENAI_API_ENDPOINT})
const models = await openaiClient.models.list();
return Response.json(models.data.map(model => model.id));
}
return Response.json(SUPPORTED_MODELS);
};
const createStreamParams = async (
streamConfig: any,
dynamicContext: any,
durableObject: any
): Promise<StreamParams> => {
return {
env: self.env,
openai: self.openai,
@@ -112,6 +122,7 @@ const ChatService = types
};
return {
getSupportedModels,
setActiveStream(streamId: string, stream: any) {
const validStream = {
name: stream?.name || "Unnamed Stream",
@@ -129,10 +140,18 @@ const ChatService = types
},
setEnv(env: Env) {
self.env = env;
self.openai = new OpenAI({
apiKey: self.openAIApiKey,
baseURL: self.openAIBaseURL,
});
if(env.OPENAI_API_ENDPOINT.includes("localhost")) {
self.openai = new OpenAI({
apiKey: self.env.OPENAI_API_KEY,
baseURL: self.env.OPENAI_API_ENDPOINT,
});
} else{
self.openai = new OpenAI({
apiKey: self.openAIApiKey,
baseURL: self.openAIBaseURL,
});
}
},
handleChatRequest: async (request: Request) => {
@@ -154,12 +173,12 @@ const ChatService = types
}) {
const {streamConfig, streamParams, controller, encoder, streamId} = params;
const modelFamily = getModelFamily(streamConfig.model);
const modelFamily = !self.env.OPENAI_API_ENDPOINT.includes("localhost") ? getModelFamily(streamConfig.model) : "openai";
const handler = !self.env.OPENAI_API_ENDPOINT.includes("localhost") ? modelHandlers[modelFamily as ModelFamily] : modelHandlers.openai;
const handler = modelHandlers[modelFamily as ModelFamily];
if (handler) {
try {
await handler(streamParams, handleStreamData(controller, encoder));
} catch (error) {