init

.dockerignore

@@ -0,0 +1,10 @@
+*
+!packages/genaiscript/genaisrc
+!src
+!Cargo.lock
+!Cargo.toml
+!package.json
+!package-lock.json
+!assets
+!bun.lockb
+!packages

.env

@@ -0,0 +1,12 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.openai.com/v1"
+GENAISCRIPT_MODEL_LARGE=""
+GENAISCRIPT_MODEL_SMALL=""
+SEARXNG_API_BASE_URL=""
+SEARXNG_PASSWORD=
+
+BING_SEARCH_API_KEY=
+PERIGON_API_KEY=
+TAVILY_API_KEY=
+CCC_API_KEY=
+HF_API_KEY=

.env.cerebras

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.cerebras.ai/v1"
+GENAISCRIPT_MODEL_LARGE=""
+GENAISCRIPT_MODEL_SMALL=""

.env.fireworks

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.fireworks.ai/inference/v1"
+GENAISCRIPT_MODEL_LARGE=""
+GENAISCRIPT_MODEL_SMALL=""

.env.google

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://generativelanguage.googleapis.com/v1beta/openai"
+GENAISCRIPT_MODEL_LARGE=""
+GENAISCRIPT_MODEL_SMALL=""

.env.groq

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.groq.com/openai/v1"
+GENAISCRIPT_MODEL_LARGE=""
+GENAISCRIPT_MODEL_SMALL=""

.env.openai

@@ -0,0 +1,4 @@
+OPENAI_API_KEY=""
+OPENAI_API_BASE="https://api.openai.com/v1"
+GENAISCRIPT_MODEL_LARGE="gpt-4o"
+GENAISCRIPT_MODEL_SMALL="gpt-4o-mini"

.gitignore

@@ -0,0 +1,12 @@
+/target
+/node_modules/
+/.idea
+prompt.md
+todo
+chrome
+stream_store
+data/
+web-agent-rs/
+./.env*
+/.genaiscript/
+/packages/genaiscript/node_modules/

.toak-ignore

@@ -0,0 +1,3 @@
+package.json
+genaisrc/genaiscript.d.ts
+.toak-ignore

Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "web-agent-rs"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+edition = "2021"
+name = "agent-server"
+path = "src/main.rs"
+
+[dependencies]
+axum = { version = "0.7", features = ["multipart"] }
+serde = { version = "1.0", features = ["derive"] }
+tokio = { version = "1.0", features = ["full"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+http = "1.1.0"
+tokio-stream = "0.1.16"
+uuid = { version = "1.11.0", features = ["v4"] }
+tokio-util = { version = "0.7", features = ["io"] }
+serde_json = "1.0.133"
+futures = "0.3.31"
+dotenv = "0.15.0"
+shell-escape = "0.1.5"
+rust-embed = "8.5.0"
+bytes = "1.8.0"
+lazy_static = "1.5.0"
+sled = "0.34.7"
+tower-http = { version = "0.6.2", features = ["trace"] }
+anyhow = "1.0.97"
+base64 = "0.22.1"
+fips204 = "0.4.6"
+

Dockerfile

@@ -0,0 +1,88 @@
+# Stage 1: Build the Rust agent-core binary
+FROM rust:1.73-slim-bullseye as agent-server-builder
+
+WORKDIR /build-context
+
+# Install only the minimal required build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config \
+    musl-tools \
+    && rm -rf /var/core/apt/lists/*
+
+# Build for musl to ensure static linking
+RUN rustup target add aarch64-unknown-linux-musl
+
+# Copy only necessary files for building
+COPY Cargo.toml Cargo.lock ./
+COPY src ./src
+COPY assets ./assets
+
+# Build with musl target for static linking
+RUN cargo build --release --target aarch64-unknown-linux-musl && \
+    strip /build-context/target/aarch64-unknown-linux-musl/release/agent-core
+
+# Stage 2: Build Bun dependencies
+FROM oven/bun:alpine as node-builder
+
+# Install system dependencies and node-gyp
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        unzip \
+        git \
+        python3 \
+        py3-pip \
+        build-base \
+        pkgconf \
+        cmake \
+        nodejs \
+        npm \
+        bash \
+        chromium \
+        nss \
+        freetype \
+        freetype-dev \
+        harfbuzz \
+        ca-certificates \
+        ttf-freefont \
+        font-noto-emoji \
+        nodejs \
+        wqy-zenhei \
+      && rm -rf /var/cache/* \
+      && mkdir /var/cache/apk \
+      && npm install -g node-gyp
+
+WORKDIR /app
+# dep files
+COPY packages packages
+COPY package.json package-lock.json ./
+## Install deps
+RUN bun install && bun --filter='./packages/genaiscript-rust-shim' run buildShim
+
+
+FROM node:20-bookworm as app
+
+WORKDIR /app
+
+# Install playwright
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/lib/chromium/chromium \
+  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+RUN npx -y playwright@1.49.1 install --with-deps chromium && npm install -g bun
+
+
+COPY --from=node-builder /app/node_modules ./node_modules
+COPY --from=node-builder /app/package.json .
+COPY --from=agent-server-builder /build-context/target/aarch64-unknown-linux-musl/release/agent-server ./agent-server
+
+# Ensure the binary is executable
+RUN chmod +x ./agent-core
+
+# copy agent source files
+COPY packages/genaiscript-rust-shim/dist ./dist
+COPY genaisrc ./genaisrc
+
+# Expose the required port
+EXPOSE 3006
+
+# Set the entrypoint to the Rust binary
+ENTRYPOINT ["./agent-server"]

Local.Dockerfile

@@ -0,0 +1,88 @@
+# Stage 1: Build the Rust agent-core binary
+FROM rust:1-slim-bullseye as agent-server-builder
+
+WORKDIR /build-context
+
+# Install only the minimal required build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config \
+    musl-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+# Build for musl to ensure static linking
+RUN rustup target add aarch64-unknown-linux-musl
+
+# Copy only necessary files for building
+COPY Cargo.toml Cargo.lock ./
+COPY src ./src
+COPY assets ./assets
+
+# Build with musl target for static linking
+RUN cargo build --release --target aarch64-unknown-linux-musl && \
+    strip /build-context/target/aarch64-unknown-linux-musl/release/agent-server
+
+# Stage 2: Build Bun dependencies
+FROM oven/bun:alpine as node-builder
+
+# Install system dependencies and node-gyp
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        unzip \
+        git \
+        python3 \
+        py3-pip \
+        build-base \
+        pkgconf \
+        cmake \
+        nodejs \
+        npm \
+        bash \
+        chromium \
+        nss \
+        freetype \
+        freetype-dev \
+        harfbuzz \
+        ca-certificates \
+        ttf-freefont \
+        font-noto-emoji \
+        nodejs \
+        wqy-zenhei \
+      && rm -rf /var/cache/* \
+      && mkdir /var/cache/apk \
+      && npm install -g node-gyp
+
+WORKDIR /app
+# dep files
+COPY packages packages
+COPY package.json package-lock.json bun.lockb ./
+## Install deps
+RUN bun install
+
+
+FROM node:20-bookworm as app
+
+WORKDIR /app
+
+# Install playwright
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/lib/chromium/chromium \
+  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+RUN npx -y playwright@1.49.1 install --with-deps chromium && npm install -g bun
+
+
+COPY --from=node-builder /app/node_modules ./node_modules
+COPY --from=node-builder /app/package.json .
+COPY --from=node-builder /app/packages ./packages
+COPY --from=agent-server-builder /build-context/target/aarch64-unknown-linux-musl/release/agent-server ./agent-server
+
+# Ensure the binary is executable
+RUN chmod +x ./agent-server
+
+# copy agent source files
+COPY packages/genaiscript-rust-shim/dist ./dist
+
+# Expose the required port
+EXPOSE 3006
+
+# Set the entrypoint to the Rust binary
+ENTRYPOINT ["./agent-server"]

README.md

@@ -0,0 +1,13 @@
+# web-agent-rs
+Hacky genaiscript host for integration into conversational AI applications. 
+
+### Disclaimer
+This has not undergone a formal security assessment. You should do your own evaluation before using this.
+
+### How it works
+1. A chat client specifies the URL to this host in their environment.
+2. They send a request with their credentials to create a stream resource 
+
+
+## Docs
+The [docs](./docs) folder is under construction

Remote.Dockerfile

@@ -0,0 +1,88 @@
+# Stage 1: Build the Rust agent-server binary
+FROM rust:1-slim-bullseye as agent-server-builder
+
+WORKDIR /build-context
+
+# Install only the minimal required build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    pkg-config \
+    musl-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+# Build for musl to ensure static linking
+RUN rustup target add x86_64-unknown-linux-musl
+
+# Copy only necessary files for building
+COPY Cargo.toml Cargo.lock ./
+COPY src ./src
+COPY assets ./assets
+
+# Build with musl target for static linking
+RUN cargo build --release --target x86_64-unknown-linux-musl && \
+    strip /build-context/target/x86_64-unknown-linux-musl/release/agent-server
+
+# Stage 2: Build Bun dependencies
+FROM oven/bun:alpine as node-builder
+
+# Install system dependencies and node-gyp
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        unzip \
+        git \
+        python3 \
+        py3-pip \
+        build-base \
+        pkgconf \
+        cmake \
+        nodejs \
+        npm \
+        bash \
+        chromium \
+        nss \
+        freetype \
+        freetype-dev \
+        harfbuzz \
+        ca-certificates \
+        ttf-freefont \
+        font-noto-emoji \
+        nodejs \
+        wqy-zenhei \
+      && rm -rf /var/cache/* \
+      && mkdir /var/cache/apk \
+      && npm install -g node-gyp
+
+WORKDIR /app
+# dep files
+COPY packages packages
+COPY package.json package-lock.json bun.lockb ./
+## Install deps
+RUN bun install
+
+
+FROM node:20-bookworm as app
+
+WORKDIR /app
+
+# Install playwright
+ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/lib/chromium/chromium \
+  PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
+RUN npx -y playwright@1.49.1 install --with-deps chromium && npm install -g bun
+
+
+COPY --from=node-builder /app/node_modules ./node_modules
+COPY --from=node-builder /app/package.json .
+COPY --from=node-builder /app/packages ./packages
+COPY --from=agent-server-builder /build-context/target/x86_64-unknown-linux-musl/release/agent-server ./agent-server
+
+# Ensure the binary is executable
+RUN chmod +x ./agent-server
+
+# copy agent source files
+COPY packages/genaiscript-rust-shim/dist ./dist
+
+# Expose the required port
+EXPOSE 3006
+
+# Set the entrypoint to the Rust binary
+ENTRYPOINT ["./agent-server"]

assets/index.html

@@ -0,0 +1,62 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Axum Server UI</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 40px;
+        }
+
+        h1 {
+            color: #333;
+        }
+
+        #output {
+            margin-top: 20px;
+            padding: 15px;
+            border: 1px solid #ddd;
+            background-color: #f9f9f9;
+            white-space: pre-wrap;
+            height: 300px;
+            overflow-y: auto;
+        }
+
+        input[type="text"] {
+            width: 60%;
+            padding: 10px;
+            font-size: 16px;
+        }
+
+        button {
+            padding: 11px 20px;
+            font-size: 16px;
+            margin-left: 10px;
+        }
+    </style>
+    <script>
+        async function sendRequest() {
+            const input = document.getElementById('userInput').value;
+            const response = await fetch(`/api/webhooks?resource=web-search&input=${encodeURIComponent(input)}`);
+            const reader = response.body.getReader();
+            const decoder = new TextDecoder('utf-8');
+            let result = '';
+            const outputDiv = document.getElementById('output');
+
+            while (true) {
+                const {done, value} = await reader.read();
+                if (done) break;
+                result += decoder.decode(value);
+                outputDiv.textContent = result;
+            }
+        }
+    </script>
+</head>
+<body>
+<h1>Axum Server UI</h1>
+<input type="text" id="userInput" placeholder="Enter your query here">
+<button onclick="sendRequest()">Submit</button>
+<div id="output"></div>
+</body>
+</html>

compose.yml

@@ -0,0 +1,32 @@
+services:
+  agent-server:
+    build:
+      context: .
+      dockerfile: Local.Dockerfile
+    ports:
+      - "3006:3006"
+    environment:
+      # Load environment variables from the .env file
+      # The .env file should contain OPENAI_API_KEY, BING_SEARCH_API_KEY, TAVILY_API_KEY, etc.
+      - OPENAI_API_BASE=${OPENAI_API_BASE}
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - GENAISCRIPT_MODEL_LARGE=${GENAISCRIPT_MODEL_LARGE}
+      - GENAISCRIPT_MODEL_SMALL=${GENAISCRIPT_MODEL_SMALL}
+      - BING_SEARCH_API_KEY=${BING_SEARCH_API_KEY}
+      - BING_SEARCH_API_KEY=${BING_SEARCH_API_KEY}
+      - TAVILY_API_KEY=${TAVILY_API_KEY}
+      - PERIGON_API_KEY=${PERIGON_API_KEY}
+      - SEARXNG_API_BASE_URL=${SEARXNG_API_BASE_URL}
+      - PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium-browser
+    env_file: "./.env"
+    develop:
+      watch:
+        - action: sync
+          path: ./packages/genaiscript/genaisrc
+          target: /app/web/packages/genaiscript/genaisrc
+          ignore:
+            - node_modules/
+        - action: rebuild
+          path: ./src
+        - action: rebuild
+          path: cargo.toml

docs/tokens.md

@@ -0,0 +1,77 @@
+# Authentication System Documentation
+
+## Overview
+
+This document outlines the token-based authentication system used in web-agent-rs. The system uses FIPS204 signatures to
+generate secure session tokens containing user data.
+
+## Core Components
+
+TODO: In the meantime, here's some hamfisted knowledge. 
+
+
+```javascript
+class FIPS204KeyPair {
+    constructor() {
+        this.publicKey = "FIPS204_PUBLIC_KEY"; // Placeholder
+        this.privateKey = "FIPS204_PRIVATE_KEY"; // Placeholder
+    }
+
+    sign(data) {
+        // Placeholder for actual FIPS204 signing logic
+        return `FIPS204_${data}_SIGNED`;
+    }
+
+    verify(data, signature) {
+        // Placeholder for actual FIPS204 verification
+        return true;
+    }
+}
+
+/* NOTES:
+- the public key needs to be retrievable, so it can be used to verify payload signature at the time of the request.
+- the private key is disposed so it can't be used to create more signatures
+- future tokens should use a completely new keypair
+
+
+- The fips204 authentication scheme was selected for its performance, flexibility, and key-length.
+- It would be wise to configure additional protections like ip whitelisting and rate limiting.   
+*/
+
+// User object representing token payload data
+const user = {
+    sub: "user123",
+    name: "John Doe",
+    email: "john@example.com",
+    roles: ["user"],
+    iat: Math.floor(Date.now() / 1000),
+    exp: Math.floor(Date.now() / 1000) + (60 * 60) // 1 hour from now
+};
+
+const keyPair = new FIPS204KeyPair();
+const signature = keyPair.sign(JSON.stringify(user));
+
+function createToken(payload, signature) {
+    const encodedPayload = Buffer.from(JSON.stringify(payload)).toString('base64');
+    const encodedSignature = Buffer.from(signature).toString('base64');
+    return `${encodedPayload}.${encodedSignature}`;
+}
+
+const token = createToken(user, signature);
+
+
+async function createStreamRequest(eventHost = "https://agent.example.com") {
+    
+    const requestParams = {
+        // will automagically include the session token as a cookie, where it will be parsed by the agent server
+        credentials: "include"
+    }
+
+    const response = await fetch(eventHost, requestParams);
+    
+    const {streamId} = await response.json();
+    
+    // This stream id is then supplied as a path parameter to stream, the token is validated to ensure the stream belongs to the user, and the stream is returned.
+    return streamId;
+}
+```

fly.toml

@@ -0,0 +1,54 @@
+app = "web-agent-rs"
+primary_region = "iad"
+
+[deploy]
+    strategy = "rolling"
+
+[build]
+    dockerfile = "Remote.Dockerfile"
+    deploy-target = "app"
+
+
+
+[env]
+    OPENAI_API_KEY=""
+    OPENAI_API_BASE=""
+    GENAISCRIPT_MODEL_LARGE=""
+    GENAISCRIPT_MODEL_SMALL=""
+    GENAISCRIPT_MODEL_PROVIDER=""
+
+    SEARXNG_API_BASE_URL=""
+    SEARXNG_PASSWORD=""
+    BING_SEARCH_API_KEY = ""
+    TAVILY_API_KEY = ""
+    PERIGON_API_KEY= ""
+    CEREBRAS_API_KEY = ""
+    CCC_API_KEY=""
+    HF_API_KEY=""
+
+[http_service]
+    internal_port = 3006
+    force_https = true
+    auto_stop_machines = "suspend"
+    auto_start_machines = true
+    # automatic shutdown when not in use
+    min_machines_running = 0
+
+    [http_service.http_options]
+        idle_timeout = 180
+
+
+    [[http_service.checks]]
+        interval = '30s'
+        timeout = '5s'
+        grace_period = '10s'
+        method = 'GET'
+        path = '/health'
+
+[[vm]]
+    size = "performance-1x"
+
+
+[[mounts]]
+source = "web_agent_app_mount"
+destination = "/app/data"

gitleaks-report.json

@@ -0,0 +1 @@
+[]

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "web-agent-rs",
+  "type": "module",
+  "workspaces": ["packages/*"],
+  "private": true,
+  "scripts": {
+    "dev": "bunx kill-port 3006 && bun run build && cargo watch -x 'run src/main.rs'",
+    "ai:search": "genaiscript run packages/genaiscript/genaisrc/web-search.genai.mts --vars USER_INPUT='who won the 2024 election?'",
+    "shim:ai:search": "pnpm build && ./dist/shim.js --file=genaisrc/search.genai.mts USER_INPUT=\"Who won the 2024 presidential election?\"\n",
+    "ai:news": "genaiscript run packages/genaiscript/genaisrc/news-search.genai.mts --vars USER_INPUT='What are the latest updates and developments in the Ukraine war?'",
+    "ai:url:read": "genaiscript run packages/genaiscript/genaisrc/web-scrape.genai.mts --vars USER_INPUT='{\"url\":\"https://geoff.seemueller.io/about\",\"query\":\"Describe the details of the page.\", \"action\": \"read\"}'",
+    "ai:url:scrape": "npx genaiscript run packages/genaiscript/genaisrc/web-scrape.genai.mts --vars USER_INPUT='{\"url\":\"https://www.time4learning.com/homeschool-curriculum/high-school/eleventh-grade/math.html\",\"query\":\"What is on this page?\", \"action\": \"scrape\"}'",
+    "crypto:quote": "npx genaiscript run packages/genaiscript/genaisrc/finance-query.genai.mts --vars USER_INPUT='Get a quote for BTC'",
+    "crypto:news": "npx genaiscript run packages/genaiscript/genaisrc/finance-query.genai.mts --vars USER_INPUT='What is the news for Bitcoin?'",
+    "crypto:overview": "npx genaiscript run packages/genaiscript/genaisrc/finance-query.genai.mts --vars USER_INPUT='What are the trending symbols in the market?'",
+    "compose:up": "docker compose up --build",
+    "prod:logs": "fly logs",
+    "build": "(cd packages/genaiscript-rust-shim && bun run buildShim)"
+  }
+}

packages/core/index.ts

@@ -0,0 +1,4 @@
+export * from "../genaiscript/genaisrc/_state";
+export * from "./news";
+export * from "./quotes";
+export * from "./types";

packages/core/market/index.ts

@@ -0,0 +1,14 @@
+import {ApiResponse} from "./types";
+
+export async function collect_gainers_losers(x: { apiKey: string, limit: number }): Promise<ApiResponse> {
+    const { apiKey, limit } = x;
+
+    //
+    const data: ApiResponse = await fetch(`https://pro-api.coinmarketcap.com/v1/cryptocurrency/trending/gainers-losers?limit=${limit}`, {
+        headers: {
+            "x-cmc_pro_api_key": apiKey
+        }
+    }).then((symbolDataRequest) => symbolDataRequest.json());
+
+    return data;
+}

packages/core/market/types.ts

@@ -0,0 +1,48 @@
+type Quote = {
+    price: number;
+    volume_24h: number;
+    percent_change_1h: number;
+    percent_change_24h: number;
+    percent_change_7d: number;
+    market_cap: number;
+    last_updated: string;
+};
+
+type Platform = null;
+
+type Tag = string;
+
+type Data = {
+    id: number;
+    name: string;
+    symbol: string;
+    slug: string;
+    cmc_rank?: number;
+    num_market_pairs: number;
+    circulating_supply: number;
+    total_supply: number;
+    max_supply: number;
+    last_updated: string;
+    date_added: string;
+    tags: Tag[];
+    platform: Platform;
+    quote: {
+        USD: Quote;
+        BTC?: Quote;
+        ETH?: Quote;
+    };
+};
+
+type Status = {
+    timestamp: string;
+    error_code: number;
+    error_message: string | null;
+    elapsed: number;
+    credit_count: number;
+};
+
+export type ApiResponse = {
+    data: Data[];
+    status: Status;
+};
+

packages/core/news/index.ts

@@ -0,0 +1,178 @@
+import {types, Instance} from 'mobx-state-tree';
+import {runInAction} from "mobx";
+
+
+const Article = types.model('Article', {
+    title: types.string,
+    content: types.string,
+    url: types.maybe(types.string),
+    source: types.maybe(types.string),
+    pubDate: types.maybe(types.string),
+
+    summary: types.maybe(types.string),
+    description: types.maybe(types.string),
+    authorsByline: types.maybe(types.string),
+    shortSummary: types.maybe(types.string),
+    labels: types.maybe(types.frozen()),
+    imageUrl: types.maybe(types.string),
+    score: types.maybe(types.number),
+});
+
+
+export const NewsStore = types
+    .model('NewsStore', {
+        symbolsNews: types.map(types.array(Article)),
+        isLoading: types.boolean,
+        error: types.maybe(types.string),
+        apiKey: types.string,
+    })
+    .actions((self) => ({
+
+        addNews(symbol: string, articles: any[]) {
+            if (!self.symbolsNews.has(symbol)) {
+                self.symbolsNews.set(symbol, []);
+            }
+
+
+            const mappedArticles = articles.map((article) => Article.create({
+                title: article.title || 'No Title',
+                content: article.content || 'No Content',
+                url: article.url,
+                source: article.domain,
+                pubDate: article.pubDate,
+                summary: article.summary,
+                description: article.description,
+                authorsByline: article.authorsByline,
+                shortSummary: article.shortSummary,
+                labels: article.labels,
+                imageUrl: article.imageUrl,
+                score: article.score,
+
+
+            }));
+            self.symbolsNews.get(symbol)!.push(...mappedArticles);
+            self.isLoading = false;
+        },
+
+        clearNews(symbol: string) {
+            if (self.symbolsNews.has(symbol)) {
+                self.symbolsNews.set(symbol, []);
+            }
+        },
+
+        setLoading(loading: boolean) {
+            self.isLoading = loading;
+        },
+
+        setError(message: string) {
+            self.error = message;
+            self.isLoading = false;
+        },
+
+        async fetchNewsForSymbol(symbol: string, limit: number, sort: "date" | "relevance") {
+            self.setLoading(true);
+            self.setError(undefined);
+
+            try {
+                await runInAction(async () => {
+                    const newsData = await collect_news({symbol, apiKey: self.apiKey, limit, sort});
+                    if (newsData && newsData.articles) {
+                        self.addNews(symbol, newsData.articles);
+                    } else {
+                        self.setError("Failed to fetch news or invalid response format.");
+                    }
+                })
+
+
+            } catch (err: any) {
+                console.error('Error fetching news:', err);
+                self.setError(err.message || "Failed to fetch news.");
+            }
+        },
+    }))
+    .views((self) => ({
+
+        getNewsForSymbol(symbol: string) {
+
+            return self.symbolsNews.get(symbol) || [];
+        },
+
+        getAllSymbols() {
+            return Array.from(self.symbolsNews.keys());
+        },
+
+        hasNewsForSymbol(symbol: string) {
+            return self.symbolsNews.has(symbol) && self.symbolsNews.get(symbol)!.length > 0;
+        },
+    }));
+
+export type INewsStore = Instance<typeof NewsStore>;
+
+
+export const createNewsStore = (apikey, perigon) => NewsStore.create({
+    symbolsNews: {},
+    isLoading: false,
+    error: undefined,
+    apiKey: apikey,
+});
+
+/* @collect_news return value structure
+{
+news: {
+status: 200,
+numResults: 4080,
+articles: [
+  [Object], [Object],
+  [Object], [Object],
+  [Object], [Object],
+  [Object], [Object],
+  [Object], [Object]
+]
+}
+}
+ */
+export async function collect_news(x: { symbol: string, apiKey: string, limit: number, sort: "date" | "relevance" }) {
+
+
+    const {symbol, apiKey, limit, sort} = x;
+
+    const symbolNameMap = {
+        "BTC": "Bitcoin",
+        "ETH": "Ethereum",
+        "XRP": "Ripple",
+        "LTC": "Litecoin",
+        "ADA": "Cardano",
+        "DOGE": "Dogecoin",
+        "BNB": "Binance Coin",
+        "DOT": "Polkadot",
+        "SOL": "Solana",
+        "AVAX": "Avalanche"
+    };
+
+
+    const cryptocurrencyName = symbolNameMap[symbol] ?? symbol;
+
+
+    const rawContentQuery = "scandal OR \"corporate misconduct*\" OR fraud OR \"financial irregularities*\" OR lawsuit OR \"legal action*\" OR bankruptcy OR \"financial distress*\" OR \"data breach\" OR \"security vulnerability*\" OR \"environmental impact\" OR \"ecological damage*\" OR \"labor dispute\" OR \"worker rights*\" OR \"product failure\" OR \"quality issue*\" OR \"ethical concern\" OR \"moral dilemma*\" OR \"health risk\" OR \"safety hazard*\" OR \"regulatory violation\" OR \"compliance issue*\" OR \"market manipulation\" OR \"trading irregularity*\" OR \"public relations crisis\" OR \"reputation damage*\" OR \"political controversy\" OR \"government intervention*\" OR \"consumer complaint\" OR \"customer dissatisfaction*\" OR \"supply chain disruption\" OR \"logistics problem*\" OR \"intellectual property dispute\" OR \"patent infringement*\"";
+    const contentQuery = encodeURIComponent(rawContentQuery);
+
+
+    const rawTitleQuery = `${cryptocurrencyName} OR ${symbol} OR "${cryptocurrencyName} price" OR "${cryptocurrencyName} market" OR "${cryptocurrencyName} news"`;
+    const titleQuery = encodeURIComponent(rawTitleQuery);
+
+    try {
+        const result = await allNews({
+            q: contentQuery,
+            title: titleQuery,
+            size: limit,
+            sortBy: sort,
+            apiKey: apiKey
+        });
+
+
+        return result.data;
+    } catch (err) {
+        console.error('Error fetching news:', err);
+        throw err;
+    }
+}

packages/core/news/news.test.ts

@@ -0,0 +1,75 @@
+import {describe, expect, it} from 'vitest';
+import {collect_news, createNewsStore, NewsStore} from './index';
+
+
+const testApiKey = '';
+
+describe('NewsStore', () => {
+    it('should create a NewsStore instance', () => {
+        const store = createNewsStore(testApiKey);
+        expect(store).toBeDefined();
+        expect(store.isLoading).toBe(false);
+        expect(store.error).toBeUndefined();
+        expect(store.getAllSymbols()).toEqual([]);
+    });
+
+    it('should add news articles for a symbol', () => {
+        const store = createNewsStore(testApiKey);
+        const articles = [
+            { title: 'Article 1', content: 'Content 1', url: 'http://example.com/1', source: 'Source 1', publishedAt: '2025-01-01' },
+            { title: 'Article 2', content: 'Content 2', url: 'http://example.com/2', source: 'Source 2', publishedAt: '2025-01-02' }
+        ];
+
+        store.addNews('BTC', articles);
+
+        expect(store.getNewsForSymbol('BTC')).toHaveLength(2);
+        expect(store.getNewsForSymbol('BTC')[0].title).toBe('Article 1');
+        expect(store.getNewsForSymbol('BTC')[1].title).toBe('Article 2');
+        expect(store.hasNewsForSymbol('BTC')).toBe(true);
+    });
+
+    it('should clear news articles for a symbol', () => {
+        const store = createNewsStore(testApiKey);
+        const articles = [
+            { title: 'Article 1', content: 'Content 1', url: 'http://example.com/1', source: 'Source 1', publishedAt: '2025-01-01' }
+        ];
+
+        store.addNews('BTC', articles);
+        store.clearNews('BTC');
+
+        expect(store.getNewsForSymbol('BTC')).toHaveLength(0);
+        expect(store.hasNewsForSymbol('BTC')).toBe(false);
+    });
+
+    it('should handle fetchNewsForSymbol successfully', async () => {
+        const store = createNewsStore(testApiKey);
+
+        await store.fetchNewsForSymbol('BTC', 10, 'date');
+
+        const storeNews = store.getNewsForSymbol('BTC');
+
+        console.log(storeNews);
+
+        expect(storeNews).toHaveLength(10);
+        expect(store.getNewsForSymbol('BTC')[0].title).toBeTypeOf("string");
+        expect(store.isLoading).toBe(false);
+        expect(store.error).toBeUndefined();
+    });
+
+
+
+
+    it('should throw an error for invalid symbol in collect_news', async () => {
+        await expect(collect_news({ symbol: 'INVALID', apiKey: testApiKey, limit: 10, sort: 'date' }))
+            .rejects.toThrow('Invalid symbol: INVALID. Must be one of BTC, ETH, XRP, LTC, ADA, DOGE, BNB, DOT, SOL, AVAX.');
+    });
+
+    it('should fetch news using collect_news', async () => {
+
+        const result = await collect_news({ symbol: 'BTC', apiKey: testApiKey, limit: 1, sort: 'date' });
+
+        expect(result).toBeDefined();
+        expect(result.status).toBe(200);
+        expect(result.articles).toBeDefined();
+    });
+});

packages/core/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@web-agent-rs/core",
+  "version": "1.0.0",
+  "type": "module",
+  "dependencies": {
+    "mobx-state-tree": "^7.0.2",
+    "@web-agent-rs/perigon": "workspace:*"
+  }
+}

packages/core/portfolio/index.ts

@@ -0,0 +1,137 @@
+import {getSnapshot, types} from "mobx-state-tree";
+import {NewsStore} from "../news";
+
+
+const PortfolioCashModel = types.model("PortfolioCash", {
+    amount: types.number,
+    currency: types.enumeration("Currency", ["USD", "BTC"]),
+});
+
+
+const PortfolioActionModel = types.model("PortfolioAction", {
+    action: types.enumeration("Action", ["buy", "sell", "hold"]),
+    symbol: types.string,
+    quantity: types.number,
+    timestamp: types.Date,
+});
+
+
+export const PortfolioNewsportfolioNewsModel = types.model("PortfolioNews", {
+    symbol: types.string,
+    date_created: types.string,
+    news: types.array(types.model("NewsItem", {
+        symbol: types.maybe(types.string),
+        date_created: types.maybe(types.string),
+        news: types.string,
+        timestamp: types.maybe(types.string),
+    })),
+    timestamp: types.maybe(types.string),
+});
+
+export const portfolioQuoteModel = types.model("PortfolioQuote", {
+    symbol: types.string,
+    quote: types.string,
+    date_created: types.string,
+});
+
+
+const PortfolioAssetContextModel = types.model("PortfolioAssetContext", {
+    timestamp: types.Date,
+    portfolio_snapshot: PortfolioCashModel,
+});
+
+
+const PortfolioAssetModel = types.model("PortfolioAsset", {
+    symbol: types.string,
+    quantity: types.number,
+    recommended_action: types.maybe(
+        types.enumeration("RecommendedAction", ["buy", "sell", "hold"])
+    ),
+    last_taken_action: types.optional(types.enumeration("LastAction", ["buy", "sell", "hold", "none", "never"]), "never"),
+    context: PortfolioAssetContextModel,
+});
+
+
+const PortfolioModel = types
+    .model("Portfolio", {
+        supportedSymbols: types.array(types.string),
+        liquidity: PortfolioCashModel,
+        actions: types.optional(types.array(PortfolioActionModel), []),
+        assets: types.array(PortfolioAssetModel),
+        news: types.optional(types.array(NewsStore), []),
+        quotes: types.optional(types.array(portfolioQuoteModel), []),
+    })
+    .actions((self) => ({
+
+        addAction(actionData: {
+            action: "buy" | "sell" | "hold";
+            symbol: string;
+            quantity: number;
+        }) {
+            self.actions.push({
+                ...actionData,
+                timestamp: new Date(),
+            });
+        },
+        addNews(newsData: any) {
+            self.news.push({
+                ...newsData,
+                timestamp: new Date(),
+            });
+        },
+        addQuote(quoteData: any) {
+            self.quotes
+            self.quotes.push({
+                ...quoteData,
+                timestamp: new Date(),
+            });
+        },
+
+        updateLiquidity(amount: number) {
+            self.liquidity.amount = amount;
+        },
+    }));
+
+const tokenList = [
+    "AAVE", "AVAX", "BAT", "BCH", "BTC",
+    "CRV", "DOGE", "DOT", "ETH", "GRT",
+    "LINK", "LTC", "MKR", "SHIB", "SUSHI",
+    "UNI", "USDC", "USDT", "XTZ", "YFI",
+];
+
+
+const portfolioCash = PortfolioCashModel.create({
+    amount: 10000,
+    currency: "USD",
+});
+
+
+const portfolioAssets = tokenList.map((token) =>
+    PortfolioAssetModel.create({
+        symbol: token,
+        quantity: 0,
+        recommended_action: "hold",
+        last_taken_action: undefined,
+        context: {
+            timestamp: new Date(),
+            portfolio_snapshot: getSnapshot(portfolioCash),
+        },
+    })
+);
+
+
+const portfolioActions = [];
+const portfolioNews = [];
+const portfolioQuotes = [];
+
+
+const portfolioInstance = PortfolioModel.create({
+    liquidity: portfolioCash,
+    actions: portfolioActions,
+    assets: portfolioAssets,
+    supportedSymbols: tokenList,
+    news: portfolioNews,
+    quotes: portfolioQuotes
+});
+
+export default portfolioInstance;

packages/core/quotes/index.ts

@@ -0,0 +1,13 @@
+import {ApiResponse} from "./types";
+
+export async function collect_quote(x: { symbol: string, apiKey: string }) {
+    const {symbol, apiKey} = x;
+
+    const data: ApiResponse = await fetch(`https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest?symbol=${symbol}`, {
+        headers: {
+            "x-cmc_pro_api_key": apiKey
+        }
+    }).then((symbolDataRequest) => symbolDataRequest.json());
+
+    return data;
+}

packages/core/quotes/models.ts

@@ -0,0 +1,77 @@
+import {types, flow, Instance} from "mobx-state-tree";
+import {collect_quote} from './index';
+
+
+const QuoteData = types.optional(types.frozen(), {})
+
+export const QuoteStore = types
+    .model("QuoteStore", {
+        apiKey: types.string,
+        quotes: types.map(QuoteData),
+    })
+    .views(self => ({
+        getQuote(symbol) {
+            return self.quotes.get(symbol);
+        },
+
+        hasQuote(symbol) {
+            return self.quotes.has(symbol);
+        }
+    }))
+    .actions(self => {
+
+        const extractUsefulData = (data, symbol) => {
+            return data.data[symbol].map(qd => ({
+                symbol: qd.symbol,
+                slug: qd.slug,
+                tags: qd.tags,
+                id: qd.id,
+                ...qd.quote.USD
+            })).at(0);
+        };
+
+        const fetchQuote = flow(function* (symbol) {
+            try {
+                const data = yield collect_quote({symbol, apiKey: self.apiKey});
+                const usefulData = extractUsefulData(data, symbol);
+
+
+                self.quotes.set(symbol, usefulData);
+
+                return usefulData;
+            } catch (error) {
+                console.error(`An error occurred fetching the quote for symbol: ${symbol}`, error);
+                throw error;
+            }
+        });
+
+        const fetchQuotes = flow(function* (symbols) {
+            const results = {};
+
+            for (const symbol of symbols) {
+
+                if (self.quotes.has(symbol)) {
+                    results[symbol] = self.quotes.get(symbol);
+                } else {
+
+                    const data = yield fetchQuote(symbol);
+                    results[symbol] = extractUsefulData(data, symbol);
+                }
+            }
+
+            return results;
+        });
+
+        const clearCache = () => {
+            self.quotes.clear();
+        };
+
+        return {
+            fetchQuote,
+            fetchQuotes,
+            clearCache
+        };
+    });
+
+
+export type QuoteManagerType = Instance<typeof QuoteStore>;

packages/core/quotes/quote.test.ts

@@ -0,0 +1,17 @@
+import {describe, it} from 'vitest';
+import {QuoteStore} from "./models";
+
+describe('QuoteStore', () => {
+    it('should get data for symbols using the quoteManager', async () => {
+        const testApiKey = '';
+        const quoteManager = QuoteStore.create({
+            apiKey: testApiKey,
+        });
+
+        const symbol = 'BTC';
+
+        const data = await quoteManager.fetchQuote(symbol);
+
+        console.log(JSON.stringify(data));
+    });
+});

packages/core/quotes/types.ts

@@ -0,0 +1,71 @@
+type Status = {
+    timestamp: string;
+    error_code: number;
+    error_message: string | null;
+    elapsed: number;
+    credit_count: number;
+    notice: string | null;
+};
+
+type Tag = {
+    slug: string;
+    name: string;
+    category: string;
+};
+
+type Quote = {
+    USD: {
+        price: number | null;
+        volume_24h: number;
+        volume_change_24h: number;
+        percent_change_1h: number;
+        percent_change_24h: number;
+        percent_change_7d: number;
+        percent_change_30d: number;
+        percent_change_60d: number;
+        percent_change_90d: number;
+        market_cap: number | null;
+        market_cap_dominance: number | null;
+        fully_diluted_market_cap: number | null;
+        tvl: number | null;
+        last_updated: string;
+    };
+};
+
+type Platform = {
+    id: number;
+    name: string;
+    symbol: string;
+    slug: string;
+    token_address: string;
+};
+
+type Cryptocurrency = {
+    id: number;
+    name: string;
+    symbol: string;
+    slug: string;
+    num_market_pairs: number;
+    date_added: string;
+    tags: Tag[];
+    max_supply: number | null;
+    circulating_supply: number | null;
+    total_supply: number;
+    platform: Platform | null;
+    is_active: number;
+    infinite_supply: boolean;
+    cmc_rank: number | null;
+    is_fiat: number;
+    self_reported_circulating_supply: number | null;
+    self_reported_market_cap: number | null;
+    tvl_ratio: number | null;
+    last_updated: string;
+    quote: Quote;
+};
+
+export type ApiResponse = {
+    status: Status;
+    data: {
+        [SYMBOL: string]: Cryptocurrency[];
+    };
+};

packages/core/types/index.ts

@@ -0,0 +1,69 @@
+export type CryptoDataResponse = {
+    status: {
+        timestamp: string;
+        error_code: number;
+        error_message: string | null;
+        elapsed: number;
+        credit_count: number;
+        notice: string | null;
+    };
+    data: {
+        [key: string]: CryptoAsset[];
+    };
+};
+
+export type CryptoAsset = {
+    id: number;
+    name: string;
+    symbol: string;
+    slug: string;
+    num_market_pairs: number;
+    date_added: string;
+    tags: CryptoTag[];
+    max_supply: number | null;
+    circulating_supply: number;
+    total_supply: number;
+    platform: CryptoPlatform | null;
+    is_active: number;
+    infinite_supply: boolean;
+    cmc_rank: number;
+    is_fiat: number;
+    self_reported_circulating_supply: number | null;
+    self_reported_market_cap: number | null;
+    tvl_ratio: number;
+    last_updated: string;
+    quote: {
+        [currency: string]: CryptoQuote;
+    };
+};
+
+export type CryptoTag = {
+    slug: string;
+    name: string;
+    category: string;
+};
+
+export type CryptoPlatform = {
+    id: number;
+    name: string;
+    symbol: string;
+    slug: string;
+    token_address: string;
+};
+
+export type CryptoQuote = {
+    price: number;
+    volume_24h: number;
+    volume_change_24h: number;
+    percent_change_1h: number;
+    percent_change_24h: number;
+    percent_change_7d: number;
+    percent_change_30d: number;
+    percent_change_60d: number;
+    percent_change_90d: number;
+    market_cap: number;
+    market_cap_dominance: number;
+    fully_diluted_market_cap: number;
+    tvl: number;
+    last_updated: string;
+};

packages/genaiscript-rust-shim/.gitignore

@@ -0,0 +1,175 @@
+# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
+
+# Logs
+
+logs
+_.log
+npm-debug.log_
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Caches
+
+.cache
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
+
+# Runtime data
+
+pids
+_.pid
+_.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+
+lib-cov
+
+# Coverage directory used by tools like istanbul
+
+coverage
+*.lcov
+
+# nyc test coverage
+
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+
+bower_components
+
+# node-waf configuration
+
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+
+build/Release
+
+# Dependency directories
+
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+
+web_modules/
+
+# TypeScript cache
+
+*.tsbuildinfo
+
+# Optional npm cache directory
+
+.npm
+
+# Optional eslint cache
+
+.eslintcache
+
+# Optional stylelint cache
+
+.stylelintcache
+
+# Microbundle cache
+
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+
+.node_repl_history
+
+# Output of 'npm pack'
+
+*.tgz
+
+# Yarn Integrity file
+
+.yarn-integrity
+
+# dotenv environment variable files
+
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+
+.parcel-cache
+
+# Next.js build output
+
+.next
+out
+
+# Nuxt.js build / generate output
+
+.nuxt
+dist
+
+# Gatsby files
+
+# Comment in the public line in if your project uses Gatsby and not Next.js
+
+# https://nextjs.org/blog/next-9-1#public-directory-support
+
+# public
+
+# vuepress build output
+
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+
+.temp
+
+# Docusaurus cache and generated files
+
+.docusaurus
+
+# Serverless directories
+
+.serverless/
+
+# FuseBox cache
+
+.fusebox/
+
+# DynamoDB Local files
+
+.dynamodb/
+
+# TernJS port file
+
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+
+.vscode-test
+
+# yarn v2
+
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+# IntelliJ based IDEs
+.idea
+
+# Finder (MacOS) folder config
+.DS_Store

packages/genaiscript-rust-shim/README.md

@@ -0,0 +1,15 @@
+# genaiscript-rust-shim
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+To run:
+
+```bash
+bun run index.ts
+```
+
+This project was created using `bun init` in bun v1.1.36. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

packages/genaiscript-rust-shim/genaiscript-rust-shim.ts

@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+import minimist from "minimist";
+import { run } from "genaiscript/api";
+import { RunScriptOptions } from "./shim-types";
+
+type Args = {
+    file: string;
+    vars: Record<string, unknown>;
+    options: Partial<RunScriptOptions> & {
+        envVars?: Record<string, string>;
+        signal?: AbortSignal;
+    };
+};
+
+async function wrapper(args: Args) {
+    try {
+        await run(args.file, [], { vars: args.vars }, args.options);
+    } catch (error) {
+        console.error("Error executing script:", error);
+        process.exit(1);
+    }
+}
+
+function parseCliArgs(): Args {
+    const argv = minimist(process.argv.slice(2), {
+        string: ["file"],
+        alias: { f: "file" },
+    });
+
+    if (!argv.file) {
+        console.error("Error: Missing required argument --file");
+        process.exit(1);
+    }
+
+    const keyValuePairs = argv._;
+
+    const vars: Record<string, unknown> = keyValuePairs.reduce((acc, pair) => {
+        const [key, value] = pair.split("=");
+        if (key && value !== undefined) {
+            acc[key] = value; // Retain the `unknown` type for later flexibility
+        } else {
+            console.error(`Error: Invalid key=value pair "${pair}"`);
+            process.exit(1);
+        }
+        return acc;
+    }, {} as Record<string, unknown>);
+
+    return {
+        file: argv.file,
+        vars,
+        options: {},
+    };
+}
+
+async function main() {
+
+    const args = parseCliArgs();
+    await wrapper(args);
+}
+
+main();

packages/genaiscript-rust-shim/index.ts

@@ -0,0 +1,2 @@
+import './genaiscript-rust-shim';
+export * from './genaiscript-rust-shim';

packages/genaiscript-rust-shim/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@web-agent-rs/genaiscript-rust-shim",
+  "module": "index.ts",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "buildShim": "esbuild genaiscript-rust-shim.ts --bundle --format=esm --packages=external --outdir=dist --platform=node && chmod +x dist/genaiscript-rust-shim.js",
+    "setupDev": "cp dist/genaiscript-rust-shim.js ../../dist/genaiscript-rust-shim.js"
+  },
+  "devDependencies": {
+    "@types/bun": "latest",
+    "minimist": "^1.2.8",
+    "genaiscript": "^1.95.1",
+    "esbuild": "^0.24.2"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  }
+}

packages/genaiscript-rust-shim/shim-types.ts

@@ -0,0 +1,70 @@
+type FenceFormat = "markdown" | "xml" | "none"
+
+export interface WorkspaceFile {
+    /**
+     * Name of the file, relative to project root.
+     */
+    filename: string
+
+    /**
+     * Content mime-type if known
+     */
+    type?: string
+
+    /**
+     * Encoding of the content
+     */
+    encoding?: "base64"
+
+    /**
+     * Content of the file.
+     */
+    content?: string
+}
+
+export interface RunScriptOptions {
+    excludedFiles: string[]
+    excludeGitIgnore: boolean
+    runRetry: string
+    out: string
+    retry: string
+    retryDelay: string
+    maxDelay: string
+    json: boolean
+    yaml: boolean
+    outTrace: string
+    outOutput: string
+    outAnnotations: string
+    outChangelogs: string
+    pullRequest: string
+    pullRequestComment: string | boolean
+    pullRequestDescription: string | boolean
+    pullRequestReviews: boolean
+    outData: string
+    label: string
+    temperature: string | number
+    topP: string | number
+    seed: string | number
+    maxTokens: string | number
+    maxToolCalls: string | number
+    maxDataRepairs: string | number
+    model: string
+    smallModel: string
+    visionModel: string
+    embeddingsModel: string
+    modelAlias: string[]
+    provider: string
+    csvSeparator: string
+    cache: boolean | string
+    cacheName: string
+    applyEdits: boolean
+    failOnErrors: boolean
+    removeOut: boolean
+    vars: string[] | Record<string, string | boolean | number | object>
+    fallbackTools: boolean
+    jsSource: string
+    logprobs: boolean
+    topLogprobs: number
+    fenceFormat: FenceFormat
+    workspaceFiles?: WorkspaceFile[]
+}

packages/genaiscript-rust-shim/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

packages/genaiscript/genaisrc/.gitignore

@@ -0,0 +1,4 @@
+# auto-generated
+genaiscript.d.ts
+tsconfig.json
+jsconfig.json

packages/genaiscript/genaisrc/_state/index.ts

@@ -0,0 +1,21 @@
+import {types} from "mobx-state-tree";
+import {QuoteStore} from "@web-agent-rs/core/quotes/models";
+import {NewsStore} from "@web-agent-rs/core/news";
+
+import newsStore from "./news";
+import quoteStore from "./quotes";
+
+
+const StateModel = types.model("State", {
+    symbols: types.array(types.string),
+    quotes: QuoteStore,
+    news: NewsStore,
+});
+
+
+const state = StateModel.create({
+    quotes: quoteStore,
+    news: newsStore,
+});
+
+export default state;

packages/genaiscript/genaisrc/_state/news.ts

@@ -0,0 +1,11 @@
+import {NewsStore} from "@web-agent-rs/core/news";
+import {Instance} from "mobx-state-tree";
+
+const newsStore = NewsStore.create({
+    isLoading: false,
+    apiKey: process.env.PERIGON_API_KEY
+});
+
+export type NewsStore = Instance<typeof newsStore>;
+
+export default newsStore;

packages/genaiscript/genaisrc/_state/quotes.ts

@@ -0,0 +1,7 @@
+import {QuoteStore} from "@web-agent-rs/core/quotes/models";
+
+const quoteStore = QuoteStore.create({
+    apiKey: process.env.CCC_API_KEY
+});
+
+export default quoteStore;

packages/genaiscript/genaisrc/agent.genai.mts

@@ -0,0 +1,24 @@
+import {PerigonClient as NewsSearchTool} from "@agentic/perigon";
+
+script({
+    system: ["system.tools"],
+    tools: "agent",
+    maxTokens: 8192
+})
+
+const newSearchTool = new NewsSearchTool();
+
+defTool(newSearchTool)
+
+$`You are a chat assistant that uses agent tools to solve problems.
+
+while true:
+    - ask the user for a question using the agent_user_input
+    - make a plan to answer the question step by step
+    - answer the question
+end while
+
+## guidance:
+    - use the agent tools to help you
+    - do NOT try to ask the user questions directly, use the agent_user_input tool instead.
+`

packages/genaiscript/genaisrc/deep-research.genai.mts

@@ -0,0 +1,281 @@
+import {task, entrypoint, interrupt, MemorySaver} from "@langchain/langgraph"
+import "./tools/searxng.genai.mjs"
+import {SearxngClient} from "@agentic/searxng";
+
+
+script({
+    title: "Deep Research Program",
+    description: "Researchers can use this program to conduct deep research on a topic",
+    model: "large",
+    cache: "ephemeral",
+})
+const {output, vars} = env
+
+
+const breakdownResearch = task(
+    "breakdown_research",
+    async (question: string) => {
+        const result = await runPrompt(
+            async (ctx) => {
+                ctx.$`You are an expert research strategist.
+
+Task: Break down the following research question into 3-5 focused sub-questions that would help comprehensively answer the main question.
+
+Research question: ${question}
+
+For each sub-question:
+1. Assign a unique ID (e.g., SQ1, SQ2)
+2. Explain the rationale for why this sub-question is important
+3. Ensure the sub-questions collectively cover the main research question
+
+Output the breakdown as a JSON object.`
+            },
+            {
+                label: "breakdown research",
+                responseSchema: {
+                    type: "object",
+                    properties: {
+                        mainQuestion: {type: "string"},
+                        subQuestions: {
+                            type: "array",
+                            items: {
+                                type: "object",
+                                properties: {
+                                    id: {type: "string"},
+                                    question: {type: "string"},
+                                    rationale: {type: "string"},
+                                },
+                            },
+                        },
+                    },
+                },
+            }
+        )
+
+        return result.json
+    }
+)
+
+const globalCtx = this;
+
+
+const researchSubQuestion = task(
+    "research_subquestion",
+    async (subQuestion: { id: string; question: string }) => {
+
+        const searxng = new SearxngClient({apiBaseUrl: "https://search-engine-gsio.fly.dev"});
+
+        const {text} = await runPrompt(
+            (_) => {
+                _.defTool(searxng)
+                _.$`You are an expert researcher with access to comprehensive information.
+    
+Task: Thoroughly research the following question and provide a detailed answer.
+
+Question ID: ${subQuestion.id}
+Question: ${subQuestion.question}
+
+Provide your findings in a structured format that includes:
+- Your answer to the sub-question
+- Relevant sources that support your answer
+- Your confidence level in the answer (0-1)`
+            },
+            {
+                model: "small",
+                label: `research subquestion ${subQuestion.id}`,
+                maxDataRepairs: 2,
+                responseSchema: {
+                    type: "object",
+                    properties: {
+                        subQuestionId: {type: "string"},
+                        answer: {type: "string"},
+                        sources: {
+                            type: "array",
+                            items: {
+                                type: "object",
+                                properties: {
+                                    title: {type: "string"},
+                                    url: {type: "string"},
+                                    relevance: {type: "string"},
+                                },
+                            },
+                        },
+                        confidence: {type: "number"},
+                    },
+                },
+            }
+        )
+        return text
+    }
+)
+
+
+const synthesizeFindings = task(
+    "synthesize_findings",
+    async (mainQuestion: string, findings: any[]) => {
+        const result = await runPrompt(
+            async (ctx) => {
+                ctx.$`You are an expert research synthesizer.
+    
+Task: Synthesize the following research findings into a coherent response to the main research question.
+
+Main Research Question: ${mainQuestion}
+
+Findings:
+${JSON.stringify(findings, null, 2)}
+
+Provide a synthesis that:
+1. Directly answers the main research question
+2. Integrates the findings from all sub-questions
+3. Identifies limitations in the current research
+4. Suggests next steps for further investigation`
+            },
+            {
+                label: "synthesize findings",
+                responseType: "markdown",
+                responseSchema: {
+                    type: "object",
+                    properties: {
+                        summary: {type: "string"},
+                        findings: {type: "array", items: {type: "string"}},
+                        limitations: {
+                            type: "array",
+                            items: {type: "string"},
+                        },
+                        nextSteps: {type: "array", items: {type: "string"}},
+                    },
+                },
+            }
+        )
+
+        return result.json
+    }
+)
+
+
+const summarizeAndIdentifyGaps = task(
+    "summarize_and_identify_gaps",
+    async (synthesis: any, findings: any[]) => {
+        const result = await runPrompt(
+            async (ctx) => {
+                ctx.$`You are an expert research evaluator.
+                        
+Task: Review the research synthesis and identify any gaps or areas that need deeper investigation.
+
+Current synthesis:
+${JSON.stringify(synthesis, null, 2)}
+
+Research findings:
+${JSON.stringify(findings, null, 2)}
+
+Please provide:
+1. A concise summary of current findings
+2. Identify 2-3 specific knowledge gaps
+3. Formulate follow-up questions to address these gaps`
+            },
+            {
+                label: "identify research gaps",
+                responseSchema: {
+                    type: "object",
+                    properties: {
+                        summary: {type: "string"},
+                        gaps: {
+                            type: "array",
+                            items: {type: "string"},
+                        },
+                        followUpQuestions: {
+                            type: "array",
+                            items: {
+                                type: "object",
+                                properties: {
+                                    id: {type: "string"},
+                                    question: {type: "string"},
+                                },
+                            },
+                        },
+                    },
+                },
+            }
+        )
+        return result.json
+    }
+)
+
+
+const researchWorkflow = entrypoint(
+    {checkpointer: new MemorySaver(), name: "research_workflow"},
+    async (input: { question: string; context?: string }) => {
+
+        const breakdown = await breakdownResearch(input.question)
+
+
+        const subQuestionFindings = []
+
+        for (const sq of breakdown.subQuestions) {
+            const analysis = await researchSubQuestion(sq);
+            console.log(analysis);
+            subQuestionFindings.push(analysis);
+        }
+
+
+        let synthesis = await synthesizeFindings(
+            input.question,
+            subQuestionFindings
+        )
+
+        const gapAnalysis = await summarizeAndIdentifyGaps(
+            synthesis,
+            subQuestionFindings
+        )
+
+
+        const followUpFindings = [];
+        for (const fq of gapAnalysis.followUpQuestions) {
+            const anwser = await researchSubQuestion(fq);
+            console.log(anwser);
+            followUpFindings.push(anwser);
+        }
+
+
+        const allFindings = [...subQuestionFindings, ...followUpFindings]
+        const finalSynthesis = await synthesizeFindings(
+            input.question,
+            allFindings
+        )
+
+
+        return {
+            question: input.question,
+            breakdown: breakdown,
+            initialFindings: subQuestionFindings,
+            gapAnalysis: gapAnalysis,
+            followUpFindings: followUpFindings,
+            synthesis: finalSynthesis,
+        }
+    }
+)
+
+
+const researchQuestion =
+    env.vars.question ||
+    "What are the most promising approaches to climate change mitigation?"
+
+
+const threadId = `research-${Date.now()}`
+
+
+const config = {
+    configurable: {
+        thread_id: threadId,
+    },
+}
+
+
+const results = await researchWorkflow.invoke(
+    {
+        question: researchQuestion,
+        context: vars.context || "",
+    },
+    config
+)
+output.fence(results, "json")

packages/genaiscript/genaisrc/finance-query.genai.mts

@@ -0,0 +1,80 @@
+import state from "./_state/index.js";
+import {getSnapshot} from "mobx-state-tree";
+import {collect_gainers_losers} from "@web-agent-rs/core/market";
+
+def("QUERY", env.vars.user_input);
+
+
+defTool(
+    "get_quote",
+    "Fetch quote for symbol",
+    {
+        "symbol": {
+            type: "string",
+            default: "BTC"
+        }
+    },
+    async (args) => {
+        const { symbol } = args;
+        await state.quotes.fetchQuote(symbol);
+        
+
+        const quote = await state.quotes.getQuote(symbol);
+        
+        return JSON.stringify(quote)
+    }
+);
+
+defTool(
+    "get_news",
+    "Fetches news for symbol",
+    {
+        "symbol": {
+            type: "string",
+            default: "BTC"
+        }
+    },
+    async (args) => {
+        const { symbol } = args;
+        await state.news.fetchNewsForSymbol(symbol, 5, "date");
+        
+        const news = await state.news.getNewsForSymbol(symbol).map(i => getSnapshot(i));
+        
+        return news
+    }
+);
+
+
+defTool(
+    "get_market",
+    "Fetches trending symbols of market",
+    {
+        "limit": {
+            type: "number",
+            default: "25"
+        }
+    },
+    async (args) => {
+
+        const { limit } = args;
+
+        
+        const marketOverviewRequest = await collect_gainers_losers({apiKey: process.env.CCC_API_KEY, limit: parseInt(limit) })
+        
+        return marketOverviewRequest.data.map(item => ({
+            symbol: item.symbol,
+            name: item.name,
+            
+            
+            change_1h: item.quote.USD.percent_change_1h,
+            price: item.quote.USD.price,
+            volume_24h: item.quote.USD.volume_24h
+        }))
+    }
+);
+
+
+
+$`You are a market data assistant specializing in financial analysis. Respond to QUERIES with accurate, clear, and concise information relevant to professionals in the finance sector. Use available tools efficiently to gather and present quantitative data.`;
+
+

packages/genaiscript/genaisrc/image-generator.genai.mts

@@ -0,0 +1,11 @@
+console.log("Generating image")
+
+def("USER_INPUT", env.vars.user_input);
+
+
+const inputs = {
+    host: JSON.parse(env.vars.user_input).host,
+    imageId: JSON.parse(env.vars.user_input).imageId
+}
+
+console.log(`![Generated Image](/generated/image/${inputs.imageId})`);

packages/genaiscript/genaisrc/incubator/firecrawl.genai.mts

@@ -0,0 +1,26 @@
+script({
+    title: "Stock Market News Scraper",
+    tools: ["searxng"],
+})
+
+defTool({
+    "mcp-server-firecrawl": {
+        command: "npx",
+        args: ["-y", "firecrawl-mcp"],
+    },
+})
+
+def("QUERY_NEWS", "Latest news on AAPL")
+def("QUERY_SENTIMENT", "Market sentiment for technology sector")
+
+
+$`Search the query with searxng: QUERY_NEWS`
+
+
+$`Scrape the top search result with firecrawl`
+
+
+$`Search the query with searxng: QUERY_SENTIMENT`
+
+
+$`Scrape the top search result with firecrawl`

packages/genaiscript/genaisrc/news-search.genai.mts

@@ -0,0 +1,24 @@
+import {SearxngClient} from "@agentic/searxng";
+import "./tools/searxng.genai.mjs"
+
+script({
+    title: "news_search_agent",
+    tools: ["searxng"],
+    maxToolCalls: 2,
+    cache: false,
+});
+
+def("USER_INPUT", env.vars.user_input);
+def("TODAY", new Date().toISOString().split("T")[0]);
+def("LINK_FORMAT", "[Link](url)");
+
+
+$`You are an assistant searching for news using complex queries to pinpoint results.
+
+ 
+- tailor search to answer the question in USER_INPUT
+- perform 2 searches in parallel sorted by relevance and date respectively
+- create a markdown table of <=5 results of both searches
+- header row: Date, Title, Summary, and Link
+ 
+Respond with a single table, no extra text.`

packages/genaiscript/genaisrc/tools/searxng.genai.mts

@@ -0,0 +1,18 @@
+script({
+    isSystem: true
+})
+
+import {SearxngClient} from "@agentic/searxng";
+import ky from 'ky';
+
+const kyWithHeaders = ky.create({
+    referrerPolicy: "unsafe-url",
+
+    headers: {
+        'Authorization': 'Basic ' + btoa(`admin:${process.env.SEARXNG_PASSWORD}`),
+    }
+});
+
+const searxng = new SearxngClient({ky: kyWithHeaders});
+
+defTool(searxng)

packages/genaiscript/genaisrc/web-scrape.genai.mts

@@ -0,0 +1,88 @@
+import {Window} from 'happy-dom';
+import {platform} from 'os';
+
+script({
+    title: "scrape",
+    cache: false,
+});
+
+/*
+  "url": "Full URL in the conversation that references the URL being interacted with. No trailing slash!",
+  "query": "Implied question about the resources at the URL.",
+  "action": "read | scrape | crawl"
+*/
+
+try {
+    const {url, query, action} = JSON.parse(env.vars.user_input);
+} catch (e) {
+    throw "Sorry! Something went wrong.";
+}
+
+const {url, query, action} = JSON.parse(env.vars.user_input);
+
+def("URL", url);
+
+def("QUERY", query);
+
+def("ACTION", action);
+
+// console.log({url, query, action});
+
+if(!(new URL(url) ?? undefined)) {
+    throw "Bad URL. Maybe try again?"
+}
+
+function getBrowser(): "webkit" | "chromium" | "firefox" {
+    if (platform() === 'darwin') {
+        return "webkit"; // macOS is identified by 'darwin'
+    }
+    return "chromium"; // default to chromium for other platforms
+}
+
+const {text} = await host.fetchText(new URL(url).toString());
+
+// const browser = getBrowser();
+
+// const page = await host.browse(new URL(url).toString(), {
+//     browser: getBrowser(),
+//     headless: true,
+//     javaScriptEnabled: browser !== "chromium",
+//     // timeout: 3000,
+//     // bypassCSP: true,
+//     // baseUrl: new URL(url).origin,
+// });
+//
+// const html = (await page.content());
+// const title = (await page.title());
+
+// console.log({html});
+
+const window = new Window({
+    // url: "http://localhost:8080",
+    height: 1920,
+    width: 1080,
+    settings: {
+        navigator: {
+            userAgent: 'Mozilla/5.0 (compatible; GeoffsAI/1.0; +https://geoff.seemueller.io)',
+        },
+    }
+});
+
+window.document.body.innerHTML = text;
+
+const textContent = window.document.body.textContent;
+
+def("PAGE_TEXT", textContent);
+
+$`You a helpful assistant interacting with resources found at the URL.
+
+- markdown table is concise representation of PAGE_TEXT relevant to the QUERY
+
+### Respond Example:
+### Data from ${url}:
+| Header 1 | Header 2 | Header 3 |
+|----------|----------|----------|
+| Data 1   | Data 2   | Data 3   |
+\n---[Example explanation of data significance to query.]
+---
+Respond with the markdown table and an explanation of significance. Do not include extra text.`;

packages/genaiscript/genaisrc/web-search.genai.mts

@@ -0,0 +1,28 @@
+import {SearxngClient} from "@agentic/searxng";
+import "./tools/searxng.genai.mjs"
+
+
+script({
+    title: "web_search_agent",
+    maxTokens: 8192,
+    cache: false,
+    tools: ["searxng"],
+});
+
+
+
+def("USER_INPUT", env.vars.user_input);
+
+
+
+def("LINK_FORMAT", "[Link](url)");
+
+$`You are an assistant searching for web content using complex queries to pinpoint results.
+
+ 
+- tailor search to answer the question in USER_INPUT
+- perform 2 searches in parallel sorted by relevance and date respectively
+- create a markdown table of <=5 results of both searches
+- header row: Title, Description, and Link
+ 
+Respond with a single table, no extra text.`

packages/genaiscript/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@web-agent-rs/genaiscript",
+  "type": "module",
+  "workspaces": ["packages/*"],
+  "private": true,
+  "scripts": {
+    "dev": "cargo watch -x 'run src/main.rs'",
+    "ai:search": "genaiscript run genaisrc/web-search.genai.mts --vars USER_INPUT='who won the 2024 election?'",
+    "shim:ai:search": "pnpm build && ./dist/shim.js --file=genaisrc/search.genai.mts USER_INPUT=\"Who won the 2024 presidential election?\"\n",
+    "ai:news": "genaiscript run genaisrc/news-search.genai.mts --vars USER_INPUT='What are the latest updates and developments in the Ukraine war?'",
+    "ai:url:read": "genaiscript run genaisrc/web-scrape.genai.mts --vars USER_INPUT='{\"url\":\"https://geoff.seemueller.io/about\",\"query\":\"Describe the details of the page.\", \"action\": \"read\"}'",
+    "ai:url:scrape": "npx genaiscript run genaisrc/web-scrape.genai.mts --vars USER_INPUT='{\"url\":\"https://www.time4learning.com/homeschool-curriculum/high-school/eleventh-grade/math.html\",\"query\":\"What is on this page?\", \"action\": \"scrape\"}'",
+    "crypto:quote": "npx genaiscript run genaisrc/finance-query.genai.mts --vars USER_INPUT='Get a quote for BTC'",
+    "crypto:news": "npx genaiscript run genaisrc/finance-query.genai.mts --vars USER_INPUT='What is the news for Bitcoin?'",
+    "crypto:overview": "npx genaiscript run genaisrc/finance-query.genai.mts --vars USER_INPUT='What are the trending symbols in the market?'"
+  },
+  "dependencies": {
+    "@agentic/perigon": "^7.2.0",
+    "@agentic/searxng": "7.5.3",
+    "@kevinwatt/mcp-server-searxng": "^0.3.9",
+    "@types/node": "^22.10.2",
+    "genaiscript": "^1.95.1",
+    "happy-dom": "^16.0.1",
+    "@web-agent-rs/perigon": "workspace:*",
+    "@web-agent-rs/core": "workspace:*",
+    "ky": "^1.8.0"
+  }
+}

packages/perigon/index.ts

@@ -0,0 +1,106 @@
+import type * as types from './types';
+import type { ConfigOptions, FetchResponse } from 'api/dist/core'
+import Oas from 'oas';
+import APICore from 'api/dist/core';
+import definition from './openapi.json';
+
+class SDK {
+  spec: Oas;
+  core: APICore;
+
+  constructor() {
+    this.spec = Oas.init(definition);
+    this.core = new APICore(this.spec, 'perigon/unknown (api/6.1.3)');
+  }
+
+  /**
+   * Optionally configure various options that the SDK allows.
+   *
+   * @param config Object of supported SDK options and toggles.
+   * @param config.timeout Override the default `fetch` request timeout of 30 seconds. This number
+   * should be represented in milliseconds.
+   */
+  config(config: ConfigOptions) {
+    this.core.setConfig(config);
+  }
+
+  /**
+   * If the API you're using requires authentication you can supply the required credentials
+   * through this method and the library will magically determine how they should be used
+   * within your API request.
+   *
+   * With the exception of OpenID and MutualTLS, it supports all forms of authentication
+   * supported by the OpenAPI specification.
+   *
+   * @example <caption>HTTP Basic auth</caption>
+   * sdk.auth('username', 'password');
+   *
+   * @example <caption>Bearer tokens (HTTP or OAuth 2)</caption>
+   * sdk.auth('myBearerToken');
+   *
+   * @example <caption>API Keys</caption>
+   * sdk.auth('myApiKey');
+   *
+   * @see {@link https://spec.openapis.org/oas/v3.0.3#fixed-fields-22}
+   * @see {@link https://spec.openapis.org/oas/v3.1.0#fixed-fields-22}
+   * @param values Your auth credentials for the API; can specify up to two strings or numbers.
+   */
+  auth(...values: string[] | number[]) {
+    this.core.setAuth(...values);
+    return this;
+  }
+
+  /**
+   * If the API you're using offers alternate server URLs, and server variables, you can tell
+   * the SDK which one to use with this method. To use it you can supply either one of the
+   * server URLs that are contained within the OpenAPI definition (along with any server
+   * variables), or you can pass it a fully qualified URL to use (that may or may not exist
+   * within the OpenAPI definition).
+   *
+   * @example <caption>Server URL with server variables</caption>
+   * sdk.server('https://{region}.api.example.com/{basePath}', {
+   *   name: 'eu',
+   *   basePath: 'v14',
+   * });
+   *
+   * @example <caption>Fully qualified server URL</caption>
+   * sdk.server('https://eu.api.example.com/v14');
+   *
+   * @param url Server URL
+   * @param variables An object of variables to replace into the server URL.
+   */
+  server(url: string, variables = {}) {
+    this.core.setServer(url, variables);
+  }
+
+  /**
+   * Search and filter all news articles available via the Perigon API. The result includes a
+   * list of individual articles that were matched to your specific criteria.
+   *
+   * @summary All Articles
+   * @throws FetchError<400, types.AllNewsResponse400> 400
+   * @throws FetchError<401, types.AllNewsResponse401> 401
+   * @throws FetchError<403, types.AllNewsResponse403> 403
+   * @throws FetchError<404, types.AllNewsResponse404> 404
+   * @throws FetchError<500, types.AllNewsResponse500> 500
+   */
+  allNews(metadata: types.AllNewsMetadataParam): Promise<FetchResponse<200, types.AllNewsResponse200>> {
+    return this.core.fetch('/v1/all', 'get', metadata);
+  }
+
+  /**
+   * Stories
+   *
+   * @throws FetchError<400, types.Stories1Response400> 400
+   */
+  stories1(metadata: types.Stories1MetadataParam): Promise<FetchResponse<200, types.Stories1Response200>> {
+    return this.core.fetch('/v1/stories/all', 'get', metadata);
+  }
+}
+
+const createSDK = (() => { return new SDK(); })()
+;
+
+export default createSDK;
+
+export type { AllNewsMetadataParam, AllNewsResponse200, AllNewsResponse400, AllNewsResponse401, AllNewsResponse403, AllNewsResponse404, AllNewsResponse500, Stories1MetadataParam, Stories1Response200, Stories1Response400 } from './types';

packages/perigon/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@web-agent-rs/perigon",
+  "version": "0.0.0",
+  "main": "index.ts",
+  "types": "./index.d.ts",
+  "type": "module",
+  "dependencies": {
+    "api": "^6.1.3",
+    "json-schema-to-ts": "^2.8.0-beta.0",
+    "oas": "^20.11.0"
+  }
+}

packages/perigon/types.ts

@@ -0,0 +1,13 @@
+import type { FromSchema } from 'json-schema-to-ts';
+import * as schemas from './schemas';
+
+export type AllNewsMetadataParam = FromSchema<typeof schemas.AllNews.metadata>;
+export type AllNewsResponse200 = FromSchema<typeof schemas.AllNews.response['200']>;
+export type AllNewsResponse400 = FromSchema<typeof schemas.AllNews.response['400']>;
+export type AllNewsResponse401 = FromSchema<typeof schemas.AllNews.response['401']>;
+export type AllNewsResponse403 = FromSchema<typeof schemas.AllNews.response['403']>;
+export type AllNewsResponse404 = FromSchema<typeof schemas.AllNews.response['404']>;
+export type AllNewsResponse500 = FromSchema<typeof schemas.AllNews.response['500']>;
+export type Stories1MetadataParam = FromSchema<typeof schemas.Stories1.metadata>;
+export type Stories1Response200 = FromSchema<typeof schemas.Stories1.response['200']>;
+export type Stories1Response400 = FromSchema<typeof schemas.Stories1.response['400']>;

searxng_tester.mts

@@ -0,0 +1,29 @@
+import { SearxngService, type SearxngServiceConfig, type SearxngSearchResult } from 'searxng';
+
+const config: SearxngServiceConfig = {
+    baseURL: 'https://search-engine-gsio.fly.dev',
+    defaultSearchParams: {
+        format: 'json',
+        lang: 'auto',
+    },
+    defaultRequestHeaders: {
+        'Content-Type': 'application/json',
+    },
+};
+
+const searxngService = new SearxngService(config);
+
+async function performSearch(query) {
+    try {
+        const results = await searxngService.search(query);
+        console.log(results);
+        return results;
+    } catch (error) {
+        console.error('Search failed:', error);
+    }
+}
+
+
+const results = await performSearch('dogs');
+
+console.log(JSON.stringify(results));

src/agents/crypto_market.rs

@@ -0,0 +1,28 @@
+use tokio::process::Child;
+use tracing;
+
+use crate::utils::utils::run_agent;
+
+pub async fn finance_query_agent(stream_id: &str, input: &str) -> Result<Child, String> {
+    run_agent(stream_id, input, "./packages/genaiscript/genaisrc/finance-query.genai.mts").await
+}
+
+
+// #[cfg(test)]
+// mod tests {
+//     use std::fmt::Debug;
+//     use crate::agents::search::search_agent;
+//
+//     #[tokio::test] // Mark the test function as async
+//     async fn test_search_execution() {
+//         let input = "Who won the 2024 presidential election?";
+//
+//         let mut command = search_agent("test-stream", input).await.unwrap();
+//
+//         // command.stdout.take().unwrap().read_to_string(&mut String::new()).await.unwrap();
+//         // Optionally, you can capture and inspect stdout if needed:
+//         let output = command.wait_with_output().await.expect("Failed to wait for output");
+//         println!("Stdout: {}", String::from_utf8_lossy(&output.stdout));
+//         println!("Stderr: {}", String::from_utf8_lossy(&output.stderr));
+//     }
+// }

src/agents/image_generator.rs

@@ -0,0 +1,10 @@
+use crate::utils::utils::run_agent;
+use tokio::process::Child;
+
+pub async fn image_generator(stream_id: &str, input: &str) -> Result<Child, String> {
+    tracing::debug!(
+                "Running image generator, \ninput: {}",
+                input
+            );
+    run_agent(stream_id, input, "./packages/genaiscript/genaisrc/image-generator.genai.mts").await
+}

src/agents/mod.rs

@@ -0,0 +1,5 @@
+pub mod news;
+pub mod scrape;
+pub mod search;
+pub mod image_generator;
+pub mod crypto_market;

src/agents/news.rs

@@ -0,0 +1,6 @@
+use crate::utils::utils::run_agent;
+use tokio::process::Child;
+
+pub async fn news_agent(stream_id: &str, input: &str) -> Result<Child, String> {
+    run_agent(stream_id, input, "./packages/genaiscript/genaisrc/news-search.genai.mts").await
+}

src/agents/scrape.rs

@@ -0,0 +1,6 @@
+use crate::utils::utils::run_agent;
+use tokio::process::Child;
+
+pub async fn scrape_agent(stream_id: &str, input: &str) -> Result<Child, String> {
+    run_agent(stream_id, input, "./packages/genaiscript/genaisrc/web-scrape.genai.mts").await
+}

src/agents/search.rs

@@ -0,0 +1,28 @@
+use tokio::process::Child;
+use tracing;
+
+use crate::utils::utils::run_agent;
+
+pub async fn search_agent(stream_id: &str, input: &str) -> Result<Child, String> {
+    run_agent(stream_id, input, "./packages/genaiscript/genaisrc/web-search.genai.mts").await
+}
+
+
+#[cfg(test)]
+mod tests {
+    use std::fmt::Debug;
+    use crate::agents::search::search_agent;
+
+    #[tokio::test] 
+    async fn test_search_execution() {
+        let input = "Who won the 2024 presidential election?";
+
+        let mut command = search_agent("test-stream", input).await.unwrap();
+
+        // command.stdout.take().unwrap().read_to_string(&mut String::new()).await.unwrap();
+        // Optionally, you can capture and inspect stdout if needed:
+        let output = command.wait_with_output().await.expect("Failed to wait for output");
+        println!("Stdout: {}", String::from_utf8_lossy(&output.stdout));
+        println!("Stderr: {}", String::from_utf8_lossy(&output.stderr));
+    }
+}

src/config.rs

@@ -0,0 +1,30 @@
+// src/config.rs
+pub struct AppConfig {
+    pub env_vars: Vec<String>,
+}
+
+
+impl AppConfig {
+    pub fn new() -> Self {
+        // Load .env file if it exists
+        match dotenv::dotenv() {
+            Ok(_) => tracing::debug!("Loaded .env file successfully"),
+            Err(e) => tracing::debug!("No .env file found or error loading it: {}", e),
+        }
+
+        Self {
+            env_vars: vec![
+                "OPENAI_API_KEY".to_string(),
+                "BING_SEARCH_API_KEY".to_string(),
+                "TAVILY_API_KEY".to_string(),
+                "GENAISCRIPT_MODEL_LARGE".to_string(),
+                "GENAISCRIPT_MODEL_SMALL".to_string(),
+                "SEARXNG_API_BASE_URL".to_string(),
+            ],
+        }
+    }
+
+    pub fn get_env_var(&self, key: &str) -> String {
+        std::env::var(key).unwrap_or_default()
+    }
+}

src/genaiscript.rs

@@ -0,0 +1,90 @@
+use std::collections::HashMap;
+use std::path::PathBuf;
+use tokio::process::{Child, Command};
+use tracing;
+
+const DEFAULT_ENV_VARS: [&str; 4] = [
+    "OPENAI_API_KEY",
+    "OPENAI_API_BASE",
+    "GENAISCRIPT_MODEL_LARGE",
+    "GENAISCRIPT_MODEL_SMALL",
+];
+
+pub struct GenAIScriptConfig {
+    script_path: PathBuf,
+    output_dir: PathBuf,
+    stream_id: String,
+    user_input: String,
+    retry_count: u32,
+    env_vars: HashMap<String, String>,
+}
+
+impl GenAIScriptConfig {
+    pub fn new(script_path: impl Into<PathBuf>, stream_id: impl Into<String>, user_input: impl Into<String>) -> Self {
+        let mut env_vars = HashMap::new();
+
+        // Initialize with default environment variables
+        for var in DEFAULT_ENV_VARS {
+            if let Ok(value) = std::env::var(var) {
+                env_vars.insert(var.to_string(), value);
+            }
+        }
+
+        Self {
+            script_path: script_path.into(),
+            output_dir: PathBuf::from("./web-agent-rs/output"),
+            stream_id: stream_id.into(),
+            user_input: user_input.into(),
+            retry_count: 0,
+            env_vars,
+        }
+    }
+
+    pub fn with_output_dir(mut self, dir: impl Into<PathBuf>) -> Self {
+        self.output_dir = dir.into();
+        self
+    }
+
+    pub fn with_retry_count(mut self, count: u32) -> Self {
+        self.retry_count = count;
+        self
+    }
+
+    pub fn with_additional_env_vars(mut self, vars: HashMap<String, String>) -> Self {
+        self.env_vars.extend(vars);
+        self
+    }
+}
+
+pub async fn run_genaiscript(config: GenAIScriptConfig) -> Result<Child, String> {
+    tracing::debug!("Initiating GenAIScript for stream {}", config.stream_id);
+
+    let output_path = config.output_dir.join(&config.stream_id);
+
+    let mut command = Command::new("bunx");
+    command
+        .arg("genaiscript")
+        .arg("run")
+        .arg(&config.script_path)
+        // .arg("--fail-on-errors")
+        .arg("—out-trace")
+        .arg(output_path)
+        .arg("--retry")
+        .arg(config.retry_count.to_string())
+        .arg("--vars")
+        .arg(format!("USER_INPUT='{}'", config.user_input));
+
+    // Add environment variables
+    for (key, value) in config.env_vars {
+        command.env(key, value);
+    }
+
+    command
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::inherit())
+        .spawn()
+        .map_err(|e| {
+            tracing::error!("Failed to spawn genaiscript process: {}", e);
+            e.to_string()
+        })
+}

src/handlers/error.rs

@@ -0,0 +1,17 @@
+// src/handlers/error.rs
+use axum::{
+    http::StatusCode,
+    Json,
+    response::IntoResponse,
+};
+
+pub async fn handle_not_found() -> impl IntoResponse {
+    tracing::warn!("404 Not Found error occurred");
+
+    let error_response = serde_json::json!({
+        "error": "Route Not Found",
+        "status": 404
+    });
+
+    (StatusCode::NOT_FOUND, Json(error_response))
+}

src/handlers/mod.rs

@@ -0,0 +1,7 @@
+
+// src/handlers/mod.rs
+pub mod error;
+pub mod status;
+pub mod stream;
+pub mod ui;
+pub mod webhooks;

src/handlers/status.rs

@@ -0,0 +1,5 @@
+// src/handlers/status.rs
+pub async fn handle_status() -> &'static str {
+    tracing::debug!("Status check requested");
+    "Server is running"
+}

src/handlers/stream.rs

@@ -0,0 +1,82 @@
+use axum::{
+    body::Body,
+    http::StatusCode,
+    response::{IntoResponse, Response},
+};
+use futures::StreamExt;
+use tokio_util::io::ReaderStream;
+
+pub async fn handle_stream() -> impl IntoResponse {
+    use tokio::process::Command;
+
+    let user_input = "Who won the 2024 election?";
+    tracing::debug!("Handling stream request with input: {}", user_input);
+
+    // Check environment variables
+    for env_var in ["OPENAI_API_KEY", "BING_SEARCH_API_KEY", "TAVILY_API_KEY"] {
+        if std::env::var(env_var).is_ok() {
+            tracing::debug!("{} is set", env_var);
+        } else {
+            tracing::warn!("{} is not set", env_var);
+        }
+    }
+
+    let mut cmd = match Command::new("genaiscript")
+        .arg("run")
+        .arg("genaisrc/web-search.genai.mts")
+        .arg("--vars")
+        .arg(format!("USER_INPUT='{}'", user_input))
+        .env("OPENAI_API_KEY", std::env::var("OPENAI_API_KEY").unwrap_or_default())
+        .env("BING_SEARCH_API_KEY", std::env::var("BING_SEARCH_API_KEY").unwrap_or_default())
+        .env("TAVILY_API_KEY", std::env::var("TAVILY_API_KEY").unwrap_or_default())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::null())
+        .spawn() {
+        Ok(cmd) => {
+            tracing::debug!("Successfully spawned genaiscript process");
+            cmd
+        }
+        Err(e) => {
+            tracing::error!("Failed to spawn genaiscript process: {}", e);
+            return Response::builder()
+                .status(StatusCode::INTERNAL_SERVER_ERROR)
+                .body(Body::from("Failed to start process"))
+                .unwrap();
+        }
+    };
+
+    let stdout = match cmd.stdout.take() {
+        Some(stdout) => {
+            tracing::debug!("Successfully captured stdout from process");
+            stdout
+        }
+        None => {
+            tracing::error!("Failed to capture stdout from process");
+            return Response::builder()
+                .status(StatusCode::INTERNAL_SERVER_ERROR)
+                .body(Body::from("Failed to capture process output"))
+                .unwrap();
+        }
+    };
+
+    let reader = tokio::io::BufReader::new(stdout);
+    let stream = ReaderStream::new(reader);
+    let mapped_stream = stream.map(|r| {
+        match r {
+            Ok(bytes) => {
+                tracing::trace!("Received {} bytes from stream", bytes.len());
+                Ok(bytes)
+            }
+            Err(e) => {
+                tracing::error!("Error reading from stream: {}", e);
+                Err(e)
+            }
+        }
+    });
+
+    tracing::debug!("Setting up SSE response");
+    Response::builder()
+        .header("Content-Type", "text/event-stream")
+        .body(Body::from_stream(mapped_stream))
+        .unwrap()
+}

src/handlers/ui.rs

@@ -0,0 +1,34 @@
+use axum::{
+    body::Body,
+    http::{StatusCode, header::CONTENT_TYPE},
+    response::{IntoResponse, Response},
+};
+use rust_embed::RustEmbed;
+use tracing::{debug, error};
+
+#[derive(RustEmbed)]
+#[folder = "assets/"]
+struct Asset;
+
+pub async fn serve_ui() -> impl IntoResponse {
+    debug!("Serving UI request");
+
+    // Attempt to retrieve the embedded "index.html"
+    match Asset::get("index.html") {
+        Some(content) => {
+            debug!("Successfully retrieved index.html");
+            Response::builder()
+                .status(StatusCode::OK)
+                .header(CONTENT_TYPE, "text/html")
+                .body(Body::from(content.data))
+                .unwrap()
+        }
+        None => {
+            error!("index.html not found in embedded assets");
+            Response::builder()
+                .status(StatusCode::NOT_FOUND)
+                .body(Body::from("404 Not Found"))
+                .unwrap()
+        }
+    }
+}

src/handlers/webhooks.rs

@@ -0,0 +1,261 @@
+use crate::agents;
+use crate::agents::news::news_agent;
+use crate::agents::scrape::scrape_agent;
+use crate::agents::search::search_agent;
+use axum::response::Response;
+use axum::{
+    body::Body, extract::Path, extract::Query, http::StatusCode, response::IntoResponse, Json,
+};
+use bytes::Bytes;
+use futures::stream::{Stream, StreamExt};
+use lazy_static::lazy_static;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use sled;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::io::{AsyncBufReadExt, BufReader};
+use tokio::process::Command;
+use tokio::sync::Mutex;
+use crate::agents::crypto_market::finance_query_agent;
+use crate::agents::image_generator::image_generator;
+
+// init sled
+lazy_static! {
+    static ref DB: Arc<Mutex<sled::Db>> = Arc::new(Mutex::new(
+        sled::open("./web-agent-rs/db/stream_store").expect("Failed to open sled database")
+    ));
+}
+
+pub async fn handle_webhooks(Path(stream_id): Path<String>) -> impl IntoResponse {
+    let db = DB.lock().await;
+    match db.get(&stream_id) {
+        Ok(Some(data)) => {
+
+            let mut info: StreamInfo = match serde_json::from_slice(&data) {
+                Ok(info) => info,
+                Err(e) => {
+                    tracing::error!("Failed to deserialize StreamInfo: {}", e);
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+
+            // Increment the call_count in the database
+            info.call_count += 1;
+            let updated_info_bytes = match serde_json::to_vec(&info) {
+                Ok(data) => data,
+                Err(e) => {
+                    tracing::error!("Failed to serialize updated StreamInfo: {}", e);
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+            match db.insert(&stream_id, updated_info_bytes) {
+                Ok(_) => {
+                    if let Err(e) = db.flush_async().await {
+                        tracing::error!("Failed to persist updated call_count to the database: {}", e);
+                        return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                    }
+                }
+                Err(e) => {
+                    tracing::error!("Failed to update call_count in the database: {}", e);
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+            let info: StreamInfo = match db.get(&stream_id) {
+                Ok(Some(updated_data)) => match serde_json::from_slice(&updated_data) {
+                    Ok(info) => info,
+                    Err(e) => {
+                        tracing::error!("Failed to deserialize updated StreamInfo: {}", e);
+                        return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                    }
+                },
+                Ok(None) => {
+                    tracing::error!("Stream ID not found after update: {}", stream_id);
+                    return StatusCode::NOT_FOUND.into_response();
+                }
+                Err(e) => {
+                    tracing::error!("Failed to fetch updated record from DB: {}", e);
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+            if(info.call_count > 1) {
+                return StatusCode::OK.into_response();
+            }
+
+            let resource = info.resource;
+            let input = serde_json::to_string(&info.payload.input).unwrap_or_default();
+
+            tracing::debug!(
+                "Processing webhook - Resource: {}, Stream ID: {}",
+                resource,
+                stream_id
+            );
+            
+            let cmd = match resource.as_str() {
+                "web-search" => search_agent(stream_id.as_str(), &*input).await,
+                "news-search" => news_agent(stream_id.as_str(), &*input).await,
+                "image-generator" => image_generator(stream_id.as_str(), &*input).await,
+                "finance-query" => finance_query_agent(stream_id.as_str(), &*input).await,
+                "web-scrape" => scrape_agent(stream_id.as_str(), &*input).await,
+                _ => {
+                    tracing::error!("Unsupported resource type: {}", resource);
+                    return StatusCode::BAD_REQUEST.into_response();
+                }
+            };
+
+            let mut cmd = match cmd {
+                Ok(cmd) => cmd,
+                Err(e) => {
+                    tracing::error!("Agent execution failed: {}", e);
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+            let stdout = match cmd.stdout.take() {
+                Some(stdout) => stdout,
+                None => {
+                    tracing::error!("No stdout available for the command.");
+                    return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+                }
+            };
+
+            let reader = BufReader::new(stdout);
+            let sse_stream = reader_to_stream(reader, stream_id.clone());
+
+            return Response::builder()
+                .header("Content-Type", "text/event-stream")
+                .header("Cache-Control", "no-cache, no-transform")
+                .header("Connection", "keep-alive")
+                .header("X-Accel-Buffering", "yes")
+                .body(Body::from_stream(sse_stream))
+                .unwrap()
+        }
+        Ok(None) => {
+            tracing::error!("Stream ID not found: {}", stream_id);
+            StatusCode::NOT_FOUND.into_response()
+        }
+        Err(e) => {
+            tracing::error!("Failed to fetch from DB: {}", e);
+            StatusCode::INTERNAL_SERVER_ERROR.into_response()
+        }
+    }
+}
+
+fn reader_to_stream<R>(
+    reader: BufReader<R>,
+    stream_id: String,
+) -> Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send>>
+where
+    R: tokio::io::AsyncRead + Unpin + Send + 'static,
+{
+    let stream = futures::stream::unfold(reader, move |mut reader| async move {
+        let mut line = String::new();
+        match reader.read_line(&mut line).await {
+            Ok(0) => None,
+            Ok(_) => Some((
+                Ok(Bytes::from(format!("data: {}\n\n", line.trim()))),
+                reader,
+            )),
+            Err(e) => Some((Err(e), reader)),
+        }
+    });
+
+    let stream_with_done = stream.chain(futures::stream::once(async {
+        Ok(Bytes::from("data: [DONE]\n\n"))
+    }));
+
+    Box::pin(stream_with_done)
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+struct Payload {
+    input: Value,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct StreamInfo {
+    resource: String,
+    payload: Payload,
+    parent: String,
+    call_count: i32,
+}
+
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct WebhookPostRequest {
+    id: String,
+    resource: String,
+    payload: Payload,
+    parent: String,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+struct WebhookPostResponse {
+    stream_url: String,
+}
+
+pub async fn handle_webhooks_post(Json(payload): Json<WebhookPostRequest>) -> impl IntoResponse {
+    let db = DB.lock().await;
+
+    tracing::info!("Received webhook post request with ID: {}", payload.id);
+
+    let stream_id = payload.id.clone();
+    let info = StreamInfo {
+        resource: payload.resource.clone(),
+        payload: payload.payload,
+        parent: payload.parent.clone(),
+        call_count: 0
+    };
+
+    let info_bytes = match serde_json::to_vec(&info) {
+        Ok(data) => data,
+        Err(e) => {
+            tracing::error!("Failed to serialize StreamInfo: {}", e);
+            return StatusCode::INTERNAL_SERVER_ERROR.into_response();
+        }
+    };
+
+    // Use atomic compare_and_swap operation
+    match db.compare_and_swap(
+        &stream_id,
+        None as Option<&[u8]>,
+        Some(info_bytes.as_slice()),
+    ) {
+        Ok(_) => {
+            // Force an immediate sync to disk
+            match db.flush_async().await {
+                Ok(_) => {
+                    // Verify the write by attempting to read it back
+                    match db.get(&stream_id) {
+                        Ok(Some(_)) => {
+                            let stream_url = format!("/webhooks/{}", stream_id);
+                            tracing::info!("Successfully created and verified stream URL: {}", stream_url);
+                            Json(WebhookPostResponse { stream_url }).into_response()
+                        },
+                        Ok(None) => {
+                            tracing::error!("Failed to verify stream creation: {}", stream_id);
+                            StatusCode::INTERNAL_SERVER_ERROR.into_response()
+                        },
+                        Err(e) => {
+                            tracing::error!("Error verifying stream creation: {}", e);
+                            StatusCode::INTERNAL_SERVER_ERROR.into_response()
+                        }
+                    }
+                },
+                Err(e) => {
+                    tracing::error!("Failed to flush DB: {}", e);
+                    StatusCode::INTERNAL_SERVER_ERROR.into_response()
+                }
+            }
+        }
+        Err(e) => {
+            tracing::error!("Failed to insert stream info: {}", e);
+            StatusCode::INTERNAL_SERVER_ERROR.into_response()
+        }
+    }
+}

src/main.rs

@@ -0,0 +1,43 @@
+// src/main.rs
+use crate::config::AppConfig;
+use crate::routes::create_router;
+use crate::setup::init_logging;
+
+mod config;
+mod routes;
+mod setup;
+mod handlers;
+mod agents;
+mod genaiscript;
+mod utils;
+mod session_identify;
+
+#[tokio::main]
+async fn main() {
+    // Initialize logging
+    init_logging();
+
+    // Load configuration
+    let config = AppConfig::new();
+
+    // Create router with all routes
+    let app = create_router();
+
+    // Start core
+    let addr = "0.0.0.0:3006";
+    tracing::info!("Attempting to bind core to {}", addr);
+
+    let listener = match tokio::net::TcpListener::bind(addr).await {
+        Ok(l) => {
+            tracing::info!("Successfully bound to {}", l.local_addr().unwrap());
+            l
+        }
+        Err(e) => {
+            tracing::error!("Failed to bind to {}: {}", addr, e);
+            panic!("Server failed to start");
+        }
+    };
+
+    tracing::info!("Server starting on {}", listener.local_addr().unwrap());
+    axum::serve(listener, app.into_make_service()).await.unwrap();
+}

src/routes.rs

@@ -0,0 +1,105 @@
+use crate::handlers::webhooks::handle_webhooks_post;
+use crate::handlers::{
+    error::handle_not_found,
+    ui::serve_ui
+    ,
+    webhooks::handle_webhooks,
+};
+use crate::session_identify::session_identify;
+use axum::extract::Request;
+use axum::response::Response;
+use axum::routing::post;
+// src/routes.rs
+use axum::routing::{get, Router};
+use http::header::AUTHORIZATION;
+use http::StatusCode;
+use serde::{Deserialize, Serialize};
+use serde_json::Number;
+use std::fmt;
+use tower_http::trace::{self, TraceLayer};
+use tracing::Level;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CurrentUser {
+    pub(crate) sub: String,
+    pub name: String,
+    pub email: String,
+    pub exp: Number,
+    pub id: String,
+    pub aud: String,
+}
+
+impl fmt::Display for CurrentUser {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "CurrentUser {{ id: {}, name: {}, email: {}, sub: {}, aud: {}, exp: {} }}",
+            self.id, self.name, self.email, self.sub, self.aud, self.exp
+        )
+    }
+}
+
+pub fn create_router() -> Router {
+
+    Router::new()
+        .route("/", get(serve_ui))
+        // request a stream resource
+        .route("/api/webhooks", post(handle_webhooks_post))
+        // consume a stream resource
+        .route("/webhooks/:stream_id", get(handle_webhooks))
+        .route_layer(axum::middleware::from_fn(auth))
+        .route("/health", get(health))
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(trace::DefaultMakeSpan::new().level(Level::INFO))
+                .on_response(trace::DefaultOnResponse::new().level(Level::INFO)),
+        )
+        // left for smoke testing
+        // .route("/api/status", get(handle_status))
+        .fallback(handle_not_found)
+}
+
+async fn health() -> String {
+    return "ok".to_string();
+}
+
+async fn auth(mut req: Request, next: axum::middleware::Next) -> Result<Response, StatusCode> {
+    let session_token_header = req
+        .headers()
+        .get(AUTHORIZATION)
+        .and_then(|header_value| header_value.to_str().ok());
+
+    let session_token_parts= session_token_header.expect("No credentials").split(" ").collect::<Vec<&str>>();
+
+    let session_token = session_token_parts.get(1);
+
+
+    // log::info!("session_token: {:?}", session_token);
+
+    let session_token = session_token.expect("Unauthorized: No credentials supplied");
+
+    let result =
+        if let Some(current_user) = authorize_current_user(&*session_token).await {
+            // info!("current user: {}", current_user);
+            // insert the current user into a request extension so the handler can
+            // extract it
+            req.extensions_mut().insert(current_user);
+            Ok(next.run(req).await)
+        } else {
+            Err(StatusCode::UNAUTHORIZED)
+        };
+    result
+}
+
+
+async fn authorize_current_user(
+    session_token: &str,
+) -> Option<CurrentUser> {
+    let session_identity = session_identify(session_token)
+        .await
+        .unwrap();
+
+    // println!("current_user: {:?}", session_identity.user);
+
+    Some(serde_json::from_value::<CurrentUser>(session_identity.user).unwrap())
+}

src/session_identify.rs

@@ -0,0 +1,55 @@
+use anyhow::Result;
+use serde_json::Value;
+use serde_json::json;
+use base64::Engine;
+use fips204::ml_dsa_44::{PrivateKey, PublicKey};
+use fips204::traits::{SerDes, Signer, Verifier};
+use crate::utils::base64::B64_ENCODER;
+
+pub struct SessionIdentity {
+    pub message: String,
+    pub signature: String,
+    pub target: String,
+    pub session_id: String,
+    pub user: Value
+}
+
+pub async fn session_identify(session_token: &str) -> Result<SessionIdentity> {
+    let session_data_base64 = session_token.split('.').nth(0).ok_or_else(|| anyhow::anyhow!("Invalid session data format"))?;
+    // println!("session_data_base64: {}", session_data_base64);
+    let session_data: Value = serde_json::de::from_slice(&*B64_ENCODER.b64_decode_payload(session_data_base64).map_err(|e| anyhow::anyhow!("Failed to decode session data: {}", e))?).map_err(|e| anyhow::anyhow!("Failed to parse session data: {}", e))?;
+    // println!("session_data: {:?}", session_data);
+
+
+    let signature_base64 = session_token.split('.').nth(1).ok_or_else(|| anyhow::anyhow!("Invalid session token format"))?;
+    // println!("signature_base64: {}", signature_base64);
+
+    let target = session_data.get("aud")
+        .and_then(|e| e.as_str())
+        .ok_or_else(|| anyhow::anyhow!("Session data missing audience"))?;
+
+    let target = target.parse::<String>().map_err(|e| anyhow::anyhow!("Failed to parse target to String: {}", e))?;
+
+    let session_id = session_data.get("id")
+        .and_then(|e| e.as_str())
+        .ok_or_else(|| anyhow::anyhow!("Session data missing id"))?;
+
+    let session_id = session_id.parse::<String>().map_err(|e| anyhow::anyhow!("Failed to parse session_id to String: {}", e))?;
+
+    // let request_payload: Value = json!({
+    //     "message": session_data_base64,
+    //     "signature": signature_base64,
+    //     "target": target,
+    //     "session_id": session_id,
+    // });
+
+    let result = SessionIdentity {
+        message: session_data_base64.to_string(),
+        signature: signature_base64.to_string(),
+        target,
+        session_id,
+        user: session_data.clone()
+    };
+
+    Ok(result)
+}

src/setup.rs

@@ -0,0 +1,10 @@
+// src/setup.rs
+pub fn init_logging() {
+    tracing_subscriber::fmt()
+        .with_max_level(tracing::Level::DEBUG)
+        .with_target(true)
+        .with_thread_ids(true)
+        .with_file(true)
+        .with_line_number(true)
+        .init();
+}

src/utils/base64.rs

@@ -0,0 +1,65 @@
+use base64::Engine;
+use base64::engine::GeneralPurpose;
+use base64::engine::general_purpose::STANDARD;
+use base64::engine::general_purpose::STANDARD_NO_PAD;
+
+pub struct Base64Encoder {
+    payload_engine: GeneralPurpose,
+    signature_engine: GeneralPurpose,
+    public_key_engine: GeneralPurpose,
+    secret_key_engine: GeneralPurpose,
+}
+
+impl Base64Encoder {
+    pub(crate) fn b64_encode(&self, p0: &[u8]) -> String {
+        self.payload_engine.encode(p0)
+    }
+    pub(crate) fn b64_decode(&self, p0: String) -> Result<Vec<u8>, base64::DecodeError> {
+        self.payload_engine.decode(p0)
+    }
+}
+
+pub const B64_ENCODER: &Base64Encoder = &Base64Encoder::new();
+
+impl Base64Encoder {
+    pub const fn new() -> Self { // Made new() a const fn
+        Base64Encoder {
+            payload_engine: STANDARD,
+            signature_engine: STANDARD,
+            public_key_engine: STANDARD,
+            secret_key_engine: STANDARD,
+        }
+    }
+
+    pub fn b64_encode_payload<T: AsRef<[u8]>>(&self, input: T) -> String { // Added trait bound
+        self.payload_engine.encode(input)
+    }
+
+    pub fn b64_decode_payload<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, base64::DecodeError> { // Added trait bound
+        self.payload_engine.decode(input)
+    }
+
+    pub fn b64_decode_signature<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, base64::DecodeError> { // Added trait bound
+        self.signature_engine.decode(input)
+    }
+
+    pub fn b64_encode_signature<T: AsRef<[u8]>>(&self, input: T) -> String { // Added trait bound
+        self.signature_engine.encode(input)
+    }
+
+    pub fn b64_encode_public_key<T: AsRef<[u8]>>(&self, input: T) -> String { // Added trait bound
+        self.public_key_engine.encode(input)
+    }
+
+    pub fn b64_decode_public_key<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, base64::DecodeError> { // Added trait bound
+        self.public_key_engine.decode(input)
+    }
+
+    pub fn b64_encode_secret_key<T: AsRef<[u8]>>(&self, input: T) -> String { // Added trait bound
+        self.secret_key_engine.encode(input)
+    }
+
+    pub fn b64_decode_secret_key<T: AsRef<[u8]>>(&self, input: T) -> Result<Vec<u8>, base64::DecodeError> { // Added trait bound
+        self.secret_key_engine.decode(input)
+    }
+}

src/utils/mod.rs

@@ -0,0 +1,2 @@
+pub mod utils;
+pub mod base64;

src/utils/utils.rs

@@ -0,0 +1,80 @@
+// utils.rs
+use tokio::process::{Child, Command}; // Use tokio::process::Child and Command
+use std::env;
+use tokio::time::{timeout, Duration};
+use tracing;
+
+
+pub struct ShimBinding {
+    user_input: String,
+    file_path: String, // Add new field for the file path
+    openai_api_key: String,
+    openai_api_base: String,
+    bing_search_api_key: String,
+    perigon_api_key: String,
+    tavily_api_key: String,
+    genaiscript_model_large: String,
+    genaiscript_model_small: String,
+    searxng_api_base_url: String,
+    searxng_password: String,
+}
+
+impl ShimBinding {
+    pub fn new(user_input: String, file_path: String) -> Self { // Update constructor to take file path
+        Self {
+            user_input,
+            file_path, // Initialize the new field
+            openai_api_key: env::var("OPENAI_API_KEY").unwrap_or_default(),
+            openai_api_base: env::var("OPENAI_API_BASE").unwrap_or_default(),
+            bing_search_api_key: env::var("BING_SEARCH_API_KEY").unwrap_or_default(),
+            tavily_api_key: env::var("TAVILY_API_KEY").unwrap_or_default(),
+            genaiscript_model_large: env::var("GENAISCRIPT_MODEL_LARGE").unwrap_or_default(),
+            genaiscript_model_small: env::var("GENAISCRIPT_MODEL_SMALL").unwrap_or_default(),
+            perigon_api_key: env::var("PERIGON_API_KEY").unwrap_or_default(),
+            searxng_api_base_url: env::var("SEARXNG_API_BASE_URL").unwrap_or_default(),
+            searxng_password: env::var("SEARXNG_PASSWORD").unwrap_or_default(),
+        }
+    }
+
+    pub fn execute(&self) -> std::io::Result<Child> {
+        let mut command = Command::new("./dist/genaiscript-rust-shim.js");
+        command
+            .arg("--file")
+            .arg(&self.file_path) // Use the file_path field instead of hardcoded value
+            .arg(format!("USER_INPUT={}", self.user_input))
+            .env("OPENAI_API_KEY", &self.openai_api_key)
+            .env("OPENAI_API_BASE", &self.openai_api_base)
+            .env("BING_SEARCH_API_KEY", &self.bing_search_api_key)
+            .env("TAVILY_API_KEY", &self.tavily_api_key)
+            .env("GENAISCRIPT_MODEL_LARGE", &self.genaiscript_model_large)
+            .env("GENAISCRIPT_MODEL_SMALL", &self.genaiscript_model_small)
+            .env("PERIGON_API_KEY", &self.perigon_api_key)
+            .env("SEARXNG_API_BASE_URL", &self.searxng_api_base_url)
+            .env("SEARXNG_PASSWORD", &self.searxng_password)
+            .stdout(std::process::Stdio::piped()) // Use tokio::io::Stdio::piped()
+            .stderr(std::process::Stdio::inherit()); // Use tokio::io::Stdio::piped()
+
+        command.spawn()
+    }
+}
+
+
+ /// Generic helper to execute a ShimBinding-based agent with a timeout
+pub async fn run_agent(stream_id: &str, input: &str, file_path: &str) -> Result<Child, String> {
+    tracing::debug!("Initiating agent for stream {} with file path {}", stream_id, file_path);
+
+    let shim_binding = ShimBinding::new(input.to_string(), file_path.to_string());
+    let spawn_future = async move {
+        match shim_binding.execute() {
+            Ok(child) => Ok(child),
+            Err(e) => {
+                tracing::error!("Failed to spawn shim process: {}", e);
+                Err(e.to_string())
+            }
+        }
+    };
+
+    timeout(Duration::from_secs(10), spawn_future)
+        .await
+        .unwrap_or_else(|_| Err("Command timed out after 10 seconds".to_string()))
+}

tsconfig.json

@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+//    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  },
+  "include": [
+    "./packages/genaiscript/genaisrc/genaiscript.d.ts"
+  ]
+}