--amend

.github/workflows/main.yml

@@ -10,13 +10,88 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  build:
+  # Job for building on macOS (with Metal)
-
+  build-linux:
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: Jimver/cuda-toolkit@v0.2.24
+        id: cuda-toolkit
+        with:
+          cuda: '12.5.0'
+      - name: Setup Python virtual environment
+        shell: bash
+        run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
+          python3 -m pip install --upgrade pip
+          python3 -m pip install huggingface-hub
+      - name: Login to HuggingFace Hub
+        env:
+          HF_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+        run: |
+          source .venv/bin/activate
+          if [ -n "$HF_TOKEN" ]; then
+            echo "Logging into Hugging Face Hub..."
+            huggingface-cli login --token $HF_TOKEN
+          else
+            echo "No HuggingFace token found, skipping login"
+          fi
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install
+      # If you intend to use CUDA on Linux (highly recommended for performance with Candle)
+      # You'll need a runner with GPU support or use an action that sets up CUDA.
+      # GitHub's default ubuntu-latest runners are CPU-only.
+      # For GPU runners, you'd typically use self-hosted runners or a service like vast.ai.
+      # For CI, you might build a CPU-only version for faster checks, or use a GPU-enabled runner for full testing.
+
+      # Option 1: Build with CPU backend (no CUDA, no Metal)
+      - name: Build (Linux - CPU only)
+        run: cargo build --release --no-default-features
+
+      # Option 2: Build with CUDA backend (requires CUDA setup and often a GPU runner)
+      # - name: Setup CUDA (example - specific action might vary)
+      #   uses: jimmy-shian/setup-cuda@v1 # This is just an example, research appropriate action
+      #   with:
+      #     cuda-version: '12.0' # Or your desired CUDA version
+      # - name: Build (Linux - CUDA)
+      #   # Explicitly enable cudarc and candle-core's cuda feature, and potentially intel-mkl-src
+      #   run: cargo build --release --features "cudarc candle-core/cuda intel-mkl-src"
+
+      - name: Run tests (Linux - CPU only)
+        # If your tests are CPU-only, they'll work with the CPU build.
+        # If they require CUDA, they'd need the CUDA build and runner.
+        run: cargo test --no-default-features
+
+  build-macos:
+    runs-on: macos-latest
+
+
 
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
-    - name: Build
+      - name: Setup Python virtual environment
-      run: cargo build
+        shell: bash
-    - name: Run tests
+        run: |
-      run: cargo test
+          python3 -m venv .venv
+          source .venv/bin/activate
+          python3 -m pip install --upgrade pip
+          python3 -m pip install huggingface-hub
+      - name: Login to HuggingFace Hub
+        env:
+          HF_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+        run: |
+          source .venv/bin/activate
+          if [ -n "$HF_TOKEN" ]; then
+            echo "Logging into Hugging Face Hub..."
+            huggingface-cli login --token $HF_TOKEN
+          else
+            echo "No HuggingFace token found, skipping login"
+          fi
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install
+      - name: Build (macOS - Metal)
+        # The `metal` feature will be automatically picked up due to target_os = "macos" in Cargo.toml
+        run: cargo build --release
+      - name: Run tests (macOS)
+        run: cargo test

Cargo.lock

@@ -782,9 +782,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1980d53280c8f9e2c6cbe1785855d7ff8010208b46e21252b978badf13ad69d"
 dependencies = [
  "candle-core",
- "candle-metal-kernels",
  "half",
- "metal 0.27.0",
  "num-traits",
  "rayon",
  "safetensors",
@@ -1381,7 +1379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -2639,7 +2637,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.6",
+ "windows-targets 0.53.0",
 ]
 
 [[package]]
@@ -4313,7 +4311,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5106,7 +5104,7 @@ dependencies = [
  "getrandom 0.3.3",
  "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5925,7 +5923,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]

Cargo.toml

@@ -4,4 +4,4 @@ members = [
     "crates/inference-engine",
 ]
 
-resolver = "2"
+resolver = "2"

crates/inference-engine/Cargo.toml

@@ -6,8 +6,8 @@ edition = "2021"
 [dependencies]
 accelerate-src = { version = "0.3.2", optional = true }
 candle-datasets = { version = "=0.9.1", optional = true }
-candle-nn = { version = "=0.9.1", features = ["metal"] }
+candle-nn = { version = "=0.9.1" }
-candle-transformers = { version = "=0.9.1", features = ["metal"] }
+candle-transformers = { version = "=0.9.1" }
 candle-flash-attn = { version = "=0.9.1", optional = true }
 candle-onnx = { version = "=0.9.1", optional = true }
 
@@ -21,16 +21,15 @@ num-traits = { version = "0.2.15" }
 palette = { version = "0.7.6", optional = true }
 enterpolation = { version = "0.2.1", optional = true}
 pyo3 = { version = "0.22.0", features = ["auto-initialize", "abi3-py311"], optional = true }
-rayon = { version = "1.7.0" }
+rayon = "1.7.0"
 rubato = { version = "0.15.0", optional = true }
-safetensors = { version = "0.4.1" }
+safetensors = "0.4.1"
 serde = { version = "1.0.171", features = ["derive"] }
-serde_json = { version = "1.0.99" }
+serde_json = "1.0.99"
 symphonia = { version = "0.5.3", features = ["all"], optional = true }
 tokenizers = { version = "0.21.0", default-features = false, features = ["onig", "http"] }
 cpal = { version = "0.15.2", optional = true }
 pdf2image = { version = "0.1.2" , optional = true}
-candle-core = { version = "=0.9.1", features = ["metal"] }
 anyhow = "1.0.98"
 clap= { version = "4.2.4", features = ["derive"] }
 tracing = "0.1.37"
@@ -45,6 +44,18 @@ utoipa = { version = "4.2.0", features = ["axum_extras"] }
 uuid = { version = "1.7.0", features = ["v4"] }
 reborrow = "0.5.5"
 
+# --- Add this section for conditional compilation ---
+[target.'cfg(target_os = "macos")'.dependencies]
+candle-core = { version = "=0.9.1", features = ["metal"] }
+
+[target.'cfg(not(target_os = "macos"))'.dependencies]
+# For Linux or other non-macOS systems, you likely want the CPU backend or CUDA
+# If you're building on Linux with a CUDA-enabled GPU:
+candle-core = { version = "=0.9.1", features = ["cuda"], default-features = false } # Or just "cuda" if not using default features
+
+# If you're building on Linux with only CPU:
+# candle-core = { version = "=0.9.1", default-features = false } # CPU is often the default, but good to be explicit
+# --- End of conditional compilation section ---
 
 [dev-dependencies]
 anyhow = { version = "1", features = ["backtrace"] }
@@ -62,4 +73,4 @@ tokio = "1.43.0"
 
 [build-dependencies]
 anyhow = { version = "1", features = ["backtrace"] }
-bindgen_cuda = { version = "0.1.1", optional = true }
+bindgen_cuda = { version = "0.1.1", optional = true }