Import ducklm runtime

app/models/__init__.py

@@ -0,0 +1,32 @@
+LLM_AVAILABLE = False
+EMBEDDINGS_AVAILABLE = False
+
+try:
+    from app.models.adapters import create_adapter, create_llama_adapter
+    from app.models.orchestrator import OrchestratorAdapter
+    from app.models.coder import CoderAdapter
+    from app.models.critic import CriticAdapter
+    LLM_AVAILABLE = True
+except ImportError:
+    create_adapter = None
+    create_llama_adapter = None
+    OrchestratorAdapter = None
+    CoderAdapter = None
+    CriticAdapter = None
+
+try:
+    from app.models.embeddings import EmbeddingsAdapter
+    EMBEDDINGS_AVAILABLE = True
+except ImportError:
+    EmbeddingsAdapter = None
+
+__all__ = [
+    "create_adapter",
+    "create_llama_adapter",
+    "OrchestratorAdapter",
+    "CoderAdapter",
+    "CriticAdapter",
+    "EmbeddingsAdapter",
+    "LLM_AVAILABLE",
+    "EMBEDDINGS_AVAILABLE",
+]

app/models/adapters.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Protocol, Iterator
+import os
+
+try:
+    from llama_cpp import Llama
+    LLAMA_AVAILABLE = True
+except ImportError:
+    Llama = None
+    LLAMA_AVAILABLE = False
+
+
+class BaseModelAdapter(Protocol):
+    async def generate(self, prompt: str, **kwargs: Any) -> str: ...
+    def stream(self, prompt: str, **kwargs: Any) -> Iterator[str]: ...
+
+
+def create_llama_adapter(
+    model_path: str,
+    backend: str = "cpu",
+    n_gpu_layers: int = 0,
+    max_tokens: int = 2048,
+    temperature: float = 0.2,
+    base_dir: Path | None = None,
+) -> "Llama":
+    if not LLAMA_AVAILABLE:
+        raise RuntimeError("llama-cpp-python not installed")
+
+    if base_dir:
+        model_path = str(base_dir / model_path)
+    else:
+        model_path = str(Path.cwd() / model_path)
+
+    return Llama(
+        model_path=model_path,
+        n_gpu_layers=n_gpu_layers,
+        n_ctx=4096,
+        n_threads=int(os.environ.get("DUCKLM_N_THREADS", max(4, min((os.cpu_count() or 4) // 2, 20)))),
+        n_threads_batch=-1,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        verbose=False,
+    )
+
+
+def create_adapter(
+    model_type: str,
+    config: dict[str, Any],
+    base_dir: Path | None = None,
+) -> "Llama":
+    if not LLAMA_AVAILABLE:
+        raise RuntimeError("llama-cpp-python not installed")
+
+    model_path = config.get("path", "")
+    backend = config.get("backend", "cpu")
+    n_gpu_layers = config.get("n_gpu_layers", 0)
+    max_tokens = config.get("max_tokens", 2048)
+    temperature = config.get("temperature", 0.2)
+
+    if backend == "vulkan" and n_gpu_layers != 0:
+        n_gpu_layers = -1
+
+    return create_llama_adapter(
+        model_path=model_path,
+        backend=backend,
+        n_gpu_layers=n_gpu_layers,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        base_dir=base_dir,
+    )

app/models/async_adapters.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, AsyncIterator
+
+from app.models.orchestrator import OrchestratorAdapter as SyncOrchestrator
+
+
+class AsyncOrchestratorAdapter:
+    """Async wrapper for orchestrator - runs in executor to avoid blocking event loop."""
+
+    def __init__(self, sync_adapter: SyncOrchestrator) -> None:
+        self._sync = sync_adapter
+
+    async def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            None,
+            lambda: self._sync.generate(prompt, max_tokens)
+        )
+
+    async def stream(self, prompt: str, max_tokens: int | None = None) -> AsyncIterator[str]:
+        loop = asyncio.get_event_loop()
+        
+        async def gen():
+            return list(self._sync.stream(prompt, max_tokens))
+        
+        result = await loop.run_in_executor(None, gen)
+        for chunk in result:
+            yield chunk
+
+
+class AsyncCoderAdapter:
+    """Async wrapper for coder."""
+
+    def __init__(self, sync_adapter) -> None:
+        self._sync = sync_adapter
+
+    async def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            None,
+            lambda: self._sync.generate(prompt, max_tokens)
+        )
+
+
+class AsyncCriticAdapter:
+    """Async wrapper for critic."""
+
+    def __init__(self, sync_adapter) -> None:
+        self._sync = sync_adapter
+
+    async def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            None,
+            lambda: self._sync.generate(prompt, max_tokens)
+        )

app/models/coder.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from threading import RLock
+from typing import Any, Iterator
+from llama_cpp import Llama
+
+
+class CoderAdapter:
+    def __init__(self, llm: Llama, system_prompt: str | None = None, lock: RLock | None = None) -> None:
+        self._llm = llm
+        self._lock = lock or RLock()
+        self._system_prompt = system_prompt or (
+            "You are an expert code generation model."
+        )
+        self._temperature = 0.2
+
+    def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            output = self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 1024,
+                temperature=self._temperature,
+            )
+        return output["choices"][0]["message"]["content"]
+
+    def stream(self, prompt: str, max_tokens: int | None = None) -> Iterator[str]:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            for chunk in self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 1024,
+                temperature=self._temperature,
+                stream=True,
+            ):
+                content = chunk["choices"][0].get("delta", {}).get("content")
+                if content:
+                    yield content

app/models/critic.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from threading import RLock
+from typing import Any, Iterator
+from llama_cpp import Llama
+
+
+class CriticAdapter:
+    def __init__(self, llm: Llama, system_prompt: str | None = None, lock: RLock | None = None) -> None:
+        self._llm = llm
+        self._lock = lock or RLock()
+        self._system_prompt = system_prompt or (
+            "You are a critic model. Evaluate tool results and respond with JSON."
+        )
+        self._temperature = 0.1
+
+    def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            output = self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 512,
+                temperature=self._temperature,
+            )
+        return output["choices"][0]["message"]["content"]
+
+    def stream(self, prompt: str, max_tokens: int | None = None) -> Iterator[str]:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            for chunk in self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 512,
+                temperature=self._temperature,
+                stream=True,
+            ):
+                content = chunk["choices"][0].get("delta", {}).get("content")
+                if content:
+                    yield content

app/models/embeddings.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+
+class EmbeddingsAdapter:
+    def __init__(
+        self,
+        model_path: str | Path | None = None,
+        model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+        embedding_dim: int = 384,
+    ) -> None:
+        self._embedding_dim = embedding_dim
+        if model_path and Path(model_path).exists():
+            self._model = SentenceTransformer(str(model_path))
+        else:
+            self._model = SentenceTransformer(model_name)
+
+    def encode(self, texts: str | list[str]) -> np.ndarray:
+        is_single = isinstance(texts, str)
+        if is_single:
+            texts = [texts]
+        embeddings = self._model.encode(texts, convert_to_numpy=True)
+        if is_single:
+            return embeddings[0]
+        return embeddings
+
+    def encode_batch(self, texts: list[str], batch_size: int = 32) -> np.ndarray:
+        return self._model.encode(texts, batch_size=batch_size, convert_to_numpy=True)
+
+    @property
+    def embedding_dim(self) -> int:
+        return self._embedding_dim

app/models/orchestrator.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from threading import RLock
+from typing import Any, Iterator
+from llama_cpp import Llama
+
+
+class OrchestratorAdapter:
+    def __init__(self, llm: Llama, system_prompt: str | None = None, lock: RLock | None = None) -> None:
+        self._llm = llm
+        self._lock = lock or RLock()
+        self._system_prompt = system_prompt or (
+            "You are an expert orchestrator for a local AI agent system. "
+            "Your role is to analyze the user's task, decide whether planning is needed."
+        )
+        self._temperature = 0.2
+
+    def generate(self, prompt: str, max_tokens: int | None = None) -> str:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            output = self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 512,
+                temperature=self._temperature,
+            )
+        return output["choices"][0]["message"]["content"]
+
+    def stream(self, prompt: str, max_tokens: int | None = None) -> Iterator[str]:
+        messages = [
+            {"role": "system", "content": self._system_prompt},
+            {"role": "user", "content": prompt},
+        ]
+        with self._lock:
+            for chunk in self._llm.create_chat_completion(
+                messages=messages,
+                max_tokens=max_tokens or 512,
+                temperature=self._temperature,
+                stream=True,
+            ):
+                content = chunk["choices"][0].get("delta", {}).get("content")
+                if content:
+                    yield content