commit cbfddf128e699d24a883112d50f613da4d1bf99e Author: admin Date: Mon Feb 23 15:59:34 2026 +0100 first commit diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e273002 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,108 @@ +# ===== Base met CUDA11.8 + cuDNN + conda ===== +FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime + +WORKDIR /app + +# Zorg dat conda libs altijd eerst gevonden worden +ENV LD_LIBRARY_PATH=/opt/conda/lib:${LD_LIBRARY_PATH} + +# P5000 = Pascal SM 6.1; handig voor (eventueel) on-the-fly builds +ENV TORCH_CUDA_ARCH_LIST="6.1" + +# ===== Model caches op vaste paden ===== +ENV HF_HOME=/opt/hf \ + HUGGINGFACE_HUB_CACHE=/opt/hf \ + TRANSFORMERS_CACHE=/opt/hf \ + SENTENCE_TRANSFORMERS_HOME=/opt/sentence-transformers \ + XDG_CACHE_HOME=/opt/cache \ + STT_MODEL=small + +ARG RAG_EMBEDDINGS=gte-multilingual +ARG STT_MODEL_ARG=small +ENV RAG_EMBEDDINGS=${RAG_EMBEDDINGS} +ENV STT_MODEL=${STT_MODEL_ARG} + +# directories +RUN mkdir -p /opt/hf /opt/cache /opt/sentence-transformers /opt/whisper && \ + chmod -R a+rX /opt/hf /opt/cache /opt/sentence-transformers /opt/whisper + +# ===== Alleen minimale apt utils (géén multimedia libs!) ===== +RUN apt-get update && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + git curl build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# ===== Multimedia via conda-forge (alles uit één ecosysteem) ===== +# - av 10 + ffmpeg<7 (past goed bij pyAV) +# - cairo/pango/gdk-pixbuf/pixman voor cairosvg stack +# VERVANG de vorige conda multimedia regel door deze: +# Tooling voor PyAV build +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config git curl ffmpeg libcairo2 libpango-1.0-0 libgdk-pixbuf2.0-0 apt-utils pkg-config libavformat-dev libavcodec-dev libavdevice-dev libavutil-dev libavfilter-dev libswscale-dev libswresample-dev build-essential && rm -rf /var/lib/apt/lists/* + +# FFmpeg via conda-forge (zodat je recente headers/libs hebt) +RUN conda config --system --set channel_priority flexible \ + && conda install -y -c conda-forge "ffmpeg>=6,<8" \ + && conda clean -afy + +# Later in je pip stap: +# ... faster-whisper==1.0.0 zal av==11.* trekken en nu WEL kunnen bouwen tegen conda’s FFmpeg 6 + +# ===== Python deps ===== +COPY requirements.txt . +RUN pip install --upgrade pip +# jouw requirements +RUN pip install --no-cache-dir -r requirements.txt +# losse extras (let op: av via conda, niet via pip!) +RUN pip install --no-cache-dir \ + PyPDF2 python-multipart gitpython chromadb httpx meilisearch \ + pandas openpyxl python-pptx faster-whisper==1.0.0 \ + cairosvg sentence-transformers rank-bm25 + +# ===== Prefetch modellen ===== + +# 1) SentenceTransformers +RUN python - <<'PY' +import os +from sentence_transformers import SentenceTransformer +mapping = { + "gte-multilingual": "Alibaba-NLP/gte-multilingual-base", + "bge-small": "BAAI/bge-small-en-v1.5", + "e5-small": "intfloat/e5-small-v2", + "gte-base-en": "thenlper/gte-base", +} +choice = os.environ.get("RAG_EMBEDDINGS","gte-multilingual").lower() +hf_id = mapping.get(choice, "BAAI/bge-small-en-v1.5") +cache_root = os.environ.get("SENTENCE_TRANSFORMERS_HOME", "/opt/sentence-transformers") +local_dir = os.path.join(cache_root, "embedder") +os.makedirs(cache_root, exist_ok=True) +print("Downloading SentenceTransformer:", hf_id) +model = SentenceTransformer(hf_id, cache_folder=cache_root, device="cpu") # download only +model.save(local_dir) +print("Prefetched SentenceTransformer:", hf_id) +PY + +# 2) faster-whisper (prefetch CPU-kant; runtime kan je device kiezen) +RUN python - <<'PY' +import os +from faster_whisper import WhisperModel +name = os.environ.get("STT_MODEL","small") +cache_root = os.path.join(os.environ.get("XDG_CACHE_HOME","/opt/cache"), "whisper") +os.makedirs(cache_root, exist_ok=True) +_ = WhisperModel(name, device="cpu", compute_type="int8", download_root=cache_root) +print("Prefetched faster-whisper:", name, "->", cache_root) +PY + +# (optioneel) piper skip ik hier; kan later + +# ===== App code ===== +COPY app.py . +COPY queue_helper.py . +COPY agent_repo.py . +COPY windowing_utils.py . +COPY smart_rag.py . +COPY llm_client.py . +COPY web_search.py . + +EXPOSE 8080 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..190568b --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# Toolserver repo toolset v2 + +Deze patch voegt een duidelijkere repo-toolset toe, met een macro-tool die het "pas repo aan en push naar test-branch" proces in 1 toolcall doet. + +## Nieuwe (aanbevolen) tools + +- `repo_open(repo_url, branch="main") -> {workspace_id,...}` +- `repo_search(query, mode="auto|grep|rag", workspace_id? / repo_url?, n_results=20, ...)` +- `repo_read(path, workspace_id? / repo_url?, start_line?, end_line?)` +- `repo_apply(workspace_id? / repo_url?, patch_b64|patch|files, dry_run=false)` +- `repo_push(workspace_id? / repo_url?, base_branch="main", new_branch?, branch_prefix="test", commit_message=...)` +- `repo_pr_create(repo_url, head_branch, base_branch="main", title, body?)` +- `repo_change_to_branch(repo_url, base_branch="main", patch_b64|patch|files, new_branch?/branch_prefix, commit_message, create_pr?, pr_title?, pr_body?)` + +## Deprecated (blijft werken) + +- `repo_grep` -> alias naar `repo_search(mode="grep")` +- `rag_index_repo` -> meestal niet meer nodig; `repo_search` indexeert automatisch +- `rag_query` -> gebruik `repo_search(mode="rag")` + +## Belangrijke env vars + +- `LLM_PROXY_URL=http://192.168.100.1:8081/v1/completions` (of `/v1/chat/completions` als je proxy dat biedt) +- `GITEA_TOKEN=...` (alleen nodig voor PR create, en optioneel voor push auth als je repo_url geen creds bevat) +- `GITEA_USER=oauth2` (optioneel; default is oauth2 bij token-auth) +- `ALLOWED_GIT_HOSTS=10.25.138.40,192.168.100.1` (aanrader; leeg = alles toegestaan) + +## Aanbevolen flow voor LLM + +1) `repo_search(mode="auto")` om relevante bestanden te vinden +2) `repo_read` om de files te lezen +3) (LLM maakt patch) +4) `repo_change_to_branch` met `patch_b64` om te pushen naar een test-branch (+ evt PR) diff --git a/README_TOOLSET_V2.md b/README_TOOLSET_V2.md new file mode 100644 index 0000000..190568b --- /dev/null +++ b/README_TOOLSET_V2.md @@ -0,0 +1,33 @@ +# Toolserver repo toolset v2 + +Deze patch voegt een duidelijkere repo-toolset toe, met een macro-tool die het "pas repo aan en push naar test-branch" proces in 1 toolcall doet. + +## Nieuwe (aanbevolen) tools + +- `repo_open(repo_url, branch="main") -> {workspace_id,...}` +- `repo_search(query, mode="auto|grep|rag", workspace_id? / repo_url?, n_results=20, ...)` +- `repo_read(path, workspace_id? / repo_url?, start_line?, end_line?)` +- `repo_apply(workspace_id? / repo_url?, patch_b64|patch|files, dry_run=false)` +- `repo_push(workspace_id? / repo_url?, base_branch="main", new_branch?, branch_prefix="test", commit_message=...)` +- `repo_pr_create(repo_url, head_branch, base_branch="main", title, body?)` +- `repo_change_to_branch(repo_url, base_branch="main", patch_b64|patch|files, new_branch?/branch_prefix, commit_message, create_pr?, pr_title?, pr_body?)` + +## Deprecated (blijft werken) + +- `repo_grep` -> alias naar `repo_search(mode="grep")` +- `rag_index_repo` -> meestal niet meer nodig; `repo_search` indexeert automatisch +- `rag_query` -> gebruik `repo_search(mode="rag")` + +## Belangrijke env vars + +- `LLM_PROXY_URL=http://192.168.100.1:8081/v1/completions` (of `/v1/chat/completions` als je proxy dat biedt) +- `GITEA_TOKEN=...` (alleen nodig voor PR create, en optioneel voor push auth als je repo_url geen creds bevat) +- `GITEA_USER=oauth2` (optioneel; default is oauth2 bij token-auth) +- `ALLOWED_GIT_HOSTS=10.25.138.40,192.168.100.1` (aanrader; leeg = alles toegestaan) + +## Aanbevolen flow voor LLM + +1) `repo_search(mode="auto")` om relevante bestanden te vinden +2) `repo_read` om de files te lezen +3) (LLM maakt patch) +4) `repo_change_to_branch` met `patch_b64` om te pushen naar een test-branch (+ evt PR) diff --git a/README_repo_push.txt b/README_repo_push.txt new file mode 100644 index 0000000..f312593 --- /dev/null +++ b/README_repo_push.txt @@ -0,0 +1,26 @@ +Toolserver patch + repo_push tool + +Dit zipje bevat: +- app.py (toolserver-only + repo_push) +- llm_client.py + +Nieuwe tool: repo_push +- Maakt een nieuwe branch (default: test/-) vanaf base_branch +- Past een unified diff toe (git apply) +- Commit + push naar origin +- Optioneel: create_pr (best-effort) via Gitea API + +Belangrijke env vars: +- LLM_PROXY_URL="http://192.168.100.1:8081/v1/completions" (voor tools die LLM nodig hebben) +- ALLOWED_GIT_HOSTS="192.168.100.1,localhost,127.0.0.1" (veiligheidscheck; pas aan indien nodig) + +Voor push-auth (HTTP): +- GITEA_USER="..." (of GIT_HTTP_USER) +- GITEA_TOKEN="..." (of GIT_HTTP_TOKEN) + +Git identity voor commits: +- GIT_AUTHOR_NAME="toolserver" +- GIT_AUTHOR_EMAIL="toolserver@local" + +Gebruik: +- tools worden automatisch beschikbaar via /openapi/repo_push en /v1/tools. diff --git a/README_toolserver_patch.txt b/README_toolserver_patch.txt new file mode 100644 index 0000000..65f7433 --- /dev/null +++ b/README_toolserver_patch.txt @@ -0,0 +1,19 @@ +Toolserver patch + +Wat verandert dit? +- app.py draait nu standaard als TOOLSERVER_ONLY=1: + * expose: /openapi.json + /openapi/* + /v1/tools + /v1/tools/call + /healthz + /metrics + * verwijdert alle overige routes (zoals /v1/chat/completions) uit de FastAPI router. +- Alle LLM-calls die tools/agents doen, gaan via llm_client -> QueueManager -> LLM_PROXY_URL. + +Config: + export TOOLSERVER_ONLY=1 + export LLM_PROXY_URL="http://192.168.100.1:8081/v1/completions" + export LLM_MODEL="mistral-medium" # of wat je proxy verwacht + +Startvoorbeeld: + uvicorn app:app --host 0.0.0.0 --port 8080 + +Opmerking: +- Interne LLM streaming is uitgezet (tools zijn non-stream). + Als je later streaming nodig hebt voor agents, dan moeten we llm_client uitbreiden. diff --git a/agent_repo.py b/agent_repo.py new file mode 100644 index 0000000..eb14552 --- /dev/null +++ b/agent_repo.py @@ -0,0 +1,4887 @@ +# agent_repo.py +# ===================================================================== +# Hybrid RAG + LLM edit-plans met: veilige fallback, anti-destructie guard, +# en EXPLICIETE UITLEG per diff. +# ===================================================================== +# agent_repo.py (bovenin) + +from __future__ import annotations +from smart_rag import enrich_intent, expand_queries, hybrid_retrieve, assemble_context +import os, re, time, uuid, difflib, hashlib, logging, json, fnmatch +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Tuple, Optional, Any +from urllib.parse import urlparse, urlunparse +import requests +import base64 +from windowing_utils import approx_token_count +from starlette.concurrency import run_in_threadpool +import asyncio +from collections import defaultdict +from llm_client import _llm_call + +# --- Async I/O executors (voorkom event-loop blocking) --- +from concurrent.futures import ThreadPoolExecutor + +_IO_POOL = ThreadPoolExecutor(max_workers=int(os.getenv("AGENT_IO_WORKERS", "8"))) +_CPU_POOL = ThreadPoolExecutor(max_workers=int(os.getenv("AGENT_CPU_WORKERS", "2"))) +_CLONE_SEMA = asyncio.Semaphore(int(os.getenv("AGENT_MAX_CONCURRENT_CLONES", "2"))) + +BACKEND = (os.getenv("VECTOR_BACKEND") or "CHROMA").upper().strip() + +#PATH_RE = re.compile(r"(?new) +# --------------------------------------------------------- +_Q = r"[\"'“”‘’`]" +_PATH_PATS = [ + r"[\"“”'](resources\/[A-Za-z0-9_\/\.-]+\.blade\.php)[\"”']", + r"(resources\/[A-Za-z0-9_\/\.-]+\.blade\.php)", + r"[\"“”'](app\/[A-Za-z0-9_\/\.-]+\.php)[\"”']", + r"(app\/[A-Za-z0-9_\/\.-]+\.php)", +] +_TRANS_WRAPPERS = [ + r"__\(\s*{q}(.+?){q}\s*\)".format(q=_Q), + r"@lang\(\s*{q}(.+?){q}\s*\)".format(q=_Q), + r"trans\(\s*{q}(.+?){q}\s*\)".format(q=_Q), +] + +def _clean_repo_arg(x): + """Zet lege/sentinel repo-waarden om naar None (geen filter).""" + if x is None: + return None + s = str(x).strip().lower() + return None if s in ("", "-", "none") else x + + +def _extract_repo_branch_from_text(txt: str) -> Tuple[Optional[str], str]: + repo_url, branch = None, "main" + m = re.search(r"\bRepo\s*:\s*(\S+)", txt, flags=re.I) + if m: repo_url = m.group(1).strip() + mb = re.search(r"\bbranch\s*:\s*([A-Za-z0-9._/-]+)", txt, flags=re.I) + if mb: branch = mb.group(1).strip() + return repo_url, branch + +def _extract_explicit_paths(txt: str) -> List[str]: + out = [] + for pat in _PATH_PATS: + for m in re.finditer(pat, txt): + p = m.group(1) + if p and p not in out: + out.append(p) + return out + +def _extract_replace_pair(txt: str) -> Tuple[Optional[str], Optional[str]]: + # NL/EN varianten + “slimme” quotes + pats = [ + rf"Vervang\s+de\s+tekst\s*{_Q}(.+?){_Q}[^.\n]*?(?:in|naar|verander(?:en)?\s+in)\s*{_Q}(.+?){_Q}", + rf"Replace(?:\s+the)?\s+text\s*{_Q}(.+?){_Q}\s*(?:to|with)\s*{_Q}(.+?){_Q}", + ] + for p in pats: + m = re.search(p, txt, flags=re.I|re.S) + if m: + return m.group(1), m.group(2) + mm = re.search(r"(Vervang|Replace)[\s\S]*?"+_Q+"(.+?)"+_Q+"[\s\S]*?"+_Q+"(.+?)"+_Q, txt, flags=re.I) + if mm: + return mm.group(2), mm.group(3) + return None, None + +def _looks_like_unified_diff_request(txt: str) -> bool: + if re.search(r"\bunified\s+diff\b", txt, flags=re.I): return True + if re.search(r"\b(diff|patch)\b", txt, flags=re.I) and _extract_explicit_paths(txt): + return True + return False + +# zet dit dicht bij de andere module-consts +async def _call_get_git_repo(repo_url: str, branch: str): + """ + Veilig wrapper: ondersteunt zowel sync als async implementaties van _get_git_repo. + """ + if asyncio.iscoroutinefunction(_get_git_repo): + return await _get_git_repo(repo_url, branch) + # sync: draai in IO pool + return await run_io_blocking(_get_git_repo, repo_url, branch) + + +async def run_io_blocking(func, *args, pool=None, **kwargs): + """Draai sync/blokkerende I/O in threadpool zodat de event-loop vrij blijft.""" + loop = asyncio.get_running_loop() + executor = pool or _IO_POOL + return await loop.run_in_executor(executor, lambda: func(*args, **kwargs)) + +async def run_cpu_blocking(func, *args, pool=None, **kwargs): + """Voor CPU-zwaardere taken (bv. index bouwen).""" + loop = asyncio.get_running_loop() + executor = pool or _CPU_POOL + return await loop.run_in_executor(executor, lambda: func(*args, **kwargs)) + +# Lazy imports +_chroma = None +_qdrant = None +_qdrant_models = None +try: + if BACKEND == "CHROMA": + import chromadb # type: ignore + _chroma = chromadb +except Exception: + _chroma = None +try: + if BACKEND == "QDRANT": + from qdrant_client import QdrantClient # type: ignore + from qdrant_client.http.models import Filter, FieldCondition, MatchValue # type: ignore + _qdrant = QdrantClient + _qdrant_models = (Filter, FieldCondition, MatchValue) +except Exception: + _qdrant = None + _qdrant_models = None + + +try: + from rank_bm25 import BM25Okapi +except Exception: + BM25Okapi = None + +logger = logging.getLogger("agent_repo") + +# ---------- Omgeving / Config ---------- +GITEA_URL = os.environ.get("GITEA_URL", "http://10.25.138.40:30085").rstrip("/") +GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "8bdbe18dd2ec93ecbf9cd0a8f01a6eadf9cfa87d") +GITEA_API = os.environ.get("GITEA_API", f"{GITEA_URL}/api/v1").rstrip("/") +AGENT_DEFAULT_BRANCH = os.environ.get("AGENT_DEFAULT_BRANCH", "main") +AGENT_MAX_QUESTIONS = int(os.environ.get("AGENT_MAX_QUESTIONS", "3")) +MAX_FILES_DRYRUN = int(os.environ.get("AGENT_MAX_FILES_DRYRUN", "27")) +RAG_TOPK = int(os.environ.get("AGENT_RAG_TOPK", "24")) # grotere kandidaatpool helpt de reranker +AGENT_DISCOVER_MAX_REPOS = int(os.environ.get("AGENT_DISCOVER_MAX_REPOS", "200")) +AGENT_AUTOSELECT_THRESHOLD = float(os.environ.get("AGENT_AUTOSELECT_THRESHOLD", "0.80")) # 0..1 +REPO_CATALOG_MEILI_INDEX = os.environ.get("REPO_CATALOG_MEILI_INDEX", "repo-catalog") +AGENT_ENABLE_GOAL_REFINE = os.environ.get("AGENT_ENABLE_GOAL_REFINE", "1").lower() in ("1","true","yes") +AGENT_CLARIFY_THRESHOLD = float(os.environ.get("AGENT_CLARIFY_THRESHOLD", "0.6")) + + +# Meilisearch (optioneel) +MEILI_URL = os.environ.get("MEILI_URL", "http://192.168.100.1:7700").strip() +MEILI_KEY = os.environ.get("MEILI_KEY", "0xipOmfgi_zMgdFplSdv7L8mlx0RPMQCNxVTNJc54lQ").strip() +MEILI_INDEX_PREFIX = os.environ.get("MEILI_INDEX_PREFIX", "code").strip() + +# optioneel: basic auth injectie voor HTTP clone (private repos) +GITEA_HTTP_USER = os.environ.get("GITEA_HTTP_USER", "Mistral-llm") +GITEA_HTTP_TOKEN = os.environ.get("GITEA_HTTP_TOKEN", "8bdbe18dd2ec93ecbf9cd0a8f01a6eadf9cfa87d") + +# Geen destructive edits. (geen complete inhoud van files verwijderen.) +AGENT_DESTRUCTIVE_RATIO = float(os.environ.get("AGENT_DESTRUCTIVE_RATIO", "0.50")) + +# Alleen relevante code/tekst-extensies (geen binaire/caches) +ALLOWED_EXTS = { + ".php",".blade.php",".vue",".js",".ts",".jsx",".tsx",".css",".scss", + ".html",".htm",".json",".md",".ini",".cfg",".yml",".yaml",".toml", + ".py",".go",".rb",".java",".cs",".txt" +} +INTERNAL_EXCLUDE_DIRS = { + ".git",".npm","node_modules","vendor","storage","dist","build",".next", + "__pycache__",".venv","venv",".mypy_cache",".pytest_cache", + "target","bin","obj","logs","cache","temp",".cache" +} +_LIST_FILES_CACHE: dict[str, tuple[float, List[str]]] = {} # path -> (ts, files) +# ---------- Injectie vanuit app.py ---------- +_app = None +_get_git_repo = None +_rag_index_repo_internal = None +_rag_query_internal = None +_llm_call = None +_extract_code_block = None +_read_text_file = None +_client_ip = None +_PROFILE_EXCLUDE_DIRS: set[str] = set() +_get_chroma_collection = None +_embed_query_fn = None +_embed_documents = None + + +# === SMART LLM WRAPPER: budget + nette afronding + auto-continue === +# Past binnen jouw GPU-cap (typisch 13027 tokens totale context). +# Non-invasief: behoudt hetzelfde response-shape als _llm_call. + +# Harde cap van jouw Mistral-LLM docker (zoals je aangaf) +_MODEL_BUDGET = int(os.getenv("LLM_TOTAL_TOKEN_BUDGET", "42000")) +# Veiligheidsmarge voor headers/EOS/afwijkingen in token-raming +_BUDGET_SAFETY = int(os.getenv("LLM_BUDGET_SAFETY_TOKENS", "512")) +# Max aantal vervolgstappen als het net afgekapt lijkt +_MAX_AUTO_CONTINUES = int(os.getenv("LLM_MAX_AUTO_CONTINUES", "2")) + +def _est_tokens(text: str) -> int: + # Ruwe schatting: ~4 chars/token (conservatief genoeg voor budgettering) + if not text: return 0 + return max(1, len(text) // 4) + +def _concat_messages_text(messages: list[dict]) -> str: + parts = [] + for m in messages or []: + c = m.get("content") + if isinstance(c, str): parts.append(c) + return "\n".join(parts) + +def _ends_neatly(s: str) -> bool: + if not s: return False + t = s.rstrip() + return t.endswith((".", "!", "?", "…", "”", "’")) + +def _append_assistant_and_continue_prompt(base_messages: list[dict], prev_text: str) -> list[dict]: + """ + Bouw een minimale vervolgprompt zonder opnieuw de hele context te sturen. + Dit beperkt prompt_tokens en voorkomt dat we opnieuw de cap raken. + """ + tail_words = " ".join(prev_text.split()[-60:]) # laatste ±60 woorden als anker + cont_user = ( + "Ga verder waar je stopte. Herhaal niets. " + "Vervolg direct de laatste zin met hetzelfde formaat.\n\n" + "Vorige woorden:\n" + tail_words + ) + # We sturen *niet* de volledige history opnieuw; alleen een korte instructie + return [ + {"role": "system", "content": "Vervolg exact en beknopt; geen herhaling van eerder gegenereerde tekst."}, + {"role": "user", "content": cont_user}, + ] + +def _merge_choice_text(resp_a: dict, resp_b: dict) -> dict: + """ + Plak de content van choices[0] aan elkaar zodat callsites één 'content' blijven lezen. + """ + a = (((resp_a or {}).get("choices") or [{}])[0].get("message") or {}).get("content","") + b = (((resp_b or {}).get("choices") or [{}])[0].get("message") or {}).get("content","") + merged = (a or "") + (b or "") + out = resp_a.copy() + if "choices" in out and out["choices"]: + out["choices"] = [{ + "index": 0, + "finish_reason": "length" if (out.get("choices",[{}])[0].get("finish_reason") in (None, "length")) else out.get("choices",[{}])[0].get("finish_reason"), + "message": {"role":"assistant","content": merged} + }] + return out + +# Voorbeeld: Chroma client/init – vervang door jouw eigen client +# from chromadb import Client +# chroma = Client(...) + +def _build_where_filter(repo: Optional[str], path_contains: Optional[str], profile: Optional[str]) -> Dict[str, Any]: + """ + Bouw een simpele metadata-filter voor de vector-DB. Pas aan naar jouw DB. + """ + where: Dict[str, Any] = {} + if repo: + where["repo"] = repo + if profile: + where["profile"] = profile + if path_contains: + # Als je DB geen 'contains' ondersteunt: filter achteraf (post-filter) + where["path_contains"] = path_contains + return where + +def _to_distance_from_similarity(x: Optional[float]) -> float: + """ + Converteer een 'similarity' (1=identiek, 0=ver weg) naar distance (lager = beter). + """ + if x is None: + return 1.0 + try: + xv = float(x) + except Exception: + return 1.0 + # Veiligheids-net: clamp + if xv > 1.0 or xv < 0.0: + # Sommige backends geven cosine distance al (0=identiek). Als >1, treat as distance passthrough. + return max(0.0, xv) + # Standaard: cosine similarity → distance + return 1.0 - xv + +def _post_filter_path_contains(items: List[Dict[str,Any]], path_contains: Optional[str]) -> List[Dict[str,Any]]: + if not path_contains: + return items + key = (path_contains or "").lower() + out = [] + for it in items: + p = ((it.get("metadata") or {}).get("path") or "").lower() + if key in p: + out.append(it) + return out + +def _chroma_query(collection_name: str, query: str, n_results: int, where: Dict[str,Any]) -> Dict[str,Any]: + global _chroma + if _chroma is None: + raise RuntimeError("Chroma backend niet beschikbaar (module niet geïnstalleerd).") + # Gebruik dezelfde collection-factory als de indexer, zodat versie/suffix consistent is + if _get_chroma_collection is None: + client = _chroma.Client() + coll = client.get_or_create_collection(collection_name) + else: + coll = _get_chroma_collection(collection_name) + # Chroma: use 'where' only for exact fields (repo/profile) + where_exact = {k:v for k,v in where.items() if k in ("repo","profile")} + qr = coll.query( + query_texts=[query], + n_results=max(1, n_results), + where=where_exact, + include=["documents","metadatas","distances"] + ) + docs = qr.get("documents", [[]])[0] or [] + metas = qr.get("metadatas", [[]])[0] or [] + dists = qr.get("distances", [[]])[0] or [] + # Chroma 'distances': lager = beter (ok) + items: List[Dict[str,Any]] = [] + for doc, meta, dist in zip(docs, metas, dists): + items.append({ + "document": doc, + "metadata": { + "repo": meta.get("repo",""), + "path": meta.get("path",""), + "chunk_index": meta.get("chunk_index", 0), + "symbols": meta.get("symbols", []), + "profile": meta.get("profile",""), + }, + "distance": float(dist) if dist is not None else 1.0, + }) + return {"results": items} + +def _qdrant_query(collection_name: str, query: str, n_results: int, where: Dict[str,Any]) -> Dict[str,Any]: + global _qdrant, _qdrant_models + if _qdrant is None or _qdrant_models is None: + raise RuntimeError("Qdrant backend niet beschikbaar (module niet geïnstalleerd).") + Filter, FieldCondition, MatchValue = _qdrant_models + # Let op: je hebt hier *ook* een embedder nodig (client-side). In dit skeleton verwachten we dat + # je server-side search by text hebt geconfigureerd. Anders: voeg hier je embedder toe. + client = _qdrant(host=os.getenv("QDRANT_HOST","192.168.100.1"), port=int(os.getenv("QDRANT_PORT","6333"))) + # Eenvoudig: text search (als ingeschakeld). Anders: raise en laat de mock fallback pakken. + try: + must: List[Any] = [] + if where.get("repo"): + must.append(FieldCondition(key="repo", match=MatchValue(value=where["repo"]))) + if where.get("profile"): + must.append(FieldCondition(key="profile", match=MatchValue(value=where["profile"]))) + flt = Filter(must=must) if must else None + # NB: Qdrant 'score' is vaak cosine similarity (hoog=goed). Converteer naar distance. + res = client.search( + collection_name=collection_name, + query=query, + limit=max(1, n_results), + query_filter=flt, + with_payload=True, + ) + except Exception as e: + raise RuntimeError(f"Qdrant text search niet geconfigureerd: {e}") + + items: List[Dict[str,Any]] = [] + for p in res: + meta = (p.payload or {}) + sim = getattr(p, "score", None) + items.append({ + "document": meta.get("document",""), + "metadata": { + "repo": meta.get("repo",""), + "path": meta.get("path",""), + "chunk_index": meta.get("chunk_index", 0), + "symbols": meta.get("symbols", []), + "profile": meta.get("profile",""), + }, + "distance": _to_distance_from_similarity(sim), + }) + return {"results": items} + +async def rag_query_internal_fn( + *, query: str, n_results: int, collection_name: str, + repo: Optional[str], path_contains: Optional[str], profile: Optional[str] +) -> Dict[str, Any]: + """ + Adapter die zoekt in je vector-DB en *exact* het verwachte formaat teruggeeft: + { + "results": [ + {"document": str, "metadata": {...}, "distance": float} + ] + } + """ + # 1) Haal collectie op (pas aan naar jouw client) + # coll = chroma.get_or_create_collection(collection_name) + + # 2) Bouw where/filter (optioneel afhankelijk van jouw DB) + where = _build_where_filter(repo, path_contains, profile) + + # ?2?) Router naar backend + try: + if BACKEND == "CHROMA": + res = _chroma_query(collection_name, query, n_results, where) + elif BACKEND == "QDRANT": + res = _qdrant_query(collection_name, query, n_results, where) + else: + raise RuntimeError(f"Onbekende VECTOR_BACKEND={BACKEND}") + + except Exception as e: + # Mock fallback zodat je app bruikbaar blijft + qr = { + "documents": [["(mock) no DB connected"]], + "metadatas": [[{"repo": repo or "", "path": "README.md", "chunk_index": 0, "symbols": []}]], + "distances": [[0.99]], + } + docs = qr.get("documents", [[]])[0] or [] + metas = qr.get("metadatas", [[]])[0] or [] + dists = qr.get("distances", [[]])[0] or [] + + items: List[Dict[str, Any]] = [] + for doc, meta, dist in zip(docs, metas, dists): + # Post-filter op path_contains als je DB dat niet ondersteunt + if path_contains: + p = (meta.get("path") or "").lower() + if (path_contains or "").lower() not in p: + continue + items.append({ + "document": doc, + "metadata": { + "repo": meta.get("repo",""), + "path": meta.get("path",""), + "chunk_index": meta.get("chunk_index", 0), + "symbols": meta.get("symbols", []), + "profile": meta.get("profile",""), + }, + "distance": float(dist) if dist is not None else 1.0, + }) + res = {"results": items[:max(1, n_results)]} + # 3) Post-filter path_contains (indien nodig) + res["results"] = _post_filter_path_contains(res.get("results", []), path_contains) + # 4) Trim + res["results"] = res.get("results", [])[:max(1, n_results)] + return res + +async def _smart_llm_call_base( + llm_call_fn, + messages: list[dict], + *, + stop: list[str] | None = None, + max_tokens: int | None = None, + temperature: float = 0.2, + top_p: float = 0.9, + stream: bool = False, + **kwargs +): + """ + 1) Dwing max_tokens af binnen totale budget (prompt + output ≤ cap). + 2) Voeg milde stop-sequenties toe voor nette afronding. + 3) Auto-continue als het lijkt afgekapt en we ruimte willen voor een vervolg. + """ + # 1) Budget berekenen op basis van huidige prompt omvang + prompt_text = _concat_messages_text(messages) + prompt_tokens = _est_tokens(prompt_text) + room = max(128, _MODEL_BUDGET - prompt_tokens - _BUDGET_SAFETY) + eff_max_tokens = max(1, min(int(max_tokens or 900), room)) + + # 2) Stop-sequenties (mild, niet beperkend voor code) + default_stops = ["\n\n", "###"] + stops = list(dict.fromkeys((stop or []) + default_stops)) + + # eerste call + try: + resp = await llm_call_fn( + messages, + stream=stream, + temperature=temperature, + top_p=top_p, + max_tokens=eff_max_tokens, + stop=stops, + **kwargs + ) + except TypeError as e: + # backend accepteert geen 'stop' → probeer opnieuw zonder stop + resp = await llm_call_fn( + messages, + stream=stream, + temperature=temperature, + top_p=top_p, + max_tokens=eff_max_tokens, + **kwargs + ) + text = (((resp or {}).get("choices") or [{}])[0].get("message") or {}).get("content","") + # Heuristiek: bijna vol + niet netjes eindigen → waarschijnlijk afgekapt + near_cap = (_est_tokens(text) >= int(0.92 * eff_max_tokens)) + needs_more = (near_cap and not _ends_neatly(text)) + + continues = 0 + merged = resp + while needs_more and continues < _MAX_AUTO_CONTINUES: + continues += 1 + cont_msgs = _append_assistant_and_continue_prompt(messages, text) + # Herbereken budget voor vervolg (nieuwe prompt is veel kleiner) + cont_prompt_tokens = _est_tokens(_concat_messages_text(cont_msgs)) + cont_room = max(128, _MODEL_BUDGET - cont_prompt_tokens - _BUDGET_SAFETY) + cont_max = max(1, min(int(max_tokens or 900), cont_room)) + try: + cont_resp = await llm_call_fn( + cont_msgs, + stream=False, + temperature=temperature, + top_p=top_p, + max_tokens=cont_max, + stop=stops, + **kwargs + ) + except TypeError: + cont_resp = await llm_call_fn( + cont_msgs, + stream=False, + temperature=temperature, + top_p=top_p, + max_tokens=cont_max, + **kwargs + ) + merged = _merge_choice_text(merged, cont_resp) + text = (((merged or {}).get("choices") or [{}])[0].get("message") or {}).get("content","") + near_cap = (_est_tokens(text.split()[-800:]) >= int(0.9 * cont_max)) # check op laatst stuk + needs_more = (near_cap and not _ends_neatly(text)) + + return merged + +def initialize_agent(*, app, get_git_repo_fn, rag_index_repo_internal_fn, rag_query_internal_fn, + llm_call_fn, extract_code_block_fn, read_text_file_fn, client_ip_fn, + profile_exclude_dirs, chroma_get_collection_fn, embed_query_fn, embed_documents_fn, + search_candidates_fn=None, repo_summary_get_fn=None, meili_search_fn=None): + global DEF_INJECTS + DEF_INJECTS.update({ + "app": app, + "get_git_repo_fn": get_git_repo_fn, + "rag_index_repo_internal_fn": rag_index_repo_internal_fn, + "rag_query_internal_fn": rag_query_internal_fn, + "llm_call_fn": llm_call_fn, + "extract_code_block_fn": extract_code_block_fn, + "read_text_file_fn": read_text_file_fn, + "client_ip_fn": client_ip_fn, + "profile_exclude_dirs": profile_exclude_dirs, + "chroma_get_collection_fn": chroma_get_collection_fn, + "embed_query_fn": embed_query_fn, + "embed_documents_fn": embed_documents_fn, + }) + global _search_candidates_fn, _repo_summary_get_fn, _meili_search_fn + _search_candidates_fn = search_candidates_fn + _repo_summary_get_fn = repo_summary_get_fn + _meili_search_fn = meili_search_fn + global _get_chroma_collection, _embed_query_fn + global _app, _get_git_repo, _rag_index_repo_internal, _rag_query_internal, _llm_call + global _extract_code_block, _read_text_file, _client_ip, _PROFILE_EXCLUDE_DIRS + _app = app + _get_git_repo = get_git_repo_fn + _rag_index_repo_internal = rag_index_repo_internal_fn + _rag_query_internal = rag_query_internal_fn + # Bewaar de originele en wrap met budget + auto-continue + _llm_call_original = llm_call_fn + async def _wrapped_llm_call(messages, **kwargs): + return await _smart_llm_call_base(_llm_call_original, messages, **kwargs) + globals()["_llm_call"] = _wrapped_llm_call + _extract_code_block = extract_code_block_fn + _read_text_file = read_text_file_fn + _client_ip = client_ip_fn + _PROFILE_EXCLUDE_DIRS = set(profile_exclude_dirs) | INTERNAL_EXCLUDE_DIRS + _get_chroma_collection = chroma_get_collection_fn + _embed_query_fn = embed_query_fn + _embed_documents = embed_documents_fn + if not hasattr(_app.state, "AGENT_SESSIONS"): + _app.state.AGENT_SESSIONS: Dict[str, AgentState] = {} + logger.info("INFO:agent_repo:init GITEA_URL=%s GITEA_API=%s MEILI_URL=%s", GITEA_URL, GITEA_API, MEILI_URL or "-") + +# ---------- Helpers ---------- +def extract_explicit_paths(text: str) -> List[str]: + """ + Robuuste extractor: + - negeert urls (http/https) + - vereist minstens één '/' en een extensie + - dedupe, behoud originele volgorde + """ + if not text: + return [] + # normaliseer “slimme” quotes naar gewone quotes (kan later handig zijn) + t = (text or "").replace("“","\"").replace("”","\"").replace("’","'").replace("\\","/").strip() + cands = PATH_RE.findall(t) + seen = set() + out: List[str] = [] + for p in cands: + if p not in seen: + seen.add(p) + out.append(p) + logger.info("EXPLICIT PATHS parsed: %s", out) # <— log + return out + +async def _llm_recovery_plan(user_goal: str, observed_candidates: list[str], last_reason: str = "") -> dict: + """ + Vraag de LLM om gerichte herstel-zoekpatronen en trefwoorden wanneer we 'geen voorstel' kregen. + Output JSON: { "patterns":[{"glob"| "regex": str},...], "keywords":[str,...], "note": str } + """ + sys = ("Return ONLY compact JSON. Schema:\n" + "{\"patterns\":[{\"glob\":str}|{\"regex\":str},...],\"keywords\":[str,...],\"note\":str}\n" + "Prefer Laravel-centric paths (resources/views/**.blade.php, routes/*.php, app/Http/Controllers/**.php, " + "config/*.php, .env, database/migrations/**.php). Max 12 patterns, 8 keywords.") + usr = (f"User goal:\n{user_goal}\n\n" + f"Candidates we tried (may be irrelevant):\n{json.dumps(observed_candidates[-12:], ensure_ascii=False)}\n\n" + f"Failure reason (if any): {last_reason or '(none)'}\n" + "Propose minimal extra patterns/keywords to find the exact files.") + try: + resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.0, top_p=1.0, max_tokens=280 + ) + raw = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + m = re.search(r"\{[\s\S]*\}", raw or "") + obj = json.loads(m.group(0)) if m else {} + except Exception: + obj = {} + # sanitize + pats = [] + for it in (obj.get("patterns") or []): + if isinstance(it, dict): + if "glob" in it and isinstance(it["glob"], str) and it["glob"].strip(): + pats.append({"glob": it["glob"].strip()[:200]}) + elif "regex" in it and isinstance(it["regex"], str) and it["regex"].strip(): + pats.append({"regex": it["regex"].strip()[:200]}) + if len(pats) >= 16: break + kws = [str(x).strip()[:64] for x in (obj.get("keywords") or []) if str(x).strip()][:8] + note = str(obj.get("note",""))[:400] + return {"patterns": pats, "keywords": kws, "note": note} + +def _extend_candidates_with_keywords(root: Path, all_files: list[str], keywords: list[str], cap: int = 24) -> list[str]: + """ + Deterministische keyword-scan (lichtgewicht). Gebruikt dezelfde text loader. + """ + out: list[str] = []; seen: set[str] = set() + kws = [k for k in keywords if k] + if not kws: return out + for rel in all_files: + if len(out) >= cap: break + try: + txt = _read_text_file(Path(root)/rel) + except Exception: + txt = "" + if not txt: continue + low = txt.lower() + if any(k.lower() in low for k in kws): + if rel not in seen: + seen.add(rel); out.append(rel) + return out + +async def _recovery_expand_candidates(root: Path, all_files: list[str], user_goal: str, + current: list[str], *, last_reason: str = "") -> tuple[list[str], dict]: + """ + 1) vraag LLM om recovery plan → patterns + keywords + 2) scan deterministisch met _scan_repo_for_patterns + 3) keyword-scan als tweede spoor + Retourneert (nieuwe_kandidaten_lijst, debug_info) + """ + plan = await _llm_recovery_plan(user_goal, current, last_reason=last_reason) + added: list[str] = [] + # patterns → scan + if plan.get("patterns"): + hits = _scan_repo_for_patterns(root, all_files, plan["patterns"], max_hits=int(os.getenv("LLM_RECOVERY_MAX_HITS","24"))) + for h in hits: + if h not in current and h not in added: + added.append(h) + # keywords → scan + if len(added) < int(os.getenv("LLM_RECOVERY_MAX_HITS","24")) and plan.get("keywords"): + khits = _extend_candidates_with_keywords(root, all_files, plan["keywords"], + cap=int(os.getenv("LLM_RECOVERY_MAX_HITS","24")) - len(added)) + for h in khits: + if h not in current and h not in added: + added.append(h) + new_list = (current + added)[:MAX_FILES_DRYRUN] + debug = {"recovery_plan": plan, "added": added[:12]} + return new_list, debug + +def _scan_repo_for_patterns(root: Path, all_files: list[str], patterns: list[dict], max_hits: int = 40) -> list[str]: + """ + patterns: [{"glob": "resources/views/**.blade.php"}, {"regex": "Truebeam\\s*foutcode"}, ...] + Retourneert unieke bestands-paden met 1+ hits. Deterministisch (geen LLM). + """ + hits: list[str] = [] + seen: set[str] = set() + def _match_glob(pat: str) -> list[str]: + try: + pat = pat.strip().lstrip("./") + return [f for f in all_files if fnmatch.fnmatch(f, pat)] + except Exception: + return [] + for spec in patterns or []: + if len(hits) >= max_hits: break + if "glob" in spec and isinstance(spec["glob"], str): + for f in _match_glob(spec["glob"]): + if f not in seen: + seen.add(f); hits.append(f) + if len(hits) >= max_hits: break + elif "regex" in spec and isinstance(spec["regex"], str): + try: + rx = re.compile(spec["regex"], re.I|re.M) + except Exception: + continue + for f in all_files: + if f in seen: continue + try: + txt = _read_text_file(Path(root)/f) + if rx.search(txt or ""): + seen.add(f); hits.append(f) + if len(hits) >= max_hits: break + except Exception: + continue + return hits + +async def _llm_make_search_specs(user_goal: str, framework: str = "laravel") -> list[dict]: + """ + LLM bedenkt globs/regexen. Output ONLY JSON: {patterns:[{glob|regex: str},...]} + We voeren daarna een deterministische scan uit met _scan_repo_for_patterns. + """ + if not (user_goal or "").strip(): + return [] + sys = ("Return ONLY JSON matching: {\"patterns\":[{\"glob\":str}|{\"regex\":str}, ...]}\n" + "For Laravel, prefer globs like resources/views/**.blade.php, routes/*.php, app/Http/Controllers/**.php, " + "config/*.php, .env, database/migrations/**.php. Keep regexes simple and safe.") + usr = f"Framework: {framework}\nUser goal:\n{user_goal}\nReturn ≤ 12 items." + try: + resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.0, top_p=1.0, max_tokens=280 + ) + raw = (resp.get('choices',[{}])[0].get('message',{}) or {}).get('content','') + m = re.search(r"\{[\s\S]*\}", raw or "") + obj = json.loads(m.group(0)) if m else {} + arr = obj.get("patterns") or [] + out = [] + for it in arr: + if isinstance(it, dict): + if "glob" in it and isinstance(it["glob"], str) and it["glob"].strip(): + out.append({"glob": it["glob"].strip()[:200]}) + elif "regex" in it and isinstance(it["regex"], str) and it["regex"].strip(): + out.append({"regex": it["regex"].strip()[:200]}) + if len(out) >= 16: break + return out + except Exception: + return [] + +def _with_preview(text: str, st: "AgentState", *, limit: int = 1200, header: str = "--- SMART-RAG quick scan (preview) ---") -> str: + """Plak een compacte SMART-RAG preview onderaan het antwoord, als die er is.""" + sp = getattr(st, "smart_preview", "") or "" + sp = sp.strip() + if not sp: + return text + if limit > 0 and len(sp) > limit: + sp = sp[:limit].rstrip() + "\n…" + return text + "\n\n" + header + "\n" + sp + + +def _now() -> int: + return int(time.time()) + +def _gitea_headers(): + return {"Authorization": f"token {GITEA_TOKEN}"} if GITEA_TOKEN else {} + +def add_auth_to_url(url: str, user: str | None = None, token: str | None = None) -> str: + if not url or not (user and token): + return url + u = urlparse(url) + if u.scheme not in ("http", "https") or "@" in u.netloc: + return url + netloc = f"{user}:{token}@{u.netloc}" + return urlunparse((u.scheme, netloc, u.path, u.params, u.query, u.fragment)) + +def ensure_git_suffix(url: str) -> str: + try: + u = urlparse(url) + if not u.path.endswith(".git") and "/api/" not in u.path: + return urlunparse((u.scheme, u.netloc, u.path.rstrip("/") + ".git", u.params, u.query, u.fragment)) + return url + except Exception: + return url + +def parse_owner_repo(hint: str) -> tuple[str | None, str | None]: + m = re.match(r"^([A-Za-z0-9_.\-]+)/([A-Za-z0-9_.\-]+)$", (hint or "").strip()) + if not m: + return None, None + return m.group(1), m.group(2) + +def gitea_get_repo(owner: str, repo: str) -> dict | None: + try: + r = requests.get(f"{GITEA_API}/repos/{owner}/{repo}", headers=_gitea_headers(), timeout=10) + if r.status_code == 404: + return None + r.raise_for_status() + return r.json() + except Exception as e: + logger.warning("WARN:agent_repo:gitea_get_repo %s/%s failed: %s", owner, repo, e) + return None + +def gitea_search_repos(q: str, limit: int = 5) -> List[dict]: + try: + r = requests.get(f"{GITEA_API}/repos/search", + params={"q": q, "limit": limit}, + headers=_gitea_headers(), timeout=10) + r.raise_for_status() + data = r.json() or {} + if isinstance(data, dict) and "data" in data: return data["data"] + if isinstance(data, list): return data + if isinstance(data, dict) and "ok" in data and "data" in data: return data["data"] + return [] + except Exception as e: + logger.warning("WARN:agent_repo:/repos/search failed: %s", e) + return [] + +def resolve_repo(hint: str) -> tuple[dict | None, str | None]: + hint = (hint or "").strip() + logger.info("INFO:agent_repo:resolve_repo hint=%s", hint) + if hint.startswith("http://") or hint.startswith("https://"): + url = add_auth_to_url(ensure_git_suffix(hint), GITEA_HTTP_USER, GITEA_HTTP_TOKEN) + owner, repo = owner_repo_from_url(url) + rd = {"full_name": f"{owner}/{repo}" if owner and repo else None, "clone_url": url} + logger.info("INFO:agent_repo:resolved direct-url %s", rd.get("full_name")) + return rd, "direct-url" + owner, repo = parse_owner_repo(hint) + if owner and repo: + meta = gitea_get_repo(owner, repo) + if meta: + url = meta.get("clone_url") or f"{GITEA_URL}/{owner}/{repo}.git" + url = add_auth_to_url(ensure_git_suffix(url), GITEA_HTTP_USER, GITEA_HTTP_TOKEN) + meta["clone_url"] = url + logger.info("INFO:agent_repo:resolved owner-repo %s", meta.get("full_name")) + return meta, "owner-repo" + url = add_auth_to_url(ensure_git_suffix(f"{GITEA_URL}/{owner}/{repo}.git"), GITEA_HTTP_USER, GITEA_HTTP_TOKEN) + rd = {"full_name": f"{owner}/{repo}", "clone_url": url} + logger.info("INFO:agent_repo:resolved owner-repo-fallback %s", rd.get("full_name")) + return rd, "owner-repo-fallback" + found = gitea_search_repos(hint, limit=5) + if found: + found[0]["clone_url"] = add_auth_to_url(ensure_git_suffix(found[0].get("clone_url") or ""), GITEA_HTTP_USER, GITEA_HTTP_TOKEN) + logger.info("INFO:agent_repo:resolved search %s", found[0].get("full_name")) + return found[0], "search" + logger.error("ERROR:agent_repo:repo not found for hint=%s", hint) + return None, "not-found" + +def extract_context_hints_from_prompt(user_goal: str) -> dict: + """ + Haal dynamisch hints uit de prompt: + - tag_names: HTML/XML tags die genoemd zijn (, <h1>, <button> ...) + - attr_names: genoemde HTML attributen (value, placeholder, title, aria-label ...) + """ + tag_names = set() + for m in re.finditer(r"<\s*([A-Za-z][A-Za-z0-9:_-]*)\s*>", user_goal): + tag_names.add(m.group(1).lower()) + attr_names = set() + for m in re.finditer(r"\b(value|placeholder|title|aria-[a-z-]+|alt|label)\b", user_goal, flags=re.IGNORECASE): + attr_names.add(m.group(1).lower()) + return {"tag_names": tag_names, "attr_names": attr_names} + +def gitea_list_all_repos(limit: int = AGENT_DISCOVER_MAX_REPOS) -> List[dict]: + """ + Haal zo veel mogelijk repos op die de token kan zien. + Probeert /repos/search paginated; valt terug op lege lijst bij problemen. + """ + out = [] + page = 1 + per_page = 50 + try: + while len(out) < limit: + r = requests.get( + f"{GITEA_API}/repos/search", + params={"q":"", "limit": per_page, "page": page}, + headers=_gitea_headers(), timeout=10 + ) + r.raise_for_status() + data = r.json() + items = data.get("data") if isinstance(data, dict) else (data if isinstance(data, list) else []) + if not items: + break + out.extend(items) + if len(items) < per_page: + break + page += 1 + except Exception as e: + logger.warning("WARN:agent_repo:gitea_list_all_repos failed: %s", e) + # Normaliseer velden + norm = [] + for it in out[:limit]: + full = it.get("full_name") or (f"{it.get('owner',{}).get('login','')}/{it.get('name','')}".strip("/")) + clone = it.get("clone_url") or (f"{GITEA_URL}/{full}.git" if full else None) + default_branch = it.get("default_branch") or "main" + norm.append({ + "full_name": full, + "name": it.get("name"), + "owner": (it.get("owner") or {}).get("login"), + "description": it.get("description") or "", + "language": it.get("language") or "", + "topics": it.get("topics") or [], + "default_branch": default_branch, + "clone_url": add_auth_to_url(ensure_git_suffix(clone), GITEA_HTTP_USER, GITEA_HTTP_TOKEN) if clone else None, + }) + return [n for n in norm if n.get("full_name")] + +def gitea_fetch_readme(owner: str, repo: str, ref: str = "main") -> str: + """Probeer README via API; dek meerdere varianten af; decode base64 als nodig.""" + candidates = [ + f"{GITEA_API}/repos/{owner}/{repo}/readme", + f"{GITEA_API}/repos/{owner}/{repo}/contents/README.md", + f"{GITEA_API}/repos/{owner}/{repo}/contents/README", + f"{GITEA_API}/repos/{owner}/{repo}/contents/readme.md", + ] + for url in candidates: + try: + r = requests.get(url, params={"ref": ref}, headers=_gitea_headers(), timeout=10) + if r.status_code == 404: + continue + r.raise_for_status() + js = r.json() + # content in base64? + if isinstance(js, dict) and "content" in js: + try: + return base64.b64decode(js["content"]).decode("utf-8", errors="ignore") + except Exception: + pass + # sommige Gitea versies hebben 'download_url' + dl = js.get("download_url") if isinstance(js, dict) else None + if dl: + rr = requests.get(dl, timeout=10, headers=_gitea_headers()) + rr.raise_for_status() + return rr.text + except Exception: + continue + return "" + +def gitea_repo_exists(owner: str, name: str) -> bool: + """Controleer via de Gitea API of owner/name bestaat (en je token rechten heeft).""" + try: + r = requests.get(f"{GITEA_API}/repos/{owner}/{name}", + headers=_gitea_headers(), timeout=5) + return r.status_code == 200 + except Exception: + return False + +def owner_repo_from_url(url: str) -> tuple[str|None, str|None]: + """ + Probeer owner/repo uit een http(s) .git URL te halen. + Voorbeeld: http://host:3080/owner/repo.git -> ('owner', 'repo') + """ + try: + from urllib.parse import urlparse + p = urlparse(url) + parts = [x for x in (p.path or "").split("/") if x] + if len(parts) >= 2: + repo = parts[-1] + if repo.endswith(".git"): + repo = repo[:-4] + owner = parts[-2] + return owner, repo + except Exception: + pass + return None, None + + +# === Repo-catalogus indexeren in Meili (optioneel) en Chroma === +def meili_get_index(name: str): + cli = get_meili() + if not cli: return None + try: + return cli.index(name) + except Exception: + try: + return cli.create_index(uid=name, options={"primaryKey":"id"}) + except Exception: + return None + +def meili_catalog_upsert(docs: List[dict]): + idx = meili_get_index(REPO_CATALOG_MEILI_INDEX) + if not idx or not docs: return + try: + idx.add_documents(docs) + try: + idx.update_searchable_attributes(["full_name","name","description","readme","topics","language"]) + idx.update_filterable_attributes(["full_name","owner","language","topics"]) + except Exception: + pass + except Exception as e: + logger.warning("WARN:agent_repo:meili_catalog_upsert: %s", e) + +def meili_catalog_search(q: str, limit: int = 10) -> List[dict]: + idx = meili_get_index(REPO_CATALOG_MEILI_INDEX) + if not idx: return [] + try: + res = idx.search(q, {"limit": limit}) + # Gebruik ALTIJD de injectie: + #res = meili_search_fn( + # q, + # limit=limit, + # filter={"repo_full": st.owner_repo, "branch": st.branch_base} + #) + return res.get("hits", []) + except Exception as e: + logger.warning("WARN:agent_repo:meili_catalog_search: %s", e) + return [] + +def chroma_catalog_upsert(docs: List[dict]): + """Indexeer/upsérten van repo-catalogus in Chroma met de GEÏNJECTEERDE embedding_function. (bij HTTP-mode embeddings client-side meezenden.)""" + try: + if not docs or _get_chroma_collection is None: + return + col = _get_chroma_collection("repo_catalog") # naam wordt in app.py gesuffixed met __<slug>__v<ver> + ids = [d["id"] for d in docs] + texts = [d["doc"] for d in docs] + metas = [d["meta"] for d in docs] + # schoon oud weg, best-effort + try: + col.delete(ids=ids) + except Exception: + pass + if _embed_documents: + embs = _embed_documents(texts) + col.add(ids=ids, documents=texts, embeddings=embs, metadatas=metas) + else: + col.add(ids=ids, documents=texts, metadatas=metas) + except Exception as e: + logger.warning("WARN:agent_repo:chroma_catalog_upsert: %s", e) + +def chroma_catalog_search(q: str, n: int = 8) -> List[dict]: + try: + if _get_chroma_collection is None or _embed_query_fn is None: + return [] + col = _get_chroma_collection("repo_catalog") + q_emb = _embed_query_fn(q) + res = col.query(query_embeddings=[q_emb], n_results=n, include=["documents","metadatas","distances"]) + docs = (res.get("documents") or [[]])[0] + metas = (res.get("metadatas") or [[]])[0] + dists = (res.get("distances") or [[]])[0] + out = [] + for doc, meta, dist in zip(docs, metas, dists): + if isinstance(meta, dict): + sim = 1.0 / (1.0 + float(dist or 0.0)) # simpele afstand→similarity + out.append({"full_name": meta.get("full_name"), "score": float(sim), "preview": doc}) + return out + except Exception as e: + logger.warning("WARN:agent_repo:chroma_catalog_search: %s", e) + return [] + + +# === Documenten maken voor catalogus === +def build_repo_catalog_doc(meta: dict, readme: str) -> dict: + full_name = meta.get("full_name","") + name = meta.get("name","") + desc = meta.get("description","") + lang = meta.get("language","") + topics = " ".join(meta.get("topics") or []) + preview = (readme or "")[:2000] + doc = ( + f"{full_name}\n" + f"{name}\n" + f"{desc}\n" + f"language: {lang}\n" + f"topics: {topics}\n" + f"README:\n{preview}" + ) + return { + "id": f"repo:{full_name}", + "doc": doc, + "meta": { + "full_name": full_name, + "name": name, + "description": desc, + "language": lang, + "topics": topics, + } + } + +# === Heuristische (lexicale) score als fallback === +def lexical_repo_score(q: str, meta: dict, readme: str) -> float: + qtokens = re.findall(r"[A-Za-z0-9_]{2,}", q.lower()) + text = " ".join([ + meta.get("full_name",""), + meta.get("name",""), + meta.get("description",""), + " ".join(meta.get("topics") or []), + (readme or "")[:4000], + ]).lower() + if not qtokens or not text: + return 0.0 + score = 0 + for t in set(qtokens): + score += text.count(t) + # kleine bonus als 'mainten', 'admin', 'viewer' etc tegelijk voorkomen in naam + name = (meta.get("name") or "").lower() + for t in set(qtokens): + if t in name: + score += 2 + return float(score) + +# === LLM-rerank voor repo's (hergebruik van je bestaande reranker) === +async def llm_rerank_repos(user_goal: str, candidates: List[dict], topk: int = 5) -> List[dict]: + if not candidates: + return [] + pack = [] + for i, c in enumerate(candidates[:12], 1): + pv = c.get("preview","")[:700] + pack.append(f"{i}. REPO: {c['full_name']}\nDESC: {c.get('description','')}\nPREVIEW:\n{pv}") + prompt = ( + "Rangschik onderstaande repositories op geschiktheid voor het doel. " + "Geef een geldige JSON-array met objecten: {\"full_name\":\"...\",\"score\":0-100}.\n\n" + "DOEL:\n" + user_goal + "\n\nCANDIDATES:\n" + "\n\n".join(pack) + ) + try: + resp = await _llm_call( + [{"role":"system","content":"Alleen geldige JSON."}, + {"role":"user","content":prompt}], + stream=False, temperature=0.0, top_p=0.9, max_tokens=600 + ) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","") + arr = safe_json_loads(raw) + if not isinstance(arr, list): + return candidates[:topk] + smap = {} + for d in (arr or []): + if not isinstance(d, dict): + continue + fn = d.get("full_name"); sc = d.get("score") + try: + if isinstance(fn, str): + smap[fn] = float(sc) + except Exception: + continue + + #smap = {d.get("full_name"): float(d.get("score",0)) for d in arr if isinstance(d, dict) and "full_name" in d} + resc = [] + for c in candidates: + resc.append({**c, "score": smap.get(c["full_name"], 0.0)/100.0}) + resc.sort(key=lambda x: x.get("score",0.0), reverse=True) + return resc[:topk] + except Exception as e: + logger.warning("WARN:agent_repo:llm_rerank_repos failed: %s", e) + return candidates[:topk] + +# --- Intent/goal refine --- +async def llm_refine_goal(raw_goal: str) -> tuple[str, List[str], float]: + """ + Laat LLM een compacte, concrete 'refined_goal' maken + max 2 verduidelijkingsvragen. + Retourneert (refined_goal, clarifying_questions, confidence(0..1)). + """ + SYSTEM = "Geef uitsluitend geldige JSON; geen uitleg." + USER = ( + "Vat de bedoeling van deze opdracht ultra-kort en concreet samen als 'refined_goal'. " + "Als er kritieke onduidelijkheden zijn: geef max 2 korte 'clarifying_questions'. " + "Geef ook 'confidence' (0..1). JSON:\n" + "{ \"refined_goal\": \"...\", \"clarifying_questions\": [\"...\"], \"confidence\": 0.0 }\n\n" + f"RAW_GOAL:\n{raw_goal}" + ) + try: + resp = await _llm_call( + [{"role":"system","content":SYSTEM},{"role":"user","content":USER}], + stream=False, temperature=0.0, top_p=0.9, max_tokens=300 + ) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","") + js = safe_json_loads(raw) or {} + rg = (js.get("refined_goal") or "").strip() or raw_goal + qs = [q.strip() for q in (js.get("clarifying_questions") or []) if isinstance(q, str) and q.strip()][:2] + cf = float(js.get("confidence", 0.0) or 0.0) + cf = max(0.0, min(1.0, cf)) + return rg, qs, cf + except Exception as e: + logger.warning("WARN:agent_repo:llm_refine_goal failed: %s", e) + return raw_goal, [], 0.0 + + +# === Discovery pipeline === +async def discover_candidate_repos(user_goal: str) -> List[dict]: + """Zoek een passende repo puur op basis van de vraag (zonder hint).""" + #repos = gitea_list_all_repos(limit=AGENT_DISCOVER_MAX_REPOS) + repos = await run_io_blocking(gitea_list_all_repos, limit=AGENT_DISCOVER_MAX_REPOS) + if not repos: + return [] + + # Concurrerende fetch (beperk paralleliteit licht voor stabiliteit) + sem = asyncio.Semaphore(int(os.getenv("AGENT_DISCOVER_README_CONCURRENCY", "8"))) + + async def _fetch_readme(m): + async with sem: + return await run_io_blocking( + gitea_fetch_readme, + m.get("owner",""), m.get("name",""), m.get("default_branch","main") + ) + + readmes = await asyncio.gather(*[_fetch_readme(m) for m in repos], return_exceptions=True) + + + # Verzamel README's (kort) en bouw catalogus docs + docs_meili = [] + docs_chroma = [] + cands = [] + for i, m in enumerate(repos): + #readme = gitea_fetch_readme(m.get("owner",""), m.get("name",""), m.get("default_branch","main")) + readme = "" if isinstance(readmes[i], Exception) else (readmes[i] or "") + doc = build_repo_catalog_doc(m, readme) + docs_chroma.append(doc) + docs_meili.append({ + "id": m["full_name"], + "full_name": m["full_name"], + "name": m.get("name",""), + "owner": m.get("owner",""), + "description": m.get("description",""), + "language": m.get("language",""), + "topics": " ".join(m.get("topics") or []), + "readme": (readme or "")[:5000], + }) + cands.append({ + "full_name": m["full_name"], + "description": m.get("description",""), + "clone_url": m.get("clone_url"), + "preview": (readme or "")[:1200], + "base_score": 0.0, # vullen we zo + }) + + # Indexeer catalogus (best effort) + if MEILI_URL: + meili_catalog_upsert(docs_meili) + chroma_catalog_upsert(docs_chroma) + + # Multi-query expand + queries = await llm_expand_queries(user_goal, extract_quotes(user_goal), extract_word_hints(user_goal), k=5) + + # Heuristische score + Meili/Chroma boosts + score_map: Dict[str, float] = {c["full_name"]: 0.0 for c in cands} + for q in queries: + # lexicale score + for i, m in enumerate(repos): + score_map[m["full_name"]] += 0.2 * lexical_repo_score(q, m, (docs_meili[i].get("readme") if i < len(docs_meili) else "")) + + # Meili boost + if MEILI_URL: + hits = meili_catalog_search(q, limit=10) + for h in hits: + fn = h.get("full_name") + if fn in score_map: + score_map[fn] += 2.0 + + # Chroma boost + chroma_hits = chroma_catalog_search(q, n=6) + for h in chroma_hits: + fn = h.get("full_name") + if fn in score_map: + score_map[fn] += 1.2 + + # Combineer in kandidaten + for c in cands: + c["score"] = score_map.get(c["full_name"], 0.0) + + # Snelle preselectie + cands.sort(key=lambda x: x["score"], reverse=True) + pre = cands[:8] + + # LLM rerank met uitleg-score + top = await llm_rerank_repos(user_goal, pre, topk=5) + return top + + +# ---------- Chroma collection naam ---------- +def sanitize_collection_name(s: str) -> str: + s = re.sub(r"[^A-Za-z0-9._-]+", "-", s).strip("-")[:128] + return s or "code_docs" + +def repo_collection_name(owner_repo: str | None, branch: str) -> str: + return sanitize_collection_name(f"code_docs-{owner_repo or 'repo'}-{branch}") + +def _get_session_id(messages: List[dict], request) -> str: + for m in messages: + if m.get("role") == "system" and str(m.get("content","")).startswith("session:"): + return str(m["content"]).split("session:",1)[1].strip() + key = (messages[0].get("content","") + "|" + _client_ip(request)).encode("utf-8", errors="ignore") + return hashlib.sha256(key).hexdigest()[:16] + +# ---------- Files & filters ---------- +def allowed_file(p: Path) -> bool: + lo = p.name.lower() + return any(lo.endswith(ext) for ext in ALLOWED_EXTS) + +def list_repo_files(repo_root: Path) -> List[str]: + # lichte TTL-cache om herhaalde rglob/IO te beperken (sneller bij multi-queries) + ttl = float(os.getenv("AGENT_LIST_CACHE_TTL", "20")) + key = str(repo_root.resolve()) + now = time.time() + if key in _LIST_FILES_CACHE: + ts, cached = _LIST_FILES_CACHE[key] + if now - ts <= ttl: + return list(cached) + + files: List[str] = [] + for p in repo_root.rglob("*"): + if p.is_dir(): continue + if any(part in _PROFILE_EXCLUDE_DIRS for part in p.parts): continue + try: + if p.stat().st_size > 2_000_000: continue + except Exception: + continue + if not allowed_file(p): continue + files.append(str(p.relative_to(repo_root))) + _LIST_FILES_CACHE[key] = (now, files) + return files + +# ---------- Query parsing ---------- +def extract_quotes(text: str) -> List[str]: + if not text: return [] + t = (text or "").replace("“","\"").replace("”","\"").replace("’","'").strip() + return re.findall(r"['\"]([^'\"]{2,})['\"]", t) + + +def extract_word_hints(text: str) -> List[str]: + if not text: return [] + words = set(re.findall(r"[A-Za-z_][A-Za-z0-9_]{1,}", text)) + blacklist = {"de","het","een","and","the","voor","naar","op","in","of","to","is","are","van","met","die","dat"} + return [w for w in words if w.lower() not in blacklist] + +# ---------- SAFE JSON loader ---------- +def safe_json_loads(s: str): + if not s: return None + t = s.strip() + if t.startswith("```"): + t = re.sub(r"^```(?:json)?", "", t.strip(), count=1).strip() + if t.endswith("```"): t = t[:-3].strip() + try: + return json.loads(t) + except Exception: + return None + +# ---------- Meilisearch (optioneel) ---------- +_meili_client = None +def get_meili(): + global _meili_client + if _meili_client is not None: + return _meili_client + if not MEILI_URL: + return None + try: + from meilisearch import Client + _meili_client = Client(MEILI_URL, MEILI_KEY or None) + return _meili_client + except Exception as e: + logger.warning("WARN:agent_repo:Meilisearch not available: %s", e) + return None + +def meili_index_name(owner_repo: Optional[str], branch: str) -> str: + base = sanitize_collection_name((owner_repo or "repo") + "-" + branch) + return sanitize_collection_name(f"{MEILI_INDEX_PREFIX}-{base}") + +# --- Slimmere, taalbewuste chunker --- + +_LANG_BY_EXT = { + ".php": "php", ".blade.php": "blade", ".js": "js", ".ts": "ts", + ".jsx": "js", ".tsx": "ts", ".py": "py", ".go": "go", + ".rb": "rb", ".java": "java", ".cs": "cs", + ".css": "css", ".scss": "css", + ".html": "html", ".htm": "html", ".md": "md", + ".yml": "yaml", ".yaml": "yaml", ".toml": "toml", ".ini": "ini", + ".json": "json", +} + +def _detect_lang_from_path(path: str) -> str: + lo = path.lower() + for ext, lang in _LANG_BY_EXT.items(): + if lo.endswith(ext): + return lang + return "txt" + +def _find_breakpoints(text: str, lang: str) -> list[int]: + """ + Retourneer lijst met 'mooie' breekposities (char indices) om chunks te knippen. + We houden het conservatief; false-positives zijn OK (we kiezen toch dichtstbij). + """ + bps = set() + # Altijd: lege-regelblokken en paragrafen + for m in re.finditer(r"\n\s*\n\s*", text): + bps.add(m.end()) + + if lang in ("php", "js", "ts", "java", "cs", "go", "rb", "py"): + # Functie/klasse boundaries + pats = [ + r"\n\s*(class|interface|trait)\s+[A-Za-z_][A-Za-z0-9_]*\b", + r"\n\s*(public|private|protected|static|\s)*\s*function\b", + r"\n\s*def\s+[A-Za-z_][A-Za-z0-9_]*\s*\(", # py + r"\n\s*func\s+[A-Za-z_][A-Za-z0-9_]*\s*\(", # go + r"\n\s*[A-Za-z0-9_<>\[\]]+\s+[A-Za-z_][A-Za-z0-9_]*\s*\(", # java/cs method-ish + r"\n\}", # sluitende brace op kolom 0 → goed eind + ] + for p in pats: + for m in re.finditer(p, text): + bps.add(m.start()) + + if lang == "blade": + for p in [r"\n\s*@section\b", r"\n\s*@endsection\b", r"\n\s*@if\b", r"\n\s*@endif\b", r"\n\s*<\w"]: + for m in re.finditer(p, text, flags=re.I): + bps.add(m.start()) + + if lang in ("html", "css"): + for p in [r"\n\s*<\w", r"\n\s*</\w", r"\n\s*}\s*\n"]: + for m in re.finditer(p, text): + bps.add(m.start()) + + if lang in ("md",): + for p in [r"\n#+\s", r"\n\-{3,}\n", r"\n\*\s", r"\n\d+\.\s"]: + for m in re.finditer(p, text): + bps.add(m.start()) + + if lang in ("yaml", "toml", "ini"): + # secties/keys aan kolom 0 + for m in re.finditer(r"\n[A-Za-z0-9_\-]+\s*[:=]", text): + bps.add(m.start()) + + # JSON: split op object/array boundaries (conservatief: op { of [ aan kolom 0-ish) + if lang == "json": + for m in re.finditer(r"\n\s*[\{\[]\s*\n", text): + bps.add(m.start()) + + # Altijd: regelgrenzen + for m in re.finditer(r"\n", text): + bps.add(m.start()+1) + + # sorteer & filter binnen range + out = sorted([bp for bp in bps if 0 < bp < len(text)]) + return out + +def smart_chunk_text(text: str, path_hint: str, target_chars: int = 1800, + hard_max: int = 2600, min_chunk: int = 800) -> List[str]: + """ + Chunk op ~target_chars, maar breek op dichtstbijzijnde semantische breakpoint. + - Als geen goed breakpoint: breek op dichtstbijzijnde newline. + - Adaptieve overlap: 200 bij nette break, 350 bij 'ruwe' break. + """ + if not text: + return [] + lang = _detect_lang_from_path(path_hint or "") + bps = _find_breakpoints(text, lang) + if not bps: + # fallback: vaste stappen met overlap + chunks = [] + i, n = 0, len(text) + step = max(min_chunk, target_chars - 300) + while i < n: + j = min(n, i + target_chars) + chunks.append(text[i:j]) + i = min(n, i + step) + return chunks + + chunks = [] + i, n = 0, len(text) + while i < n: + # streef naar i+target_chars, maar zoek 'mooie' breakpoints tussen [i+min_chunk, i+hard_max] + ideal = i + target_chars + lo = i + min_chunk + hi = min(n, i + hard_max) + # kandidaten = bps in range + candidates = [bp for bp in bps if lo <= bp <= hi] + if not candidates: + # geen mooie; breek grof op ideal of n + j = min(n, ideal) + chunk = text[i:j] + chunks.append(chunk) + # grotere overlap (ruw) + i = j - 350 if j - 350 > i else j + continue + # kies dichtstbij het ideaal + j = min(candidates, key=lambda bp: abs(bp - ideal)) + chunk = text[i:j] + chunks.append(chunk) + # nette break → kleine overlap + i = j - 200 if j - 200 > i else j + + # schoon lege/te-kleine staarten + out = [c for c in chunks if c and c.strip()] + return out + + +def meili_index_repo(repo_root: Path, owner_repo: Optional[str], branch: str): + cli = get_meili() + if not cli: return + idx_name = meili_index_name(owner_repo, branch) + try: + idx = cli.index(idx_name) + except Exception: + idx = cli.create_index(uid=idx_name, options={"primaryKey":"id"}) + docs = [] + bm25_docs = [] # ← verzamel hier voor BM25 + count = 0 + for rel in list_repo_files(repo_root): + p = repo_root / rel + try: + txt = _read_text_file(p) or "" + except Exception: + continue + for ci, chunk in enumerate(smart_chunk_text(txt, rel, target_chars=int(os.getenv("CHUNK_TARGET_CHARS","1800")),hard_max=int(os.getenv("CHUNK_HARD_MAX","2600")),min_chunk=int(os.getenv("CHUNK_MIN_CHARS","800")))): + doc_id = f"{owner_repo}:{branch}:{rel}:{ci}" + item = {"id": doc_id, "path": rel, "repo": owner_repo, "branch": branch, "content": chunk} + docs.append(item) + bm25_docs.append(item) # ← ook hier + count += 1 + if len(docs) >= 1000: + idx.add_documents(docs); docs.clear() + if docs: + idx.add_documents(docs) + try: + idx.update_searchable_attributes(["content","path","repo","branch"]) + idx.update_filterable_attributes(["repo","branch","path"]) + except Exception: + pass + logger.info("INFO:agent_repo:meili indexed ~%d chunks into %s", count, idx_name) + + # Lokale BM25 cache opbouwen uit bm25_docs (niet uit docs dat intussen leeg kan zijn) + try: + if BM25Okapi and bm25_docs: + toks = [re.findall(r"[A-Za-z0-9_]+", d["content"].lower()) for d in bm25_docs] + bm = BM25Okapi(toks) if toks else None + if bm: + _BM25_CACHE[idx_name] = {"bm25": bm, "docs": bm25_docs} + except Exception as e: + logger.warning("WARN:agent_repo:bm25 build failed: %s", e) + + +def meili_search(owner_repo: Optional[str], branch: str, q: str, limit: int = 10) -> List[dict]: + cli = get_meili() + if not cli: return [] + try: + idx = cli.index(meili_index_name(owner_repo, branch)) + res = idx.search(q, {"limit": limit}) + # Gebruik ALTIJD de injectie: + #res = meili_search_fn( + # q, + # limit=limit, + # filter={"repo_full": st.owner_repo, "branch": st.branch_base} + #) + return res.get("hits", []) + except Exception as e: + logger.warning("WARN:agent_repo:meili_search failed: %s", e) + return [] + +# ---------- BM25 fallback ---------- +_BM25_CACHE: Dict[str, dict] = {} + +# module-scope +_BM25_BY_REPO: dict[str, tuple[BM25Okapi, list[dict]]] = {} +def _tok(s: str) -> list[str]: + return re.findall(r"[A-Za-z0-9_]+", s.lower()) + +# --- Lightweight symbol index (in-memory, per repo collection) --- +_SYMBOL_INDEX: dict[str, dict[str, dict[str, int]]] = {} +# structuur: { collection_name: { symbol_lower: { path: count } } } + + +def bm25_index_name(owner_repo: Optional[str], branch: str) -> str: + return meili_index_name(owner_repo, branch) # dezelfde naam, andere cache + +def bm25_build_index(repo_root: Path, owner_repo: Optional[str], branch: str): + # hergebruik meili_index_repo’s docs-opbouw om dubbele IO te vermijden? Hier snel en lokaal: + if not BM25Okapi: + return + idx_name = bm25_index_name(owner_repo, branch) + docs = [] + for rel in list_repo_files(repo_root): + p = repo_root / rel + try: + txt = _read_text_file(p) or "" + except Exception: + continue + for ci, chunk in enumerate(smart_chunk_text(txt, rel, + target_chars=int(os.getenv("CHUNK_TARGET_CHARS","1800")), + hard_max=int(os.getenv("CHUNK_HARD_MAX","2600")), + min_chunk=int(os.getenv("CHUNK_MIN_CHARS","800")))): + docs.append({"id": f"{owner_repo}:{branch}:{rel}:{ci}", "path": rel, "repo": owner_repo, "branch": branch, "content": chunk}) + toks = [re.findall(r"[A-Za-z0-9_]+", d["content"].lower()) for d in docs] + if toks: + _BM25_CACHE[idx_name] = {"bm25": BM25Okapi(toks), "docs": docs} + +def bm25_search(owner_repo: Optional[str], branch: str, q: str, limit: int = 10) -> List[dict]: + idx = _BM25_CACHE.get(bm25_index_name(owner_repo, branch)) + if not idx: + return [] + bm = idx.get("bm25"); docs = idx.get("docs") or [] + if not bm: + return [] + toks = re.findall(r"[A-Za-z0-9_]+", (q or "").lower()) + if not toks: + return [] + scores = bm.get_scores(toks) + order = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:limit] + return [docs[i] for i in order] + +def _extract_symbols_generic(path: str, text: str) -> list[str]: + """ + Ultra-simpele symbol scraper (taal-agnostisch): + - class/interface/trait namen + - function foo(...), Foo::bar, "Controller@method" + - Laravel: ->name('route.name') + - React-ish: function Foo(...) { return ( ... ) }, export default function Foo(...) + - Blade-ish: @section('...'), @component('...'), <x-foo-bar> + - Basename van file als pseudo-symbool + """ + if not text: + return [] + syms = set() + + for m in re.finditer(r"\b(class|interface|trait)\s+([A-Za-z_][A-Za-z0-9_\\]*)", text): + syms.add(m.group(2)) + + for m in re.finditer(r"\bfunction\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", text): + syms.add(m.group(1)) + + for m in re.finditer(r"([A-Za-z_][A-Za-z0-9_\\]*)::([A-Za-z_][A-Za-z0-9_]*)", text): + syms.add(m.group(1) + "::" + m.group(2)) + + for m in re.finditer(r"['\"]([A-Za-z0-9_\\]+)@([A-Za-z0-9_]+)['\"]", text): + syms.add(m.group(1) + "@" + m.group(2)) + + for m in re.finditer(r"->\s*name\s*\(\s*['\"]([^'\"]+)['\"]\s*\)", text): + syms.add(m.group(1)) + + for m in re.finditer(r"\bfunction\s+([A-Z][A-Za-z0-9_]*)\s*\(", text): + syms.add(m.group(1)) + + for m in re.finditer(r"export\s+default\s+function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", text): + syms.add(m.group(1)) + + for m in re.finditer(r"@\s*(section|component|slot)\s*\(\s*['\"]([^'\"]+)['\"]\s*\)", text): + syms.add(m.group(2)) + for m in re.finditer(r"<\s*x-([a-z0-9\-:]+)", text, flags=re.IGNORECASE): + syms.add("x-" + m.group(1).lower()) + + base = os.path.basename(path) + if base: + syms.add(base) + + return list(syms) + +def _symbol_index_name(owner_repo: Optional[str], branch: str) -> str: + return repo_collection_name(owner_repo, branch) + +def symbol_index_repo(repo_root: Path, owner_repo: Optional[str], branch: str): + """Best-effort: bouw/refresh symbol index voor dit repo/branch.""" + try: + coll = _symbol_index_name(owner_repo, branch) + store: dict[str, dict[str, int]] = {} + for rel in list_repo_files(repo_root): + p = repo_root / rel + try: + if p.stat().st_size > 500_000: + continue + txt = _read_text_file(p) or "" + except Exception: + continue + for s in _extract_symbols_generic(rel, txt): + k = s.strip().lower() + if not k: + continue + bucket = store.setdefault(k, {}) + bucket[rel] = bucket.get(rel, 0) + 1 + _SYMBOL_INDEX[coll] = store + except Exception as e: + logger.warning("WARN:agent_repo:symbol_index_repo: %s", e) + +def symbol_search(owner_repo: Optional[str], branch: str, q: str, limit: int = 10) -> list[tuple[str, int]]: + """Eenvoudige symbol-zoeker -> [(path, score)].""" + coll = _symbol_index_name(owner_repo, branch) + idx = _SYMBOL_INDEX.get(coll) or {} + if not idx or not q: + return [] + quoted = re.findall(r"['\"]([^'\"]{2,})['\"]", q) + words = re.findall(r"[A-Za-z0-9_:\\.\-]{2,}", q) + seen = set(); tokens = [] + for t in quoted + words: + tl = t.lower() + if tl not in seen: + seen.add(tl); tokens.append(tl) + + scores: dict[str, int] = {} + # exact + for t in tokens[:12]: + if t in idx: + for path, c in idx[t].items(): + scores[path] = scores.get(path, 0) + 3 * c + # zachte substring + for sym, paths in idx.items(): + if t in sym: + for path, c in paths.items(): + scores[path] = scores.get(path, 0) + 1 + + return sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit] + + +# ---------- Signal-first scan ---------- +def glob_match(rel: str, patterns: List[str]) -> bool: + for pat in patterns or []: + if fnmatch.fnmatch(rel, pat): + return True + return False + +def scan_with_signals(repo_root: Path, files: List[str], sig: dict, phrase_boosts: List[str], hint_boosts: List[str], limit: int = 20) -> List[Tuple[str,int,dict]]: + file_globs = sig.get("file_globs") or [] + must = [s.lower() for s in (sig.get("must_substrings") or [])] + maybe = [s.lower() for s in (sig.get("maybe_substrings") or [])] + regexes = sig.get("regexes") or [] + path_hints = [s.lower() for s in (sig.get("path_hints") or [])] + exclude_dirs = set(sig.get("exclude_dirs") or []) + + maybe = list(set(maybe + [p.lower() for p in phrase_boosts]))[:20] + path_hints = list(set(path_hints + [h.lower() for h in hint_boosts]))[:20] + + scored: List[Tuple[str,int,dict]] = [] + for rel in files: + if any(part in exclude_dirs for part in Path(rel).parts): continue + if file_globs and not glob_match(rel, file_globs): continue + score = 0 + meta = {"must_hits":0,"maybe_hits":0,"regex_hits":0,"path_hits":0,"phrase_hits":0} + rel_lo = rel.lower() + for h in path_hints: + if h and h in rel_lo: meta["path_hits"] += 1; score += 1 + try: + txt = _read_text_file(repo_root / rel) or "" + except Exception: + continue + txt_lo = txt.lower() + if any(m and (m not in txt_lo) for m in must): + continue + meta["must_hits"] = len([m for m in must if m and m in txt_lo]); score += 3*meta["must_hits"] + meta["maybe_hits"] = len([m for m in maybe if m and m in txt_lo]); score += meta["maybe_hits"] + for rp in regexes: + try: + if re.search(rp, txt, flags=re.IGNORECASE|re.DOTALL): + meta["regex_hits"] += 1; score += 2 + except re.error: + pass + phrase_hits = 0 + for ph in phrase_boosts: + if ph and ph.lower() in txt_lo: + phrase_hits += 1 + if phrase_hits: + meta["phrase_hits"] = phrase_hits + score += 2*phrase_hits + if score > 0: + scored.append((rel, score, meta)) + scored.sort(key=lambda x: x[1], reverse=True) + return scored[:limit] + +# ---------- Simple keyword fallback ---------- +def simple_keyword_search(repo_root: Path, files: List[str], query: str, limit: int = 8) -> List[Tuple[str,int]]: + toks = set(re.findall(r"[A-Za-z0-9_]{2,}", (query or "").lower())) + scores: List[Tuple[str,int]] = [] + for rel in files: + score = 0 + lo = rel.lower() + for t in toks: + if t in lo: score += 1 + if score == 0: + try: + txt = _read_text_file(Path(repo_root) / rel) or "" + txt_lo = txt.lower() + score += sum(txt_lo.count(t) for t in toks) + except Exception: + pass + if score > 0: scores.append((rel, score)) + scores.sort(key=lambda x: x[1], reverse=True) + return scores[:limit] + +# ---------- Expliciete paden ---------- + + +def best_path_by_basename(all_files: List[str], hint: str) -> str | None: + base = os.path.basename(hint) + if not base: return None + hint_tokens = set(re.findall(r"[A-Za-z0-9_]+", hint.lower())) + scored = [] + for rel in all_files: + if os.path.basename(rel).lower() == base.lower(): + score = 1 + lo = rel.lower() + for t in hint_tokens: + if t in lo: score += 1 + scored.append((rel, score)) + if not scored: return None + scored.sort(key=lambda x: x[1], reverse=True) + return scored[0][0] + +# ---------- Hybrid RAG ---------- +def _append_ctx_preview(answer: str, chunks: list[dict], limit: int = 12) -> str: + paths = [] + for h in chunks: + meta = h.get("metadata") or {} + p = meta.get("path"); + if p and p not in paths: paths.append(p) + if not paths: return answer + head = paths[:limit] + return answer + "\n\n--- context (paths) ---\n" + "\n".join(f"- {p}" for p in head) + +async def smart_rag_answer(messages: list[dict], *, n_ctx: int = 8, + owner_repo: Optional[str] = None, + branch: Optional[str] = None, + collection_name: Optional[str] = None, + add_preview: bool = True) -> str: + # 1) intent + spec = await enrich_intent(_llm_call, messages) + task = (spec.get("task") or "").strip() + if not task: + return "Geen vraag gedetecteerd." + + # 2) queries + variants = await expand_queries(_llm_call, task, k=3) + + # 3) hybrid retrieve (let op: gebruik dezelfde collectie als index; 'code_docs' wordt in app.py al versied via _collection_versioned) + # resolve collection: expliciet > (owner_repo,branch) > default + coll = collection_name or (repo_collection_name(owner_repo, branch or AGENT_DEFAULT_BRANCH) if owner_repo else "code_docs") + all_hits = [] + for q in variants: + hits = await hybrid_retrieve( + _rag_query_internal, + q, + n_results=n_ctx, + per_query_k=max(30, n_ctx * 6), + alpha=0.6, + # expliciet doorgeven om zeker de juiste (versie-gesufficede) collectie te raken: + collection_name=coll, + ) + all_hits.extend(hits) + + # dedup op path + chunk_index + seen = set() + uniq = [] + for h in sorted(all_hits, key=lambda x: x.get("score", 0), reverse=True): + meta = h.get("metadata") or {} + key = (meta.get("path"), meta.get("chunk_index")) + if key in seen: + continue + seen.add(key) + uniq.append(h) + if len(uniq) >= n_ctx: + break + + # 4) context + ctx, top = assemble_context(uniq, max_chars=int(os.getenv("REPO_AGENT_CONTEXT_CHARS","640000"))) + if not ctx: + return "Geen context gevonden." + + # 5) laat LLM antwoorden + sys = "Beantwoord concreet en kort. Citeer relevante paths. Als iets onzeker is: zeg dat." + usr = f"Vraag: {task}\n\n--- CONTEXT ---\n{ctx}" + resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.2, top_p=0.9, max_tokens=700 + ) + ans = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + return _append_ctx_preview(ans, uniq) if (add_preview and os.getenv("REPO_AGENT_PREVIEW","1") not in ("0","false")) else ans + + + + + + +async def llm_expand_queries(user_goal: str, quotes: List[str], hints: List[str], k: int = 5, extra_seeds: Optional[List[str]] = None) -> List[str]: # already defined above + # (duplicate name kept intentionally — Python allows redef; using the latest one) + + seed = [] + if quotes: seed += quotes + if hints: seed += hints[:6] + if extra_seeds: seed += extra_seeds[:6] + seed = list(dict.fromkeys(seed))[:8] + prompt = ( + f"Maak {k} alternatieve zoekqueries (kort, divers). Mix NL/EN, synoniemen, veldnamen." + " Alleen geldige JSON-array met strings.\n" + f"Doel:\n{user_goal}\n\nHints:\n" + ", ".join(seed) + ) + try: + resp = await _llm_call( + [{"role":"system","content":"Alleen geldige JSON, geen uitleg."}, + {"role":"user","content":prompt}], + stream=False, temperature=0.3, top_p=0.9, max_tokens=400 + ) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","") + arr = safe_json_loads(raw) + base = [user_goal] + if isinstance(arr, list): + base += [s for s in arr if isinstance(s, str) and s.strip()] + out = [] + for q in base: + qn = re.sub(r"\s+", " ", q.strip()) + if qn and qn not in out: out.append(qn) + return out[:1+k] + except Exception as e: + logger.warning("WARN:agent_repo:llm_expand_queries failed: %s", e) + return [user_goal] + +def get_file_preview(repo_root: Path, rel: str, terms: List[str], window: int = 180) -> str: + try: + txt = _read_text_file(repo_root / rel) or "" + except Exception: + return "" + if not txt: return "" + if not terms: return txt[:window*2] + lo = txt.lower() + for t in terms: + i = lo.find(t.lower()) + if i >= 0: + a = max(0, i - window); b = min(len(txt), i + len(t) + window) + return txt[a:b] + return txt[:window*2] + +async def llm_rerank_candidates(user_goal: str, candidates: List[dict], topk: int = 8) -> List[dict]: + if not candidates: return [] + pack = [] + for i, c in enumerate(candidates[:20], 1): + pv = c.get("preview","")[:600] + pth = c["path"] + base = os.path.basename(pth) + dr = os.path.dirname(pth) + pack.append(f"{i}. PATH: {pth}\nDIR: {dr}\nBASENAME: {base}\nPREVIEW:\n{pv}") + + prompt = ( + "Rangschik de onderstaande codefragmenten op relevantie om het doel te behalen. " + "Geef een JSON-array met objecten: {\"path\":\"...\",\"score\":0-100}." + "\n\nDOEL:\n" + user_goal + "\n\nFRAGMENTEN:\n" + "\n\n".join(pack) + ) + try: + resp = await _llm_call( + [{"role":"system","content":"Alleen geldige JSON zonder uitleg."}, + {"role":"user","content":prompt}], + stream=False, temperature=0.0, top_p=0.9, max_tokens=600 + ) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","") + arr = safe_json_loads(raw) + if not isinstance(arr, list): + return candidates[:topk] + score_map = {d.get("path"): float(d.get("score",0)) for d in arr if isinstance(d, dict) and "path" in d} + rescored = [] + for c in candidates: + rescored.append({**c, "score": score_map.get(c["path"], 0.0)}) + rescored.sort(key=lambda x: x.get("score",0.0), reverse=True) + return rescored[:topk] + except Exception as e: + logger.warning("WARN:agent_repo:llm_rerank_candidates failed: %s", e) + return candidates[:topk] + +def _rrf_fuse_paths(*ordered_lists: List[str], k: int = int(os.getenv("RRF_K","60"))) -> List[str]: + """ + Neem meerdere geordende padlijsten (beste eerst) en geef een RRF-fusie. + """ + acc = defaultdict(float) + for lst in ordered_lists: + for i, p in enumerate(lst): + acc[p] += 1.0 / (k + i + 1) + # path prior + def _prior(p: str) -> float: + return ( + (0.35 if p.lower().startswith("routes/") else 0.0) + + (0.30 if p.lower().startswith("app/http/controllers/") else 0.0) + + (0.25 if p.lower().startswith("resources/views/") or p.lower().endswith(".blade.php") else 0.0) + + (0.12 if p.lower().startswith(("src/","app/","lib/","pages/","components/")) else 0.0) + + (0.05 if p.lower().endswith((".php",".ts",".tsx",".js",".jsx",".py",".go",".rb",".java",".cs",".vue",".html",".md")) else 0.0) - + (0.10 if ("/tests/" in p.lower() or p.lower().startswith(("tests/","test/"))) else 0.0) - + (0.10 if p.lower().endswith((".lock",".map",".min.js",".min.css")) else 0.0) + ) + for p in list(acc.keys()): + acc[p] += float(os.getenv("RRF_PATH_PRIOR_WEIGHT","0.25")) * _prior(p) + return [p for p,_ in sorted(acc.items(), key=lambda t: t[1], reverse=True)] + +async def hybrid_rag_select_paths(repo_root: Path, + owner_repo: Optional[str], + branch: str, + user_goal: str, + all_files: List[str], + max_out: int = 8) -> List[str]: + quotes = extract_quotes(user_goal) + hints = extract_word_hints(user_goal) + # signals + sig_messages = [ + {"role":"system","content":"Produceer alleen geldige JSON zonder uitleg."}, + {"role":"user","content":( + "Bedenk een compacte zoekstrategie als JSON om relevante bestanden te vinden (globs/must/maybe/regex/path_hints/excludes). Wijziging:\n" + + user_goal + )} + ] + try: + resp = await _llm_call(sig_messages, stream=False, temperature=0.1, top_p=0.9, max_tokens=384) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","").strip() + sig = safe_json_loads(raw) or {} + except Exception as e: + logger.warning("WARN:agent_repo:signals LLM failed: %s", e) + sig = {} + # Tweepassig: eerst lenient (recall), dan strict (precision) + sig_lenient = dict(sig or {}) + sig_lenient["must_substrings"] = [] + sig_lenient["regexes"] = [] + scan_hits_lenient = scan_with_signals( + repo_root, all_files, sig_lenient, + phrase_boosts=quotes, hint_boosts=hints, limit=24 + ) + scan_hits_strict = scan_with_signals( + repo_root, all_files, sig, + phrase_boosts=quotes, hint_boosts=hints, limit=20 + ) + # combineer met voorkeur voor strict + seen_paths_local = set() + prepicked = [] + for rel, _sc, _m in scan_hits_strict + scan_hits_lenient: + if rel not in seen_paths_local: + seen_paths_local.add(rel); prepicked.append(rel) + + # --- NIEUW: expliciete pad-hints uit de user prompt voorrang geven --- + try: + explicit = extract_explicit_paths(user_goal) + except Exception: + explicit = [] + explicit_resolved: List[str] = [] + for ep in explicit: + if ep in all_files: + explicit_resolved.append(ep) + else: + bp = best_path_by_basename(all_files, ep) + if bp: explicit_resolved.append(bp) + # plaats expliciete paden vooraan met dedupe + for ep in reversed(explicit_resolved): + if ep not in seen_paths_local: + prepicked.insert(0, ep); seen_paths_local.add(ep) + + # lichte stack-seeds + seeds = [] + if (repo_root / "artisan").exists() or (repo_root / "composer.json").exists(): + seeds += ["Route::get", "Controller", "blade", "resources/views", "routes/web.php", "app/Http/Controllers"] + if (repo_root / "package.json").exists(): + seeds += ["component", "pages", "src/components", "useState", "useEffect"] + queries = await llm_expand_queries(user_goal, quotes, hints, k=5, extra_seeds=seeds) + + + chroma_paths: List[str] = [] + for q in queries: + try: + rag_res = await _rag_query_internal( + query=q, n_results=RAG_TOPK, + # zoek in de versie-consistente collectie: + collection_name=repo_collection_name(owner_repo, branch), + repo=None, path_contains=None, profile=None + ) + for item in rag_res.get("results", []): + meta = item.get("metadata") or {} + pth = meta.get("path") + if pth and pth in all_files: + chroma_paths.append(pth) + except Exception as e: + logger.warning("WARN:agent_repo:Chroma query failed: %s", e) + + meili_paths: List[str] = [] + if MEILI_URL: + for q in queries: + hits = meili_search(owner_repo, branch, q, limit=RAG_TOPK) + for h in hits: + p = h.get("path") + if p and p in all_files: + meili_paths.append(p) + else: + # BM25 fallback wanneer Meili uit staat + # zorg dat er een (eenmalige) index is + try: + if bm25_index_name(owner_repo, branch) not in _BM25_CACHE: + bm25_build_index(repo_root, owner_repo, branch) + except Exception: + pass + for q in queries: + hits = bm25_search(owner_repo, branch, q, limit=RAG_TOPK) + for h in hits: + p = h.get("path") + if p and p in all_files: + meili_paths.append(p) + + + try: + laravel_picks = laravel_signal_candidates(repo_root, user_goal, all_files, max_out=6) + except Exception: + laravel_picks = [] + + + # --- NIEUW: Symbol-driven candidates --- + sym_hits = symbol_search(owner_repo, branch, user_goal, limit=12) + sym_paths = [p for p, _sc in sym_hits if p in all_files] + + # RRF-fusie van bronnen + Laravel-picks + #fused = _rrf_fuse_paths(prepicked, chroma_paths, meili_paths, laravel_picks) + + # --- Optionele RRF-fusie van kanalen (standaard UIT) --- + use_rrf = str(os.getenv("RRF_ENABLE", "1")).lower() in ("1","true","yes") + if use_rrf: + k = int(os.getenv("RRF_K", "30")) + # eenvoudige gewichten per kanaal (pas aan via env) + w_signals = float(os.getenv("RRF_W_SIGNALS", "1.0")) + w_chroma = float(os.getenv("RRF_W_CHROMA", "1.0")) + w_meili = float(os.getenv("RRF_W_MEILI", "0.8")) + w_sym = float(os.getenv("RRF_W_SYMBOLS", "1.3")) + w_lara = float(os.getenv("RRF_W_LARAVEL", "1.2")) + + sources = [ + ("signals", prepicked, w_signals), + ("chroma", chroma_paths, w_chroma), + ("meili", meili_paths, w_meili), + ("symbols", sym_paths, w_sym), + ("laravel", laravel_picks,w_lara), + ] + + rrf_scores: dict[str, float] = {} + seen_any = set() + for _name, paths, w in sources: + for rank, p in enumerate(paths, start=1): + if p not in all_files: + continue + seen_any.add(p) + rrf_scores[p] = rrf_scores.get(p, 0.0) + (w * (1.0 / (k + rank))) + + # kies top op basis van RRF; val terug op union als leeg + fused_paths = [p for p, _ in sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)] + base_pool = fused_paths[: max_out*3] if fused_paths else [] + + # bouw pool (met dedupe) + vul aan met de oude volgorde indien nodig + pool, seen = [], set() + def add(p): + if p not in seen and p in all_files: + seen.add(p); pool.append(p) + + for p in base_pool: add(p) + if len(pool) < max_out: + for lst in (prepicked, chroma_paths, meili_paths, sym_paths, laravel_picks): + for p in lst: + add(p) + else: + # oude (jouw huidige) manier zonder RRF + pool, seen = [], set() + def add(p): + if p not in seen and p in all_files: + seen.add(p); pool.append(p) + for lst in (prepicked, chroma_paths, meili_paths, sym_paths, laravel_picks): + for p in lst: + add(p) + + # LLM-rerank blijft identiek: + cands = [{"path": p, "preview": get_file_preview(repo_root, p, quotes+hints)} for p in pool[:20]] + ranked = await llm_rerank_candidates(user_goal, cands, topk=max_out) + + # symbol-boost (licht) ná LLM-rerank (ongewijzigd) + sym_map = {p: sc for p, sc in sym_hits} + boost = float(os.getenv("SYMBOL_LIGHT_BOOST", "0.15")) + rescored = [] + for c in ranked: + base = float(c.get("score", 0.0)) + s = sym_map.get(c["path"], 0) + adj = base + (boost if s > 0 else 0.0) + rescored.append({**c, "score": adj}) + rescored.sort(key=lambda x: x["score"], reverse=True) + return [c["path"] for c in rescored[:max_out]] + +# ---------- Focus-snippets ---------- +def extract_focus_snippets(text: str, needles: List[str], window: int = 240, max_snippets: int = 3) -> str: + if not text or not needles: return (text[:window*2] if text else "") + lo = text.lower() + hits = [] + for n in needles: + nlo = (n or "").lower() + if not nlo: continue + start = 0 + for _ in range(4): + idx = lo.find(nlo, start) + if idx < 0: break + a = max(0, idx - window) + b = min(len(text), idx + len(nlo) + window) + hits.append(text[a:b]); start = idx + len(nlo) + uniq = [] + for h in hits: + # de-dupe met wederzijdse containment (voorkom overlap/ingebed) + if all((h not in u) and (u not in h) for u in uniq): + uniq.append(h) + if len(uniq) >= max_snippets: break + return "\n----- CONTEXT SPLIT -----\n".join(uniq) if uniq else text[:window*2] + +# ---------- LLM edit-plan ---------- +async def llm_plan_edits_for_file(user_goal: str, rel: str, focus_snippet: str) -> dict | None: + SYSTEM = "Produceer uitsluitend geldige JSON; geen verdere uitleg. Minimaliseer edits; raak zo min mogelijk regels." + # (optioneel) korte tree-hint in de prompt – zet AGENT_TREE_PROMPT=1 om te activeren + # Tree-hint standaard aan: korte mapoverzicht + samenvattingen van nabije files + tree_block = globals().get("_LLM_EDIT_TREE_HINT", "") + tree_hint = os.getenv("AGENT_TREE_PROMPT","1").lower() not in ("0","false") + try: + if tree_hint: + # NB: eenvoudige, lokale context: alleen siblings + map info om tokens te sparen + # (Vereist repo_root hier normaal gesproken; als niet beschikbaar, laat leeg) + if not tree_block: + tree_block = "\n(Tree-overzicht niet beschikbaar in deze context)\n" + except Exception: + pass + USER = ( + "Doel:\n" + user_goal + "\n\n" + + f"Bestand: {rel}\n" + + "Relevante contextfragmenten:\n----- BEGIN SNIPPETS -----\n" + + focus_snippet + "\n----- EIND SNIPPETS -----\n\n" + + ("Korte tree-hint:\n" + tree_block + "\n") + + "JSON schema:\n" + + "{ \"allow_destructive\": false, \"edits\": [\n" + + " {\"type\":\"regex_replace\",\"pattern\":\"...\",\"replacement\":\"...\",\"flags\":\"ims\",\"count\":1,\"explain\":\"...\"},\n" + + " {\"type\":\"string_replace\",\"find\":\"...\",\"replace\":\"...\",\"count\":1,\"explain\":\"...\"},\n" + + " {\"type\":\"insert_after\",\"anchor_regex\":\"...\",\"text\":\"...\",\"occur\":\"first|last\",\"flags\":\"ims\",\"explain\":\"...\"},\n" + + " {\"type\":\"insert_before\",\"anchor_regex\":\"...\",\"text\":\"...\",\"occur\":\"first|last\",\"flags\":\"ims\",\"explain\":\"...\"},\n" + + " {\"type\":\"replace_between_anchors\",\"start_regex\":\"...\",\"end_regex\":\"...\",\"replacement\":\"...\",\"flags\":\"ims\",\"explain\":\"...\"},\n" + + " {\"type\":\"delete_between_anchors\",\"start_regex\":\"...\",\"end_regex\":\"...\",\"keep_anchors\":false,\"flags\":\"ims\",\"explain\":\"...\"},\n" + + " {\"type\":\"conditional_insert\",\"absent_regex\":\"...\",\"anchor_regex\":\"...\",\"text\":\"...\",\"occur\":\"first|last\",\"flags\":\"ims\",\"explain\":\"...\"},\n" + + " {\"type\":\"insert_at_top\",\"text\":\"...\",\"explain\":\"...\"},\n" + + " {\"type\":\"insert_at_bottom\",\"text\":\"...\",\"explain\":\"...\"}\n" + + "]}\n" + + "Maximaal 4 edits. Geef bij elke edit een korte 'explain'." + ) + try: + resp = await _llm_call( + [{"role":"system","content":SYSTEM},{"role":"user","content":USER}], + stream=False, temperature=0.1, top_p=0.9, max_tokens=800 + ) + raw = resp.get("choices",[{}])[0].get("message",{}).get("content","").strip() + plan = safe_json_loads(raw) + if isinstance(plan, dict) and isinstance(plan.get("edits"), list): + return plan + return None + except Exception as e: + logger.warning("WARN:agent_repo:llm_plan_edits_for_file failed for %s: %s", rel, e) + return None + +# ---------- Apply helpers ---------- +def _regex_flags(flag_str: str) -> int: + flags = 0 + if not flag_str: return flags + for ch in flag_str.lower(): + if ch == 'i': flags |= re.IGNORECASE + if ch == 'm': flags |= re.MULTILINE + if ch == 's': flags |= re.DOTALL + return flags + +def apply_edit_plan(original: str, plan: dict) -> tuple[str, int, List[str], bool]: + """ + Returns: (modified, changes_count, explains[], allow_destructive) + """ + if not original or not plan or not isinstance(plan.get("edits"), list): + return original, 0, [], False + txt = original + changes = 0 + explains: List[str] = [] + for ed in plan["edits"]: + try: + et = (ed.get("type") or "").lower() + ex = ed.get("explain") or et + if et == "string_replace": + find = ed.get("find") or ""; rep = ed.get("replace") or "" + cnt = int(ed.get("count") or 0) or 1 + if find: + new = txt.replace(find, rep, cnt) + if new != txt: changes += 1; txt = new; explains.append(f"string_replace: {ex}") + elif et == "regex_replace": + pat = ed.get("pattern") or ""; rep = ed.get("replacement") or "" + flags = _regex_flags(ed.get("flags") or ""); cnt = int(ed.get("count") or 0) or 1 + if pat: + new, n = re.subn(pat, rep, txt, count=cnt, flags=flags) + if n > 0: changes += 1; txt = new; explains.append(f"regex_replace: {ex}") + elif et in ("insert_after","insert_before"): + anchor = ed.get("anchor_regex") or ""; ins = ed.get("text") or "" + occur = (ed.get("occur") or "first").lower(); flags = _regex_flags(ed.get("flags") or "") + if not anchor or not ins: continue + matches = list(re.finditer(anchor, txt, flags)) + if not matches: continue + m = matches[0] if occur != "last" else matches[-1] + pos = m.end() if et == "insert_after" else m.start() + # idempotentie: voeg niet opnieuw in als de tekst al vlakbij staat + win_a, win_b = max(0, pos-200), min(len(txt), pos+200) + if ins in txt[win_a:win_b]: + continue + txt = txt[:pos] + ins + txt[pos:]; changes += 1; explains.append(f"{et}: {ex}") + elif et in ("replace_between_anchors","delete_between_anchors"): + srx = ed.get("start_regex") or ""; erx = ed.get("end_regex") or "" + flags = _regex_flags(ed.get("flags") or ""); keep_anchors = bool(ed.get("keep_anchors")) if et == "delete_between_anchors" else True + repl = ed.get("replacement") or "" + if not srx or not erx: continue + s_matches = list(re.finditer(srx, txt, flags)) + e_matches = list(re.finditer(erx, txt, flags)) + if not s_matches or not e_matches: continue + s0 = s_matches[0] + # Kies de eerste end-anker ná het start-anker + e0 = next((em for em in e_matches if em.start() >= s0.end()), None) + if not e0: continue + a = s0.end(); b = e0.start() + if et == "replace_between_anchors": + txt = txt[:a] + repl + txt[b:]; changes += 1; explains.append(f"replace_between_anchors: {ex}") + else: + if keep_anchors: txt = txt[:a] + txt[b:] + else: txt = txt[:s0.start()] + txt[e0.end():] + changes += 1; explains.append(f"delete_between_anchors: {ex}") + elif et == "conditional_insert": + absent = ed.get("absent_regex") or ""; anchor = ed.get("anchor_regex") or "" + occur = (ed.get("occur") or "first").lower(); ins = ed.get("text") or "" + flags = _regex_flags(ed.get("flags") or "") + if not anchor or not ins: continue + if absent and re.search(absent, txt, flags): continue + matches = list(re.finditer(anchor, txt, flags)) + if not matches: continue + m = matches[0] if occur != "last" else matches[-1] + pos = m.end() + # idempotentie: lokale window-check + win_a, win_b = max(0, pos-200), min(len(txt), pos+200) + if ins in txt[win_a:win_b]: + continue + txt = txt[:pos] + ins + txt[pos:]; changes += 1; explains.append(f"conditional_insert: {ex}") + elif et == "insert_at_top": + ins = ed.get("text") or "" + if ins: txt = ins + txt; changes += 1; explains.append(f"insert_at_top: {ex}") + elif et == "insert_at_bottom": + ins = ed.get("text") or "" + if ins: txt = txt + ins; changes += 1; explains.append(f"insert_at_bottom: {ex}") + except Exception as e: + logger.warning("WARN:agent_repo:apply_edit_plan step failed: %s", e) + continue + allow_destructive = bool(plan.get("allow_destructive")) + return txt, changes, explains, allow_destructive + +# ==== BEGIN PATCH A: destructiviteit op diff-basis + drempel via env ==== +# Veilige default voor AGENT_DESTRUCTIVE_RATIO (voorkom NameError als niet gedefinieerd) +try: + AGENT_DESTRUCTIVE_RATIO +except NameError: + AGENT_DESTRUCTIVE_RATIO = float(os.getenv("AGENT_DESTRUCTIVE_RATIO", "0.45")) + +def _deletion_ratio(original: str, modified: str) -> float: + """Schat welk deel van de originele regels als deletions wegvalt.""" + ol = original.splitlines() + ml = modified.splitlines() + if not ol: + return 0.0 + # ndiff: regels met prefix '- ' tellen we als deletions + dels = 0 + for line in difflib.ndiff(ol, ml): + if line.startswith("- "): + dels += 1 + return dels / max(1, len(ol)) + +def is_destructive(original: str, modified: str, allow_destructive: bool) -> bool: + """Blokkeer alleen als er aantoonbaar veel deletions zijn.""" + if allow_destructive: + return False + # heel kleine files: laat door, we willen niet te streng zijn + if len(original.splitlines()) < 6: + return False + ratio = _deletion_ratio(original, modified) + return ratio > AGENT_DESTRUCTIVE_RATIO + +# ==== END PATCH A ==== + +def list_sibling_files(repo_root: Path, rel: str, limit: int = 12) -> List[str]: + d = (repo_root / rel).parent + if not d.exists(): + # directory kan nog niet bestaan; kies dichtstbijzijnde bestaande ouder + d = repo_root / os.path.dirname(rel) + while not d.exists() and d != repo_root: + d = d.parent + outs = [] + if d.exists(): + for p in d.iterdir(): + if p.is_file() and allowed_file(p) and p.stat().st_size < 500_000: + outs.append(str(p.name)) + # stabiele output i.p.v. FS-volgorde + outs.sort(key=str.lower) + return outs[:limit] + + +def read_snippet(p: Path, max_chars: int = 2000) -> str: + try: + t = _read_text_file(p) or "" + return t[:max_chars] + except Exception: + return "" + +async def propose_new_file(repo_root: Path, rel: str, user_goal: str) -> tuple[Optional[str], str]: + """ + Vraag de LLM om een *volledig nieuwe file* te genereren op pad `rel` + met minimale aannames. Geeft (content, reason). + """ + ext = os.path.splitext(rel)[1].lower() + siblings = list_sibling_files(repo_root, rel) + sibling_snippets = [] + for name in siblings[:3]: + snippet = read_snippet(repo_root / os.path.join(os.path.dirname(rel), name), max_chars=1600) + if snippet: + sibling_snippets.append({"name": name, "snippet": snippet[:1600]}) + + SYSTEM = "Je bent een zorgvuldige codegenerator. Lever exact één compleet bestand. Geen extra refactors." + USER = ( + f"Doel (nieuwe file aanmaken):\n{user_goal}\n\n" + f"Bestandspad: {rel}\n" + f"Directory siblings: {', '.join(siblings) if siblings else '(geen)'}\n\n" + "Enkele nabije referenties (indien aanwezig):\n" + + "\n".join([f"--- {s['name']} ---\n{s['snippet']}" for s in sibling_snippets]) + + "\n\nEisen:\n" + "- Maak een minimal-werkende versie van dit bestand die past bij de context hierboven.\n" + "- Raak geen andere paden aan; geen includes naar niet-bestaande bestanden.\n" + "- Gebruik hetzelfde framework/stack als de referenties suggereren (indien duidelijk).\n" + "- Output: alleen de VOLLEDIGE bestandinformatie in één codeblok, niets anders." + ) + try: + resp = await _llm_call( + [{"role":"system","content":SYSTEM},{"role":"user","content":USER}], + stream=False, temperature=0.2, top_p=0.9, max_tokens=2048 + ) + content = _extract_code_block( + resp.get("choices",[{}])[0].get("message",{}).get("content","") + ) or "" + content = content.strip() + if not content: + return None, "LLM gaf geen inhoud terug." + # simpele sanity-limit + if len(content) > 200_000: + content = content[:200_000] + return content, "Nieuw bestand voorgesteld op basis van directory-context en doel." + except Exception as e: + logger.warning("WARN:agent_repo:propose_new_file failed for %s: %s", rel, e) + return None, f"Kon geen nieuwe file genereren: {e}" + + + + +# ---------- Diff helper ---------- +def make_diffs(original: str, modified: str, filename: str, max_lines: int = 200) -> str: + diff = list(difflib.unified_diff( + original.splitlines(keepends=True), + modified.splitlines(keepends=True), + fromfile=f"a/{filename}", + tofile=f"b/{filename}", + lineterm="" + )) + if len(diff) > max_lines: + return "".join(diff[:max_lines]) + "\n... (diff ingekort)" + return "".join(diff) + +def make_new_file_diff(filename: str, content: str, max_lines: int = 400) -> str: + new_lines = content.splitlines(keepends=True) + diff = list(difflib.unified_diff( + [], new_lines, + fromfile="/dev/null", + tofile=f"b/{filename}", + lineterm="" + )) + if len(diff) > max_lines: + return "".join(diff[:max_lines]) + "\n... (diff ingekort)" + return "".join(diff) + +# ---------- Lightweight Laravel Graph helpers ---------- +def _view_name_to_path(repo_root: Path, view_name: str) -> Optional[str]: + """ + 'users.index' -> resources/views/users/index.blade.php (als bestaand) + 'users/index' -> idem. Return relatieve path of None als niet gevonden. + """ + if not view_name: + return None + cand = view_name.replace(".", "/").strip("/ ") + for ext in [".blade.php", ".php"]: + rel = f"resources/views/{cand}{ext}" + if (repo_root / rel).exists(): + return rel + return None + +def _controller_extract_views(text: str, repo_root: Path) -> list[str]: + """ + Zoek 'return view("x.y")' en map naar blade-bestanden. + Ondersteunt ook: View::make('x.y'), Inertia::render('X/Y') -> best effort naar blade. + """ + outs: list[str] = [] + # view('foo.bar') + for m in re.finditer(r"(?:return\s+)?view\s*\(\s*['\"]([^'\"]+)['\"]", text, flags=re.I): + rel = _view_name_to_path(repo_root, m.group(1)) + if rel: + outs.append(rel) + # View::make('foo.bar') + for m in re.finditer(r"View::make\s*\(\s*['\"]([^'\"]+)['\"]", text, flags=re.I): + rel = _view_name_to_path(repo_root, m.group(1)) + if rel: + outs.append(rel) + # Inertia::render('Foo/Bar') -> probeer view pad heuristisch + for m in re.finditer(r"Inertia::render\s*\(\s*['\"]([^'\"]+)['\"]", text, flags=re.I): + rel = _view_name_to_path(repo_root, m.group(1)) + if rel: + outs.append(rel) + # dedupe + seen=set(); uniq=[] + for r in outs: + if r not in seen: + uniq.append(r); seen.add(r) + return uniq + +def _blade_extract_lang_keys(text: str) -> list[str]: + """ + Haal vertaalkeys uit Blade/PHP: __('x.y'), @lang('x.y'), trans('x.y') + """ + keys = [] + for rx in [ + r"__\(\s*['\"]([^'\"]+)['\"]\s*\)", + r"@lang\(\s*['\"]([^'\"]+)['\"]\s*\)", + r"trans\(\s*['\"]([^'\"]+)['\"]\s*\)" + ]: + for m in re.finditer(rx, text): + keys.append(m.group(1)) + # dedupe + seen=set(); out=[] + for k in keys: + if k not in seen: + out.append(k); seen.add(k) + return out + +def _grep_lang_files_for_key(repo_root: Path, key: str, limit: int = 6) -> list[str]: + """ + Zoek in resources/lang/**/*.(json|php) naar KEY. Best-effort, klein limiet. + """ + base = repo_root / "resources/lang" + if not base.exists(): + return [] + hits=[] + try: + for p in base.rglob("*"): + if p.is_dir(): + continue + if not (str(p).endswith(".json") or str(p).endswith(".php")): + continue + if p.stat().st_size > 300_000: + continue + txt = p.read_text(encoding="utf-8", errors="ignore") + if key in txt: + hits.append(str(p.relative_to(repo_root))) + if len(hits) >= limit: + break + except Exception: + pass + return hits + +def _build_laravel_graph(repo_root: Path) -> dict[str, set[str]]: + """ + Maak een lichte ongerichte graaf: + - routes/web.php|api.php ↔ controller-bestanden + - controller ↔ views (via return view(...)) + - view ↔ lang-bestanden (voor keys die in de view voorkomen) + Node-labels = relatieve padnamen; edges zijn ongericht (buren). + """ + g: dict[str, set[str]] = {} + def _add(a: str, b: str): + g.setdefault(a, set()).add(b) + g.setdefault(b, set()).add(a) + + # 1) routes → controllers (reeds beschikbare scanner hergebruiken) + routes = laravel_scan_routes(repo_root) + for r in routes: + rp = r.get("file") or "" + ctrl = r.get("controller") or "" + if not ctrl: + continue + for cpath in _candidate_paths_for_controller(repo_root, ctrl): + _add(rp, cpath) + # 2) controllers → views (parse controller file) + try: + txt = _read_text_file(repo_root / cpath) or "" + except Exception: + txt = "" + for vrel in _controller_extract_views(txt, repo_root): + _add(cpath, vrel) + # 3) views → lang-files (op basis van keys) + try: + vtxt = _read_text_file(repo_root / vrel) or "" + except Exception: + vtxt = "" + for key in _blade_extract_lang_keys(vtxt): + for lrel in _grep_lang_files_for_key(repo_root, key, limit=4): + _add(vrel, lrel) + return g + +def _graph_bfs_boosts(graph: dict[str, set[str]], seeds: list[str], max_depth: int = 3) -> dict[str, tuple[int, str]]: + """ + BFS vanaf seed-nodes. Return: {node: (distance, via)} met via=eerste buur of route. + """ + from collections import deque + dist: dict[str, int] = {} + via: dict[str, str] = {} + q = deque() + for s in seeds: + if s in graph: + dist[s] = 0 + via[s] = s + q.append(s) + while q: + cur = q.popleft() + if dist[cur] >= max_depth: + continue + for nb in graph.get(cur, ()): + if nb not in dist: + dist[nb] = dist[cur] + 1 + via[nb] = cur if via.get(cur) == cur else via.get(cur, cur) + q.append(nb) + return {n: (d, via.get(n, "")) for n, d in dist.items()} + +def _get_graph_cached(repo_root: Path, memo_key: str) -> dict[str, set[str]]: + if os.getenv("AGENT_GRAPH_ENABLE", "1").lower() in ("0", "false"): + return {} + g = _GRAPH_CACHE.get(memo_key) + if g is not None: + return g + try: + g = _build_laravel_graph(repo_root) + except Exception: + g = {} + _GRAPH_CACHE[memo_key] = g + return g + +# ---------- Tree summaries (korte per-file beschrijving) ---------- +def _summarize_file_for_tree(path: Path) -> str: + """ + Heuristische mini-samenvatting (<=160 chars): + - eerste docblock / commentregel / heading + - anders eerste niet-lege regel + """ + try: + txt = path.read_text(encoding="utf-8", errors="ignore") + except Exception: + return "" + head = txt[:1200] + # PHP docblock + m = re.search(r"/\*\*([\s\S]{0,400}?)\*/", head) + if m: + s = re.sub(r"[*\s]+", " ", m.group(1)).strip() + return (s[:160]) + # single-line comments / headings + for rx in [r"^\s*//\s*(.+)$", r"^\s*#\s*(.+)$", r"^\s*<!--\s*(.+?)\s*-->", r"^\s*<h1[^>]*>([^<]+)</h1>", r"^\s*<title[^>]*>([^<]+)"]: + mm = re.search(rx, head, flags=re.M|re.I) + if mm: + return mm.group(1).strip()[:160] + # first non-empty line + for line in head.splitlines(): + ln = line.strip() + if ln: + return ln[:160] + return "" + +def _build_tree_summaries(repo_root: Path, all_files: list[str], max_files: int = 2000) -> dict[str, str]: + out: dict[str, str] = {} + count = 0 + for rel in all_files: + if count >= max_files: + break + p = repo_root / rel + try: + if p.stat().st_size > 200_000: + continue + except Exception: + continue + s = _summarize_file_for_tree(p) + if s: + out[rel] = s + count += 1 + return out + +def _get_tree_cached(repo_root: Path, memo_key: str, all_files: list[str]) -> dict[str, str]: + if os.getenv("AGENT_TREE_ENABLE", "1").lower() in ("0","false"): + return {} + t = _TREE_SUM_CACHE.get(memo_key) + if t is not None: + return t + try: + t = _build_tree_summaries(repo_root, all_files) + except Exception: + t = {} + _TREE_SUM_CACHE[memo_key] = t + return t + +# ---------- Mini tree-hint voor LLM edit-plannen ---------- +def _make_local_tree_hint(repo_root: Path, rel: str, max_siblings: int = 14) -> str: + """ + Bouw een compact overzicht van de map van 'rel' met 10–14 nabije files en korte samenvattingen. + Houd het kort en voorspelbaar voor de LLM. + """ + try: + base_dir = (repo_root / rel).parent + except Exception: + return "" + lines = [] + try: + folder = str(base_dir.relative_to(repo_root)) + except Exception: + folder = base_dir.name + lines.append(f"Map: {folder or '.'}") + + items = [] + try: + for p in sorted(base_dir.iterdir(), key=lambda x: x.name.lower()): + if not p.is_file(): + continue + try: + if not allowed_file(p) or p.stat().st_size > 200_000: + continue + except Exception: + continue + summ = _summarize_file_for_tree(p) + name = p.name + if summ: + items.append(f"- {name}: {summ[:120]}") + else: + items.append(f"- {name}") + if len(items) >= max_siblings: + break + except Exception: + pass + lines.extend(items) + return "\n".join(lines) + +# ---------- Basic syntax guards ---------- +def _write_tmp(content: str, suffix: str) -> Path: + import tempfile + fd, path = tempfile.mkstemp(suffix=suffix) + os.close(fd) + p = Path(path) + p.write_text(content, encoding="utf-8") + return p + +def _php_lint_ok(tmp_path: Path) -> bool: + # disable via AGENT_SYNTAX_GUARD=0 + if os.getenv("AGENT_SYNTAX_GUARD","1").lower() in ("0","false"): + return True + try: + import subprocess + res = subprocess.run(["php","-l",str(tmp_path)], capture_output=True, text=True, timeout=8) + return res.returncode == 0 + except Exception: + return True + +def _blade_balance_ok(text: str) -> bool: + # Zeer conservatieve balans-check voor veelvoorkomende Blade directives + tl = (text or "").lower() + pairs = [("section","endsection"),("if","endif"),("foreach","endforeach"),("isset","endisset"),("php","endphp")] + for a,b in pairs: + if tl.count("@"+a) != tl.count("@"+b): + return False + return True + + +# ---------- Gerichte, veilige literal fallback ---------- +# === PATCH: generieke HTML-scope vervanging === + +def html_scoped_literal_replace(html: str, old: str, new: str, tag_names: set[str]) -> tuple[str, bool, str]: + """ + Probeer 'old' -> 'new' te vervangen, maar ALLEEN binnen de genoemde tags. + Werkt zonder externe libs; gebruikt conservatieve regex (DOTALL). + Retour: (modified, changed, rationale) + """ + if not html or not old or not tag_names: + return html, False, "" + changed = False + rationale = [] + result = html + + for tag in sorted(tag_names): + # ... (greedy genoeg per blok, maar beperkt via DOTALL) + tag_re = re.compile(rf"(<\s*{re.escape(tag)}\b[^>]*>)(.*?)()", + flags=re.IGNORECASE | re.DOTALL) + def _one(m): + nonlocal changed + open_tag, inner, close_tag = m.group(1), m.group(2), m.group(3) + if old in inner: + # maximaal 1 vervanging per tag-blok (conform docstring) + new_inner = inner.replace(old, new, 1) + if new_inner != inner: + changed = True + rationale.append(f"'{old}' vervangen binnen <{tag}> (1x)") + return open_tag + new_inner + close_tag + return m.group(0) + result_new = tag_re.sub(_one, result) + result = result_new + + return result, changed, "; ".join(rationale) if changed else "" + +# === PATCH: veilige, algemene string-literal vervanging === + +def quoted_literal_replace(original: str, old: str, new: str, max_occurrences: int = 2) -> tuple[str, bool, str]: + """ + Vervang 'old' of "old" als string-literal, maximaal 'max_occurrences' keer. + Dit is taalagnostisch en wijzigt geen identifiers, enkel stringwaarden. + Return: (modified, changed, rationale) + """ + if not original or not old: + return original, False, "" + pat = re.compile(rf"(?P['\"])({re.escape(old)})(?P=q)") + cnt = 0 + def _repl(m): + nonlocal cnt + if cnt >= max_occurrences: + return m.group(0) + cnt += 1 + q = m.group("q") + return q + new + q + new_text = pat.sub(_repl, original) + if new_text != original and cnt > 0: + return new_text, True, f"'{old}' → '{new}' als string-literal ({cnt}x, limiet {max_occurrences})" + return original, False, "" + + +# ==== BEGIN PATCH B: per-bestand oud/nieuw bepalen + generieke fallback ==== +def _literal_matches_with_context(src: str, needle: str, window: int = 160): + """Vind alle posities waar 'needle' als literal voorkomt en geef de operator-context terug.""" + escaped = re.escape(needle) + pat = re.compile(r"(?P['\"])(" + escaped + r")(?P=q)") + for m in pat.finditer(src): + a, b = m.span() + before = src[max(0, a - window):a] + op = None + if re.search(r"\?\?\s*$", before): + op = "??" + elif re.search(r"\?\s*[^:\n]{0,120}:\s*$", before): + op = "?:" + elif re.search(r"\|\|\s*$", before): + op = "||" + elif re.search(r"\bor\b\s*$", before, flags=re.IGNORECASE): + op = "or" + yield (a, b, op) + +def deduce_old_new_literals(user_goal: str, original: str) -> tuple[Optional[str], Optional[str], str]: + """ + Kies 'old' als de quoted string uit de prompt die ook in de file staat + én het vaakst in fallback-context (??, ?:, ||, or) voorkomt. + Kies 'new' als een andere quoted string uit de prompt (liefst die níet in de file voorkomt). + Retourneer (old, new, rationale). + """ + quotes = extract_quotes(user_goal) + if not quotes: + return None, None, "Geen quoted strings in prompt gevonden." + # Score candidates for OLD + scores = [] + for q in quotes: + hits = list(_literal_matches_with_context(original, q)) + if hits: + # gewicht: aantal hits + bonus als er operator context is + ctx_hits = sum(1 for _,_,op in hits if op) + score = 2 * ctx_hits + len(hits) + scores.append((q, score, ctx_hits)) + if not scores: + # Geen van de quotes komt in de file voor; dan geen gerichte fallback + return None, None, "Geen van de quotes uit prompt kwam in de file voor." + scores.sort(key=lambda x: (x[1], x[2]), reverse=True) + old = scores[0][0] + + # Kies NEW uit overige quotes: bij voorkeur eentje die niet in de file voorkomt + rest = [q for q in quotes if q != old] + if not rest: + return old, None, f"OLD='{old}' gekozen; geen 'new' gevonden." + prefer = [q for q in rest if q not in original] + new = (prefer[0] if prefer else rest[0]) + + why = f"OLD='{old}' (meeste fallback-contexthits), NEW='{new}'." + return old, new, why + +def targeted_fallback_replace(original: str, old: str, new: str) -> tuple[str, bool, str]: + """ + Vervang uitsluitend de literal OLD als die duidelijk fallback is nabij ??, ?:, || of 'or'. + Retourneer (modified, changed_bool, rationale). + """ + if not original or not old: + return original, False, "" + window = 160 + escaped_old = re.escape(old) + pat = re.compile(r"(?P['\"])(" + escaped_old + r")(?P=q)") + text = original + for m in pat.finditer(text): + q = m.group("q") + a, b = m.span() + before = text[max(0, a - window):a] + op = None + if re.search(r"\?\?\s*$", before): + op = "??" + elif re.search(r"\?\s*[^:\n]{0,120}:\s*$", before): + op = "?:" + elif re.search(r"\|\|\s*$", before): + op = "||" + elif re.search(r"\bor\b\s*$", before, flags=re.IGNORECASE): + op = "or" + if not op: + continue + new_text = text[:a] + q + new + q + text[b:] + reason = f"Gerichte vervanging van fallback-literal nabij operator '{op}'" + return new_text, True, reason + return original, False, "" + +# ==== END PATCH B ==== + +# === Repo-QA: vraag-antwoord over 1 specifieke repository === +_LARAVEL_CREATE_HINTS = { + "verbs": ["create", "store", "new", "aanmaken", "aanmaak", "nieuw", "toevoegen", "add"], + "nouns": ["melding", "incident", "ticket", "aanvraag", "report", "issue", "storingen", "storing"] +} + +def _read_file_safe(p: Path) -> str: + try: + return _read_text_file(p) or "" + except Exception: + return "" + +def laravel_scan_routes(repo_root: Path) -> list[dict]: + out = [] + for rp in ["routes/web.php", "routes/api.php"]: + p = repo_root / rp + if not p.exists(): + continue + txt = _read_file_safe(p) + for m in re.finditer(r"Route::(get|post|put|patch|delete|match)\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*([^)]+)\)", txt, flags=re.I): + verb, uri, target = m.group(1).lower(), m.group(2), m.group(3) + ctrl = None; method = None; name = None + # controller@method + m2 = re.search(r"['\"]([A-Za-z0-9_\\]+)@([A-Za-z0-9_]+)['\"]", target) + if m2: + ctrl, method = m2.group(1), m2.group(2) + else: + # ['Foo\\BarController::class', 'index'] of [Foo\\BarController::class, 'index'] + m2b = re.search(r"\[\s*([A-Za-z0-9_\\]+)::class\s*,\s*['\"]([A-Za-z0-9_]+)['\"]\s*\]", target) + if m2b: + ctrl, method = m2b.group(1), m2b.group(2) + # ->name('...') + tail = txt[m.end(): m.end()+140] + m3 = re.search(r"->\s*name\s*\(\s*['\"]([^'\"]+)['\"]\s*\)", tail) + if m3: name = m3.group(1) + out.append({"file": rp, "verb": verb, "uri": uri, "target": target, "controller": ctrl, "method": method, "name": name}) + # Route::resource + for m in re.finditer(r"Route::resource\s*\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)", txt, flags=re.I): + res, ctrl = m.group(1), m.group(2) + out.append({"file": rp, "verb": "resource", "uri": res, "target": ctrl, "controller": ctrl, "method": None, "name": None}) + return out + +def _candidate_paths_for_controller(repo_root: Path, controller_fqcn: str) -> list[str]: + """ + Probeer Controller-bestand + views te vinden vanuit FQCN zoals App\\Http\\Controllers\\Foo\\BarController. + """ + rels = [] + # controller pad + base = controller_fqcn.replace("\\\\","/").replace("\\","/") + name = base.split("/")[-1] + ctrl_guess = [ + f"app/Http/Controllers/{base}.php", + f"app/Http/Controllers/{name}.php" + ] + for g in ctrl_guess: + if (repo_root / g).exists(): + rels.append(g) + # view dir guesses (resource-achtig) + view_roots = ["resources/views", "resources/views/livewire", "resources/views/components"] + stem = re.sub(r"Controller$", "", name, flags=re.I) + for vr in view_roots: + for hint in [stem, stem.lower()]: + dp = repo_root / f"{vr}/{hint}" + if dp.exists() and dp.is_dir(): + for bp in dp.rglob("*.blade.php"): + if bp.stat().st_size < 500000: + rels.append(str(bp.relative_to(repo_root))) + return list(dict.fromkeys(rels))[:8] + +def laravel_signal_candidates(repo_root: Path, user_goal: str, all_files: list[str], max_out: int = 6) -> list[str]: + """ + Heuristische preselectie voor Laravel 'aanmaken/nieuw' use-cases: + - zoekt in routes naar 'create|store' of semantic hints + - projecteert naar controllers + blade views + """ + # snelle exit als er geen laravel markers zijn + if not (repo_root / "artisan").exists() and not (repo_root / "composer.json").exists(): + return [] + + goal = (user_goal or "").lower() + verbs = _LARAVEL_CREATE_HINTS["verbs"] + nouns = _LARAVEL_CREATE_HINTS["nouns"] + + def _goal_hits(s: str) -> int: + lo = s.lower() + v = sum(1 for w in verbs if w in lo) + n = sum(1 for w in nouns if w in lo) + return v*2 + n # verbs wegen iets zwaarder + + routes = laravel_scan_routes(repo_root) + scored = [] + for r in routes: + base_s = f"{r.get('uri','')} {r.get('name','')} {r.get('controller','') or ''} {r.get('method','') or ''}" + score = _goal_hits(base_s) + # bonus als expliciet create/store + if (r.get("method") or "").lower() in ("create","store"): + score += 3 + if r.get("verb") == "resource": + # resource → heeft impliciet create/store routes + score += 2 + if score > 0: + scored.append((score, r)) + + if not scored: + return [] + + scored.sort(key=lambda x: x[0], reverse=True) + picks: list[str] = [] + for _score, r in scored[:8]: + # controller + vermoedelijke views + if r.get("controller"): + for rel in _candidate_paths_for_controller(repo_root, r["controller"]): + if rel in all_files and rel not in picks: + picks.append(rel) + # view guess als padnaam “melding*create.blade.php” + for rel in all_files: + name = os.path.basename(rel).lower() + dirname = os.path.dirname(rel).lower() + if any(n in dirname for n in nouns) and ("create" in name or "form" in name): + if rel not in picks: + picks.append(rel) + if len(picks) >= max_out: + break + return picks[:max_out] + + +def _detect_stack_summary(repo_root: Path) -> dict: + """Heuristieken: taal/vermoed framework, routes/migraties/DB hints.""" + summary = { + "languages": {}, + "framework": [], + "entrypoints": [], + "routes": [], + "db": [], + "notable_dirs": [], + } + # talen tellen (globaal) + ext_map = {} + for rel in list_repo_files(repo_root): + ext = os.path.splitext(rel)[1].lower() + ext_map[ext] = ext_map.get(ext, 0) + 1 + summary["languages"] = dict(sorted(ext_map.items(), key=lambda x: x[1], reverse=True)[:8]) + + # PHP/Laravel hints + comp = repo_root / "composer.json" + if comp.exists(): + try: + import json as _json + js = _json.loads(comp.read_text(encoding="utf-8", errors="ignore")) + req = (js.get("require") or {}) | (js.get("require-dev") or {}) + if any("laravel/framework" in k for k in req.keys()): + summary["framework"].append("Laravel") + except Exception: + pass + if (repo_root / "artisan").exists(): + summary["entrypoints"].append("artisan (Laravel CLI)") + # Node hints + pkg = repo_root / "package.json" + if pkg.exists(): + try: + import json as _json + js = _json.loads(pkg.read_text(encoding="utf-8", errors="ignore")) + deps = list((js.get("dependencies") or {}).keys()) + list((js.get("devDependencies") or {}).keys()) + if any(x in deps for x in ["next", "nuxt", "react", "vue", "vite"]): + summary["framework"].append("Node/Frontend") + except Exception: + pass + + # Routes (Laravel) + for rp in ["routes/web.php", "routes/api.php"]: + p = repo_root / rp + if p.exists(): + txt = _read_text_file(p) or "" + for m in re.finditer(r"Route::(get|post|put|patch|delete)\s*\(\s*['\"]([^'\"]+)['\"]", txt): + summary["routes"].append(f"{rp}: {m.group(1).upper()} {m.group(2)}") + # DB hints (Laravel/vanilla PHP) + for rp in ["config/database.php", ".env", ".env.example", "app/config/database.php"]: + p = repo_root / rp + if p.exists(): + txt = _read_text_file(p) or "" + if "DB_" in txt or "mysql" in txt or "sqlite" in txt or "pgsql" in txt: + snippet = txt[:800].replace("\r"," ") + summary["db"].append(f"{rp}: {snippet}") + # Notable dirs + for d in ["app", "app/admin", "app/public", "public", "resources", "storage", "config", "routes", "src", "docs", "tests"]: + if (repo_root / d).exists(): + summary["notable_dirs"].append(d) + return summary + +def _format_stack_summary_text(s: dict) -> str: + lines = [] + if s.get("framework"): + lines.append("Frameworks (heuristiek): " + ", ".join(sorted(set(s["framework"])))) + if s.get("languages"): + langs = ", ".join([f"{k or '∅'}×{v}" for k,v in s["languages"].items()]) + lines.append("Talen (bestandext): " + langs) + if s.get("notable_dirs"): + lines.append("Mappen: " + ", ".join(s["notable_dirs"])) + if s.get("entrypoints"): + lines.append("Entrypoints: " + ", ".join(s["entrypoints"])) + if s.get("routes"): + sample = "; ".join(s["routes"][:8]) + lines.append("Routes (sample): " + sample) + if s.get("db"): + # toon alleen paden, geen volledige secrets + lines.append("DB-config aanwezig in: " + ", ".join([d.split(":")[0] for d in s["db"]])) + return "\n".join(lines) + +def _collect_repo_context(repo_root: Path, owner_repo: Optional[str], branch: str, question: str, n_ctx: int = 8) -> list[dict]: + """Kies relevante paden + snippets via hybrid RAG/keywords, voor QA.""" + # Deze sync helper is bewust niet geïmplementeerd om misbruik te voorkomen. + # Gebruik altijd de async-variant: _collect_repo_context_async(...) + raise NotImplementedError("_collect_repo_context is niet beschikbaar; gebruik _collect_repo_context_async") + all_files = list_repo_files(repo_root) + # explicit paths uit vraag + picked: List[str] = [] + for pth in extract_explicit_paths(question): + if pth in all_files and pth not in picked: + picked.append(pth) + else: + best = best_path_by_basename(all_files, pth) + if best and best not in picked: picked.append(best) + # hybrid rag + loop = asyncio.get_event_loop() + # NB: call hybrag via run_until_complete buiten async? we zitten al in async in hoofdhandler; hier helper sync → laat caller het async deel doen + return [] # placeholder; deze helper niet direct gebruiken buiten async + +async def _collect_repo_context_async(repo_root: Path, owner_repo: Optional[str], branch: str, question: str, n_ctx: int = 8) -> list[dict]: + all_files = list_repo_files(repo_root) + picked: List[str] = [] + for pth in extract_explicit_paths(question): + if pth in all_files and pth not in picked: + picked.append(pth) + else: + best = best_path_by_basename(all_files, pth) + if best and best not in picked: picked.append(best) + # DB-vragen: seed eerst met bekende DB-artefacten zodat recall direct goed is + def _db_seed_paths() -> list[str]: + prefer: list[str] = [] + # 1) directe, bekende locaties + for rel in [ + ".env", ".env.example", "config/database.php", "config/database.yml", + "database/database.sqlite" + ]: + if (repo_root / rel).exists() and rel in all_files: + prefer.append(rel) + # 2) migrations / seeders / modellen + for rel in all_files: + lo = rel.lower() + if lo.startswith("database/migrations/") or lo.startswith("database/seeders/"): + prefer.append(rel) + elif lo.startswith(("app/models/", "app/model/", "app/Models/")) and lo.endswith(".php"): + prefer.append(rel) + elif lo.endswith(".sql"): + prefer.append(rel) + # 3) ruwe heuristiek: bestanden met Schema::, DB::, select/insert/update + hits = [] + for rel in all_files: + try: + txt = _read_text_file(repo_root / rel) or "" + except Exception: + continue + tlo = txt.lower() + if any(x in tlo for x in ["schema::create(", "schema::table(", "db::table(", "db::select(", "select ", "insert into ", "create table "]): + hits.append(rel) + # dedupe en cap + seen = set(); out = [] + for rel in prefer + hits: + if rel not in seen: + seen.add(rel); out.append(rel) + if len(out) >= n_ctx: + break + return out + + if _db_intent(question): + for p in _db_seed_paths(): + if p in all_files and p not in picked: + picked.append(p) + + hybrid = await hybrid_rag_select_paths(repo_root, owner_repo, branch, question, all_files, max_out=n_ctx) + + for p in hybrid: + if p not in picked: picked.append(p) + # keyword fallback als nodig + if len(picked) < n_ctx: + for rel, _s in simple_keyword_search(repo_root, all_files, question, limit=n_ctx): + if rel not in picked: picked.append(rel) + # maak snippets + quotes = extract_quotes(question) + hints = extract_word_hints(question) + out = [] + for rel in picked[:n_ctx]: + txt = _read_text_file(repo_root / rel) or "" + snippet = extract_focus_snippets(txt, (quotes + hints)[:6], window=320, max_snippets=2) + out.append({"path": rel, "snippet": snippet}) + return out + +def _trim_text_to_tokens(text: str, max_tokens: int, tok_len=approx_token_count) -> str: + if tok_len(text) <= max_tokens: + return text + # ruwe char-slice obv 4 chars/token + max_chars = max(200, max_tokens * 4) + return text[:max_chars] + +def _jaccard_tokens(a: str, b: str) -> float: + ta = set(re.findall(r"[A-Za-z0-9_]+", (a or "").lower())) + tb = set(re.findall(r"[A-Za-z0-9_]+", (b or "").lower())) + if not ta or not tb: + return 0.0 + return len(ta & tb) / max(1, len(ta | tb)) + +def _db_intent(text: str) -> bool: + """Detecteer of de vraag over DB-verbindingen/schema/queries gaat.""" + t = (text or "").lower() + keys = [ + "database", "sql", "microsoft sql", "ms sql", "mssql", "sql server", + "schema", "tabel", "tabellen", "migratie", "migrations", + "query", "queries", "select", "insert", "update", "delete", + "db_", "connection string", "dsn", "driver", "host", "poort", "poortnummer", + "database.php", ".env" + ] + return any(k in t for k in keys) + + +def _prepare_contexts_under_budget( + contexts: List[dict], + question: str, + stack_summary_text: str, + *, + budget_tokens: int = int(os.getenv("AGENT_QA_CTX_BUDGET_TOKENS", "7000")), + tok_len=approx_token_count +) -> List[dict]: + """ + Slimme budgetverdeler: + - dedup & near-dedup + - novelty-gewicht t.o.v. reeds gekozen snippets + - adaptieve toekenningsstrategie met min/max per snippet + """ + if not contexts: + return contexts + + # Tunables (mil de default iets conservatiever): + MIN_PER = int(os.getenv("QA_MIN_PER_SNIPPET", "180")) # hard min + MAX_PER = int(os.getenv("QA_MAX_PER_SNIPPET", "900")) # hard max + KEEP_TOP = int(os.getenv("QA_KEEP_TOP_K", "8")) # cap op #snippets + NOVELTY_THRESH = float(os.getenv("QA_NOVELTY_DROP", "0.25")) # onder deze novelty laten we vallen + DEDUP_THRESH = float(os.getenv("QA_DEDUP_JACCARD", "0.85")) # zeer hoge overlap => drop + + # 0) cap aantal snippets alvast (caller leverde al gerankt) + contexts = contexts[:KEEP_TOP] + + # 1) brute dedup op pad + near-dup op tekst (Jaccard) + unique: List[dict] = [] + seen_paths = set() + for c in contexts: + p = c.get("path","") + s = str(c.get("snippet","")) + if p in seen_paths: + continue + # near-dup check tegen al gekozen snippets + is_dup = False + for u in unique: + if _jaccard_tokens(u["snippet"], s) >= DEDUP_THRESH: + is_dup = True + break + if not is_dup: + unique.append({"path": p, "snippet": s}) + seen_paths.add(p) + contexts = unique + + if not contexts: + return contexts + + # Overhead raming zoals voorheen (headers + vraag + stack) + header = ( + "Beantwoord de vraag over deze codebase. Wees concreet, kort, en noem bronnen (padnamen).\n" + "Als zekerheid laag is, stel max 2 verduidelijkingsvragen.\n\n" + f"VRAAG:\n{question}\n\n" + f"REPO SAMENVATTING:\n{stack_summary_text or '(geen)'}\n\n" + "RELEVANTE FRAGMENTEN:\n" + ) + frag_headers = "\n\n".join([f"{i+1}) PATH: {c['path']}\nFRAGMENT:\n" for i, c in enumerate(contexts)]) + overhead_tokens = tok_len(header) + tok_len(frag_headers) + 200 + + # Beschikbaar voor echte snippet-inhoud + remain = max(300, budget_tokens - overhead_tokens) + n = len(contexts) + + # 2) Schat "relevance proxy" = overlap tussen vraag en snippet + def rel(sn: str) -> float: + return _jaccard_tokens(question, sn) + + # 3) Greedy novelty: per snippet extra score voor info die nog niet gedekt is + chosen_text = "" # cumulatieve "coverage" + scores = [] + for i, c in enumerate(contexts): + s = c["snippet"] + r = rel(s) + # novelty = 1 - overlap met reeds gekozen tekst + nov = 1.0 - _jaccard_tokens(chosen_text, s) if chosen_text else 1.0 + # filter extreem lage novelty: helpt ruis te schrappen + if nov < NOVELTY_THRESH and i > 0: + # Markeer als zwak; we geven ‘m een heel lage score (kan later afvallen) + scores.append((i, r * 0.05, nov)) + else: + # na 3 snippets weeg novelty zwaarder + if i >= 3: + scores.append((i, r * (0.35 + 0.65 * nov), nov)) + else: + scores.append((i, r * (0.5 + 0.5 * nov), nov)) + # update coverage grof: voeg tokens toe (beperkt) om drift te vermijden + if tok_len(chosen_text) < 4000: + chosen_text += "\n" + s[:1200] + + # 4) Als totaal-minima al boven budget → kap staart + total_min = n * MIN_PER + if total_min > remain: + # Sorteer op score aflopend, en hou zoveel als past met MIN_PER + ranked_idx = sorted(range(n), key=lambda i: scores[i][1], reverse=True) + keep_idx = ranked_idx[: max(1, remain // MIN_PER)] + contexts = [contexts[i] for i in keep_idx] + scores = [scores[i] for i in keep_idx] + n = len(keep_idx) + + # 5) Verdeel budget: iedereen MIN_PER, rest proportioneel op score; cap op MAX_PER + base = n * MIN_PER + extra = max(0, remain - base) + # normaliseer score-gewichten + raw = [max(0.0, sc) for (_i, sc, _nov) in scores] + ssum = sum(raw) or 1.0 + weights = [x / ssum for x in raw] + + alloc = [MIN_PER + int(extra * w) for w in weights] + # enforce MAX_PER; redistribueer overschot grofweg + overshoot = 0 + for i in range(n): + if alloc[i] > MAX_PER: + overshoot += alloc[i] - MAX_PER + alloc[i] = MAX_PER + if overshoot > 0: + # verdeel overschot naar anderen die nog onder MAX_PER zitten + holes = [i for i in range(n) if alloc[i] < MAX_PER] + if holes: + plus = overshoot // len(holes) + for i in holes: + alloc[i] = min(MAX_PER, alloc[i] + plus) + + # 6) Trim snippet-tekst op toegekend budget + trimmed = [] + for i, c in enumerate(contexts): + sn = str(c.get("snippet","")) + sn = _trim_text_to_tokens(sn, alloc[i], tok_len) + trimmed.append({"path": c["path"], "snippet": sn}) + return trimmed + + +async def _llm_qa_answer(question: str, stack_summary_text: str, contexts: List[dict]) -> str: + """ + Laat de LLM een bondig antwoord formuleren met bronverwijzingen. + - Antwoord in NL + - Noem paden als bronnen + - Stel max 2 verduidelijkingsvragen als informatie ontbreekt + """ + # --- NIEUW: trim contexts onder tokenbudget --- + contexts = _prepare_contexts_under_budget( + contexts, question, stack_summary_text, + budget_tokens=int(os.getenv("AGENT_QA_CTX_BUDGET_TOKENS", "7000")), + tok_len=approx_token_count + ) + + ctx_blocks = [] + for i, c in enumerate(contexts, 1): + ctx_blocks.append(f"{i}) PATH: {c['path']}\nFRAGMENT:\n{c['snippet'][:1200]}") # laat 1200 char-cap staan; _prepare_contexts_ kapt al eerder af + USER = ( + "Beantwoord de vraag over deze codebase. Wees concreet, kort, en noem bronnen (padnamen).\n" + "Als zekerheid laag is, stel max 2 verduidelijkingsvragen.\n\n" + f"VRAAG:\n{question}\n\n" + "REPO SAMENVATTING:\n" + (stack_summary_text or "(geen)") + "\n\n" + "RELEVANTE FRAGMENTEN:\n" + ("\n\n".join(ctx_blocks) if ctx_blocks else "(geen)") + "\n\n" + "FORMAT:\n" + "- Antwoord (kort en feitelijk)\n" + "- Bronnen: lijst van paden die je gebruikt hebt\n" + "- (optioneel) Vervolgvragen als iets onduidelijk is\n" + ) + resp = await _llm_call( + [{"role":"system","content":"Je bent een zeer precieze, nuchtere code-assistent. Antwoord in het Nederlands."}, + {"role":"user","content": USER}], + stream=False, temperature=0.2, top_p=0.9, max_tokens=900 + ) + return resp.get("choices",[{}])[0].get("message",{}).get("content","").strip() + +# heuristics: iets kleinere chunks voor Laravel/Blade/Routes, anders iets groter +def _chunk_params_for_repo(root: Path) -> tuple[int,int]: + # simpele stack detectie: + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + if is_laravel: + return int(os.getenv("CHUNK_CHARS_LARAVEL","1800")), int(os.getenv("CHUNK_OVERLAP_LARAVEL","300")) + return int(os.getenv("CHUNK_CHARS_DEFAULT","2600")), int(os.getenv("CHUNK_OVERLAP_DEFAULT","350")) + + +# ---------- QA repo agent ---------- +async def repo_qa_answer(repo_hint: str, question: str, branch: str = "main", n_ctx: int = 8) -> str: + """ + High-level QA over een specifieke repo: + - resolve + clone/update + - (re)index RAG collectie + - stack summary + - context ophalen + - LLM antwoord met bronnen + """ + meta, _reason = resolve_repo(repo_hint) + if not meta: + # Als hint owner/repo is: meteen bestaan-check + if re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", repo_hint): + owner, name = repo_hint.split("/", 1) + if not gitea_repo_exists(owner, name): + return f"Repo `{repo_hint}` niet gevonden of geen rechten. Controleer naam/URL/token." + return f"Kon repo niet vinden voor hint: {repo_hint}" + + repo_url = meta.get("clone_url") or repo_hint + owner_repo = meta.get("full_name") + + # clone/checkout + try: + async with _CLONE_SEMA: + repo_path = await _call_get_git_repo(repo_url, branch) + except Exception as e: + # fallback naar master + branch = "master" + try: + async with _CLONE_SEMA: + repo_path = await _call_get_git_repo(repo_url, branch) + except Exception as e: + return (f"Clonen mislukte voor `{owner_repo or repo_hint}`: {e}. " + "Controleer repo-naam/URL of je toegangsrechten.") + root = Path(repo_path) + + # (re)index collectie voor deze repo + collection = repo_collection_name(owner_repo, branch) + chunk_chars, overlap = _chunk_params_for_repo(Path(repo_path)) + try: + await _rag_index_repo_internal( + repo_url=repo_url, branch=branch, profile="auto", + include="", exclude_dirs="", chunk_chars=chunk_chars, overlap=overlap, + collection_name=collection + ) + except Exception as e: + logger.warning("WARN:agent_repo:rag_index for QA failed (%s), fallback 'code_docs': %s", collection, e) + collection = "code_docs" + await _rag_index_repo_internal( + repo_url=repo_url, branch=branch, profile="auto", + include="", exclude_dirs="", chunk_chars=chunk_chars, overlap=overlap, + collection_name=collection + ) + + # stack summary + stack = _detect_stack_summary(root) + stack_txt = _format_stack_summary_text(stack) + + try: + symbol_index_repo(root, owner_repo, branch) + except Exception as e: + logger.warning("WARN:agent_repo:symbol index build (QA) failed: %s", e) + + # context + contexts = await _collect_repo_context_async(root, owner_repo, branch, question, n_ctx=n_ctx) + + # antwoord + answer = await _llm_qa_answer(question, stack_txt, contexts) + return answer + + + +# ---------- Dry-run voorstel ---------- +async def propose_patches_without_apply(repo_path: str, candidates: List[str], user_goal: str) -> Tuple[Dict[str,str], Dict[str,str], Dict[str,str]]: + """ + Returns: proposed, diffs, reasons + - reasons[pad] bevat korte uitleg over de wijziging/keuze + """ + proposed, diffs, reasons = {}, {}, {} + root = Path(repo_path) + token_steps = [1536, 1024, 768, 512] + quotes = extract_quotes(user_goal) + hints = extract_word_hints(user_goal) + old_new = (quotes[0], quotes[1]) if len(quotes) >= 2 else (None, None) + + + # Bepaal taaktype lokaal (lichtgewicht, 1 LLM-call; framework-heuristiek) + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + try: + _route = await _llm_task_route(user_goal, framework=("laravel" if is_laravel else "generic")) + _task_type = (_route.get("task_type") or "").lower() + except Exception: + _task_type = "" + + def _is_view_or_lang(path: str) -> bool: + return path.endswith(".blade.php") or path.startswith("resources/lang/") + + + for rel in candidates: + p = root / rel + # als het pad nog niet bestaat probeer een create-voorstel + if not p.exists(): + content, because = await propose_new_file(root, rel, user_goal) + if content: + proposed[rel] = content + diffs[rel] = make_new_file_diff(rel, content, max_lines=300) + reasons[rel] = because + else: + logger.info("INFO:agent_repo:no create-proposal for missing file %s", rel) + continue + + try: + original = _read_text_file(p) + except Exception: + original = "" + if not original: + logger.info("INFO:agent_repo:skip unreadable/empty %s", rel) + continue + + # 0) Gerichte, veilige fallback-literal replace (alleen bij oud->nieuw) + old, new, why_pair = deduce_old_new_literals(user_goal, original) + if old and new: + tmp, ok, because = targeted_fallback_replace(original, old, new) + if ok and tmp != original: + # anti-destructie niet nodig: minimale vervanging + proposed[rel] = tmp + diffs[rel] = make_diffs(original, tmp, rel, max_lines=200) + reasons[rel] = f"{because}. ({why_pair})" + continue + + # 1) HTML-scope als prompt tags noemt + ctx = extract_context_hints_from_prompt(user_goal) + if old and new and ctx["tag_names"]: + scoped, ok, because = html_scoped_literal_replace(original, old, new, ctx["tag_names"]) + if ok and scoped != original and not is_destructive(original, scoped, allow_destructive=False): + proposed[rel] = scoped + diffs[rel] = make_diffs(original, scoped, rel, max_lines=200) + reasons[rel] = (because + (f" ({why_pair})" if why_pair else "")) + continue + + # 2) Fallback-literal (??,?:, "", or) - volledig generiek + #if old and new: + # tmp, ok, because = targeted_fallback_replace(original, old, new) + # if ok and tmp != original and not is_destructive(original, tmp, allow_destructive=False): + # proposed[rel] = tmp + # diffs[rel] = make_diffs(original, tmp, rel, max_lines=200) + # reasons[rel] = (because + (f" ({why_pair})" if why_pair else "")) + # continue + # Zit al in stap 0) + + # 3) Algemene quoted-literal (taalagnostisch, behoud minimaliteit) + if old and new: + qrep, ok, because = quoted_literal_replace(original, old, new, max_occurrences=2) + if ok and qrep != original and not is_destructive(original, qrep, allow_destructive=False): + proposed[rel] = qrep + diffs[rel] = make_diffs(original, qrep, rel, max_lines=200) + reasons[rel] = (because + (f" ({why_pair})" if why_pair else "")) + continue + + + # 4) Focus-snippets + LLM edit-plan + needles = [] + if quotes: needles += quotes + if hints: needles += hints[:6] + focus = extract_focus_snippets(original, needles, window=240, max_snippets=3) + + # Tree-hint standaard aan: maak compacte map-tree en zet in globale var voor de prompt + try: + globals()["_LLM_EDIT_TREE_HINT"] = _make_local_tree_hint(root, rel, max_siblings=14) + except Exception: + globals()["_LLM_EDIT_TREE_HINT"] = "" + plan = await llm_plan_edits_for_file(user_goal, rel, focus) + if plan: + patched, change_count, explains, allow_destructive = apply_edit_plan(original, plan) + if change_count > 0 and patched.strip() != original.strip(): + if is_destructive(original, patched, allow_destructive): + logger.warning("WARN:agent_repo:destructive patch blocked for %s", rel) + else: + proposed[rel] = patched + diffs[rel] = make_diffs(original, patched, rel, max_lines=200) + reasons[rel] = "LLM edit-plan: " + "; ".join(explains[:4]) + continue + + # 5) Volledige rewrite fallback (met guard) + # Bij UI-label taken verbieden we volledige rewrites op NIET-view/lang bestanden. + if _task_type == "ui_label_change" and not _is_view_or_lang(rel): + logger.info("INFO:agent_repo:skip full rewrite for non-view/lang during ui_label_change: %s", rel) + # sla deze stap over; ga door naar volgende kandidaat + continue + last_err = None + for mx in [2048]: + try: + messages = [ + {"role":"system","content":"Voer exact de gevraagde wijziging uit. GEEN extra refactors/best practices. Lever de volledige, werkende bestandinformatie als 1 codeblok."}, + {"role":"user","content": f"Doel:\n{user_goal}\n\nBestand ({rel}) huidige inhoud:\n```\n{original}\n```"} + ] + resp = await _llm_call(messages, stream=False, temperature=0.2, top_p=0.9, max_tokens=mx) + newc = _extract_code_block(resp.get("choices",[{}])[0].get("message",{}).get("content","")) or original + if newc.strip() != original.strip(): + if is_destructive(original, newc, allow_destructive=False): + logger.warning("WARN:agent_repo:destructive rewrite blocked for %s (ratio>%.2f)", rel, AGENT_DESTRUCTIVE_RATIO) + break # early-exit: geen extra pogingen + proposed[rel] = newc + diffs[rel] = make_diffs(original, newc, rel, max_lines=200) + reasons[rel] = "Full rewrite (guarded): minimale aanpassing om het doel te halen." + break + except Exception as e: + last_err = e + logger.warning("WARN:agent_repo:LLM rewrite fail %s mx=%d: %s", rel, mx, repr(e)) + #continue + if rel not in proposed and last_err: + logger.error("ERROR:agent_repo:give up on %s after retries: %s", rel, repr(last_err)) + # --- Syntax guard filtering (laatste stap) --- + drop: List[str] = [] + for rel, content in proposed.items(): + try: + if rel.endswith(".php"): + tmp = _write_tmp(content, ".php") + ok = _php_lint_ok(tmp) + try: tmp.unlink(missing_ok=True) + except Exception: pass + if not ok: + reasons[rel] = (reasons.get(rel,"") + " [PHP lint failed]").strip() + drop.append(rel) + elif rel.endswith(".blade.php"): + if not _blade_balance_ok(content): + reasons[rel] = (reasons.get(rel,"") + " [Blade balance failed]").strip() + drop.append(rel) + except Exception: + # in twijfel: laat de patch door (fail-open), maar log upstream + pass + for rel in drop: + proposed.pop(rel, None); diffs.pop(rel, None) + return proposed, diffs, reasons + + +# ---------- Agent state ---------- +@dataclass +class AgentState: + stage: str = "TRIAGE" + questions_asked: int = 0 + user_goal: str = "" + repo_hint: str = "" + selected_repo: dict | None = None + repo_url: str = "" + branch_base: str = AGENT_DEFAULT_BRANCH + repo_path: str = "" + owner_repo: str | None = None + collection_name: str = "" + candidate_paths: List[str] = field(default_factory=list) + proposed_patches: Dict[str, str] = field(default_factory=dict) + reasons: Dict[str, str] = field(default_factory=dict) + new_branch: str = "" + dry_run: bool = True + repo_candidates: List[dict] = field(default_factory=list) + smart_preview: str = "" + recovery_attempted: bool = False + +# --- bootstrap op echte repo-inhoud ------------------------------------------------ +async def _detect_repo_url(text: str) -> str | None: + m = re.search(r"(https?://\S+?\.git)\b", text or "") + return m.group(1) if m else None + +async def _ensure_indexed(repo_url: str, *, branch: str = "main", profile: str = "auto", + rag_index_repo_internal_fn=None, get_git_repo_fn=None): + # clone/update (best-effort) om failures vroeg te vangen + if get_git_repo_fn: + try: + loop = asyncio.get_running_loop() + await loop.run_in_executor(None, get_git_repo_fn, repo_url, branch) + except Exception: + pass + if rag_index_repo_internal_fn: + await rag_index_repo_internal_fn( + repo_url=repo_url, branch=branch, profile=profile, + include="", exclude_dirs="", + chunk_chars=int(os.getenv("RAG_CHUNK_CHARS","3000")), + overlap=int(os.getenv("RAG_CHUNK_OVERLAP","400")), + collection_name=os.getenv("RAG_COLLECTION","code_docs"), + ) + +async def _bootstrap_overview(repo_url: str, rag_query_internal_fn, *, collection="code_docs") -> str: + """Haalt echte passages op en maakt een compacte context.""" + # Bij per-repo collections is een extra repo-filter contraproductief. + # Gebruik daarom repo=None zodra we een collection doorgeven. + owner, name = owner_repo_from_url(repo_url) + repo_full = f"{owner}/{name}" if (owner and name) else None + wants = [ + {"q": "project overview readme", "path_contains": "README"}, + {"q": "install setup configuration", "path_contains": "README"}, + {"q": "composer dependencies autoload", "path_contains": "composer.json"}, + {"q": "npm dependencies scripts", "path_contains": "package.json"}, + {"q": "routes definitions", "path_contains": "routes"}, + {"q": "controllers overview", "path_contains": "app/Http/Controllers"}, + {"q": "views templates blade", "path_contains": "resources/views"}, + {"q": "env example", "path_contains": ".env"}, + ] + chunks = [] + for w in wants: + res = await rag_query_internal_fn( + query=w["q"], n_results=3, + collection_name=collection, # per-repo collectie al gebruikt + repo=None, # voorkom dubbele/te strikte scoping + path_contains=w["path_contains"], profile=None + ) + chunks.extend((res or {}).get("results", [])) + + seen = set(); buf = [] + for r in chunks[:18]: + meta = r.get("metadata") or {} + key = (meta.get("path",""), meta.get("chunk_index")) + if key in seen: + continue + seen.add(key) + body = (r.get("document") or "").strip()[:1200] + buf.append(f"### {meta.get('path','')}\n{body}") + return "\n\n".join(buf[:8]).strip() + +def _extract_explicit_paths_robust(text: str) -> list[str]: + """ + Haalt bestands-paden uit vrije tekst robuust op. + Herkent tokens met minimaal één '/' en één '.' (extensie), + negeert trailing leestekens. + """ + if not text: + return [] + pats = re.findall(r"[A-Za-z0-9_./\\-]+\\.[A-Za-z0-9_.-]+", text) + out = [] + for p in pats: + # normaliseer Windows backslashes → unix + p = p.replace("\\", "/") + # strip algemene trailing chars + p = p.strip().strip(",.;:)]}>'\"") + if "/" in p and "." in p: + out.append(p) + # de-dup behoud volgorde + seen = set(); uniq = [] + for p in out: + if p not in seen: + uniq.append(p); seen.add(p) + return uniq + +def _sanitize_path_hints(hints: list[str], all_files: list[str]) -> list[str]: + """ + Filter pseudo-paden zoals 'tool.list' weg. Sta alleen echte projectpaden of + bekende extensies toe en vereis een '/' om pure tokens te weren. + """ + if not hints: + return [] + ALLOWED_SUFFIXES = ( + ".blade.php",".php",".js",".ts",".json",".yml",".yaml",".py",".md",".env", + ".sql",".css",".vue",".jsx",".tsx" + ) + BAD_BASENAMES = {"tool","tools","list","search","update","create","store","index"} + out, seen = [], set() + for h in hints: + if not h: + continue + h = h.strip().lstrip("./").replace("\\","/") + if "/" not in h: + continue + base = os.path.basename(h) + stem = base.split(".",1)[0].lower() + if h not in all_files and not any(h.endswith(suf) for suf in ALLOWED_SUFFIXES): + continue + if stem in BAD_BASENAMES and h not in all_files: + continue + if h not in seen: + seen.add(h); out.append(h) + return out + +def _grep_repo_for_literal(root: Path, needle: str, limit: int = 12) -> list[str]: + """ + Heel snelle, ruwe literal-zoeker over tekstbestanden in de repo. + Retourneert lijst met relatieve paden waar 'needle' voorkomt (top 'limit'). + """ + if not needle or len(needle) < 2: + return [] + hits = [] + try: + for p in root.rglob("*"): + if p.is_dir(): + continue + # respecteer uitgesloten directories en grootte-limiet + if any(part in _PROFILE_EXCLUDE_DIRS for part in p.parts): + continue + try: + if p.stat().st_size > 500_000: + continue + except Exception: + continue + # alleen tekst-achtige extensies volgens allowed_file() + if not allowed_file(p): + continue + # lees als tekst (met best-effort fallback) + try: + txt = p.read_text(encoding="utf-8", errors="ignore") + except Exception: + try: + txt = p.read_text(encoding="latin-1", errors="ignore") + except Exception: + continue + if needle in txt: + try: + rel = str(p.relative_to(root)) + except Exception: + rel = str(p) + hits.append(rel) + if len(hits) >= limit: + break + except Exception: + pass + return hits + +def _laravel_priors_from_prompt(user_goal: str, root: Path, all_files: list[str], max_k: int = 8) -> list[str]: + """ + Geef een lijst met waarschijnlijke Laravel-bestanden op basis van conventies + prompt-keywords. + Neem ALLEEN paden op die daadwerkelijk bestaan in de repo (all_files). + """ + text = (user_goal or "").lower() + exists = set(all_files) + priors: list[str] = [] + + def add_if_present(paths: list[str]): + for p in paths: + if p in exists and p not in priors: + priors.append(p) + + # Altijd nuttige ankerpunten in Laravel repos + add_if_present([ + "routes/web.php", + "routes/api.php", + "config/app.php", + "config/database.php", + ".env", + ".env.example", + "resources/lang/en.json", + "resources/lang/nl.json", + ]) + + # Prompt-gestuurde hints + if any(k in text for k in ("api ", "endpoint", "jwt", "sanctum", "api-route")): + add_if_present(["routes/api.php"]) + if any(k in text for k in ("route", "router", "web", "pagina", "page", "url ")): + add_if_present(["routes/web.php"]) + if any(k in text for k in ("controller", "actie", "action", "handler", "store(", "update(", "create(", "edit(")): + # neem de meest voorkomende controllers-map mee + # (geen directory listing; we kiezen alleen de indexerende anchor-files) + for p in exists: + if p.startswith("app/Http/Controllers/") and p.endswith(".php"): + priors.append(p) + if len(priors) >= max_k: + break + if any(k in text for k in ("view", "blade", "template", "pagina", "page", "formulier", "form")): + # bekende view-locaties + add_if_present([ + "resources/views/layouts/app.blade.php", + "resources/views/welcome.blade.php", + "resources/views/dashboard.blade.php", + ]) + # heuristisch: als prompt een padfragment noemt (b.v. 'log/create'), pak views daaronder + m = re.search(r"resources/views/([A-Za-z0-9_/\-]+)/", user_goal) + if m: + base = f"resources/views/{m.group(1).strip('/')}/" + for p in exists: + if p.startswith(base) and p.endswith(".blade.php") and p not in priors: + priors.append(p) + if len(priors) >= max_k: + break + if any(k in text for k in ("validatie", "validation", "formrequest", "request class", "rules(")): + # vaak custom FormRequest classes + for p in exists: + if p.startswith("app/Http/Requests/") and p.endswith(".php"): + priors.append(p) + if len(priors) >= max_k: + break + if any(k in text for k in ("database", "db", "sql", "sqlserver", "mssql", "mysql", "pgsql", "connection", "migratie", "migration", "schema")): + add_if_present(["config/database.php", ".env", ".env.example"]) + # migrations en models zijn vaak relevant + for p in exists: + if (p.startswith("database/migrations/") and p.endswith(".php")) or \ + (p.startswith("app/Models/") and p.endswith(".php")): + priors.append(p) + if len(priors) >= max_k: + break + if any(k in text for k in ("taal", "language", "vertaling", "translation", "lang", "i18n")): + # neem json én php lang packs mee + for p in exists: + if p.startswith("resources/lang/") and (p.endswith(".json") or p.endswith(".php")): + priors.append(p) + if len(priors) >= max_k: + break + + # dedupe + cap + uniq: list[str] = [] + seen = set() + for p in priors: + if p not in seen: + uniq.append(p); seen.add(p) + if len(uniq) >= max_k: + break + return uniq + +async def _llm_framework_priors(user_goal: str, all_files: list[str], framework: str = "laravel", max_k: int = 10) -> list[str]: + """ + Laat de LLM kansrijke BESTAANDE bestanden/globs voorstellen op basis van framework-conventies. + - Output MOET JSON zijn: {"files":[...]} met relatieve paden of simpele globs. + - We filteren op echt-bestaande paden (match tegen all_files), globs toegestaan. + - Geen netwerk I/O; 1 kleine LLM-call. + """ + text = (user_goal or "").strip() + if not text: + return [] + # Bescheiden token budget + sys = ("You are a precise code navigator. Output ONLY compact JSON with likely file paths for the task.\n" + "Rules:\n- Return: {\"files\":[\"relative/path/or/glob\", ...]}\n" + "- Use framework conventions (e.g., Laravel routes/controllers/views, config, .env, migrations, lang).\n" + "- Do NOT invent files that cannot exist; prefer generic globs (e.g., resources/views/**/create*.blade.php).\n" + "- No explanations, no prose.") + usr = (f"Framework: {framework}\n" + f"Task/prompt:\n{text}\n" + "Return at most 15 items.\n" + "Examples for Laravel (if applicable): routes/web.php, app/Http/Controllers/**.php, " + "resources/views/**.blade.php, config/database.php, .env, database/migrations/**.php, resources/lang/**") + try: + resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.0, top_p=1.0, max_tokens=300 + ) + raw = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","").strip() + except Exception: + return [] + # Haal eventuele ```json fences weg + m = re.search(r"\{[\s\S]*\}", raw) + if not m: + return [] + try: + obj = json.loads(m.group(0)) + except Exception: + return [] + items = obj.get("files") or [] + if not isinstance(items, list): + return [] + # Glob -> concrete bestanden; filter op bestaande paden + exists = set(all_files) + out: list[str] = [] + def _match(pat: str) -> list[str]: + # simpele glob: **, *, ?. We matchen tegen all_files. + try: + pat_norm = pat.strip().lstrip("./") + return [f for f in all_files if fnmatch.fnmatch(f, pat_norm)] + except Exception: + return [] + for it in items: + if not isinstance(it, str) or not it.strip(): + continue + it = it.strip().lstrip("./") + if it in exists: + if it not in out: + out.append(it) + else: + for hit in _match(it): + if hit not in out: + out.append(hit) + if len(out) >= max_k: + break + return out[:max_k] + +async def _llm_task_route(user_goal: str, framework: str = "laravel") -> dict: + """ + Laat de LLM expliciet kiezen: {task_type, categories[], hints[]} + Voorbeelden task_type: + - "ui_label_change", "db_credentials", "db_queries", "routes_to_views", "config_env", "generic_code_change" + categories: welke mappen/artefacten zijn relevant (bv. ["views","controllers","routes","migrations","config",".env"]) + hints: korte trefwoorden of view/controller namen. + """ + if not (user_goal or "").strip(): + return {} + sys = ("You are a precise task router. Return ONLY compact JSON.\n" + "Schema: {\"task_type\":str, \"categories\":[str,...], \"hints\":[str,...]}\n" + "Use framework conventions (e.g., Laravel). No explanations.") + usr = f"Framework: {framework}\nUser goal:\n{user_goal}\nReturn at most 6 categories and 8 hints." + try: + resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.0, top_p=1.0, max_tokens=250 + ) + raw = (resp.get('choices',[{}])[0].get('message',{}) or {}).get('content','') + m = re.search(r"\{[\s\S]*\}", raw or "") + obj = json.loads(m.group(0)) if m else {} + # sanitize + obj["task_type"] = (obj.get("task_type") or "generic_code_change")[:64] + obj["categories"] = [str(x)[:32] for x in (obj.get("categories") or [])][:8] + obj["hints"] = [str(x)[:64] for x in (obj.get("hints") or [])][:8] + return obj + except Exception: + return {"task_type":"generic_code_change","categories":[],"hints":[]} + +# ---------- Hoofd-handler ---------- +async def handle_repo_agent(messages: List[dict], request) -> str: + """ + Uitbreiding: fast-path voor unified diffs op expliciete bestanden met tekstvervanging. + Als niet van toepassing, valt automatisch terug op de bestaande flow. + """ + # 1) Combineer user/system content om opdracht te parsen + try: + full_txt = "\n".join([m.get("content","") for m in messages if m.get("role") in ("system","user")]) + except Exception: + full_txt = "" + + # 2) Herken fast-path + try_fast = _looks_like_unified_diff_request(full_txt) + paths_fp = _extract_explicit_paths(full_txt) if try_fast else [] + old_txt, new_txt = _extract_replace_pair(full_txt) if try_fast else (None, None) + + # NB: we gebruiken de injecties die via initialize_agent zijn gezet: + # - get_git_repo_fn (async) + # - read_text_file_fn (sync) + # Deze symbolen worden onderin initialize_agent aan globals() gehangen. + get_git_repo_fn = globals().get("get_git_repo_fn") + read_text_file_fn = globals().get("read_text_file_fn") + + if try_fast and paths_fp and old_txt and new_txt and callable(get_git_repo_fn) and callable(read_text_file_fn): + # 3) repo + branch bepalen + repo_url, branch = _extract_repo_branch_from_text(full_txt) + if not repo_url: + # fallback: probeer repo uit eerdere agent-state (optioneel), anders stop fast-path + repo_url = globals().get("_last_repo_url") + branch = globals().get("_last_branch", "main") + if repo_url: + try: + repo_root = await get_git_repo_fn(repo_url, branch or "main") + root = Path(repo_root) + lang_path = root / "resources" / "lang" / "nl.json" + lang_before = lang_path.read_text(encoding="utf-8", errors="ignore") if lang_path.exists() else "{}" + lang_data = {} + try: + lang_data = json.loads(lang_before or "{}") + except Exception: + lang_data = {} + + diffs_out = [] + lang_changed = False + + def _make_udiff(a: str, b: str, rel: str) -> str: + return "".join(difflib.unified_diff( + a.splitlines(keepends=True), + b.splitlines(keepends=True), + fromfile=f"a/{rel}", tofile=f"b/{rel}", n=3 + )) + + # 4) per bestand: ofwel inline replace, ofwel vertaling bijwerken + for rel in paths_fp: + p = root / rel + if not p.exists(): + continue + before = read_text_file_fn(p) + if not before: + continue + # Als de 'oude' tekst voorkomt BINNEN een vertaalwrapper, dan géén blade-edit + found_in_wrapper = False + for pat in _TRANS_WRAPPERS: + for m in re.finditer(pat, before): + inner = m.group(1) + if inner == old_txt: + found_in_wrapper = True + break + if found_in_wrapper: + break + if found_in_wrapper: + # update nl.json: {"oude": "nieuwe"} + if lang_data.get(old_txt) != new_txt: + lang_data[old_txt] = new_txt + lang_changed = True + continue + + # anders: directe, exacte vervanging (conservatief) + after = before.replace(old_txt, new_txt) + if after != before: + diff = _make_udiff(before, after, rel) + if diff.strip(): + diffs_out.append(("blade", rel, diff)) + + # 5) indien vertaling gewijzigd: diff voor nl.json toevoegen + if lang_changed: + new_lang = json.dumps(lang_data, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + diff_lang = _make_udiff(lang_before if isinstance(lang_before, str) else "", new_lang, "resources/lang/nl.json") + if diff_lang.strip(): + diffs_out.append(("lang", "resources/lang/nl.json", diff_lang)) + if diffs_out: + parts = ["### Unified diffs"] + for kind, rel, d in diffs_out: + parts.append(f"**{rel}**") + parts.append("```diff\n" + d + "```") + return "\n\n".join(parts) + else: + return "Dry-run: geen wijzigbare treffers gevonden in opgegeven bestanden (of reeds actueel)." + except Exception as e: + # mislukt → val terug op bestaande discover/agent flow + pass + + # === GEEN fast-path → ga door met de bestaande flow hieronder === + + sid = _get_session_id(messages, request) + st = _app.state.AGENT_SESSIONS.get(sid) or AgentState() + _app.state.AGENT_SESSIONS[sid] = st + user_last = next((m["content"] for m in reversed(messages) if m.get("role")=="user"), "").strip() + user_last_lower = user_last.lower() + logger.info("INFO:agent_repo:[%s] stage=%s", sid, st.stage) + from smart_rag import ( + enrich_intent, + expand_queries, + hybrid_retrieve, + _laravel_pairs_from_route_text, + _laravel_guess_view_paths_from_text, + ) + # Als user een .git URL meegeeft: zet state en ga via de state-machine verder + user_txt = next((m.get("content","") for m in reversed(messages) if m.get("role")=="user"), "") + repo_url = await _detect_repo_url(user_txt) + + if repo_url: + st.repo_hint = repo_url + st.stage = "SELECT_REPO" + logger.info("INFO:agent_repo:[%s] direct SELECT_REPO via .git url: %s", sid, repo_url) + # LET OP: geen vroegtijdige return hier; de SELECT_REPO tak hieronder handelt DISCOVER/INDEX etc. af. + + + # === SMART-RAG: opt-in pad (alleen als er nog GEEN repo is) === + smart_enabled = str(os.getenv("REPO_AGENT_SMART","1")).lower() not in ("0","false") + if smart_enabled and not st.repo_hint and st.stage in ("TRIAGE","ASK"): + # 1) intent → plan + spec = await enrich_intent(_llm_call, messages) + task = spec.get("task","").strip() + file_hints = spec.get("file_hints") or [] + keywords = spec.get("keywords") or [] + constraints= spec.get("constraints") or [] + acceptance = spec.get("acceptance") or [] + ask = spec.get("ask") + + # 2) query expansion (kort) en hybride retrieval + variants = await expand_queries(_llm_call, task, k=int(os.getenv("RAG_EXPAND_K","3"))) + merged: list[dict] = [] + for i, qv in enumerate(variants): + partial = await hybrid_retrieve( + _rag_query_internal, + qv, + repo= None, + profile= None, + path_contains=(file_hints[0] if file_hints else None), + per_query_k=int(os.getenv("RAG_PER_QUERY_K","30")), + n_results=int(os.getenv("RAG_N_RESULTS","18")), + alpha=float(os.getenv("RAG_EMB_WEIGHT","0.6")), + ) + merged.extend(partial) + # dedupe op path+chunk + seen = set(); uniq = [] + for r in sorted(merged, key=lambda x: x["score"], reverse=True): + meta = r.get("metadata") or {} + key = (meta.get("path",""), meta.get("chunk_index","")) + if key in seen: continue + seen.add(key); uniq.append(r) + + # 3) context + confidence + ctx_text, top_score = assemble_context(uniq, max_chars=int(os.getenv("REPO_AGENT_CONTEXT_CHARS","640000"))) + # heel simpele confidence: als top_score erg laag is en vragen toegestaan → stel 1 verhelderingsvraag + if ask and float(os.getenv("REPO_AGENT_ASK_CLARIFY","1")) and top_score < float(os.getenv("REPO_AGENT_ASK_THRESHOLD","0.35")): + return f"Snelle check: {ask}" + + # 4) finale prompt samenstellen + sys = ( + "Je bent een senior code-assistent. " + "Lees de contextfragmenten (met padheaders). " + "Beantwoord taakgericht, concreet en veilig. " + "Als je verbeteringen doet, geef dan eerst een kort plan en daarna exacte, toepasbare wijzigingen." + ) + user = ( + f"TAKEN:\n{task}\n\n" + f"CONSTRAINTS: {', '.join(constraints) or '-'}\n" + f"ACCEPTANCE: {', '.join(acceptance) or '-'}\n" + f"KEYWORDS: {', '.join(keywords) or '-'}\n" + f"FILE HINTS: {', '.join(file_hints) or '-'}\n\n" + f"--- CONTEXT (gedeeltelijk) ---\n{ctx_text}\n--- EINDE CONTEXT ---\n\n" + "Geef eerst een kort, puntsgewijs plan (max 6 bullets). " + "Daarna de concrete wijzigingen per bestand met codeblokken. " + "Geen herhaling van hele bestanden als dat niet nodig is." + ) + llm_resp = await _llm_call( + [{"role":"system","content":sys},{"role":"user","content":user}], + stream=False, temperature=0.2, top_p=0.9, max_tokens=2048 + ) + out = (llm_resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + if out.strip(): + # niet returnen — maar bijvoorbeeld loggen of meesturen als “quick analysis” + st.smart_preview = out + logger.info("SMART-RAG preview gemaakt (geen vroegtijdige exit)") + # === /SMART-RAG === + + + if any(k in user_last_lower for k in ["dry-run","dryrun","preview"]): st.dry_run = True + if "apply" in user_last_lower and ("akkoord" in user_last_lower or "ga door" in user_last_lower): st.dry_run = False + + if st.stage == "TRIAGE": + logger.info("Stage TRIAGE") + st.user_goal = user_last + # Optioneel: intent refine + verduidelijkingsvragen + if AGENT_ENABLE_GOAL_REFINE and st.user_goal: + try: + refined, questions, conf = await llm_refine_goal(st.user_goal) + if refined and refined != st.user_goal: + st.user_goal = refined + if questions and conf < AGENT_CLARIFY_THRESHOLD: + st.stage = "ASK" + qtxt = "\n".join([f"- {q}" for q in questions]) + return ("Om zeker de juiste bestanden te kiezen, beantwoord kort:\n" + qtxt) + except Exception: + pass + st.stage = "ASK" + base = ("Ik verken de code en doe een voorstel. Geef de repo (bv. `admin/image-viewing-website` of " + "`http://10.25.138.40:30085/admin/image-viewing-website.git`). " + "Of zeg: **'zoek repo'** als ik zelf moet zoeken.") + return _with_preview(base, st) + + if st.stage == "ASK": + logger.info("Stage ASK ") + # 1) check of er een repo-hint in de zin zit + hint = None + m = re.search(r"(https?://\S+)", user_last) + if m: hint = m.group(1) + elif "/" in user_last: + for p in user_last.split(): + if re.match(r"^[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+$", p): hint = p; break + # 2) Als expliciete vraag om repo te zoeken óf geen hint → auto-discovery + if (not hint) and ("zoek repo" in user_last_lower): + # Probeer auto-discovery + st.repo_candidates = await discover_candidate_repos(st.user_goal) + if not st.repo_candidates: + st.questions_asked += 1 + return _with_preview("Ik kon geen repos vinden. Geef de Gitea repo (owner/repo) of volledige .git-URL.", st) + # Normalize scores naar 0..1 + maxs = max((c.get("score",0.0) for c in st.repo_candidates), default=0.0) or 1.0 + for c in st.repo_candidates: + c["score"] = min(1.0, c["score"]/maxs) if maxs else 0.0 + best = st.repo_candidates[0] + # Als hoogste score duidelijk is, auto-select + if best.get("score",0.0) >= AGENT_AUTOSELECT_THRESHOLD and best.get("clone_url"): + st.repo_hint = best["clone_url"] + st.stage = "SELECT_REPO" + return _with_preview(f"Repo automatisch gekozen: **{best['full_name']}** (score {best['score']:.2f}).", st) + # Anders: laat top-3 zien en vraag keuze + st.stage = "CONFIRM_REPO" + lines = [] + for i, c in enumerate(st.repo_candidates[:3], 1): + lines.append(f"{i}. {c['full_name']} — score {c.get('score',0.0):.2f}") + base = "Ik vond deze passende repos:\n" + "\n".join(lines) + "\nKies een nummer, of typ de naam/URL." + return _with_preview(base, st) + + # 3) Er is wel een hint - ga door + if hint: + st.repo_hint = hint + st.stage = "SELECT_REPO" + else: + st.questions_asked += 1 + if st.questions_asked <= AGENT_MAX_QUESTIONS: + return _with_preview("Graag de Gitea repo (owner/repo) of volledige .git-URL.", st) + return _with_preview("Ik heb de repo-naam of URL nodig om verder te gaan.", st) + + + if st.stage == "CONFIRM_REPO": + logger.info("Stage CONFIRM_REPO") + # parse keuze + pick = None + m = re.match(r"^\s*([1-5])\s*$", user_last) + if m: + idx = int(m.group(1)) - 1 + if 0 <= idx < len(st.repo_candidates): + pick = st.repo_candidates[idx] + if not pick: + # probeer naam match + for c in st.repo_candidates: + if c["full_name"].lower() in user_last_lower or (c.get("clone_url","") and c["clone_url"] in user_last): + pick = c; break + if not pick: + return _with_preview("Typ een nummer (1..3) of de naam/URL van de repo.", st) + + st.repo_hint = pick.get("clone_url") or (f"{GITEA_URL}/{pick['full_name']}.git") + st.stage = "SELECT_REPO" + return _with_preview(f"Repo gekozen: **{pick['full_name']}**.", st) + + if st.stage == "SELECT_REPO": + logger.info("Stage SELECT_REPO") + repo_meta, reason = resolve_repo(st.repo_hint) + if not repo_meta: + return (f"Geen repo gevonden voor “{st.repo_hint}”. Probeer volledige URL: {GITEA_URL}//.git") + st.selected_repo = repo_meta + st.repo_url = repo_meta.get("clone_url") or "" + st.owner_repo = repo_meta.get("full_name") + if not st.repo_url: + return f"Geen clone URL voor “{st.repo_hint}”." + progress = [f"Repo ({reason}): {st.owner_repo or st.repo_url}"] + + # DISCOVER + logger.info("DISCOVER") + try: + try: + st.repo_path = await _call_get_git_repo(st.repo_url, st.branch_base) + except Exception as e_main: + logger.warning("WARN:agent_repo:get_git_repo %s failed: %s; fallback master", st.branch_base, e_main) + st.branch_base = "master" + st.repo_path = await _call_get_git_repo(st.repo_url, st.branch_base) + + + st.collection_name = repo_collection_name(st.owner_repo, st.branch_base) + chunk_chars, overlap = _chunk_params_for_repo(Path(st.repo_path)) + + # ── Fast-path: check HEAD en sla index over als ongewijzigd ── + try: + import git + head_sha = await run_in_threadpool(lambda: git.Repo(st.repo_path).head.commit.hexsha) + except Exception: + head_sha = "" + #memo_key = f"{st.repo_url}|{st.branch_base}|{st.collection_name}" + # ‘Brede’ key (repo+branch) voorkomt dubbele index runs bij dezelfde HEAD, + # ook als collection_name varieert. + memo_key = f"{st.repo_url}|{st.branch_base}" + + if _INDEX_HEAD_MEMO.get(memo_key) == head_sha and head_sha: + progress.append(f"Index overslaan: HEAD ongewijzigd ({head_sha[:7]}).") + else: + try: + res = await _rag_index_repo_internal( + repo_url=st.repo_url, branch=st.branch_base, + profile="auto", include="", exclude_dirs="", + chunk_chars=chunk_chars, overlap=overlap, collection_name=st.collection_name + ) + # alleen updaten als index call succesvol was + _INDEX_HEAD_MEMO[memo_key] = head_sha or _INDEX_HEAD_MEMO.get(memo_key, "") + + if isinstance(res, dict) and res.get("status") == "skipped": + progress.append(f"Index: skip (cache) — HEAD {head_sha[:7]}.") + else: + progress.append("Index: bijgewerkt.") + except Exception as e_idx: + logger.warning("WARN:agent_repo:rag index failed '%s': %s; fallback 'code_docs'", st.collection_name, e_idx) + st.collection_name = "code_docs" + res = await _rag_index_repo_internal( + repo_url=st.repo_url, branch=st.branch_base, + profile="auto", include="", exclude_dirs="", + chunk_chars=chunk_chars, overlap=overlap, collection_name=st.collection_name + ) + _INDEX_HEAD_MEMO[memo_key] = head_sha or _INDEX_HEAD_MEMO.get(memo_key, "") + + + + # na succesvolle _rag_index_repo_internal(...) en meili/bm25: + logger.info("Symbol index repo") + try: + symbol_index_repo(Path(st.repo_path), st.owner_repo, st.branch_base) + except Exception as e: + logger.warning("WARN:agent_repo:symbol index build failed: %s", e) + + + logger.info("Meili part") + if MEILI_URL: + try: + # Skip Meili herindex als HEAD ongewijzigd + if _MEILI_HEAD_MEMO.get(memo_key) == head_sha and head_sha: + progress.append("Meili: overslaan (HEAD ongewijzigd).") + else: + await run_cpu_blocking(meili_index_repo, Path(st.repo_path), st.owner_repo, st.branch_base) + _MEILI_HEAD_MEMO[memo_key] = head_sha or _MEILI_HEAD_MEMO.get(memo_key, "") + + except Exception as e: + logger.warning("WARN:agent_repo:meili_index_repo failed: %s", e) + else: + try: + if _BM25_HEAD_MEMO.get(memo_key) == head_sha and head_sha: + progress.append("BM25: overslaan (HEAD ongewijzigd).") + else: + await run_cpu_blocking(bm25_build_index, Path(st.repo_path), st.owner_repo, st.branch_base) + _BM25_HEAD_MEMO[memo_key] = head_sha or _BM25_HEAD_MEMO.get(memo_key, "") + except Exception as e: + logger.warning("WARN:agent_repo:bm25_build_index failed: %s", e) + + + progress.append("DISCOVER klaar.") + logger.info("DISCOVER klaar.") + except Exception as e: + logger.exception("ERROR:agent_repo:DISCOVER failed") + st.stage = "ASK" + return _with_preview("\n".join(progress + [f"DISCOVER mislukte: {e}"]), st) + + + # RANK via hybrid RAG + logger.info("RANK via hybrid RAG") + root = Path(st.repo_path) + all_files = list_repo_files(root) + # Precompute graph + tree (per HEAD) voor ranking-boost en explain + graph = _get_graph_cached(root, memo_key=(f"{st.repo_url}|{st.branch_base}|{head_sha or ''}")) + tree_summ = _get_tree_cached(root, memo_key=(f"{st.repo_url}|{st.branch_base}|{head_sha or ''}"), all_files=all_files) + + + picked: List[str] = [] + # 1) expliciete paden uit de prompt (bestaande extractor) + explicit = _sanitize_path_hints(list(extract_explicit_paths(st.user_goal) or []), all_files) + # 2) robuuste fallback extractor + robust = _sanitize_path_hints(_extract_explicit_paths_robust(st.user_goal), all_files) + for pth in explicit + [p for p in robust if p not in explicit]: + norm = pth.replace("\\", "/").strip() + if norm in all_files and norm not in picked: + picked.append(norm) + continue + best = best_path_by_basename(all_files, norm) + if best and best not in picked: + picked.append(best) + continue + # Als het niet bestaat: toch opnemen (voor create-flow) + if norm not in picked: + picked.append(norm) + + # Laravel priors (alleen bestaande paden), vóór RAG + try: + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + except Exception: + is_laravel = False + if is_laravel: + priors = _laravel_priors_from_prompt(st.user_goal, root, all_files, max_k=int(os.getenv("LARAVEL_PRIORS_K","8"))) + for p in priors: + if p not in picked: + picked.append(p) + + # ---- LLM-PRIORS (optioneel via env, standaard aan) ---- + use_llm_priors = os.getenv("LLM_PRIORS_ENABLE", "1").lower() not in ("0","false","no") + if use_llm_priors: + try: + # Hint framework adhv repo + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + except Exception: + is_laravel = False + fw = "laravel" if is_laravel else "generic" + llm_hits = await _llm_framework_priors(st.user_goal, all_files, framework=fw, max_k=int(os.getenv("LLM_PRIORS_K","12"))) + for p in llm_hits: + if p not in picked: + picked.append(p) + + # ---- Rules fallback (alleen als nog mager) ---- + try: + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + except Exception: + is_laravel = False + if is_laravel and len(picked) < max(4, int(os.getenv("LLM_PRIORS_MIN_BEFORE_RAG","4"))): + priors = _laravel_priors_from_prompt(st.user_goal, root, all_files, max_k=int(os.getenv("LARAVEL_PRIORS_K","8"))) + for p in priors: + if p not in picked: + picked.append(p) + + # --- LLM Task Router --- + is_laravel = (root / "artisan").exists() or (root / "composer.json").exists() + route = await _llm_task_route(st.user_goal, framework=("laravel" if is_laravel else "generic")) + st.reasons["task_route"] = json.dumps(route, ensure_ascii=False) + task_type = (route.get("task_type") or "").lower() + + # --- LLM zoekpatronen → deterministische scan --- + if os.getenv("LLM_PATTERN_SCAN","1").lower() not in ("0","false","no"): + specs = await _llm_make_search_specs(st.user_goal, framework=("laravel" if is_laravel else "generic")) + scan_hits = _scan_repo_for_patterns(root, all_files, specs, max_hits=int(os.getenv("LLM_PATTERN_MAX_HITS","24"))) + for f in scan_hits: + if f not in picked: + picked.append(f) + + # --- VIEW/LANG bias voor UI-label wijzigingen --- + # Pak de eerste quote uit de prompt als "oude" literal + qs = extract_quotes(st.user_goal) or [] + old_lit = qs[0] if qs else None + + def _contains_old(rel: str) -> bool: + if not old_lit: + return True # fallback: geen filtering + try: + txt = _read_text_file(Path(st.repo_path) / rel) or "" + return old_lit in txt + except Exception: + return False + + view_files = [f for f in all_files + if f.startswith("resources/views/") and f.endswith(".blade.php")] + lang_files = [f for f in all_files + if f.startswith("resources/lang/") and (f.endswith(".json") or f.endswith(".php"))] + + # Als we de oude literal kennen: eerst de files waar die echt in staat + if old_lit: + view_hits = [f for f in view_files if _contains_old(f)] + lang_hits = [f for f in lang_files if _contains_old(f)] + else: + view_hits = view_files + lang_hits = lang_files + + # Zet de meest waarschijnlijke kandidaten vóóraan, behoud verder huidige volgorde + front = [] + for lst in (view_hits, lang_hits): + for f in lst: + if f in all_files and f not in front: + front.append(f) + picked = list(dict.fromkeys(front + picked))[:MAX_FILES_DRYRUN] + + + # --- (optioneel) priors op basis van framework (je eerdere patch A/B) --- + # LLM priors + rule-based priors kun je hier behouden zoals je eerder hebt toegevoegd. + + + + # --- NIEUW: Smart-RAG path selectie op repo-collectie --- + + # 1) intent (voor file_hints) + query-expansion + logger.info("Smart RAG path select. 1) intent") + spec = await enrich_intent(_llm_call, [{"role":"user","content": st.user_goal}]) + file_hints = (spec.get("file_hints") or []) + variants = await expand_queries(_llm_call, spec.get("task") or st.user_goal, k=2) + + # 2) retrieval per variant met repo-filter & collectie van deze repo + logger.info("Smart RAG path select. 2) retrieval") + merged = [] + for qv in variants: + use_collection = bool(st.collection_name) + part = await hybrid_retrieve( + _rag_query_internal, + qv, + repo=_clean_repo_arg(st.owner_repo) if not use_collection else None, + profile=None, + path_contains=(file_hints[0] if file_hints else None), + per_query_k=int(os.getenv("RAG_PER_QUERY_K","30")), + n_results=int(os.getenv("RAG_N_RESULTS","18")), + alpha=float(os.getenv("RAG_EMB_WEIGHT","0.6")), + collection_name=(st.collection_name if use_collection else None) + ) + merged.extend(part) + + # 3) naar unieke paden + sort op score + logger.info("Smart RAG path select. 3) unieke paden sort op score") + seen=set() + for r in sorted(merged, key=lambda x: x.get("score",0.0), reverse=True): + meta = r.get("metadata") or {} + rel = meta.get("path","") + if not rel or rel in seen: + continue + seen.add(rel) + if rel not in picked: + picked.append(rel) + # 4) Laravel neighbors (klein zetje, opt-in via env) + logger.info("Smart RAG path select. 4) Laravel neighbors") + if os.getenv("RAG_NEIGHBORS", "1").lower() not in ("0","false"): + add = [] + for rel in picked[:8]: + # routes -> controllers + if rel in ("routes/web.php","routes/api.php"): + txt = (Path(st.repo_path)/rel).read_text(encoding="utf-8", errors="ignore") + for ctrl_path, _m in _laravel_pairs_from_route_text(txt): + if ctrl_path and ctrl_path not in picked and ctrl_path not in add: + add.append(ctrl_path) + # controllers -> views + if rel.startswith("app/Http/Controllers/") and rel.endswith(".php"): + txt = (Path(st.repo_path)/rel).read_text(encoding="utf-8", errors="ignore") + for v in _laravel_guess_view_paths_from_text(txt): + if v and v not in picked and v not in add: + add.append(v) + # Extra: neem kleine nabije partials/layouts mee (zelfde dir, ≤40KB) + more = [] + for rel in (picked + add)[:8]: + if rel.endswith(".blade.php"): + d = (Path(st.repo_path) / rel).parent + try: + for bp in d.glob("*.blade.php"): + if bp.name == os.path.basename(rel): + continue + if bp.stat().st_size <= 40_000: + cand = str(bp.relative_to(Path(st.repo_path))) + if cand not in picked and cand not in add and cand not in more: + more.append(cand) + except Exception: + pass + picked = (picked + add + more)[:MAX_FILES_DRYRUN] + # 5) Literal-grep fallback: als de user een oud->nieuw wijziging impliceert, zoek de 'old' literal repo-breed + qs = extract_quotes(st.user_goal) or [] + old = qs[0].strip() if qs and qs[0].strip() else None + if old: + grep_hits = _grep_repo_for_literal(Path(st.repo_path), old, limit=16) + for rel in grep_hits: + if rel in all_files and rel not in picked: + picked.append(rel) + + # Keyword fallback alleen als we nog te weinig zeker zijn + top_conf = 0.0 + try: + top_conf = max([r.get("score",0.0) for r in merged]) if merged else 0.0 + except Exception: + pass + if len(picked) < MAX_FILES_DRYRUN and top_conf < float(os.getenv("RAG_FALLBACK_THRESHOLD","0.42")): + + for rel, _s in simple_keyword_search(root, all_files, st.user_goal, limit=MAX_FILES_DRYRUN): + if rel not in picked: picked.append(rel) + # --- Gewogen her-ranking (Meili/embeddings/heuristiek/explicit) --- + explicit_all = extract_explicit_paths(st.user_goal) + _extract_explicit_paths_robust(st.user_goal) + explicit_all = [p.replace("\\","/").strip() for p in explicit_all] + # 1) verzamel meili/embeddings scores vanuit 'merged' + meili_scores = {} + for r in merged: + meta = (r or {}).get("metadata") or {} + rel = meta.get("path","") + if rel: + try: + sc = float(r.get("score", 0.0)) + except Exception: + sc = 0.0 + meili_scores[rel] = max(meili_scores.get(rel, 0.0), sc) + # 2) weeg en motiveer + cand_scores = {} + cand_why = {} + def _boost(rel: str, amt: float, why: str): + cand_scores[rel] = cand_scores.get(rel, 0.0) + float(amt) + if amt > 0: + cand_why[rel] = (cand_why.get(rel, "") + f"{why}; ").strip() + for rel in picked: + # Meili/embeddings top-hit + if rel in meili_scores: + _boost(rel, 0.55 * meili_scores[rel], "meili") + # pad-heuristiek + lo = rel.lower() + if lo.startswith("routes/"): _boost(rel, 0.08, "routes") + if lo.startswith("app/http/controllers/"): _boost(rel, 0.06, "controller") + if lo.startswith("resources/views/"): _boost(rel, 0.06, "view") + if lo.startswith("resources/lang/"): _boost(rel, 0.05, "lang") + # expliciet genoemd door user + if rel in explicit_all: _boost(rel, 0.20, "explicit") + + # 2b) Graph-boost: BFS vanaf expliciete seeds (en evt. route-bestanden) + try: + seeds = [p for p in picked if p in explicit_all] + # heuristisch: als gebruiker over "route" praat, neem routes/web.php als seed + if any(k in st.user_goal.lower() for k in [" route", "routes", "/"]): + for rp in ["routes/web.php","routes/api.php"]: + if rp in picked and rp not in seeds: + seeds.append(rp) + if graph and seeds: + bfs = _graph_bfs_boosts(graph, seeds, max_depth=int(os.getenv("AGENT_GRAPH_MAX_DEPTH","3"))) + for rel in picked: + if rel in bfs: + d, via = bfs[rel] + # afstand → boost: 0:0.08, 1:0.06, 2:0.03, 3:0.01 + boost_map = {0:0.08, 1:0.06, 2:0.03, 3:0.01} + b = boost_map.get(min(d,3), 0.0) + if b > 0: + _boost(rel, b, f"graph:d={d} via {via}") + st.reasons[f"graph::{rel}"] = f"d={d}, via {via}" + except Exception: + pass + + # 2c) Tree-summary boost: hits van prompt-keywords in samenvatting + try: + hints = extract_word_hints(st.user_goal) or [] + if hints and tree_summ: + lo_hints = [h.lower() for h in hints[:8]] + for rel in picked: + s = (tree_summ.get(rel) or "").lower() + if not s: + continue + hits = sum(1 for h in lo_hints if h in s) + if hits: + _boost(rel, min(0.04, 0.01 * hits), f"tree:{hits}hit") + if hits >= 2: + st.reasons[f"tree::{rel}"] = tree_summ.get(rel, "")[:200] + except Exception: + pass + + # 3) sorteer op totale score (desc) + picked.sort(key=lambda p: cand_scores.get(p, 0.0), reverse=True) + # 4) leg motivatie vast voor UI/preview + for rel in picked[:MAX_FILES_DRYRUN]: + if cand_scores.get(rel, 0.0) > 0: + st.reasons[f"rank::{rel}"] = f"{cand_scores[rel]:.2f} via {cand_why.get(rel,'')}" + st.candidate_paths = picked[:MAX_FILES_DRYRUN] + logger.info("CANDIDATES (explicit first, capped=%d): %s", MAX_FILES_DRYRUN, st.candidate_paths) + if not len(st.candidate_paths)>0: + st.stage = "ASK" + return _with_preview("\n".join(progress + ["Geen duidelijke kandidaten. Noem een pagina/onderdeel of (optioneel) bestandsnaam."]), st) + + + progress.append("Kandidaten:\n" + "\n".join([f"- {rel}" for rel in st.candidate_paths])) + logger.info("Kandidaten gevonden!") + + # DRY-RUN + logger.info("dry-run") + try: + proposed, diffs, reasons = await propose_patches_without_apply(st.repo_path, st.candidate_paths, st.user_goal) + if not proposed: + # ---- T3: automatische recovery (éénmalig) ---- + if not st.recovery_attempted: + st.recovery_attempted = True + try: + new_list, dbg = await _recovery_expand_candidates( + Path(st.repo_path), list_repo_files(Path(st.repo_path)), + st.user_goal, st.candidate_paths, last_reason="no_proposal_after_dryrun" + ) + st.candidate_paths = new_list + st.reasons["recovery_note"] = dbg.get("recovery_plan",{}).get("note","") + # opnieuw proberen + proposed2, diffs2, reasons2 = await propose_patches_without_apply(st.repo_path, st.candidate_paths, st.user_goal) + if proposed2: + st.proposed_patches = proposed2 + st.reasons.update(reasons2 or {}) + st.stage = "APPLY" + preview = [] + for rel in list(diffs2.keys())[:3]: + why = st.reasons.get(rel, "") + preview.append(f"### {rel}\n```\n{diffs2[rel]}\n```\n**Waarom**: {why}") + more = "" if len(diffs2) <= 3 else f"\n(Plus {len(diffs2)-3} extra diff(s).)" + base = "\n".join(progress + [ + "**Dry-run voorstel (na recovery):**", + "\n\n".join(preview) + more, + "\nTyp **'Akkoord apply'** om te schrijven & pushen, of geef feedback." + ]) + return _with_preview(base, st, header="--- SMART-RAG + recovery notities ---") + except Exception as e: + logger.warning("WARN:agent_repo:recovery attempt failed: %s", e) + # geen succes → val terug op bestaande melding + st.stage = "PROPOSE_DIFF_DRYRUN" + return "\n".join(progress + ["Dry-run: geen bruikbaar voorstel met deze kandidaten. Geef extra hint (pagina/ term)."]) + + st.proposed_patches = proposed + st.reasons = reasons + st.stage = "APPLY" + preview = [] + for rel in list(diffs.keys())[:3]: + why = reasons.get(rel, "") + preview.append(f"### {rel}\n```\n{diffs[rel]}\n```\n**Waarom**: {why}") + more = "" if len(diffs) <= 3 else f"\n(Plus {len(diffs)-3} extra diff(s).)" + base= "\n".join(progress + [ + "**Dry-run voorstel (geen writes):**", + "\n\n".join(preview) + more, + "\nTyp **'Akkoord apply'** om te schrijven & pushen, of geef feedback." + ]) + return _with_preview(base, st, header="--- SMART-RAG contextnotities ---") + except Exception as e: + logger.exception("ERROR:agent_repo:PROPOSE_DIFF_DRYRUN failed") + st.stage = "PROPOSE_DIFF_DRYRUN" + return "\n".join(progress + [f"Dry-run mislukte: {e}"]) + + if st.stage == "PROPOSE_DIFF_DRYRUN": + logger.info("Stage PROPOSE_DIFF_DRYRUN") + root = Path(st.repo_path) + all_files = list_repo_files(root) + added = [] + for pth in extract_explicit_paths(user_last): + if pth in all_files and pth not in st.candidate_paths: + added.append(pth) + else: + best = best_path_by_basename(all_files, pth) + if best and best not in st.candidate_paths: added.append(best) + st.candidate_paths = (added + st.candidate_paths)[:MAX_FILES_DRYRUN] + # extra: grep op 'old' literal uit user_goal om kandidaten te verrijken + qs = extract_quotes(st.user_goal) or [] + old = qs[0].strip() if qs and qs[0].strip() else None + if old: + for rel in _grep_repo_for_literal(root, old, limit=16): + if rel in all_files and rel not in st.candidate_paths: + st.candidate_paths.append(rel) + + + try: + proposed, diffs, reasons = await propose_patches_without_apply(st.repo_path, st.candidate_paths, st.user_goal) + if not proposed: + if not st.recovery_attempted: + st.recovery_attempted = True + try: + new_list, dbg = await _recovery_expand_candidates( + Path(st.repo_path), list_repo_files(Path(st.repo_path)), + st.user_goal, st.candidate_paths, last_reason="no_proposal_in_propose_diff" + ) + st.candidate_paths = new_list + st.reasons["recovery_note"] = dbg.get("recovery_plan",{}).get("note","") + # direct nog een poging + proposed2, diffs2, reasons2 = await propose_patches_without_apply(st.repo_path, st.candidate_paths, st.user_goal) + if proposed2: + st.proposed_patches = proposed2 + st.reasons.update(reasons2 or {}) + st.stage = "APPLY" + preview = [] + for rel in list(diffs2.keys())[:3]: + why = st.reasons.get(rel, "") + preview.append(f"### {rel}\n```\n{diffs2[rel]}\n```\n**Waarom**: {why}") + more = "" if len(diffs2) <= 3 else f"\n(Plus {len(diffs2)-3} extra diff(s).)" + base = ("**Dry-run voorstel (na recovery):**\n" + + "\n\n".join(preview) + more + + "\n\nTyp **'Akkoord apply'** om te schrijven & pushen, of geef feedback.") + return _with_preview(base, st, header="--- SMART-RAG + recovery notities ---") + except Exception as e: + logger.warning("WARN:agent_repo:recovery in PROPOSE_DIFF failed: %s", e) + return _with_preview("Nog geen bruikbaar voorstel. Noem exact bestand/pagina of plak relevante code.", st) + + st.proposed_patches = proposed + st.reasons = reasons + st.stage = "APPLY" + preview = [] + for rel in list(diffs.keys())[:3]: + why = reasons.get(rel, "") + preview.append(f"### {rel}\n```\n{diffs[rel]}\n```\n**Waarom**: {why}") + more = "" if len(diffs) <= 3 else f"\n(Plus {len(diffs)-3} extra diff(s).)" + base = ("**Dry-run voorstel (geen writes):**\n" + + "\n\n".join(preview) + more + + "\n\nTyp **'Akkoord apply'** om te schrijven & pushen, of geef feedback.") + return _with_preview(base, st, header="--- SMART-RAG contextnotities ---") + except Exception as e: + logger.exception("ERROR:agent_repo:PROPOSE_DIFF_DRYRUN retry failed") + return _with_preview(f"Dry-run mislukte: {e}", st) + + + + def _apply(): + if not (("akkoord" in user_last_lower) and ("apply" in user_last_lower)): + return "Typ **'Akkoord apply'** om de dry-run wijzigingen te schrijven & pushen." + try: + repo_path = _get_git_repo(st.repo_url, st.branch_base) + import git + repo = git.Repo(repo_path) + short = re.sub(r'[^a-z0-9\-]+','-', st.user_goal.lower()).strip("-") + st.new_branch = f"task/{short[:40]}-{time.strftime('%Y%m%d-%H%M%S')}" + repo.git.checkout("-b", st.new_branch) + changed = [] + for rel, content in st.proposed_patches.items(): + f = Path(repo_path) / rel + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(content, encoding="utf-8") + changed.append(str(f)) + if not changed: + return "Er waren geen wijzigingen om te commiten." + repo.index.add(changed) + msg = (f"feat: {st.user_goal}\n\nScope:\n" + + "\n".join([f"- {Path(c).relative_to(repo_path)}" for c in changed]) + + "\n\nRationale (samengevat):\n" + + "\n".join([f"- {k}: {v}" for k,v in st.reasons.items()]) + + "\n\nCo-authored-by: repo-agent\n") + repo.index.commit(msg) + repo.remotes.origin.push(refspec=f"{st.new_branch}:{st.new_branch}") + st.stage = "DONE" + return f"✅ Branch aangemaakt en gepusht: `{st.new_branch}`. Maak nu je PR in Gitea." + except Exception as e: + logger.exception("ERROR:agent_repo:APPLY failed") + st.stage = "PROPOSE_DIFF_DRYRUN" + return f"Apply/push mislukte: {e}" + if st.stage == "APPLY": + logger.info("Stage APPLY") + return await run_in_threadpool(_apply) + + if st.stage == "DONE": + logger.info("Stage DONE") + st.smart_preview = "" + return f"Klaar. Branch: `{st.new_branch}`." + return "Interne status onduidelijk; begin opnieuw of herformuleer je doel." + + diff --git a/app.py b/app.py new file mode 100644 index 0000000..e0facf3 --- /dev/null +++ b/app.py @@ -0,0 +1,5512 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +FastAPI bridge met: +- RAG (Chroma) index & query (client-side embeddings, http óf local) +- Repo cloning/updating (git) +- LLM-bridge (OpenAI-compatible /v1/chat/completions) +- Repo agent endpoints (injectie van helpers in agent_repo.py) +""" + +from __future__ import annotations +import contextlib +from contextlib import contextmanager +import os, re, json, time, uuid, hashlib, logging, asyncio, fnmatch, threading +from dataclasses import dataclass +from typing import List, Dict, Optional, Union, Any, Callable +from pathlib import Path +from io import BytesIO + +import requests +import httpx +import chromadb +import git +import base64 + +from fastapi import FastAPI, APIRouter, UploadFile, File, Form, Request, HTTPException, Body +from fastapi.responses import JSONResponse, StreamingResponse, PlainTextResponse +from fastapi.middleware.cors import CORSMiddleware +from fastapi.openapi.utils import get_openapi +from fastapi.routing import APIRoute +from starlette.concurrency import run_in_threadpool +from pydantic import BaseModel + +AUTO_CONTINUE = os.getenv("AUTO_CONTINUE", "1").lower() not in ("0","false","no") +AUTO_CONTINUE_MAX_ROUNDS = int(os.getenv("AUTO_CONTINUE_MAX_ROUNDS", "6")) +AUTO_CONTINUE_TAIL_CHARS = int(os.getenv("AUTO_CONTINUE_TAIL_CHARS", "600")) + +from llm_client import init_llm_client, _sync_model_infer +from web_search import Tools, HelpFunctions, EventEmitter + +import subprocess +import tempfile + + + +# Optionele libs voor tekst-extractie +try: + import PyPDF2 +except Exception: + PyPDF2 = None +try: + import docx # python-docx +except Exception: + docx = None +try: + import pandas as pd +except Exception: + pd = None +try: + from pptx import Presentation +except Exception: + Presentation = None + +# (optioneel) BM25 voor hybride retrieval +try: + from rank_bm25 import BM25Okapi +except Exception: + BM25Okapi = None + +# --- BM25 fallback registry (per repo) --- +_BM25_BY_REPO: dict[str, tuple[object, list[dict]]] = {} # repo_key -> (bm25, docs) +def _bm25_tok(s: str) -> list[str]: + return re.findall(r"[A-Za-z0-9_]+", s.lower()) + + +# --- Extra optional libs voor audio/vision/images --- +try: + import cairosvg +except Exception: + cairosvg = None + +import tempfile, subprocess # voor audio +import html # ← nieuw: unescape van HTML-entities in tool-call JSON + +# Forceer consistente caches zowel binnen als buiten Docker +os.environ.setdefault("HF_HOME", "/opt/hf") +os.environ.setdefault("HUGGINGFACE_HUB_CACHE", "/opt/hf") +os.environ.setdefault("TRANSFORMERS_CACHE", "/opt/hf") +os.environ.setdefault("SENTENCE_TRANSFORMERS_HOME", "/opt/sentence-transformers") +os.environ.setdefault("XDG_CACHE_HOME", "/opt/cache") +# whisper cache pad (gebruikt door faster-whisper) +os.environ.setdefault("WHISPER_CACHE_DIR", os.path.join(os.environ.get("XDG_CACHE_HOME","/opt/cache"), "whisper")) + + +# STT (Whisper-compatible via faster-whisper) — optioneel +_STT_MODEL = None +STT_MODEL_NAME = os.getenv("STT_MODEL", "small") +STT_DEVICE = os.getenv("STT_DEVICE", "auto") # "auto" | "cuda" | "cpu" + +# TTS (piper) — optioneel +PIPER_BIN = os.getenv("PIPER_BIN", "/usr/bin/piper") +PIPER_VOICE = os.getenv("PIPER_VOICE", "") + +RAG_LLM_RERANK = os.getenv("RAG_LLM_RERANK", "0").lower() in ("1","true","yes") + +# Lazy LLM import & autodetect +_openai = None +_mistral = None + + + +# Token utility / conversatie window (bestaat in je project) +try: + from windowing_utils import ( + derive_thread_id, SUMMARY_STORE, ConversationWindow, + approx_token_count, count_message_tokens + ) +except Exception: + # Fallbacks zodat dit bestand standalone blijft werken + SUMMARY_STORE = {} + def approx_token_count(s: str) -> int: + return max(1, len(s) // 4) + def count_message_tokens(messages: List[dict]) -> int: + return sum(approx_token_count(m.get("content","")) for m in messages) + def derive_thread_id(messages: List[dict]) -> str: + payload = (messages[0].get("content","") if messages else "") + "|".join(m.get("role","") for m in messages) + return hashlib.sha256(payload.encode("utf-8", errors="ignore")).hexdigest()[:16] + class ConversationWindow: + def __init__(self, *a, **k): pass + +# Queue helper (optioneel aanwezig in je project) +try: + from queue_helper import QueueManager, start_position_notifier, _Job + from queue_helper import USER_MAX_QUEUE, AGENT_MAX_QUEUE, UPDATE_INTERVAL, WORKER_TIMEOUT +except Exception: + QueueManager = None + +# Smart_rag import +from smart_rag import enrich_intent, expand_queries, hybrid_retrieve, assemble_context + +# Repo-agent (wij injecteren functies hieronder) +from agent_repo import initialize_agent, handle_repo_agent, repo_qa_answer, rag_query_internal_fn + +try: + from agent_repo import smart_chunk_text # al aanwezig in jouw agent_repo +except Exception: + def smart_chunk_text(text: str, path_hint: str, target_chars: int = 1800, + hard_max: int = 2600, min_chunk: int = 800): + # simpele fallback + chunks = [] + i, n = 0, len(text) + step = max(1, target_chars - 200) + while i < n: + chunks.append(text[i:i+target_chars]) + i += step + return chunks + + +def _build_tools_system_prompt(tools: list) -> str: + lines = [ + "You can call functions. When a function is needed, answer ONLY with a JSON object:", + '{"tool_calls":[{"name":"","arguments":{...}}]}', + "No prose. Arguments MUST be valid JSON for the function schema." + ] + for t in tools: + fn = t.get("function", {}) + lines.append(f"- {fn.get('name')}: {fn.get('description','')}") + return "\n".join(lines) + +def _extract_tool_calls_from_text(txt: str): + s = (txt or "").strip() + # Strip code fences & bekende chat-tokens + if s.startswith("```"): + s = extract_code_block(s) + s = re.sub(r"^<\|im_start\|>\s*assistant\s*", "", s, flags=re.I) + # HTML-escaped JSON (" etc.) + try: + s = html.unescape(s) + except Exception: + pass + # tolerant: pak eerste JSON object + m = re.search(r"\{[\s\S]*\}", s) + if not m: + return [] + try: + obj = json.loads(m.group(0)) + except Exception: + return [] + # === Normaliseer verschillende dialecten === + # 1) OpenAI: {"tool_calls":[{"type":"function","function":{"name":..,"arguments":..}}]} + if "tool_calls" in obj and isinstance(obj["tool_calls"], list): + tc = obj["tool_calls"] + # 2) Replit/Magistral/Mistral-achtig: {"call_tool":{"name":"...","arguments":{...}}} óf lijst + elif "call_tool" in obj: + raw = obj["call_tool"] + if isinstance(raw, dict): + raw = [raw] + if isinstance(raw, str): + print("Invalid toolcall:",str(raw)) + raw=[] + tc = [] + for it in (raw or []): + try: + name = (it or {}).get("name") + except Exception as e: + print("Error:",str(e),"raw=",str(raw),"it=",str(it)) + continue + try: + args = (it or {}).get("arguments") or {} + except Exception as e: + print("Error:",str(e),"raw=",str(raw),"it=",str(it)) + continue + if isinstance(args, str): + try: args = json.loads(args) + except Exception: pass + if name: + tc.append({"type":"function","function":{"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}) + # 3) Legacy OpenAI: {"function_call":{"name":"...","arguments":"{...}"}} + elif "function_call" in obj and isinstance(obj["function_call"], dict): + fc = obj["function_call"] + name = fc.get("name") + args = fc.get("arguments") or {} + if isinstance(args, str): + try: args = json.loads(args) + except Exception: pass + tc = [{"type":"function","function":{"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}] if name else [] + else: + tc = [] + out = [] + for c in tc: + name = (c or {}).get("name") + args = (c or {}).get("arguments") or {} + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + args = {} + # Support zowel {"name":..,"arguments":..} als {"type":"function","function":{...}} + if not name and isinstance(c.get("function"), dict): + name = c["function"].get("name") + args = c["function"].get("arguments") or args + if isinstance(args, str): + try: args = json.loads(args) + except Exception: args = {} + if name: + out.append({ + "id": f"call_{uuid.uuid4().hex[:8]}", + "type": "function", + "function": { + "name": name, + "arguments": json.dumps(args, ensure_ascii=False) + } + }) + return out + +# --- Universal toolcall detector (JSON + ReAct + HTML-escaped) --- +_TOOL_REACT = re.compile( + r"Action\s*:\s*([A-Za-z0-9_\-\.]+)\s*[\r\n]+Action\s*Input\s*:\s*(?:```json\s*([\s\S]*?)\s*```|([\s\S]*))", + re.I +) + +def detect_toolcalls_any(text: str) -> list[dict]: + """ + Retourneert altijd OpenAI-achtige tool_calls items: + [{"id":..., "type":"function", "function":{"name":..., "arguments": "{...}"}}] + """ + calls = _extract_tool_calls_from_text(text) + if calls: + return calls + s = (text or "").strip() + try: + s = html.unescape(s) + except Exception: + pass + # ReAct fallback + m = _TOOL_REACT.search(s) + if m: + name = (m.group(1) or "").strip() + raw = (m.group(2) or m.group(3) or "").strip() + # strip eventuele fences + if raw.startswith("```"): + raw = extract_code_block(raw) + try: + args = json.loads(raw) if raw else {} + except Exception: + # laatste redmiddel: wikkel als {"input": ""} + args = {"input": raw} + if name: + return [{ + "id": f"call_{uuid.uuid4().hex[:8]}", + "type": "function", + "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)} + }] + return [] + +def _coerce_text_toolcalls_to_openai(data: dict) -> dict: + """Als een upstream LLM tool-calls als tekst (bv. '[TOOL_CALLS] ...') teruggeeft, + zet dit om naar OpenAI-native choices[0].message.tool_calls zodat OpenWebUI tools kan runnen. + Laat bestaande tool_calls ongemoeid. + """ + try: + if not isinstance(data, dict): + return data + choices = data.get("choices") or [] + if not choices or not isinstance(choices, list): + return data + ch0 = choices[0] or {} + if not isinstance(ch0, dict): + return data + msg = ch0.get("message") or {} + if not isinstance(msg, dict): + return data + # native tool_calls bestaan al → niets doen + if msg.get("tool_calls"): + return data + + content = msg.get("content") + if not isinstance(content, str): + return data + s = content.strip() + if not s: + return data + + # Alleen proberen als er duidelijke signalen zijn + if ("[TOOL_CALLS]" not in s) and (not s.lstrip().startswith("[")) and ("call_tool" not in s) and ("tool_calls" not in s): + return data + + calls = detect_toolcalls_any(s) or [] + if not calls: + # vLLM/[TOOL_CALLS] stijl: vaak een JSON array na de tag + s2 = re.sub(r"^\s*\[TOOL_CALLS\]\s*", "", s, flags=re.I) + try: + s2 = html.unescape(s2) + except Exception: + pass + m = re.search(r"\[[\s\S]*\]", s2) + arr = None + if m: + try: + arr = json.loads(m.group(0)) + except Exception: + arr = None + if isinstance(arr, list): + calls = [] + for it in arr: + if not isinstance(it, dict): + continue + name = it.get("name") + args = it.get("arguments", {}) + if not name and isinstance(it.get("function"), dict): + name = it["function"].get("name") + args = it["function"].get("arguments", args) + if isinstance(args, str): + try: + args = json.loads(args) + except Exception: + args = {"input": args} + if name: + calls.append({ + "id": f"call_{uuid.uuid4().hex[:8]}", + "type": "function", + "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)} + }) + + if calls: + msg["role"] = msg.get("role") or "assistant" + msg["content"] = None + msg["tool_calls"] = calls + ch0["message"] = msg + ch0["finish_reason"] = "tool_calls" + data["choices"][0] = ch0 + return data + except Exception: + return data + + +# ----------------------------------------------------------------------------- +# App & logging +# ----------------------------------------------------------------------------- +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("app") + +def _unique_id(route: APIRoute): + # unieke operation_id op basis van naam, pad en method + method = list(route.methods)[0].lower() if route.methods else "get" + return f"{route.name}_{route.path.replace('/', '_')}_{method}" + + +app = FastAPI(title="Mistral Bridge API",openapi_url=None,generate_unique_id_function=_unique_id)#openapi_url=None, +app.add_middleware( + CORSMiddleware, + allow_credentials=True, + allow_origins=["*"], # dev only + allow_methods=["*"], + allow_headers=["*"], +) + + +# Eén centrale QueueManager voor ALLE LLM-calls +#app.state.LLM_QUEUE = QueueManager(model_infer_fn=_sync_model_infer) + +# Koppel deze queue aan llm_client zodat _llm_call dezelfde queue gebruikt +#init_llm_client(app.state.LLM_QUEUE) + +# Let op: +# - llm_call_openai_compat verwacht dat app.state.LLM_QUEUE een deque-achtige queue is +# (append/index/popleft/remove). +# - De QueueManager uit queue_helper.py is thread-based en wordt hier niet gebruikt. +# Als je QueueManager wilt inzetten, doe dat in llm_client.py of elders, +# maar niet als app.state.LLM_QUEUE, om type-conflicten te voorkomen. + + +@app.on_event("startup") +async def _startup(): + # Zorg dat lokale hosts nooit via een proxy gaan + os.environ.setdefault( + "NO_PROXY", + "localhost,127.0.0.1,::1,host.docker.internal" + ) + app.state.HTTPX = httpx.AsyncClient( + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20), + timeout=httpx.Timeout(LLM_READ_TIMEOUT, connect=LLM_CONNECT_TIMEOUT), + trust_env=False, # belangrijk: negeer env-proxy’s voor LLM + headers={"Connection": "keep-alive"} # houd verbindingen warm + ) + app.state.HTTPX_PROXY = httpx.AsyncClient( + limits=httpx.Limits(max_connections=100, max_keepalive_connections=20), + timeout=httpx.Timeout(LLM_READ_TIMEOUT, connect=LLM_CONNECT_TIMEOUT), + trust_env=True, # belangrijk: negeer env-proxy’s voor LLM + headers={"Connection": "keep-alive"} # houd verbindingen warm + ) + +@app.on_event("shutdown") +async def _shutdown(): + try: + await app.state.HTTPX.aclose() + except Exception: + pass + try: + await app.state.HTTPX_PROXY.aclose() + except Exception: + pass + + +# --- Globale LLM-concurrency & wachtrij (serieel by default) --- +LLM_MAX_CONCURRENCY = int(os.getenv("LLM_MAX_CONCURRENCY", os.getenv("LLM_CONCURRENCY", "1"))) + +if not hasattr(app.state, "LLM_SEM"): + import asyncio + app.state.LLM_SEM = asyncio.Semaphore(max(1, LLM_MAX_CONCURRENCY)) +if not hasattr(app.state, "LLM_QUEUE"): + from collections import deque + app.state.LLM_QUEUE = deque() + + +@app.middleware("http") +async def log_requests(request: Request, call_next): + logger.info("➡️ %s %s", request.method, request.url.path) + response = await call_next(request) + logger.info("⬅️ %s", response.status_code) + return response + +# ----------------------------------------------------------------------------- +# Config +# ----------------------------------------------------------------------------- +MISTRAL_MODE = os.getenv("MISTRAL_MODE", "v1").lower() +LLM_URL = os.getenv("LLM_URL", "http://192.168.100.1:8000/v1/chat/completions").strip() +RAW_URL = os.getenv("MISTRAL_URL_RAW", "http://host.docker.internal:8000/completion").strip() +LLM_CONNECT_TIMEOUT = float(os.getenv("LLM_CONNECT_TIMEOUT", "10")) +LLM_READ_TIMEOUT = float(os.getenv("LLM_READ_TIMEOUT", "1200")) + +_UPSTREAM_URLS = [u.strip() for u in os.getenv("LLM_UPSTREAMS","").split(",") if u.strip()] + +# ==== Meilisearch (optioneel) ==== +MEILI_URL = os.getenv("MEILI_URL", "http://192.168.100.1:7700").rstrip("/") +MEILI_API_KEY = os.getenv("MEILI_API_KEY", "0xipOmfgi_zMgdFplSdv7L8mlx0RPMQCNxVTNJc54lQ") +MEILI_INDEX = os.getenv("MEILI_INDEX", "code_chunks") +MEILI_ENABLED = bool(MEILI_URL) + +# --- Chat memory (hybride RAG: Meili + Chroma) --- +MEILI_MEMORY_INDEX = os.getenv("MEILI_MEMORY_INDEX", "chat_memory") +MEMORY_COLLECTION = os.getenv("MEMORY_COLLECTION", "chat_memory") + +MEMORY_CHUNK_CHARS = int(os.getenv("MEMORY_CHUNK_CHARS", "3500")) +MEMORY_OVERLAP_CHARS = int(os.getenv("MEMORY_OVERLAP_CHARS", "300")) + +MEMORY_EMB_WEIGHT = float(os.getenv("MEMORY_EMB_WEIGHT", os.getenv("RAG_EMB_WEIGHT", "0.6"))) +MEMORY_RECENCY_BIAS = float(os.getenv("MEMORY_RECENCY_BIAS", "0.10")) +MEMORY_HALF_LIFE_SEC = int(os.getenv("MEMORY_HALF_LIFE_SEC", "259200")) # 3 dagen +MEMORY_MAX_TOTAL_CHARS = int(os.getenv("MEMORY_MAX_TOTAL_CHARS", "12000")) +MEMORY_CLIP_CHARS = int(os.getenv("MEMORY_CLIP_CHARS", "1200")) + + +# Repo summaries (cache on demand) +_SUMMARY_DIR = os.path.join("/rag_db", "repo_summaries") +os.makedirs(_SUMMARY_DIR, exist_ok=True) + +@dataclass +class _Upstream: + url: str + active: int = 0 + ok: bool = True + +_UPS = [_Upstream(u) for u in _UPSTREAM_URLS] if _UPSTREAM_URLS else [] + +def _pick_upstream_sticky(key: str) -> _Upstream | None: + if not _UPS: + return None + try: + h = int(hashlib.sha1(key.encode("utf-8")).hexdigest(), 16) + except Exception: + h = 0 + idx = h % len(_UPS) + cand = _UPS[idx] + if cand.ok: + cand.active += 1 + return cand + ok_list = [u for u in _UPS if u.ok] + best = min(ok_list or _UPS, key=lambda x: x.active) + best.active += 1 + return best + + +def _pick_upstream() -> _Upstream | None: + if not _UPS: + return None + # kies de minst-belaste die ok is, anders toch de minst-belaste + ok_list = [u for u in _UPS if u.ok] + cand = min(ok_list or _UPS, key=lambda x: x.active) + cand.active += 1 + return cand + +def _release_upstream(u: _Upstream | None, bad: bool = False): + if not u: + return + u.active = max(0, u.active - 1) + if bad: + u.ok = False + # simpele cool-off: na 2 sec weer ok + asyncio.create_task(_mark_ok_later(u, 10.0)) + +async def _mark_ok_later(u: _Upstream, delay: float): + await asyncio.sleep(delay) + u.ok = True + +SYSTEM_PROMPT = ( + "Je bent een expert programmeringsassistent. Je geeft accurate, specifieke antwoorden zonder hallucinaties.\n" + "Voor code base analyse:\n" + "1. Geef eerst een samenvatting van de functionaliteit\n" + "2. Identificeer mogelijke problemen of verbeterpunten\n" + "3. Geef concrete aanbevelingen voor correcties\n" + "4. Voor verbeterde versies: geef eerst de toelichting, dan alleen het codeblok" +) + +# ==== Chroma configuratie (local/http) ==== +CHROMA_MODE = os.getenv("CHROMA_MODE", "local").lower() # "local" | "http" +CHROMA_PATH = os.getenv("CHROMA_PATH", "/rag_db") +CHROMA_HOST = os.getenv("CHROMA_HOST", "chroma") +CHROMA_PORT = int(os.getenv("CHROMA_PORT", "8005")) + +# ==== Celery (optioneel, voor async indexing) ==== +CELERY_ENABLED = os.getenv("CELERY_ENABLED", "0").lower() in ("1", "true", "yes") +celery_app = None +if CELERY_ENABLED: + try: + from celery import Celery + celery_app = Celery( + "agent_tasks", + broker=os.getenv("AMQP_URL", "amqp://guest:guest@rabbitmq:5672//"), + backend=os.getenv("CELERY_BACKEND", "redis://redis:6379/0"), + ) + celery_app.conf.update( + task_acks_late=True, + worker_prefetch_multiplier=1, + task_time_limit=60*60, + ) + except Exception as e: + logger.warning("Celery init failed (fallback naar sync): %s", e) + celery_app = None + +# Git / repos +GITEA_URL = os.environ.get("GITEA_URL", "http://10.25.138.40:30085").rstrip("/") +REPO_PATH = os.environ.get("REPO_PATH", "/tmp/repos") + +# ----------------------------------------------------------------------------- +# Models (Pydantic) +# ----------------------------------------------------------------------------- +class ChatMessage(BaseModel): + role: str + content: str + +class ChatRequest(BaseModel): + messages: List[ChatMessage] + +class RepoQARequest(BaseModel): + repo_hint: str + question: str + branch: str = "main" + n_ctx: int = 8 + +# ----------------------------------------------------------------------------- +# Embeddings (SentenceTransformers / ONNX / Default) +# ----------------------------------------------------------------------------- +from chromadb.api.types import EmbeddingFunction, Documents, Embeddings +try: + from sentence_transformers import SentenceTransformer +except Exception: + SentenceTransformer = None + +@dataclass +class _Embedder: + slug: str + family: str + model: object + device: str = "cpu" + + def _encode(self, texts: list[str]) -> list[list[float]]: + if hasattr(self.model, "encode"): + bs = int(os.getenv("RAG_EMBED_BATCH_SIZE", "64")) + # geen progressbar; grotere batches voor throughput + return self.model.encode( + texts, + normalize_embeddings=True, + batch_size=bs, + show_progress_bar=False + ).tolist() + return self.model(texts) + + def embed_documents(self, docs: list[str]) -> list[list[float]]: + if self.family == "e5": + docs = [f"passage: {t}" for t in docs] + return self._encode(docs) + + def embed_query(self, q: str) -> list[float]: + # e5 prefix blijft identiek + if self.family == "e5": + q = f"query: {q}" + # --- LRU cache (per proces) om dubbele embeds bij routed buckets te vermijden --- + key = (q or "").strip() + cache = getattr(self, "_q_cache", None) + if cache is None: + from collections import OrderedDict + self._q_cache = OrderedDict() + self._q_cache_cap = int(os.getenv("RAG_Q_EMBED_CACHE", "2048")) + cache = self._q_cache + if key in cache: + v = cache.pop(key) + cache[key] = v + return v + v = self._encode([q])[0] + cache[key] = v + if len(cache) > getattr(self, "_q_cache_cap", 2048): + cache.popitem(last=False) # evict oldest + return v + +def _build_embedder() -> _Embedder: + import inspect + # voorkom tokenizer thread-oversubscription + os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + choice = os.getenv("RAG_EMBEDDINGS", "gte-multilingual").lower().strip() + if SentenceTransformer: + mapping = { + "gte-multilingual": ("Alibaba-NLP/gte-multilingual-base", "gte", "gte-multilingual"), + "bge-small": ("BAAI/bge-small-en-v1.5", "bge", "bge-small"), + "e5-small": ("intfloat/e5-small-v2", "e5", "e5-small"), + "gte-base-en": ("thenlper/gte-base", "gte", "gte-english"), + } + if choice not in mapping: + model_name, family, slug = mapping["bge-small"] + else: + model_name, family, slug = mapping[choice] + cache_dir = os.environ.get("SENTENCE_TRANSFORMERS_HOME", "/opt/sentence-transformers") + local_dir = os.path.join(cache_dir, "embedder") + # --- device auto-select: cuda als beschikbaar, anders cpu; override met RAG_EMBED_DEVICE --- + dev_req = os.getenv("RAG_EMBED_DEVICE", "auto").strip().lower() + dev = "cpu" + try: + import torch + if dev_req in ("cuda", "gpu", "auto") and torch.cuda.is_available(): + dev = "cuda" + except Exception: + dev = "cpu" + st_kwargs = {"device": dev} + if os.path.isdir(local_dir): + # Prefetched model in image → gebruik dat + model_source = local_dir + print(f"Loading SentenceTransformer from local dir: {local_dir}") + else: + # Geen lokale dir gevonden → terugvallen op HF-ID + cache_folder + model_source = model_name + st_kwargs["cache_folder"] = cache_dir + print(f"Local dir {local_dir} not found, falling back to HF model: {model_name}") + + try: + if "trust_remote_code" in inspect.signature(SentenceTransformer).parameters: + st_kwargs["trust_remote_code"] = True + model = SentenceTransformer(model_source, **st_kwargs) + # logging: inzichtelijk maken waar embeds draaien + print(f"[embeddings] model={model_name} device={dev}") + # optioneel: CPU thread-telling forceren (fallback) + try: + thr = int(os.getenv("RAG_TORCH_THREADS", "0")) + if thr > 0: + import torch + torch.set_num_threads(thr) + except Exception: + pass + return _Embedder(slug=slug, family=family, model=model, device=dev) + except Exception as e: + print("ERROR building embedder:",str(e)) + + # Fallback via Chroma embedding functions + from chromadb.utils import embedding_functions as ef + try: + onnx = ef.ONNXMiniLM_L6_V2() + slug, family = "onnx-minilm", "minilm" + except Exception: + onnx = ef.DefaultEmbeddingFunction() + slug, family = "default", "minilm" + + class _OnnxWrapper: + def __init__(self, fun): self.fun = fun + def __call__(self, texts): return self.fun(texts) + return _Embedder(slug=slug, family=family, model=_OnnxWrapper(onnx)) + +_EMBEDDER = _build_embedder() + +class _ChromaEF(EmbeddingFunction): + """Alleen gebruikt bij local PersistentClient (niet bij http); naam wordt geborgd.""" + def __init__(self, embedder: _Embedder): + self._embedder = embedder + def __call__(self, input: Documents) -> Embeddings: + return self._embedder.embed_documents(list(input)) + def name(self) -> str: + return f"rag-ef:{self._embedder.family}:{self._embedder.slug}" + +_CHROMA_EF = _ChromaEF(_EMBEDDER) + +# === Chroma client (local of http) === +if CHROMA_MODE == "http": + _CHROMA = chromadb.HttpClient(host=CHROMA_HOST, port=CHROMA_PORT) +else: + _CHROMA = chromadb.PersistentClient(path=CHROMA_PATH) + +def _collection_versioned(base: str) -> str: + ver = os.getenv("RAG_INDEX_VERSION", "3") + return f"{base}__{_EMBEDDER.slug}__v{ver}" + +_COLLECTIONS: dict[str, any] = {} + +def _get_collection(base: str): + """Haalt collection op; bij local voegt embedding_function toe, bij http niet (embedden doen we client-side).""" + name = _collection_versioned(base) + if name not in _COLLECTIONS: + if CHROMA_MODE == "http": + _COLLECTIONS[name] = _CHROMA.get_or_create_collection(name=name) + else: + _COLLECTIONS[name] = _CHROMA.get_or_create_collection(name=name, embedding_function=_CHROMA_EF) + return _COLLECTIONS[name] + +def _collection_add(collection, documents: list[str], metadatas: list[dict], ids: list[str]): + """Altijd embeddings client-side meezenden — werkt voor local én http. + Voeg een lichte header toe (pad/taal/symbolen) om retrieval te verbeteren. + """ + def _symbol_hints(txt: str) -> list[str]: + hints = [] + # heel simpele, taalonafhankelijke patterns + for pat in [r"def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"class\s+([A-Za-z_][A-Za-z0-9_]*)\b", + r"function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"public\s+function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\("]: + try: + hints += re.findall(pat, txt[:4000]) + except Exception: + pass + # uniek en klein houden + out = [] + for h in hints: + if h not in out: + out.append(h) + if len(out) >= 6: + break + return out + + def _sanitize_meta(meta: dict) -> dict: + out = {} + for k, v in (meta or {}).items(): + if v is None: + continue # DROP None: Rust Chroma accepteert dit niet + if isinstance(v, (str, int, float, bool)): + out[k] = v + elif isinstance(v, (list, tuple, set)): + if not v: + continue + if all(isinstance(x, str) for x in v): + out[k] = ",".join(v) + else: + out[k] = json.dumps(list(v), ensure_ascii=False) + elif isinstance(v, dict): + out[k] = json.dumps(v, ensure_ascii=False) + else: + out[k] = str(v) + return out + augmented_docs = [] + metadatas_mod = [] + for doc, meta in zip(documents, metadatas): + path = (meta or {}).get("path", "") + ext = (Path(path).suffix.lower().lstrip(".") if path else "") or "txt" + syms = _symbol_hints(doc) + header = f"FILE:{path} | LANG:{ext} | SYMBOLS:{','.join(syms)}\n" + augmented_docs.append(header + (doc or "")) + raw = dict(meta or {}) + m = _sanitize_meta(dict(meta or {})) + if syms: + m["symbols"] = ",".join(syms[:8]) + metadatas_mod.append(m) + + + embs = _EMBEDDER.embed_documents(augmented_docs) + collection.add(documents=augmented_docs, embeddings=embs, metadatas=metadatas_mod, ids=ids) + +# ----------------------------------------------------------------------------- +# Repository profile + file selectie +# ----------------------------------------------------------------------------- +PROFILE_EXCLUDE_DIRS = { + ".git",".npm","node_modules","vendor","storage","dist","build",".next", + "__pycache__",".venv","venv",".mypy_cache",".pytest_cache", + "target","bin","obj","logs","cache","temp",".cache",".idea",".vscode" +} +PROFILE_INCLUDES = { + "generic": ["*.md","*.txt","*.json","*.yml","*.yaml","*.ini","*.cfg","*.toml", + "*.py","*.php","*.js","*.ts","*.jsx","*.tsx","*.css","*.scss", + "*.html","*.htm","*.vue","*.rb","*.go","*.java","*.cs","*.blade.php"], + "laravel": ["*.php","*.blade.php","*.md","*.env","*.json"], + "node": ["*.js","*.ts","*.jsx","*.tsx","*.vue","*.md","*.json","*.css","*.scss","*.html","*.htm"], +} + +def _detect_repo_profile(root: Path) -> str: + if (root / "artisan").exists() or (root / "composer.json").exists(): + return "laravel" + if (root / "package.json").exists(): + return "node" + return "generic" + +# ----------------------------------------------------------------------------- +# Tekst-extractie helpers +# ----------------------------------------------------------------------------- +TEXT_EXTS = { + ".php",".blade.php",".vue",".js",".ts",".jsx",".tsx",".css",".scss", + ".html",".htm",".json",".md",".ini",".cfg",".yml",".yaml",".toml", + ".py",".go",".rb",".java",".cs",".txt",".env",".sh",".bat",".dockerfile", +} +BINARY_SKIP = {".png",".jpg",".jpeg",".webp",".bmp",".gif",".ico",".pdf",".zip",".gz",".tar",".7z",".rar",".woff",".woff2",".ttf",".eot",".otf"} + +def _read_text_file(p: Path) -> str: + ext = p.suffix.lower() + # Skip obvious binaries quickly + if ext in BINARY_SKIP: + # PDF → probeer tekst + if ext == ".pdf" and PyPDF2: + try: + with open(p, "rb") as f: + reader = PyPDF2.PdfReader(f) + out = [] + for page in reader.pages: + try: + out.append(page.extract_text() or "") + except Exception: + pass + return "\n".join(out).strip() + except Exception: + return "" + return "" + # DOCX + if ext == ".docx" and docx: + try: + d = docx.Document(str(p)) + return "\n".join([para.text for para in d.paragraphs]) + except Exception: + return "" + # CSV/XLSX (alleen header + paar regels) + if ext in {".csv",".tsv"} and pd: + try: + df = pd.read_csv(p, nrows=200) + return df.to_csv(index=False)[:20000] + except Exception: + pass + if ext in {".xlsx",".xls"} and pd: + try: + df = pd.read_excel(p, nrows=200) + return df.to_csv(index=False)[:20000] + except Exception: + pass + # PPTX + if ext == ".pptx" and Presentation: + try: + prs = Presentation(str(p)) + texts = [] + for slide in prs.slides: + for shape in slide.shapes: + if hasattr(shape, "text"): + texts.append(shape.text) + return "\n".join(texts) + except Exception: + return "" + # Default: lees als tekst + try: + return p.read_text(encoding="utf-8", errors="ignore") + except Exception: + try: + return p.read_text(encoding="latin-1", errors="ignore") + except Exception: + return "" + +def _chunk_text(text: str, chunk_chars: int = 3000, overlap: int = 400) -> List[str]: + if not text: return [] + n = len(text); i = 0; step = max(1, chunk_chars - overlap) + chunks = [] + while i < n: + chunks.append(text[i:i+chunk_chars]) + i += step + return chunks + +# ----------------------------------------------------------------------------- +# Git helper +# ----------------------------------------------------------------------------- +def get_git_repo(repo_url: str, branch: str = "main") -> str: + """ + Clone of update repo in REPO_PATH, checkout branch. + Retourneert pad als string. + """ + os.makedirs(REPO_PATH, exist_ok=True) + # Unieke directory obv owner/repo of hash + name = None + try: + from urllib.parse import urlparse + u = urlparse(repo_url) + parts = [p for p in u.path.split("/") if p] + if parts: + name = parts[-1] + if name.endswith(".git"): name = name[:-4] + except Exception: + pass + if not name: + name = hashlib.sha1(repo_url.encode("utf-8", errors="ignore")).hexdigest()[:12] + local = os.path.join(REPO_PATH, name) + lock_path = f"{local}.lock" + with _file_lock(lock_path, timeout=float(os.getenv("GIT_LOCK_TIMEOUT","60"))): + if not os.path.exists(local): + logger.info("Cloning %s → %s", repo_url, local) + repo = git.Repo.clone_from(repo_url, local, depth=1) + else: + repo = git.Repo(local) + try: + repo.remote().fetch(depth=1, prune=True) + except Exception as e: + logger.warning("Fetch failed: %s", e) + # Checkout + logger.info("Checking out branch %s", branch) + try: + repo.git.checkout(branch) + except Exception: + # probeer origin/branch aan te maken + try: + repo.git.checkout("-B", branch, f"origin/{branch}") + except Exception: + # laatste fallback: default HEAD + logger.warning("Checkout %s failed; fallback to default HEAD", branch) + logger.info("Done with: Checking out branch %s", branch) + return local + +# ----------------------------------------------------------------------------- +# LLM call (OpenAI-compatible) +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# LLM call (OpenAI-compatible) +# ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# LLM call (OpenAI-compatible) — met seriële wachtrij (LLM_SEM + LLM_QUEUE) +# ----------------------------------------------------------------------------- +async def llm_call_openai_compat( + messages: List[dict], + *, + model: Optional[str] = None, + stream: bool = False, + temperature: float = 0.02, + top_p: float = 0.9, + max_tokens: int = 42000, + extra: Optional[dict] = None, + stop: Optional[Union[str, list[str]]] = None, + **kwargs +) -> dict | StreamingResponse: + payload: dict = { + "model": model or os.getenv("LLM_MODEL", "mistral-medium"), + "messages": messages, + "temperature": temperature, + "top_p": top_p, + "max_tokens": max_tokens, + "stream": bool(stream), + "repeat_penalty": 1.5, + } + # OpenAI-compat: optionele stop-sequenties doorgeven indien aanwezig + if stop is not None: + payload["stop"] = stop + # Eventuele andere onbekende kwargs negeren (compat met callsites die extra parameters sturen) + # (bewust geen payload.update(kwargs) om upstream niet te breken) + + if extra: + payload.update(extra) + + # kies URL + thread_key = None + try: + # Sticky per conversatie én model (voorkomt verkeerde stickiness en TypeErrors) + _model = model or os.getenv("LLM_MODEL","mistral-medium") + thread_key = f"{_model}:{derive_thread_id(messages)}" + except Exception: + thread_key = (model or os.getenv("LLM_MODEL","mistral-medium")) + ":" + json.dumps( + [m.get("role","")+":"+(m.get("content","")[:64]) for m in messages] + )[:256] + + upstream = _pick_upstream_sticky(thread_key) or _pick_upstream() + #url = (upstream.url if upstream else LLM_URL) + #upstream = _pick_upstream() + url = (upstream.url if upstream else LLM_URL) + + # --- NON-STREAM: wacht keurig op beurt en houd exclusieve lock vast + if not stream: + token = object() + app.state.LLM_QUEUE.append(token) + try: + # Fair: wacht tot je vooraan staat + # Fair: laat de eerste N (LLM_MAX_CONCURRENCY) door zodra er een permit vrij is + while True: + try: + pos = app.state.LLM_QUEUE.index(token) + 1 + except ValueError: + return # token verdwenen (client annuleerde) + free = getattr(app.state.LLM_SEM, "_value", 0) + if pos <= LLM_MAX_CONCURRENCY and free > 0: + await app.state.LLM_SEM.acquire() + break + await asyncio.sleep(0.1) + + + # ⬇️ BELANGRIJK: gebruik non-proxy client (zelfde host; vermijd env-proxy timeouts) + client = app.state.HTTPX + r = await client.post(url, json=payload) + try: + r.raise_for_status() + return r.json() + except httpx.HTTPStatusError as e: + _release_upstream(upstream, bad=True) + # geef een OpenAI-achtige fout terug + raise HTTPException(status_code=r.status_code, detail=f"LLM upstream error: {e}") # behoudt statuscode + except ValueError: + _release_upstream(upstream, bad=True) + raise HTTPException(status_code=502, detail="LLM upstream gaf geen geldige JSON.") + finally: + _release_upstream(upstream) + app.state.LLM_SEM.release() + finally: + try: + if app.state.LLM_QUEUE and app.state.LLM_QUEUE[0] is token: + app.state.LLM_QUEUE.popleft() + else: + app.state.LLM_QUEUE.remove(token) + except ValueError: + pass + + # --- STREAM: stuur wachtrij-updates als SSE totdat je aan de beurt bent + async def _aiter(): + token = object() + app.state.LLM_QUEUE.append(token) + try: + # 1) wachtrij-positie uitsturen zolang je niet mag + first = True + while True: + try: + pos = app.state.LLM_QUEUE.index(token) + 1 + except ValueError: + return + free = getattr(app.state.LLM_SEM, "_value", 0) + if pos <= LLM_MAX_CONCURRENCY and free > 0: + await app.state.LLM_SEM.acquire() + break + if first or pos <= LLM_MAX_CONCURRENCY: + data = { + "id": f"queue-info-{int(time.time())}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": payload["model"], + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": f"⏳ Plek #{pos} – vrije slots: {free}\n"}, + "finish_reason": None + }] + } + yield ("data: " + json.dumps(data, ensure_ascii=False) + "\n\n").encode("utf-8") + first = False + await asyncio.sleep(1.0) + + + # 2) nu exclusieve toegang → echte upstream streamen + try: + client = app.state.HTTPX + # timeout=None voor onbegrensde streams + async with client.stream("POST", url, json=payload, timeout=None) as r: + r.raise_for_status() + HEARTBEAT = float(os.getenv("SSE_HEARTBEAT_SEC","10")) + q: asyncio.Queue[bytes] = asyncio.Queue(maxsize=100) + + async def _reader(): + try: + async for chunk in r.aiter_bytes(): + if chunk: + await q.put(chunk) + except Exception: + pass + finally: + await q.put(b"__EOF__") + + reader_task = asyncio.create_task(_reader()) + try: + while True: + try: + chunk = await asyncio.wait_for(q.get(), timeout=HEARTBEAT) + except asyncio.TimeoutError: + # SSE comment; door UIs genegeerd → houdt verbinding warm + yield b": ping\n\n" + continue + if chunk == b"__EOF__": + break + yield chunk + finally: + reader_task.cancel() + with contextlib.suppress(Exception): + await reader_task + except (asyncio.CancelledError, httpx.RemoteProtocolError) as e: + # Client brak verbinding; upstream niet ‘bad’ markeren + _release_upstream(upstream, bad=False) + logger.info("🔌 stream cancelled/closed by client: %s", e) + return + except Exception as e: + _release_upstream(upstream, bad=True) + logger.info("🔌 stream aborted (compat): %s", e) + return + finally: + _release_upstream(upstream) + app.state.LLM_SEM.release() + + finally: + try: + if app.state.LLM_QUEUE and app.state.LLM_QUEUE[0] is token: + app.state.LLM_QUEUE.popleft() + else: + app.state.LLM_QUEUE.remove(token) + except ValueError: + pass + + return StreamingResponse( + _aiter(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache, no-transform", + "X-Accel-Buffering": "no", + "Connection": "keep-alive", + }, + ) + + +def extract_code_block(s: str) -> str: + """ + Pak de eerste ``` ``` blok-inhoud (zonder fences). Val terug naar volledige tekst als geen blok aangetroffen. + """ + if not s: return "" + m = re.search(r"```[a-zA-Z0-9_+-]*\n([\s\S]*?)```", s) + if m: + return m.group(1).strip() + return s.strip() + +_SIZE_RE = re.compile(r"^\s*(\d+)\s*x\s*(\d+)\s*$") + +def _parse_size(size_str: str) -> tuple[int,int]: + m = _SIZE_RE.match(str(size_str or "512x512")) + if not m: return (512,512) + w = max(64, min(2048, int(m.group(1)))) + h = max(64, min(2048, int(m.group(2)))) + return (w, h) + +def _sanitize_svg(svg: str) -> str: + # strip code fences + if "```" in svg: + svg = extract_code_block(svg) + # remove scripts + on* handlers + svg = re.sub(r"<\s*script\b[^>]*>.*?<\s*/\s*script\s*>", "", svg, flags=re.I|re.S) + svg = re.sub(r"\son\w+\s*=\s*['\"].*?['\"]", "", svg, flags=re.I|re.S) + return svg + +def _svg_wrap_if_needed(svg: str, w: int, h: int, bg: str="white") -> str: + s = (svg or "").strip() + if " + + SVG niet gevonden in modeloutput +''' + if 'width=' not in s or 'height=' not in s: + s = re.sub(r" str: + sys = ("Je bent een SVG-tekenaar. Geef ALLEEN raw SVG 1.1 markup terug; geen uitleg of code fences. " + "Geen externe refs of scripts.") + user = (f"Maak een eenvoudige vectorillustratie.\n- Canvas {w}x{h}, achtergrond {background}\n" + f"- Thema: {prompt}\n- Gebruik eenvoudige vormen/paths/tekst.") + resp = await llm_call_openai_compat( + [{"role":"system","content":sys},{"role":"user","content":user}], + stream=False, temperature=0.035, top_p=0.9, max_tokens=2048 + ) + svg = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + return _svg_wrap_if_needed(_sanitize_svg(svg), w, h, background) + +# --------- kleine helpers: atomic json write & simpele file lock ---------- +def _atomic_json_write(path: str, data: dict): + tmp = f"{path}.tmp.{uuid.uuid4().hex}" + Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(tmp, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) + os.replace(tmp, path) + +@contextmanager +def _file_lock(lock_path: str, timeout: float = 60.0, poll: float = 0.2): + start = time.time() + fd = None + while True: + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + break + except FileExistsError: + if time.time() - start > timeout: raise TimeoutError(f"Lock timeout for {lock_path}") + time.sleep(poll) + try: + os.write(fd, str(os.getpid()).encode("utf-8", errors="ignore")) + yield + finally: + with contextlib.suppress(Exception): + os.close(fd) if fd is not None else None + os.unlink(lock_path) + + +# -------- UploadFile → text (generiek), gebruikt je optional libs -------- +async def read_file_content(file: UploadFile) -> str: + name = (file.filename or "").lower() + raw = await run_in_threadpool(file.file.read) + + # Plain-text achtig + if name.endswith((".txt",".py",".php",".js",".ts",".jsx",".tsx",".html",".css",".json", + ".md",".yml",".yaml",".ini",".cfg",".log",".toml",".env",".sh",".bat",".dockerfile")): + return raw.decode("utf-8", errors="ignore") + + if name.endswith(".docx") and docx: + d = docx.Document(BytesIO(raw)) + return "\n".join(p.text for p in d.paragraphs) + + if name.endswith(".pdf") and PyPDF2: + reader = PyPDF2.PdfReader(BytesIO(raw)) + return "\n".join([(p.extract_text() or "") for p in reader.pages]) + + if name.endswith(".csv") and pd is not None: + df = pd.read_csv(BytesIO(raw)) + return df.head(200).to_csv(index=False) + + if name.endswith((".xlsx",".xls")) and pd is not None: + try: + sheets = pd.read_excel(BytesIO(raw), sheet_name=None, header=0) + except Exception as e: + return f"(Kon XLSX niet parsen: {e})" + out = [] + for sname, df in sheets.items(): + out.append(f"# Sheet: {sname}\n{df.head(200).to_csv(index=False)}") + return "\n\n".join(out) + + if name.endswith(".pptx") and Presentation: + pres = Presentation(BytesIO(raw)) + texts = [] + for i, slide in enumerate(pres.slides, 1): + buf = [f"# Slide {i}"] + for shape in slide.shapes: + if hasattr(shape, "has_text_frame") and shape.has_text_frame: + buf.append(shape.text.strip()) + texts.append("\n".join([t for t in buf if t])) + return "\n".join(texts) + + # Afbeeldingen of onbekend → probeer raw tekst + try: + return raw.decode("utf-8", errors="ignore") + except Exception: + return "(onbekend/beeld-bestand)" + + +def _client_ip(request: Request) -> str: + for hdr in ("cf-connecting-ip","x-real-ip","x-forwarded-for"): + v = request.headers.get(hdr) + if v: return v.split(",")[0].strip() + ip = request.client.host if request.client else "0.0.0.0" + return ip + +# --- Vision helpers: OpenAI content parts -> plain text + images (b64) --- +_JSON_FENCE_RE = re.compile(r"^```(?:json)?\s*([\s\S]*?)\s*```$", re.I) + +def _normalize_openai_vision_messages(messages: list[dict]) -> tuple[list[dict], list[str]]: + """ + Converteer OpenAI-achtige message parts (text + image_url) + naar (1) platte string met markers en (2) images=[base64,...]. + Alleen data: URI's of al-b64 doorlaten; remote http(s) URL's negeren (veilig default). + """ + imgs: list[str] = [] + out: list[dict] = [] + for m in messages: + c = m.get("content") + role = m.get("role","user") + if isinstance(c, list): + parts = [] + for item in c: + if isinstance(item, dict) and item.get("type") == "text": + parts.append(item.get("text","")) + elif isinstance(item, dict) and item.get("type") == "image_url": + u = item.get("image_url") + if isinstance(u, dict): + u = u.get("url") + if isinstance(u, str): + if u.startswith("data:image") and "," in u: + b64 = u.split(",",1)[1] + imgs.append(b64) + parts.append("") + elif re.match(r"^[A-Za-z0-9+/=]+$", u.strip()): # ruwe base64 + imgs.append(u.strip()) + parts.append("") + else: + # http(s) URL's niet meesturen (veilig), maar ook geen ruis in de prompt + pass + out.append({"role": role, "content": "\n".join([p for p in parts if p]).strip()}) + else: + out.append({"role": role, "content": c or ""}) + return out, imgs + +def _parse_repo_qa_from_messages(messages: list[dict]) -> tuple[Optional[str], str, str, int]: + """Haal repo_hint, question, branch, n_ctx grofweg uit user-tekst.""" + txts = [m.get("content","") for m in messages if m.get("role") == "user"] + full = "\n".join(txts).strip() + repo_hint = None + m = re.search(r"(https?://\S+?)(?:\s|$)", full) + if m: repo_hint = m.group(1).strip() + if not repo_hint: + m = re.search(r"\brepo\s*:\s*([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", full, flags=re.I) + if m: repo_hint = m.group(1).strip() + if not repo_hint: + m = re.search(r"\b([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)\b", full) + if m: repo_hint = m.group(1).strip() + branch = "main" + m = re.search(r"\bbranch\s*:\s*([A-Za-z0-9_.\/-]+)", full, flags=re.I) + if m: branch = m.group(1).strip() + n_ctx = 8 + m = re.search(r"\bn_ctx\s*:\s*(\d+)", full, flags=re.I) + if m: + try: n_ctx = max(1, min(16, int(m.group(1)))) + except: pass + question = full + return repo_hint, question, branch, n_ctx + +# ------------------------------ +# Unified-diff "fast path" (expliciete paden + vervangtekst) +# ------------------------------ +_Q = r"[\"'“”‘’`]" +_PATH_PATS = [ + r"[\"“”'](resources\/[A-Za-z0-9_\/\.-]+\.blade\.php)[\"”']", + r"(resources\/[A-Za-z0-9_\/\.-]+\.blade\.php)", + r"[\"“”'](app\/[A-Za-z0-9_\/\.-]+\.php)[\"”']", + r"(app\/[A-Za-z0-9_\/\.-]+\.php)", +] + +def _extract_repo_branch(text: str) -> tuple[Optional[str], str]: + repo_url = None + branch = "main" + m = re.search(r"\bRepo\s*:\s*(\S+)", text, flags=re.I) + if m: + repo_url = m.group(1).strip() + mb = re.search(r"\bbranch\s*:\s*([A-Za-z0-9._/-]+)", text, flags=re.I) + if mb: + branch = mb.group(1).strip() + return repo_url, branch + +def _extract_paths(text: str) -> list[str]: + paths: set[str] = set() + for pat in _PATH_PATS: + for m in re.finditer(pat, text): + paths.add(m.group(1)) + return list(paths) + +def _extract_replace_pair(text: str) -> tuple[Optional[str], Optional[str]]: + """ + Haalt (old,new) uit NL/EN varianten. Ondersteunt rechte en ‘slimme’ quotes. + Voorbeelden die matchen: + - Vervang de tekst “A” in “B” + - Vervang de tekst "A" veranderen in "B" + - Replace the text 'A' to 'B' + - Replace "A" with "B" + """ + pats = [ + rf"Vervang\s+de\s+tekst\s*{_Q}(.+?){_Q}[^.\n]*?(?:in|naar|verander(?:en)?\s+in)\s*{_Q}(.+?){_Q}", + rf"Replace(?:\s+the)?\s+text\s*{_Q}(.+?){_Q}\s*(?:to|with)\s*{_Q}(.+?){_Q}", + ] + for p in pats: + m = re.search(p, text, flags=re.I|re.S) + if m: + return m.group(1), m.group(2) + # fallback: eerste twee quoted strings in de buurt van 'Vervang' / 'Replace' + mm = re.search(r"(Vervang|Replace)[\s\S]*?"+_Q+"(.+?)"+_Q+"[\s\S]*?"+_Q+"(.+?)"+_Q, text, flags=re.I) + if mm: + return mm.group(2), mm.group(3) + return None, None + +def _looks_like_unified_diff_request(text: str) -> bool: + if re.search(r"\bunified\s+diff\b", text, flags=re.I): + return True + # ook accepteren bij "maak een diff" met expliciete bestanden + if re.search(r"\b(diff|patch)\b", text, flags=re.I) and any(re.search(p, text) for p in _PATH_PATS): + return True + return False + +def _make_unified_diff(a_text: str, b_text: str, path: str) -> str: + import difflib + a_lines = a_text.splitlines(keepends=True) + b_lines = b_text.splitlines(keepends=True) + return "".join(difflib.unified_diff(a_lines, b_lines, fromfile=f"a/{path}", tofile=f"b/{path}", n=3)) + +_TRANS_WRAPPERS = [ + r"__\(\s*{q}({txt}){q}\s*\)".format(q=_Q, txt=r".+?"), + r"@lang\(\s*{q}({txt}){q}\s*\)".format(q=_Q, txt=r".+?"), + r"trans\(\s*{q}({txt}){q}\s*\)".format(q=_Q, txt=r".+?"), +] + +def _find_in_translation_wrapper(content: str, needle: str) -> bool: + for pat in _TRANS_WRAPPERS: + for m in re.finditer(pat, content): + inner = m.group(1) + if inner == needle: + return True + return False + +def _update_nl_json(root: Path, key: str, value: str) -> tuple[bool, str]: + """ + Zet of update resources/lang/nl.json: { key: value } + Retourneert (gewijzigd, nieuwe_tekst). + """ + lang_path = root / "resources" / "lang" / "nl.json" + data = {} + if lang_path.exists(): + try: + data = json.loads(lang_path.read_text(encoding="utf-8", errors="ignore") or "{}") + except Exception: + data = {} + # Alleen aanpassen als nodig + prev = data.get(key) + if prev == value: + return False, lang_path.as_posix() + data[key] = value + # Mooie, stabiele JSON dump + lang_path.parent.mkdir(parents=True, exist_ok=True) + tmp = lang_path.with_suffix(".tmp.json") + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + os.replace(tmp, lang_path) + return True, lang_path.as_posix() + +async def _fast_unified_diff_task(messages: list[dict]) -> Optional[str]: + """ + Als de prompt vraagt om 'unified diff' met expliciete bestanden en vervangtekst, + voer dat direct uit (bypass handle_repo_agent) en geef diffs terug. + """ + full = "\n".join([m.get("content","") for m in messages if m.get("role") in ("system","user")]) + if not _looks_like_unified_diff_request(full): + return None + repo_url, branch = _extract_repo_branch(full) + paths = _extract_paths(full) + old, new = _extract_replace_pair(full) + if not (repo_url and paths and old and new): + return None + + # Repo ophalen + repo_path = await _get_git_repo_async(repo_url, branch) + root = Path(repo_path) + changed_files: list[tuple[str, str]] = [] # (path, diff_text) + lang_changed = False + lang_path_text = "" + + for rel in paths: + p = root / rel + if not p.exists(): + continue + before = _read_text_file_wrapper(p) + if not before: + continue + + # Als de tekst binnen een vertaal-wrapper staat, wijzig NIET de blade, + # maar zet/patch resources/lang/nl.json: { "oude": "nieuwe" }. + if _find_in_translation_wrapper(before, old): + ch, lang_path_text = _update_nl_json(root, old, new) + lang_changed = lang_changed or ch + # geen bladewijziging in dit geval + continue + + after = before.replace(old, new) + if after == before: + continue + diff = _make_unified_diff(before, after, rel) + if diff.strip(): + changed_files.append((rel, diff)) + + if not changed_files and not lang_changed: + return "Dry-run: geen wijzigbare treffers gevonden in opgegeven bestanden (of reeds actueel)." + + out = [] + if changed_files: + out.append("### Unified diffs") + for rel, d in changed_files: + out.append(f"```diff\n{d}```") + if lang_changed: + out.append(f"✅ Bijgewerkte vertaling in: `{lang_path_text}` → \"{old}\" → \"{new}\"") + return "\n\n".join(out).strip() + +# ------------------------------ +# OpenAI-compatible endpoints +# ------------------------------ +@app.get("/v1/models") +def list_models(): + base_model = os.getenv("LLM_MODEL", "mistral-medium") + # Toon ook je twee "virtuele" modellen voor de UI + return { + "object": "list", + "data": [ + {"id": base_model, "object": "model", "created": 0, "owned_by": "you"}, + {"id": "repo-agent", "object": "model", "created": 0, "owned_by": "you"}, + {"id": "repo-qa", "object": "model", "created": 0, "owned_by": "you"}, + ], + } + + +def _openai_chat_response(model: str, text: str, messages: list[dict]): + created = int(time.time()) + # heel simpele usage schatting; vermijdt None + try: + prompt_tokens = count_message_tokens(messages) if 'count_message_tokens' in globals() else approx_token_count(json.dumps(messages)) + except Exception: + prompt_tokens = approx_token_count(json.dumps(messages)) + completion_tokens = approx_token_count(text) + return { + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": created, + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens + } + } + +def _get_stt_model(): + global _STT_MODEL + if _STT_MODEL is not None: + return _STT_MODEL + try: + from faster_whisper import WhisperModel + except Exception: + raise HTTPException(status_code=500, detail="STT niet beschikbaar: faster-whisper ontbreekt.") + # device-selectie + download_root = os.getenv("STT_MODEL_DIR", os.environ.get("WHISPER_CACHE_DIR")) + os.makedirs(download_root, exist_ok=True) + if STT_DEVICE == "auto": + try: + _STT_MODEL = WhisperModel(STT_MODEL_NAME, device="cuda", compute_type="float16", download_root=download_root) + return _STT_MODEL + except Exception: + _STT_MODEL = WhisperModel(STT_MODEL_NAME, device="cpu", compute_type="int8", download_root=download_root) + return _STT_MODEL + dev, comp = ("cuda","float16") if STT_DEVICE=="cuda" else ("cpu","int8") + _STT_MODEL = WhisperModel(STT_MODEL_NAME, device=dev, compute_type=comp, download_root=download_root) + return _STT_MODEL + +def _stt_transcribe_path(path: str, lang: str | None): + model = _get_stt_model() + segments, info = model.transcribe(path, language=lang or None, vad_filter=True) + text = "".join(seg.text for seg in segments).strip() + return text, getattr(info, "language", None) + +# Initialize the Tools instance +tools_instance = Tools() + +@app.post("/web/search/xng") +async def web_search_xng( + query: str = Form(...), + SEARXNG_ENGINE_API_BASE_URL: str = tools_instance.valves.SEARXNG_ENGINE_API_BASE_URL, + IGNORED_WEBSITES: str = tools_instance.valves.IGNORED_WEBSITES, + RETURNED_SCRAPPED_PAGES_NO: int = tools_instance.valves.RETURNED_SCRAPPED_PAGES_NO, + SCRAPPED_PAGES_NO: int = tools_instance.valves.SCRAPPED_PAGES_NO, + PAGE_CONTENT_WORDS_LIMIT: int = tools_instance.valves.PAGE_CONTENT_WORDS_LIMIT, + CITATION_LINKS: bool = tools_instance.valves.CITATION_LINKS, +) -> str: + omsch=""" + Search the web using SearXNG and get the content of the relevant pages. + OpenAI-compat: { "query": "string", "SEARXNG_ENGINE_API_BASE_URL": "string", "IGNORED_WEBSITES": "string", "RETURNED_SCRAPPED_PAGES_NO": "integer", "SCRAPPED_PAGES_NO": "integer", "PAGE_CONTENT_WORDS_LIMIT": "integer", "CITATION_LINKS": "boolean" } + Return: JSON string with search results. + """ + #query = (body.get("query") or "").strip() + if not query: + raise HTTPException(status_code=400, detail="Lege query") + + # Extract parameters from the body + params = { + "SEARXNG_ENGINE_API_BASE_URL": SEARXNG_ENGINE_API_BASE_URL, + "IGNORED_WEBSITES": IGNORED_WEBSITES, + "RETURNED_SCRAPPED_PAGES_NO": RETURNED_SCRAPPED_PAGES_NO, + "SCRAPPED_PAGES_NO": SCRAPPED_PAGES_NO, + "PAGE_CONTENT_WORDS_LIMIT": PAGE_CONTENT_WORDS_LIMIT, + "CITATION_LINKS": CITATION_LINKS, + } + + # Update the valves with the provided parameters + tools_instance.valves = tools_instance.Valves(**params) + + # Call the existing search_web function from tools_instance + result = await tools_instance.search_web(query) + return JSONResponse(result) + +@app.post("/web/get_website/xng") +async def get_website_xng( + url: str = Form(...), + SEARXNG_ENGINE_API_BASE_URL: str = tools_instance.valves.SEARXNG_ENGINE_API_BASE_URL, + IGNORED_WEBSITES: str = tools_instance.valves.IGNORED_WEBSITES, + PAGE_CONTENT_WORDS_LIMIT: int = tools_instance.valves.PAGE_CONTENT_WORDS_LIMIT, + CITATION_LINKS: bool = tools_instance.valves.CITATION_LINKS, +) -> str: + omsch=""" + Web scrape the website provided and get the content of it. + OpenAI-compat: { "url": "string", "SEARXNG_ENGINE_API_BASE_URL": "string", "IGNORED_WEBSITES": "string", "PAGE_CONTENT_WORDS_LIMIT": "integer", "CITATION_LINKS": "boolean" } + Return: JSON string with website content. + """ + #url = (body.get("url") or "").strip() + if not url: + raise HTTPException(status_code=400, detail="Lege URL") + + # Extract parameters from the body + params = { + "SEARXNG_ENGINE_API_BASE_URL": SEARXNG_ENGINE_API_BASE_URL, + "IGNORED_WEBSITES": IGNORED_WEBSITES, + "PAGE_CONTENT_WORDS_LIMIT": PAGE_CONTENT_WORDS_LIMIT, + "CITATION_LINKS": CITATION_LINKS, + } + + # Update the valves with the provided parameters + tools_instance.valves = tools_instance.Valves(**params) + + # Call the existing get_website function from tools_instance + result = await tools_instance.get_website(url) + return JSONResponse(result) + +@app.post("/v1/audio/transcriptions") +async def audio_transcriptions( + file: UploadFile = File(...), + model: str = Form("whisper-1"), + language: str | None = Form(None), + prompt: str | None = Form(None) +): + data = await file.read() + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(data) + tmp_path = tmp.name + try: + text, lang = await run_in_threadpool(_stt_transcribe_path, tmp_path, language) + return {"text": text, "language": lang or "unknown"} + finally: + try: os.unlink(tmp_path) + except Exception: pass + +@app.post("/v1/audio/speech") +async def audio_speech(body: dict = Body(...)): + """ + OpenAI-compat: { "model": "...", "voice": "optional", "input": "tekst" } + Return: audio/wav + """ + if not PIPER_VOICE: + raise HTTPException(status_code=500, detail="PIPER_VOICE env var niet gezet (piper TTS).") + text = (body.get("input") or "").strip() + if not text: + raise HTTPException(status_code=400, detail="Lege input") + + def _synth_to_wav_bytes() -> bytes: + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as out: + out_path = out.name + try: + cp = subprocess.run( + [PIPER_BIN, "-m", PIPER_VOICE, "-f", out_path, "-q"], + input=text.encode("utf-8"), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True + ) + with open(out_path, "rb") as f: + return f.read() + finally: + try: os.unlink(out_path) + except Exception: pass + + audio_bytes = await run_in_threadpool(_synth_to_wav_bytes) + return StreamingResponse(iter([audio_bytes]), media_type="audio/wav") + +@app.post("/v1/images/generations") +async def images_generations(payload: dict = Body(...)): + prompt = (payload.get("prompt") or "").strip() + if not prompt: + raise HTTPException(status_code=400, detail="prompt required") + n = max(1, min(8, int(payload.get("n", 1)))) + size = payload.get("size","512x512") + w,h = _parse_size(size) + background = payload.get("background","white") + fmt = (payload.get("format") or "").lower().strip() # non-standard + if fmt not in ("png","svg",""): + fmt = "" + if not fmt: + fmt = "png" if cairosvg is not None else "svg" + + out_items = [] + for _ in range(n): + svg = await _svg_from_prompt(prompt, w, h, background) + if fmt == "png" and cairosvg is not None: + png_bytes = cairosvg.svg2png(bytestring=svg.encode("utf-8"), output_width=w, output_height=h) + b64 = base64.b64encode(png_bytes).decode("ascii") + else: + b64 = base64.b64encode(svg.encode("utf-8")).decode("ascii") + out_items.append({"b64_json": b64}) + return {"created": int(time.time()), "data": out_items} + +@app.get("/improve_web_query") +async def improve_web_query(request: Request, format: str = "proxy"): + text = (request.query_params.get("text") or "")[:int(request.query_params.get("max_chars", 20000))] + objective = request.query_params.get("objective") or "Verbeter deze search query door de kern van de tekst te identificeren en deze om te zetten in een betere query. Gebruik AND, OR, of andere logische operatoren om de query te optimaliseren." + style = request.query_params.get("style") or "Hanteer best practices, behoud inhoudelijke betekenis." + + resp = await llm_call_openai_compat( + [{"role": "user", "content": f"" + f"Je taak is om deze search query te verbeteren: {text}\n" + f"Objectief: {objective}\n" + f"Stijl: {style}\n" + f"Verbeterde query:"}], + stream=False, max_tokens=200 + ) + return resp + +@app.get("/v1/images/health") +def images_health(): + return {"svg_to_png": bool(cairosvg is not None)} + +@app.post("/present/make") +async def present_make( + prompt: str = Form(...), + file: UploadFile | None = File(None), + max_slides: int = Form(8) +): + if Presentation is None: + raise HTTPException(500, "python-pptx ontbreekt in de container.") + src_text = "" + if file: + try: + src_text = (await read_file_content(file))[:30000] + except Exception: + src_text = "" + sys = ("Je bent een presentatieschrijver. Geef ALLEEN geldige JSON terug: " + '{"title": str, "slides":[{"title": str, "bullets":[str,...]}...]}.') + user = (f"Doelpresentatie: {prompt}\nBron (optioneel):\n{src_text[:12000]}\n" + f"Max. {max_slides} dia's, 3–6 bullets per dia.") + plan = await llm_call_openai_compat( + [{"role":"system","content":sys},{"role":"user","content":user}], + stream=False, temperature=0.03, top_p=0.9, max_tokens=13021 + ) + raw = (plan.get("choices",[{}])[0].get("message",{}) or {}).get("content","{}") + try: + spec = json.loads(raw) + except Exception: + raise HTTPException(500, "Model gaf geen valide JSON voor slides.") + prs = Presentation() + title = spec.get("title") or "Presentatie" + # Titel + slide_layout = prs.slide_layouts[0] + s = prs.slides.add_slide(slide_layout) + s.shapes.title.text = title + # Content + for slide in spec.get("slides", []): + layout = prs.slide_layouts[1] + sl = prs.slides.add_slide(layout) + sl.shapes.title.text = slide.get("title","") + tx = sl.placeholders[1].text_frame + tx.clear() + for i, bullet in enumerate(slide.get("bullets", [])[:10]): + p = tx.add_paragraph() if i>0 else tx.paragraphs[0] + p.text = bullet + p.level = 0 + bio = BytesIO() + prs.save(bio); bio.seek(0) + headers = {"Content-Disposition": f'attachment; filename="deck.pptx"'} + return StreamingResponse(bio, media_type="application/vnd.openxmlformats-officedocument.presentationml.presentation", headers=headers) + +@app.get("/whoami") +def whoami(): + import socket + return {"service": "mistral-api", "host": socket.gethostname(), "LLM_URL": LLM_URL} + +IMG_EXTS = (".png",".jpg",".jpeg",".webp",".bmp",".gif") + +def _is_image_filename(name: str) -> bool: + return (name or "").lower().endswith(IMG_EXTS) + +@app.post("/vision/ask") +async def vision_ask( + file: UploadFile = File(...), + prompt: str = Form("Beschrijf kort wat je ziet."), + stream: bool = Form(False), + temperature: float = Form(0.02), + top_p: float = Form(0.9), + max_tokens: int = Form(1024), +): + raw = await run_in_threadpool(file.file.read) + img_b64 = base64.b64encode(raw).decode("utf-8") + messages = [{"role":"user","content": f" {prompt}"}] + if stream: + return await llm_call_openai_compat( + messages, + model=os.getenv("LLM_MODEL","mistral-medium"), + stream=True, + temperature=float(temperature), + top_p=float(top_p), + max_tokens=int(max_tokens), + extra={"images": [img_b64]} + ) + + + return await llm_call_openai_compat( + messages, stream=False, temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra={"images": [img_b64]} + ) + +@app.post("/file/vision-and-text") +async def vision_and_text( + files: List[UploadFile] = File(...), + prompt: str = Form("Combineer visuele analyse met de tekstcontext. Geef 5 bullets met bevindingen en 3 actiepunten."), + stream: bool = Form(False), + max_images: int = Form(6), + max_chars: int = Form(25000), + temperature: float = Form(0.02), + top_p: float = Form(0.9), + max_tokens: int = Form(2048), +): + images_b64: list[str] = [] + text_chunks: list[str] = [] + for f in files: + name = f.filename or "" + raw = await run_in_threadpool(f.file.read) + if _is_image_filename(name) and len(images_b64) < int(max_images): + images_b64.append(base64.b64encode(raw).decode("utf-8")) + else: + try: + tmp = UploadFile(filename=name, file=BytesIO(raw), headers=f.headers) + text = await read_file_content(tmp) + except Exception: + text = raw.decode("utf-8", errors="ignore") + if text: + text_chunks.append(f"### {name}\n{text.strip()}") + + text_context = ("\n\n".join(text_chunks))[:int(max_chars)].strip() + image_markers = ("\n".join([""] * len(images_b64))).strip() + user_content = (image_markers + ("\n\n" if image_markers else "") + prompt.strip()) + if text_context: + user_content += f"\n\n--- TEKST CONTEXT (ingekort) ---\n{text_context}" + messages = [{"role":"user","content": user_content}] + + if stream: + return await llm_call_openai_compat( + messages, + model=os.getenv("LLM_MODEL","mistral-medium"), + stream=True, + temperature=float(temperature), + top_p=float(top_p), + max_tokens=int(max_tokens), + extra={"images": images_b64} + ) + + + return await llm_call_openai_compat( + messages, stream=False, temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra={"images": images_b64} + ) + +@app.get("/vision/health") +async def vision_health(): + tiny_png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAOaO5nYAAAAASUVORK5CYII=" + try: + messages = [{"role":"user","content":" Beschrijf dit in één woord."}] + resp = await llm_call_openai_compat(messages, extra={"images":[tiny_png]}, max_tokens=128) + txt = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","").strip() + return {"vision": bool(txt), "sample": txt[:60]} + except Exception as e: + return {"vision": False, "error": str(e)} + +# -------- Tool registry (OpenAI-style) -------- +LLM_FUNCTION_CALLING_MODE = os.getenv("LLM_FUNCTION_CALLING_MODE", "auto").lower() # "native" | "shim" | "auto" + +OWUI_BASE_URL='http://192.168.100.1:8089' +OWUI_API_TOKEN='sk-f1b7991b054442b5ae388de905019726' +# Aliassen zodat oudere codepaths blijven werken +OWUI_BASE = OWUI_BASE_URL +OWUI_TOKEN = OWUI_API_TOKEN + + +@app.get("/tools", operation_id="get_tools_list_compat") +async def tools_compat(): + return await list_tools() + + + +async def _fetch_openwebui_file(file_id: str, dst_dir: Path) -> Path: + """ + Download raw file content from OpenWebUI and save locally. + """ + if not (OWUI_BASE_URL and OWUI_API_TOKEN): + raise HTTPException(status_code=501, detail="OpenWebUI niet geconfigureerd (OWUI_BASE_URL/OWUI_API_TOKEN).") + url = f"{OWUI_BASE_URL}/api/v1/files/{file_id}/content" + headers = {"Authorization": f"Bearer {OWUI_API_TOKEN}"} + + dst_dir.mkdir(parents=True, exist_ok=True) + out_path = dst_dir / f"{file_id}" + # Gebruik proxy-aware client uit app.state als beschikbaar + client = getattr(app.state, "HTTPX_PROXY", None) + close_after = False + if client is None: + client = httpx.AsyncClient(timeout=None, trust_env=True) + close_after = True + try: + async with client.stream("GET", url, headers=headers, timeout=None) as resp: + if resp.status_code != 200: + # lees body (beperkt) voor foutmelding + body = await resp.aread() + raise HTTPException(status_code=resp.status_code, + detail=f"OpenWebUI file fetch failed: {body[:2048]!r}") + max_mb = int(os.getenv("OWUI_MAX_DOWNLOAD_MB", "64")) + max_bytes = max_mb * 1024 * 1024 + total = 0 + with out_path.open("wb") as f: + async for chunk in resp.aiter_bytes(): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + try: f.close() + except Exception: pass + try: out_path.unlink(missing_ok=True) + except Exception: pass + raise HTTPException(status_code=413, detail=f"Bestand groter dan {max_mb}MB; download afgebroken.") + f.write(chunk) + return out_path + except HTTPException: + raise + except Exception as e: + logging.exception("download failed") + raise HTTPException(status_code=500, detail=f"download error: {e}") + finally: + if close_after: + await client.aclose() + + +def _normalize_files_arg(args: dict): + # OpenWebUI injecteert files als __files__ = [{ id, name, mime, size, ...}] + files = args.get("__files__") or args.get("files") or [] + if isinstance(files, dict): # enkel bestand + files = [files] + return files + + +@app.get("/openapi.json", include_in_schema=False) +async def openapi_endpoint(): + tool_routes=[] + for r in app.routes: + if getattr(r, "path", "").split("/")[1] in ["v1","web","openapi","vision","file","rag","repo"]: + tool_routes.append(r) + logger.info("toolwithpath: %s",getattr(r, "path", "")) + tool_routes = [ + r for r in app.routes + if isinstance(r, APIRoute) and r.path.startswith("/openapi/") + ] + logger.info("OpenAPI tool_routes=%d", len(tool_routes)) + for r in tool_routes: + logger.info(" tool route: %s", r.path) + #tool_routes = [ + # r for r in app.routes + # if getattr(r, "path", "").startswith("/openapi/") + #] + return get_openapi( + title="Tool Server", + version="0.1.0", + routes=tool_routes, + ) + +def _openai_tools_from_registry(reg: dict): + out = [] + for name, spec in reg.items(): + out.append({ + "type": "function", + "function": { + "name": name, + "description": spec.get("description",""), + "parameters": spec.get("parameters", {"type":"object","properties":{}}) + } + }) + return out + +def _visible_registry(reg: dict) -> dict: + return {k:v for k,v in reg.items() if not v.get("hidden")} + +VALIDATE_PROMPT = ( + "Je bent een code-reviewer die deze code moet valideren:\n" + "1. Controleer op syntactische fouten\n" + "2. Controleer op logische fouten en onjuiste functionaliteit\n" + "3. Check of alle vereiste functionaliteit aanwezig is\n" + "4. Zoek naar mogelijke bugs of veiligheidsrisico's\n" + "5. Geef specifieke feedback met regelnummers\n\n" + "Geef een lijst in dit formaat:\n" + "- Regels [10-12]: Beschrijving\n" + "- Regels 25: Beschrijving" +) + +def _parse_validation_results(text: str) -> list[str]: + issues = [] + for line in (text or "").splitlines(): + line = line.strip() + if line.startswith('-') and 'Regels' in line and ':' in line: + issues.append(line) + return issues + +def _norm_collection_name(x: str | None, default="code_docs") -> str: + name = (x or "").strip() + return name or default + +def _collection_effective(name: str) -> str: + name = (name or "code_docs").strip() or "code_docs" + # als je al versioned namen hebt, laat ze staan + if re.search(r"__v\d+$", name): + return name + return _collection_versioned(name) + +async def t_run_shell(args: dict) -> dict: + cmd = args["command"]; timeout = int(args.get("timeout", 600)) + BAN_RM=False + if BAN_RM and re.search(r"\brm\s+-", cmd): + return {"exit_code": 1, "stdout": "", "stderr": "rm is verboden", "duration": 0.0, "log_path": ""} + start = time.time() + proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + try: + #out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout) + # Lees stdout en stderr apart (in bytes) + out_bytes = await asyncio.wait_for(proc.stdout.read(), timeout=timeout) + err_bytes = await asyncio.wait_for(proc.stderr.read(), timeout=timeout) + + # Splits stdout en stderr naar regels + stdout_lines = out_bytes.decode(errors="ignore").splitlines() + stderr_lines = err_bytes.decode(errors="ignore").splitlines() + return { + "exit_code": proc.returncode, + "stdout_lines": stdout_lines, + "stderr_lines": stderr_lines, + "duration": time.time() - start, + "log_path": "", + "merge_stderr_used": False + } + except asyncio.TimeoutError: + try: proc.kill() + except Exception: pass + return {"exit_code": -1, "stdout": "", "stderr": "timeout", "duration": time.time()-start, "log_path": ""} + #return {"exit_code": proc.returncode, "stdout": out.decode(errors="ignore"), "stderr": err.decode(errors="ignore"), "duration": time.time()-start, "log_path": ""} + + + +# ----------------------------------------------------------------------------- +# Repo toolset v2 (clearer naming + macro tool) +# ----------------------------------------------------------------------------- +_WS_LOCK = threading.Lock() +_WORKSPACES: dict[str, dict] = {} + +def _ws_id(repo_url: str, branch: str) -> str: + key = f"{repo_url}|{branch}".encode("utf-8", errors="ignore") + return "ws_" + hashlib.sha1(key).hexdigest()[:12] + +def _ws_open(repo_url: str, branch: str = "main") -> dict: + wsid = _ws_id(repo_url, branch) + local = get_git_repo(repo_url, branch) + repo = git.Repo(local) + try: + head = repo.head.commit.hexsha + except Exception: + head = "" + info = { + "workspace_id": wsid, + "repo_url": repo_url, + "branch": branch, + "local_path": local, + "head": head, + "opened_ts": int(time.time()), + } + with _WS_LOCK: + _WORKSPACES[wsid] = info + return info + +def _ws_resolve(workspace_id: str | None, repo_url: str | None, branch: str) -> dict: + if workspace_id: + with _WS_LOCK: + ws = _WORKSPACES.get(workspace_id) + if ws: + # ensure checkout requested branch if differs + if branch and ws.get("branch") != branch and repo_url: + ws = _ws_open(repo_url, branch) + return ws + if not repo_url: + raise RuntimeError("repo_url vereist als workspace_id onbekend is.") + return _ws_open(repo_url, branch) + +def _normalize_patch_text(patch_text: str) -> str: + if patch_text is None: + return "" + # trim + s = str(patch_text) + s = s.replace("\r\n", "\n").replace("\r", "\n") + # if looks like single line with \n escapes, unescape once + if "\n" in s and "\n" not in s.splitlines()[0] and len(s.splitlines()) == 1: + s = s.replace("\\n", "\n") + # strip common fences + s = re.sub(r"^```(?:diff|patch)?\s*\n", "", s.strip(), flags=re.IGNORECASE) + s = re.sub(r"\n```\s*$", "", s.strip(), flags=re.IGNORECASE) + # drop chatter before first diff header + m = re.search(r"^(diff --git |---\s)", s, flags=re.MULTILINE) + if m and m.start() > 0: + s = s[m.start():] + return s.strip() + "\n" + +def _git_auth_remote_url(repo_url: str) -> str: + # prefer token-based auth if provided + user = os.getenv("GITEA_USER") or os.getenv("GIT_HTTP_USER") or "" + token = os.getenv("GITEA_TOKEN") or os.getenv("GIT_HTTP_TOKEN") or "" + if not token: + return repo_url + if "://" not in repo_url: + return repo_url + try: + from urllib.parse import urlparse, urlunparse + u = urlparse(repo_url) + host = u.hostname or "" + port = f":{u.port}" if u.port else "" + cred_user = user or "oauth2" + netloc = f"{cred_user}:{token}@{host}{port}" + return urlunparse((u.scheme, netloc, u.path, u.params, u.query, u.fragment)) + except Exception: + return repo_url + +def _git_safe_url(repo_url: str) -> str: + # redact credentials for logging + try: + from urllib.parse import urlparse, urlunparse + u = urlparse(repo_url) + if u.username or u.password: + netloc = u.hostname or "" + if u.port: + netloc += f":{u.port}" + return urlunparse((u.scheme, netloc, u.path, u.params, u.query, u.fragment)) + except Exception: + pass + return repo_url + +def _allowed_git_host(repo_url: str) -> bool: + allowed = (os.getenv("ALLOWED_GIT_HOSTS") or "").strip() + if not allowed: + # if not set, allow everything (but recommended to set) + return True + allowed_hosts = {h.strip().lower() for h in allowed.split(",") if h.strip()} + try: + from urllib.parse import urlparse + u = urlparse(repo_url) + host = (u.hostname or "").lower() + return host in allowed_hosts + except Exception: + return False + +def _repo_grep_hits(root: Path, query: str, max_hits: int = 50) -> list[dict]: + hits: list[dict] = [] + qlow = query.lower() + for p in root.rglob("*"): + if p.is_dir(): + continue + if set(p.parts) & PROFILE_EXCLUDE_DIRS: + continue + if p.suffix.lower() in BINARY_SKIP: + continue + # fast path: filename match + if qlow in str(p).lower(): + try: + txt = _read_text_file(p) + except Exception: + txt = "" + excerpt = (txt or "")[:400] + hits.append({"path": str(p.relative_to(root)), "line": 0, "excerpt": excerpt or "-match in filename-"}) + if len(hits) >= max_hits: + break + try: + txt = _read_text_file(p) + except Exception: + continue + if not txt: + continue + for ln, line in enumerate(txt.splitlines(), 1): + if qlow in line.lower(): + hits.append({"path": str(p.relative_to(root)), "line": ln, "excerpt": line.strip()[:400]}) + if len(hits) >= max_hits: + break + if len(hits) >= max_hits: + break + return hits + +def _repo_read_file(root: Path, rel_path: str, start_line: int | None = None, end_line: int | None = None, max_bytes: int = 200000) -> dict: + p = (root / rel_path).resolve() + if not str(p).startswith(str(root.resolve())): + raise RuntimeError("Path traversal blocked.") + if not p.exists() or not p.is_file(): + raise RuntimeError(f"File not found: {rel_path}") + data = p.read_bytes() + if len(data) > max_bytes: + data = data[:max_bytes] + txt = data.decode("utf-8", errors="replace") + lines = txt.splitlines() + if start_line is not None or end_line is not None: + s = max(1, int(start_line or 1)) + e = int(end_line or (s + 200)) + s0 = s-1 + e0 = min(len(lines), e) + slice_lines = lines[s0:e0] + return {"path": rel_path, "start_line": s, "end_line": e0, "content": "\n".join(slice_lines)} + return {"path": rel_path, "content": txt} + +def _repo_apply_patch(repo_path: str, patch_text: str, dry_run: bool = False) -> dict: + patch_text = _normalize_patch_text(patch_text) + if not patch_text.strip(): + raise RuntimeError("Empty patch.") + # quick sanity check + if "diff --git" not in patch_text and not patch_text.lstrip().startswith("---"): + raise RuntimeError("Patch lijkt geen unified diff (geen 'diff --git' / '---').") + import tempfile, subprocess + with tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") as tf: + tf.write(patch_text) + tf.flush() + patch_file = tf.name + try: + cmd = ["git", "apply"] + if dry_run: + cmd.append("--check") + else: + cmd += ["--3way", "--whitespace=nowarn"] + cmd.append(patch_file) + res = subprocess.run(cmd, cwd=repo_path, capture_output=True, text=True) + if res.returncode != 0: + # fallback without 3way + cmd2 = ["git", "apply"] + if dry_run: + cmd2.append("--check") + else: + cmd2 += ["--whitespace=nowarn"] + cmd2.append(patch_file) + res2 = subprocess.run(cmd2, cwd=repo_path, capture_output=True, text=True) + if res2.returncode != 0: + raise RuntimeError("git apply failed:\n" + (res.stderr or "") + "\n" + (res2.stderr or "")) + # list changed files + st = subprocess.run(["git","status","--porcelain"], cwd=repo_path, capture_output=True, text=True) + changed = [] + for line in (st.stdout or "").splitlines(): + if not line.strip(): + continue + changed.append(line[3:].strip()) + return {"applied": (not dry_run), "dry_run": dry_run, "changed_files": changed} + finally: + try: + os.unlink(patch_file) + except Exception: + pass + +def _repo_apply_files(repo_path: str, files: list[dict], dry_run: bool = False) -> dict: + if not isinstance(files, list) or not files: + raise RuntimeError("files[] is leeg.") + root = Path(repo_path) + changed = [] + for f in files: + path = (f or {}).get("path") + content_b64 = (f or {}).get("content_b64") + if not path or content_b64 is None: + raise RuntimeError("files[] items require path + content_b64") + p = (root / path).resolve() + if not str(p).startswith(str(root.resolve())): + raise RuntimeError("Path traversal blocked.") + data = base64.b64decode(content_b64.encode("utf-8")) + if not dry_run: + p.parent.mkdir(parents=True, exist_ok=True) + p.write_bytes(data) + changed.append(path) + return {"applied": (not dry_run), "dry_run": dry_run, "changed_files": changed} + +def _repo_push(repo_url: str, repo_path: str, base_branch: str, new_branch: str | None, branch_prefix: str = "test", commit_message: str = "Apply changes", allow_empty: bool = False) -> dict: + import subprocess + if not _allowed_git_host(repo_url): + raise RuntimeError(f"repo_url host not allowed by ALLOWED_GIT_HOSTS: {_git_safe_url(repo_url)}") + repo = git.Repo(repo_path) + # ensure remote url includes token if env is set + origin = repo.remotes.origin + auth_url = _git_auth_remote_url(repo_url) + try: + origin.set_url(auth_url) + except Exception: + pass + # fetch base + try: + origin.fetch(prune=True) + except Exception: + pass + # checkout base (keep changes if already on base) + try: + repo.git.checkout(base_branch) + except Exception: + try: + repo.git.checkout("-B", base_branch, f"origin/{base_branch}") + except Exception as e: + raise RuntimeError(f"Cannot checkout base_branch {base_branch}: {e}") + # create branch name + if not new_branch: + ts = time.strftime("%Y%m%d-%H%M%S") + new_branch = f"{branch_prefix}/{ts}" + # create/reset branch, keeping working tree changes + repo.git.checkout("-B", new_branch) + # stage and commit + repo.git.add(A=True) + # detect if anything to commit + diff_index = repo.index.diff("HEAD") + untracked = repo.untracked_files + if not diff_index and not untracked: + if not allow_empty: + raise RuntimeError("Nothing to commit (working tree clean).") + repo.git.commit("--allow-empty", "-m", commit_message) + else: + author_name = os.getenv("GIT_AUTHOR_NAME") or "toolserver" + author_email = os.getenv("GIT_AUTHOR_EMAIL") or "toolserver@local" + env = os.environ.copy() + env["GIT_AUTHOR_NAME"] = author_name + env["GIT_AUTHOR_EMAIL"] = author_email + env["GIT_COMMITTER_NAME"] = author_name + env["GIT_COMMITTER_EMAIL"] = author_email + subprocess.run(["git","commit","-m",commit_message], cwd=repo_path, env=env, capture_output=True, text=True, check=False) + # push + res = subprocess.run(["git","push","-u","origin",new_branch], cwd=repo_path, capture_output=True, text=True) + if res.returncode != 0: + raise RuntimeError("git push failed:\n" + (res.stderr or res.stdout or "")) + # head sha + head = repo.head.commit.hexsha if repo.head and repo.head.commit else "" + return {"pushed": True, "branch": new_branch, "commit": head, "repo": _git_safe_url(repo_url)} + +def _gitea_pr_create(repo_url: str, head: str, base: str, title: str, body: str = "") -> dict: + token = os.getenv("GITEA_TOKEN") or os.getenv("GIT_HTTP_TOKEN") or "" + if not token: + raise RuntimeError("GITEA_TOKEN (of GIT_HTTP_TOKEN) ontbreekt voor PR creation.") + # parse owner/repo and base url + from urllib.parse import urlparse + u = urlparse(repo_url) + base_url = f"{u.scheme}://{u.hostname}{(':'+str(u.port)) if u.port else ''}" + owner_repo = _repo_owner_repo_from_url(repo_url) + if not owner_repo or "/" not in owner_repo: + raise RuntimeError("Cannot parse owner/repo from repo_url.") + owner, repo = owner_repo.split("/",1) + api = f"{base_url}/api/v1/repos/{owner}/{repo}/pulls" + headers = {"Authorization": f"token {token}", "Content-Type": "application/json"} + payload = {"head": head, "base": base, "title": title, "body": body} + r = requests.post(api, headers=headers, json=payload, timeout=30) + if r.status_code not in (200,201): + raise RuntimeError(f"Gitea PR create failed ({r.status_code}): {r.text[:500]}") + data = r.json() + return {"pr_number": data.get("number"), "pr_url": data.get("html_url") or data.get("url")} + + +async def _execute_tool(name: str, args: dict) -> dict: + logger.info("toolcall: "+str(name)+" ("+str(args)+")") + required=[] + if name in TOOLS_REGISTRY: + required=TOOLS_REGISTRY[name]["parameters"]["required"] + else: + return {"error": f"Unknown tool '{name}'."} + if not all(k in args and args[k] for k in required): + return {"error": f"Missing required arguments for tool '{name}'. Required: {required}"} + for k in args: + if k in required: + if args[k] in ['',None]: + return {"error": f"Missing required arguments for tool '{name}'. Required: {required}"} + + # ------------------------------------------------------------------ + # Repo toolset v2 (preferred) + # ------------------------------------------------------------------ + if name == "repo_open": + repo_url = args.get("repo_url") or args.get("repo") or "" + branch = args.get("branch") or args.get("base_branch") or "main" + if not repo_url: + return {"error": "repo_open: repo_url is required"} + ws = await run_in_threadpool(_ws_open, repo_url, branch) + return { + "workspace_id": ws["workspace_id"], + "repo_url": _git_safe_url(ws["repo_url"]), + "branch": ws["branch"], + "head": ws.get("head",""), + } + + if name == "repo_search": + repo_url = args.get("repo_url") or args.get("repo") or "" + branch = args.get("branch") or "main" + workspace_id = args.get("workspace_id") + query = (args.get("query") or "").strip() + mode = (args.get("mode") or "auto").lower() + n_results = int(args.get("n_results", args.get("max_hits", 20))) + collection_name = args.get("collection_name", "code_docs") + profile = args.get("profile") + path_contains = args.get("path_contains") + if not query: + return {"error": "repo_search: query is required"} + ws = await run_in_threadpool(_ws_resolve, workspace_id, repo_url or None, branch) + root = Path(ws["local_path"]) + # auto: grep first, then rag fallback if few hits + if mode in ("grep","auto"): + hits = await run_in_threadpool(_repo_grep_hits, root, query, max(1, n_results)) + if mode == "grep" or len(hits) >= max(3, min(8, n_results)): + return {"count": len(hits[:n_results]), "mode": "grep", "hits": hits[:n_results], "workspace_id": ws["workspace_id"]} + # rag mode (or auto fallback) + try: + await run_in_threadpool(_rag_index_repo_sync, **{ + "repo_url": ws["repo_url"], + "branch": ws["branch"], + "profile": profile or "auto", + "include": "", + "exclude_dirs": "", + "chunk_chars": 3000, + "overlap": 400, + "collection_name": collection_name, + "force": False, + }) + except Exception as e: + return {"error": f"repo_search: index failed: {str(e)}"} + try: + out = await rag_query_api( + query=query, + n_results=n_results, + collection_name=_norm_collection_name(collection_name, "code_docs"), + repo=ws["repo_url"], + path_contains=path_contains, + profile=profile + ) + # normalize to hits + hits = [] + for r in out.get("results", []): + meta = r.get("metadata") or {} + hits.append({ + "path": meta.get("path",""), + "line": int(meta.get("start_line") or 0), + "excerpt": (r.get("document") or "")[:400], + "score": r.get("score"), + }) + return {"count": len(hits), "mode": "rag", "hits": hits, "workspace_id": ws["workspace_id"]} + except Exception as e: + return {"error": f"repo_search: rag_query failed: {str(e)}"} + + if name == "repo_read": + repo_url = args.get("repo_url") or args.get("repo") or "" + branch = args.get("branch") or "main" + workspace_id = args.get("workspace_id") + path = (args.get("path") or "").strip() + start_line = args.get("start_line") + end_line = args.get("end_line") + max_bytes = int(args.get("max_bytes", 200000)) + if not path: + return {"error": "repo_read: path is required"} + ws = await run_in_threadpool(_ws_resolve, workspace_id, repo_url or None, branch) + root = Path(ws["local_path"]) + try: + out = await run_in_threadpool(_repo_read_file, root, path, start_line, end_line, max_bytes) + out["workspace_id"] = ws["workspace_id"] + return out + except Exception as e: + return {"error": f"repo_read failed: {str(e)}"} + + if name == "repo_apply": + repo_url = args.get("repo_url") or args.get("repo") or "" + branch = args.get("branch") or "main" + workspace_id = args.get("workspace_id") + dry_run = bool(args.get("dry_run", False)) + patch_b64 = args.get("patch_b64") + patch = args.get("patch") + files = args.get("files") + ws = await run_in_threadpool(_ws_resolve, workspace_id, repo_url or None, branch) + repo_path = ws["local_path"] + try: + if patch_b64: + patch_text = base64.b64decode(patch_b64.encode("utf-8")).decode("utf-8", errors="replace") + out = await run_in_threadpool(_repo_apply_patch, repo_path, patch_text, dry_run) + elif patch: + out = await run_in_threadpool(_repo_apply_patch, repo_path, str(patch), dry_run) + elif files: + out = await run_in_threadpool(_repo_apply_files, repo_path, files, dry_run) + else: + return {"error": "repo_apply: provide patch_b64 or patch or files[]"} + out["workspace_id"] = ws["workspace_id"] + return out + except Exception as e: + return {"error": f"repo_apply failed: {str(e)}", "workspace_id": ws["workspace_id"]} + + if name == "repo_push": + repo_url = args.get("repo_url") or args.get("repo") or "" + branch = args.get("branch") or "main" + workspace_id = args.get("workspace_id") + base_branch = args.get("base_branch") or branch or "main" + new_branch = args.get("new_branch") or args.get("branch_name") or "" + branch_prefix = args.get("branch_prefix") or "test" + commit_message = args.get("commit_message") or args.get("message") or "Apply changes" + allow_empty = bool(args.get("allow_empty", False)) + if not repo_url and not workspace_id: + return {"error": "repo_push: repo_url or workspace_id required"} + ws = await run_in_threadpool(_ws_resolve, workspace_id, repo_url or None, base_branch) + try: + out = await run_in_threadpool(_repo_push, ws["repo_url"], ws["local_path"], base_branch, (new_branch or None), branch_prefix, commit_message, allow_empty) + out["workspace_id"] = ws["workspace_id"] + return out + except Exception as e: + return {"error": f"repo_push failed: {str(e)}", "workspace_id": ws["workspace_id"]} + + if name == "repo_pr_create": + repo_url = args.get("repo_url") or args.get("repo") or "" + head = args.get("head_branch") or args.get("head") or "" + base_branch = args.get("base_branch") or "main" + title = args.get("title") or args.get("pr_title") or "PR" + body = args.get("body") or "" + if not repo_url or not head: + return {"error": "repo_pr_create: repo_url and head_branch required"} + try: + out = await run_in_threadpool(_gitea_pr_create, repo_url, head, base_branch, title, body) + return out + except Exception as e: + return {"error": f"repo_pr_create failed: {str(e)}"} + + if name == "repo_change_to_branch": + repo_url = args.get("repo_url") or args.get("repo") or "" + base_branch = args.get("base_branch") or "main" + branch_prefix = args.get("branch_prefix") or "test" + new_branch = args.get("new_branch") or "" + commit_message = args.get("commit_message") or "Apply changes" + create_pr = bool(args.get("create_pr", False)) + pr_title = args.get("pr_title") or commit_message + pr_body = args.get("pr_body") or "" + patch_b64 = args.get("patch_b64") + patch = args.get("patch") + files = args.get("files") + if not repo_url: + return {"error": "repo_change_to_branch: repo_url required"} + ws = await run_in_threadpool(_ws_open, repo_url, base_branch) + # apply + if patch_b64 or patch or files: + a = {"workspace_id": ws["workspace_id"], "repo_url": repo_url, "branch": base_branch, "dry_run": False} + if patch_b64: a["patch_b64"] = patch_b64 + if patch: a["patch"] = patch + if files: a["files"] = files + res_apply = await _execute_tool("repo_apply", a) + if isinstance(res_apply, dict) and res_apply.get("error"): + return {"error": "repo_change_to_branch: apply failed", "details": res_apply} + else: + return {"error": "repo_change_to_branch: provide patch_b64/patch/files"} + # push + res_push = await _execute_tool("repo_push", { + "workspace_id": ws["workspace_id"], + "repo_url": repo_url, + "base_branch": base_branch, + "new_branch": new_branch, + "branch_prefix": branch_prefix, + "commit_message": commit_message + }) + if isinstance(res_push, dict) and res_push.get("error"): + return {"error": "repo_change_to_branch: push failed", "details": res_push} + out = {"workspace_id": ws["workspace_id"], "apply": res_apply, "push": res_push} + if create_pr: + head_branch = (res_push or {}).get("branch") or new_branch + pr = await _execute_tool("repo_pr_create", {"repo_url": repo_url, "head_branch": head_branch, "base_branch": base_branch, "title": pr_title, "body": pr_body}) + out["pr"] = pr + return out + + # ------------------------------------------------------------------ + # Deprecated aliases (keep for backward compatibility) + # ------------------------------------------------------------------ + if name == "repo_grep": + # alias to repo_search(mode=grep) with repo_url + return await _execute_tool("repo_search", { + "repo_url": args.get("repo_url") or args.get("repo") or "", + "branch": args.get("branch") or "main", + "query": args.get("query") or "", + "mode": "grep", + "n_results": int(args.get("max_hits", args.get("n_results", 20))) + }) + if name == "repo_grep": + repo_url = args.get("repo_url","") + branch = args.get("branch","main") + query = (args.get("query") or "").strip() + max_hits = int(args.get("max_hits",200)) + if not repo_url or not query: + raise HTTPException(status_code=400, detail="repo_url en query verplicht.") + repo_path = await _get_git_repo_async(repo_url, branch) + root = Path(repo_path) + hits = [] + qlow = query.lower() + for p in root.rglob("*"): + #logger.info(p) + if qlow in str(p).lower() and str(p).split('.')[-1] in ['txt','md','htm','html','cpp','js','json','env','py','php','c','h']: + if qlow==str(p).lower().split("/")[-1]: + max_chars=55000 + else: + max_chars=400 + file_txt=_read_text_file(p) + if len(file_txt) > max_chars: + add_str=" so first " + str(max_chars) + " chars given in excerpt" + else: + add_str=" so file content given in excerpt" + hits.append({ + "path": str(p.relative_to(root)), + "line": "-text found in filename"+str(add_str)+"- ", + "excerpt": str(file_txt)[:max_chars] + }) + elif qlow in str(p).lower(): + hits.append({ + "path": str(p.relative_to(root)), + "line": "-text found in filename- ", + "excerpt": "-text found in filename" + }) + if p.is_dir(): + continue + if set(p.parts) & PROFILE_EXCLUDE_DIRS: + continue + if p.suffix.lower() in BINARY_SKIP: + continue + try: + txt = _read_text_file(p) + except Exception: + continue + if not txt: + continue + # snelle lijngewijze scan + for ln, line in enumerate(txt.splitlines(), 1): + if qlow in line.lower(): + hits.append({ + "path": str(p.relative_to(root)), + "line": ln, + "excerpt": line.strip()[:400] + }) + if len(hits) >= max_hits: + break + if len(hits) >= max_hits: + break + return {"count": len(hits), "hits": hits, "repo": os.path.basename(repo_url), "branch": branch} + # RAG + if name == "rag_index_repo": + out = await run_in_threadpool(_rag_index_repo_sync, **{ + "repo_url": args.get("repo_url",""), + "branch": args.get("branch","main"), + "profile": args.get("profile","auto"), + "include": args.get("include",""), + "exclude_dirs": args.get("exclude_dirs",""), + "chunk_chars": int(args.get("chunk_chars",3000)), + "overlap": int(args.get("overlap",400)), + "collection_name": args.get("collection_name","code_docs"), + "force": bool(args.get("force", False)), + }) + return out + if name == "rag_query": + try: + out= await run_in_threadpool(_rag_index_repo_sync, **{ + "repo_url": args.get("repo",""), + "branch": "main", + "profile": "auto", + "include": "", + "exclude_dirs": "", + "chunk_chars": 3000, + "overlap": 400, + "collection_name": "code_docs", + "force": False, + }) + except Exception as e: + return {"error": f"Error for functioncall '{name}', while doing repo_index. errortext: {str(e)}"} + try: + out = await rag_query_api( + query=args.get("query",""), + n_results=int(args.get("n_results",5)), + collection_name=_norm_collection_name(args.get("collection_name","code_docs"), "code_docs"), + repo=args.get("repo"), + path_contains=args.get("path_contains"), + profile=args.get("profile") + ) + return out + except Exception as e: + return {"error": f"Error for functioncall '{name}', while doing repo_query. errortext: {str(e)}"} + if name == "memory_upsert": + namespace = (args.get("namespace") or "").strip() + if not namespace: + return {"error": "memory_upsert: namespace is required"} + + # accepteer: items[] of shorthand text/role + items = args.get("items") + if not items: + txt = (args.get("text") or "").strip() + if not txt: + return {"error": "memory_upsert: provide items[] or text"} + items = [{ + "role": args.get("role", "user"), + "text": txt, + "ts_unix": args.get("ts_unix"), + "source": args.get("source", "openwebui"), + "tags": args.get("tags") or [], + "meta": {} + }] + + chunk_chars = int(args.get("chunk_chars", MEMORY_CHUNK_CHARS)) + overlap = int(args.get("overlap_chars", MEMORY_OVERLAP_CHARS)) + collection_base = (args.get("collection_name") or MEMORY_COLLECTION).strip() or MEMORY_COLLECTION + collection_eff = _collection_effective(collection_base) + col = _get_collection(collection_eff) + + docs_for_meili = [] + chroma_docs = [] + chroma_metas = [] + chroma_ids = [] + + now = int(time.time()) + for it in items: + role = (it.get("role") or "user").strip() + text0 = (it.get("text") or "").strip() + if not text0: + continue + ts_unix = int(it.get("ts_unix") or now) + source = (it.get("source") or "openwebui") + turn_id = it.get("turn_id") + tags = it.get("tags") or [] + message_id = it.get("message_id") or uuid.uuid4().hex + + chunks = _chunk_text(text0, chunk_chars=chunk_chars, overlap=overlap) if len(text0) > chunk_chars else [text0] + for ci, ch in enumerate(chunks): + chunk_id = f"{message_id}:{ci}" + meta = { + "namespace": namespace, + "role": role, + "ts_unix": ts_unix, + "source": source, + "turn_id": turn_id, + "tags": tags, + "message_id": message_id, + "chunk_id": chunk_id, + "chunk_index": ci, + "id": chunk_id, + } + + chroma_docs.append(ch) + chroma_metas.append(meta) + chroma_ids.append(chunk_id) + + docs_for_meili.append({ + "id": chunk_id, + "namespace": namespace, + "role": role, + "ts_unix": ts_unix, + "source": source, + "turn_id": turn_id, + "tags": tags, + "message_id": message_id, + "chunk_index": ci, + "text": ch, + }) + + if chroma_docs: + _collection_add(col, chroma_docs, chroma_metas, chroma_ids) + + if MEILI_ENABLED and docs_for_meili: + await _meili_memory_upsert(docs_for_meili) + + return { + "status": "ok", + "namespace": namespace, + "collection_effective": collection_eff, + "chunks_indexed": len(chroma_docs), + "meili_indexed": bool(MEILI_ENABLED and docs_for_meili) + } + + if name == "memory_query": + namespace = (args.get("namespace") or "").strip() + query = (args.get("query") or "").strip() + if not namespace or not query: + return {"error": "memory_query: namespace and query are required"} + + k = int(args.get("k", 12)) + meili_limit = int(args.get("meili_limit", 40)) + chroma_limit = int(args.get("chroma_limit", 40)) + max_total = int(args.get("max_total_chars", MEMORY_MAX_TOTAL_CHARS)) + clip_chars = int(args.get("clip_chars", MEMORY_CLIP_CHARS)) + + collection_base = (args.get("collection_name") or MEMORY_COLLECTION).strip() or MEMORY_COLLECTION + collection_eff = _collection_effective(collection_base) + col = _get_collection(collection_eff) + + # --- Chroma candidates --- + q_emb = _EMBEDDER.embed_query(query) + where = {"namespace": {"$eq": namespace}} + res = col.query( + query_embeddings=[q_emb], + n_results=max(k, chroma_limit), + where=where, + include=["metadatas", "documents", "distances"] + ) + docs = (res.get("documents") or [[]])[0] + metas = (res.get("metadatas") or [[]])[0] + dists = (res.get("distances") or [[]])[0] + + cands = [] + for doc, meta, dist in zip(docs, metas, dists): + meta = meta or {} + cid = meta.get("chunk_id") or meta.get("id") or "" + emb_sim = 1.0 / (1.0 + float(dist or 0.0)) + cands.append({ + "id": cid, + "text": doc or "", + "meta": meta, + "emb_sim": emb_sim, + "bm25": 0.0, + "score": 0.0 + }) + + # --- Meili hits --- + meili_hits = await _meili_memory_search(namespace, query, limit=meili_limit) if MEILI_ENABLED else [] + bm25_map = {} + for h in meili_hits: + if h.get("id"): + bm25_map[h["id"]] = float(h.get("score") or 0.0) + + # voeg meili-only hits toe als ze niet al in chroma zitten + cand_ids = set([c["id"] for c in cands if c.get("id")]) + for h in meili_hits: + hid = h.get("id") or "" + if hid and hid not in cand_ids: + cands.append({ + "id": hid, + "text": (h.get("text") or ""), + "meta": (h.get("meta") or {}), + "emb_sim": 0.0, + "bm25": float(h.get("score") or 0.0), + "score": 0.0 + }) + cand_ids.add(hid) + + # bm25 invullen voor bestaande cands + for c in cands: + cid = c.get("id") or "" + if cid in bm25_map: + c["bm25"] = bm25_map[cid] + + # normaliseer bm25 + bm25_scores = [c["bm25"] for c in cands] + if bm25_scores: + mn, mx = min(bm25_scores), max(bm25_scores) + def _norm(s): return (s - mn) / (mx - mn) if mx > mn else 0.0 + else: + def _norm(s): return 0.0 + + alpha = float(MEMORY_EMB_WEIGHT) + now = int(time.time()) + half = max(1, int(MEMORY_HALF_LIFE_SEC)) + bias = float(MEMORY_RECENCY_BIAS) + + for c in cands: + bm25_n = _norm(c["bm25"]) + base = alpha * float(c["emb_sim"]) + (1.0 - alpha) * bm25_n + ts = int((c.get("meta") or {}).get("ts_unix") or now) + age = max(0, now - ts) + rec = half / (half + age) # 0..1 + c["score"] = base + bias * rec + + ranked = sorted(cands, key=lambda x: x["score"], reverse=True) + + # clip + max_total_chars + out = [] + used = 0 + for c in ranked: + txt = (c.get("text") or "") + if clip_chars and len(txt) > clip_chars: + txt = txt[:clip_chars] + " …" + if max_total and used + len(txt) > max_total: + continue + used += len(txt) + out.append({ + "id": c.get("id"), + "score": float(c.get("score") or 0.0), + "text": txt, + "meta": c.get("meta") or {} + }) + if len(out) >= k: + break + + return { + "namespace": namespace, + "query": query, + "k": k, + "collection_effective": collection_eff, + "results": out, + "stats": { + "meili_hits": len(meili_hits), + "chroma_hits": len(docs), + "returned_chars": used + } + } + + + # Console tools + if name == "run_shell": + # Search the web using SearXNG and get the content of the relevant pages. + out=json.dumps(await t_run_shell(args), ensure_ascii=False) + return out + + # Repo + if name == "repo_qa": + # High-level QA over een specifieke repo. + out=json.dumps(await repo_qa_answer(repo_hint=args.get("repo").replace('"','').replace("'",""),question=args.get("question"),branch=args.get("branch","main"),n_ctx=10), ensure_ascii=False) + return out + + # Web tools + if name == "web_search_xng": + # Search the web using SearXNG and get the content of the relevant pages. + out=await web_search_xng(query=args.get("query","")) + return out + + if name == "get_website_xng": + # Web scrape the website provided and get the content of it. + out=await get_website_xng( + url=args.get("url", "")) + return out + + # Tekst tools + if name == "summarize_text": + text = (args.get("text") or "")[:int(args.get("max_chars",16000))] + instruction = args.get("instruction") or "Vat samen in bullets (max 10), met korte inleiding en actiepunten." + resp = await llm_call_openai_compat( + [{"role":"system","content":"Je bent behulpzaam en exact."}, + {"role":"user","content": f"{instruction}\n\n--- BEGIN ---\n{text}\n--- EINDE ---"}], + stream=False, max_tokens=7680 + ) + return resp + + if name == "analyze_text": + text = (args.get("text") or "")[:int(args.get("max_chars",20000))] + goal = args.get("goal") or "Licht toe wat dit document doet. Benoem sterke/zwakke punten, risico’s en concrete verbeterpunten." + resp = await llm_call_openai_compat( + [{"role":"system","content":"Wees feitelijk en concreet."}, + {"role":"user","content": f"{goal}\n\n--- BEGIN ---\n{text}\n--- EINDE ---"}], + stream=False, max_tokens=7680 + ) + return resp + + if name == "improve_text": + text = (args.get("text") or "")[:int(args.get("max_chars",20000))] + objective = args.get("objective") or "Verbeter dit document. Lever een beknopte toelichting en vervolgens de volledige verbeterde versie." + style = args.get("style") or "Hanteer best practices, behoud inhoudelijke betekenis." + resp = await llm_call_openai_compat( + [{"role":"system","content": SYSTEM_PROMPT}, + {"role":"user","content": f"{objective}\nStijl/criteria: {style}\n" + "Antwoord met eerst een korte toelichting, daarna alleen de verbeterde inhoud tussen een codeblok.\n\n" + f"--- BEGIN ---\n{text}\n--- EINDE ---"}], + stream=False, max_tokens=10240 + ) + return resp + + # Code tools + if name == "validate_code_text": + code = args.get("code","") + resp = await llm_call_openai_compat( + [{"role":"system","content":"Wees strikt en concreet."}, + {"role":"user","content": f"{VALIDATE_PROMPT}\n\nCode om te valideren:\n```\n{code}\n```"}], + stream=False, max_tokens=10000 + ) + txt = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + return {"status":"issues_found" if _parse_validation_results(txt) else "valid", + "issues": _parse_validation_results(txt), "raw": txt} + + if name == "improve_code_text": + code = args.get("code","") + language = args.get("language","auto") + focus = args.get("improvement_focus","best practices") + resp = await llm_call_openai_compat( + [{"role":"system","content": SYSTEM_PROMPT}, + {"role":"user","content": f"Verbeter deze {language}-code met focus op {focus}:\n\n" + f"{code}\n\nGeef eerst een korte toelichting, dan alleen het verbeterde codeblok. Behoud functionaliteit."}], + stream=False, max_tokens=10768 + ) + return resp + + if name == "ingest_openwebui_files": + files = _normalize_files_arg(args) + if not files: + raise HTTPException(status_code=400, detail="Geen bestanden ontvangen (__files__ is leeg).") + saved = [] + tmpdir = Path("/tmp/owui_files") + for fmeta in files: + fid = fmeta.get("id") + if not fid: + continue + path = await _fetch_openwebui_file(fid, tmpdir) + saved.append({"id": fid, "path": str(path), "name": fmeta.get("name"), "mime": fmeta.get("mime")}) + + # >>> HIER je eigen pipeline aanroepen, bijv. direct indexeren: + # for s in saved: index_file_into_chroma(s["path"], collection=args.get("target_collection","code_docs"), ...) + + return {"ok": True, "downloaded": saved, "collection": args.get("target_collection","code_docs")} + + if name == "vision_analyze": + image_url = args.get("image_url","") + prompt = args.get("prompt","Beschrijf beknopt wat je ziet en noem de belangrijkste details.") + max_tokens = int(args.get("max_tokens",1024)) + b64 = None + # Alleen data: of raw base64 accepteren; http(s) niet, want die worden niet + # ingeladen in de vision-call en zouden stil falen. + if image_url.startswith(("http://","https://")): + raise HTTPException(status_code=400, detail="vision_analyze: image_url moet data: URI of raw base64 zijn.") + if image_url.startswith("data:image") and "," in image_url: + b64 = image_url.split(",",1)[1] + elif re.match(r"^[A-Za-z0-9+/=]+$", image_url.strip()): + b64 = image_url.strip() + messages = [{"role":"user","content": f" {prompt}"}] + return await llm_call_openai_compat(messages, stream=False, max_tokens=max_tokens, + extra={"images": [b64] if b64 else []}) + + raise HTTPException(status_code=400, detail=f"Unknown tool: {name}") + +""" + +""" + +TOOLS_REGISTRY = { + "repo_grep": { + "description": "Zoek een specifieke exacte tekst in een file in een git repo (fast grep-achtig).", + "parameters": { + "type":"object", + "properties":{ + "repo_url":{"type":"string"}, + "branch":{"type":"string","default":"main"}, + "query":{"type":"string"}, + "max_hits":{"type":"integer","default":10} + }, + "required":["repo_url","query"] + } + }, + "ingest_openwebui_files": { + "description": "Download aangehechte OpenWebUI-bestanden en voer ingestie/embedding uit.", + "parameters": { + "type":"object", + "properties":{ + "target_collection":{"type":"string","default":"code_docs"}, + "profile":{"type":"string","default":"auto"}, + "chunk_chars":{"type":"integer","default":3000}, + "overlap":{"type":"integer","default":400} + } + } + }, + "rag_index_repo": { + "description": "Indexeer een git-repo in ChromaDB (chunken & metadata).", + "parameters": { + "type":"object", + "properties":{ + "repo_url":{"type":"string"}, + "branch":{"type":"string","default":"main"}, + "profile":{"type":"string","default":"auto"}, + "include":{"type":"string","default":""}, + "exclude_dirs":{"type":"string","default":""}, + "chunk_chars":{"type":"integer","default":3000}, + "overlap":{"type":"integer","default":400}, + "collection_name":{"type":"string","default":"code_docs"}, + "force":{"type":"boolean","default":False} + }, + "required":["repo_url"] + } + }, + "rag_query": { + "description": "Zoek korte text in de RAG-collectie en geef top-N passages (hybride rerank). (! first call rag_index_repo !)", + "parameters": { + "type":"object", + "properties":{ + "query":{"type":"string"}, + "n_results":{"type":"integer","default":8}, + "collection_name":{"type":"string","default":"code_docs"}, + "repo":{"type":["string","null"]}, + "path_contains":{"type":["string","null"]}, + "profile":{"type":["string","null"]} + }, + "required":["query","repo"] + } + }, + "memory_upsert": { + "description": "Sla chat/gesprekstekst op in memory (hybride RAG: Meili + Chroma). Gebruik namespace=chat_id.", + "parameters": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "message_id": {"type": "string"}, + "turn_id": {"type": ["string","integer","null"]}, + "role": {"type": "string", "default": "user"}, + "text": {"type": "string"}, + "ts_unix": {"type": ["integer","null"]}, + "source": {"type": ["string","null"], "default": "openwebui"}, + "tags": {"type": "array", "items": {"type":"string"}}, + "meta": {"type": "object"} + }, + "required": ["role","text"] + } + }, + "text": {"type": "string"}, + "role": {"type": "string", "default": "user"}, + "ts_unix": {"type": ["integer","null"]}, + "source": {"type": ["string","null"], "default": "openwebui"}, + "tags": {"type": "array", "items": {"type":"string"}}, + "chunk_chars": {"type": "integer", "default": 3500}, + "overlap_chars": {"type": "integer", "default": 300}, + "collection_name": {"type": "string", "default": "chat_memory"} + }, + "required": ["namespace"] + } + }, + "memory_query": { + "description": "Zoek relevante chat-memory snippets (hybride Meili + Chroma) binnen namespace.", + "parameters": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "query": {"type": "string"}, + "k": {"type": "integer", "default": 12}, + "meili_limit": {"type": "integer", "default": 40}, + "chroma_limit": {"type": "integer", "default": 40}, + "collection_name": {"type": "string", "default": "chat_memory"}, + "max_total_chars": {"type": "integer", "default": 12000}, + "clip_chars": {"type": "integer", "default": 1200} + }, + "required": ["namespace","query"] + } + }, + "web_search_xng": { + "description": "Search the web using SearXNG and get the content of the relevant pages.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + },"required":["query"]} + }, + "get_website_xng": { + "description": "Web scrape the website provided and get the content of it.", + "parameters": { + "type": "object", + "properties": { + "url": {"type": "string"}, + },"required":["url"]} + }, + "run_shell": { + "description": "Voer een shell-commando uit (stdout/stderr). (for simple commands)", + "parameters":{ + "type":"object", + "properties":{ + "command":{"type":"string"}, + "timeout":{"type":"integer","default":600} + },"required":["command"]} + }, + "repo_qa": { + "description": "High-level QA over een specifieke repo. (incl. RAG,clone,summary,context fuctionalities).", + "parameters":{ + "type":"object", + "properties":{ + "repo":{"type":"string"}, + "question":{"type":"string"}, + "branch":{"type":"string"}, + },"required":["repo","question"]} + }, + "summarize_text": { + "description": "Vat tekst samen in bullets met inleiding en actiepunten.", + "parameters": {"type":"object","properties":{ + "text":{"type":"string"}, + "instruction":{"type":"string","default":"Vat samen in bullets (max 10), met korte inleiding en actiepunten."}, + "max_chars":{"type":"integer","default":16000} + },"required":["text"]} + }, + "analyze_text": { + "description": "Analyseer tekst: sterke/zwakke punten, risico’s en verbeterpunten.", + "parameters": {"type":"object","properties":{ + "text":{"type":"string"}, + "goal":{"type":"string","default":"Licht toe wat dit document doet. Benoem sterke/zwakke punten, risico’s en concrete verbeterpunten."}, + "max_chars":{"type":"integer","default":20000} + },"required":["text"]} + }, + "improve_text": { + "description": "Verbeter tekst: korte toelichting + volledige verbeterde versie.", + "parameters": {"type":"object","properties":{ + "text":{"type":"string"}, + "objective":{"type":"string","default":"Verbeter dit document. Lever een beknopte toelichting en vervolgens de volledige verbeterde versie."}, + "style":{"type":"string","default":"Hanteer best practices, behoud inhoudelijke betekenis."}, + "max_chars":{"type":"integer","default":20000} + },"required":["text"]} + }, + "validate_code_text": { + "description": "Valideer code; geef issues (met regelnummers).", + "parameters": {"type":"object","properties":{ + "code":{"type":"string"} + },"required":["code"]} + }, + "improve_code_text": { + "description": "Verbeter code met focus (best practices/perf/security).", + "parameters": {"type":"object","properties":{ + "code":{"type":"string"}, + "language":{"type":"string","default":"auto"}, + "improvement_focus":{"type":"string","default":"best practices"} + },"required":["code"]} + }, + "vision_analyze": { + "description": "Eén afbeelding analyseren via data: URI of base64.", + "parameters":{"type":"object","properties":{ + "image_url":{"type":"string"}, + "prompt":{"type":"string","default":"Beschrijf beknopt wat je ziet en de belangrijkste details."}, + "max_tokens":{"type":"integer","default":512} + },"required":["image_url"]} + } +} + +# Verberg OWUI-afhankelijke tool wanneer niet geconfigureerd +try: + if not (OWUI_BASE_URL and OWUI_API_TOKEN): + # bestaat altijd als key; markeer als hidden zodat /v1/tools hem niet toont + TOOLS_REGISTRY["ingest_openwebui_files"]["hidden"] = True +except Exception: + pass + + +def _tools_list_from_registry(reg: dict): + lst = [] + for name, spec in reg.items(): + lst.append({ + "name": name, + "description": spec.get("description", ""), + "parameters": spec.get("parameters", {"type":"object","properties":{}}) + }) + return {"mode": LLM_FUNCTION_CALLING_MODE if 'LLM_FUNCTION_CALLING_MODE' in globals() else "shim", + "tools": lst} + + +@app.get("/v1/tools", operation_id="get_tools_list") +async def list_tools(format: str = "proxy"): + # negeer 'format' en geef altijd OpenAI tool list terug + return { + "object": "tool.list", + "mode": "function", # <- belangrijk voor OWUI 0.6.21 + "data": _openai_tools_from_registry(_visible_registry(TOOLS_REGISTRY)) + } + +# === OpenAPI-compat: 1 endpoint per tool (operationId = toolnaam) === +# Open WebUI kijkt naar /openapi.json, leest operationId’s en maakt daar “tools” van. +def _register_openapi_tool(name: str): + # Opzet: POST /openapi/{name} met body == arguments-dict + # operation_id = name => Open WebUI toolnaam = name + # Belangrijk: injecteer requestBody schema in OpenAPI, zodat OWUI de parameters kent. + schema = (TOOLS_REGISTRY.get(name, {}) or {}).get("parameters", {"type":"object","properties":{}}) + @app.post( + f"/openapi/{name}", + operation_id=name, + summary=f"Tool: {name}", + openapi_extra={ + "requestBody": { + "required": True, + "content": { + "application/json": { + "schema": schema + } + } + } + } + ) + async def _tool_entry(payload: dict = Body(...)): + if name not in TOOLS_REGISTRY: + raise HTTPException(status_code=404, detail=f"Unknown tool: {name}") + # payload = {"arg1":..., "arg2":...} + out = await _execute_tool(name, payload or {}) + # Als _execute_tool een OpenAI-chat response teruggeeft, haal de tekst eruit + if isinstance(out, dict) and "choices" in out: + try: + txt = ((out.get("choices") or [{}])[0].get("message") or {}).get("content", "") + return {"result": txt} + except Exception: + pass + return out + + +# ----------------------------------------------------------------------------- +# Tool registry v2 additions + deprecations +# ----------------------------------------------------------------------------- +def _register_repo_toolset_v2(): + # New preferred tools + TOOLS_REGISTRY.update({ + "repo_open": { + "description": "Open/clone/pull een repo en maak een workspace_id. Gebruik dit als eerste stap voor repo-bewerkingen.", + "parameters": {"type":"object","properties":{ + "repo_url":{"type":"string","description":"Gitea git URL"}, + "branch":{"type":"string","default":"main"} + },"required":["repo_url"]} + }, + "repo_search": { + "description": "Zoek in de repo. mode=auto (default) doet eerst grep, valt terug op RAG. Gebruik dit om relevante bestanden/regels te vinden.", + "parameters": {"type":"object","properties":{ + "workspace_id":{"type":"string"}, + "repo_url":{"type":"string"}, + "branch":{"type":"string","default":"main"}, + "query":{"type":"string"}, + "mode":{"type":"string","default":"auto","enum":["auto","grep","rag"]}, + "n_results":{"type":"integer","default":20}, + "collection_name":{"type":"string","default":"code_docs"}, + "profile":{"type":"string"}, + "path_contains":{"type":"string"} + },"required":["query"]} + }, + "repo_read": { + "description": "Lees exact een bestand (of line-range) uit de repo/workspace.", + "parameters": {"type":"object","properties":{ + "workspace_id":{"type":"string"}, + "repo_url":{"type":"string"}, + "branch":{"type":"string","default":"main"}, + "path":{"type":"string"}, + "start_line":{"type":"integer"}, + "end_line":{"type":"integer"}, + "max_bytes":{"type":"integer","default":200000} + },"required":["path"]} + }, + "repo_apply": { + "description": "Pas wijzigingen toe in de workspace. Gebruik bij voorkeur patch_b64 (unified diff, base64).", + "parameters": {"type":"object","properties":{ + "workspace_id":{"type":"string"}, + "repo_url":{"type":"string"}, + "branch":{"type":"string","default":"main"}, + "dry_run":{"type":"boolean","default":False}, + "patch_b64":{"type":"string","description":"base64 van unified diff"}, + "patch":{"type":"string","description":"(legacy) unified diff als string"}, + "files":{"type":"array","items":{"type":"object","properties":{ + "path":{"type":"string"}, + "content_b64":{"type":"string"} + }}, "description":"(alternatief) files overschrijven"} + },"required":[]} + }, + "repo_push": { + "description": "Commit + push huidige workspace wijzigingen naar een nieuwe branch op origin (Gitea).", + "parameters": {"type":"object","properties":{ + "workspace_id":{"type":"string"}, + "repo_url":{"type":"string"}, + "base_branch":{"type":"string","default":"main"}, + "new_branch":{"type":"string","description":"exacte branch naam (optioneel)"}, + "branch_prefix":{"type":"string","default":"test"}, + "commit_message":{"type":"string","default":"Apply changes"}, + "allow_empty":{"type":"boolean","default":False} + },"required":[]} + }, + "repo_pr_create": { + "description": "Maak een Pull Request in Gitea (vereist GITEA_TOKEN).", + "parameters": {"type":"object","properties":{ + "repo_url":{"type":"string"}, + "head_branch":{"type":"string"}, + "base_branch":{"type":"string","default":"main"}, + "title":{"type":"string"}, + "body":{"type":"string"} + },"required":["repo_url","head_branch","title"]} + }, + "repo_change_to_branch": { + "description": "MACRO: open repo → apply patch/files → commit+push naar test branch → (optioneel) PR. Dit is de aanbevolen 'pas repo aan en upload naar branch' tool.", + "parameters": {"type":"object","properties":{ + "repo_url":{"type":"string"}, + "base_branch":{"type":"string","default":"main"}, + "branch_prefix":{"type":"string","default":"test"}, + "new_branch":{"type":"string"}, + "commit_message":{"type":"string","default":"Apply changes"}, + "patch_b64":{"type":"string"}, + "patch":{"type":"string"}, + "files":{"type":"array","items":{"type":"object","properties":{ + "path":{"type":"string"}, + "content_b64":{"type":"string"} + }}}, + "create_pr":{"type":"boolean","default":False}, + "pr_title":{"type":"string"}, + "pr_body":{"type":"string"} + },"required":["repo_url"]} + }, + }) + + # Deprecate old names (keep behavior) + if "repo_grep" in TOOLS_REGISTRY: + TOOLS_REGISTRY["repo_grep"]["description"] = "DEPRECATED: gebruik repo_search(mode='grep' of 'auto'). (blijft werken voor compat.)" + if "rag_index_repo" in TOOLS_REGISTRY: + TOOLS_REGISTRY["rag_index_repo"]["description"] = "DEPRECATED: je hoeft meestal niet handmatig te indexeren; repo_search(mode='rag'/'auto') indexeert automatisch." + if "rag_query" in TOOLS_REGISTRY: + TOOLS_REGISTRY["rag_query"]["description"] = "DEPRECATED: gebruik repo_search(mode='rag' of 'auto')." +_register_repo_toolset_v2() + +for _t in TOOLS_REGISTRY.keys(): + _register_openapi_tool(_t) + + +@app.post("/v1/tools/call", operation_id="post_tools_call") +async def tools_entry(request: Request): + #if request.method == "GET": + # return {"object":"tool.list","mode":LLM_FUNCTION_CALLING_MODE,"data": _openai_tools_from_registry(TOOLS_REGISTRY)} + data = await request.json() + name = (data.get("name") + or (data.get("tool") or {}).get("name") + or (data.get("function") or {}).get("name")) + raw_args = (data.get("arguments") + or (data.get("tool") or {}).get("arguments") + or (data.get("function") or {}).get("arguments") + or {}) + if isinstance(raw_args, str): + try: args = json.loads(raw_args) + except Exception: args = {} + else: + args = raw_args + if not name or name not in TOOLS_REGISTRY or TOOLS_REGISTRY[name].get("hidden"): + raise HTTPException(status_code=404, detail=f"Unknown tool: {name}") + try: + output = await _execute_tool(name, args) + except HTTPException: + raise + except Exception as e: + logging.exception("tool call failed") + raise HTTPException(status_code=500, detail=str(e)) + return {"id": f"toolcall-{uuid.uuid4().hex}","object":"tool.run","created": int(time.time()),"name": name,"arguments": args,"output": output} + +async def _complete_with_autocontinue( + messages: list[dict], + *, + model: str, + temperature: float, + top_p: float, + max_tokens: int, + extra_payload: dict | None, + max_autocont: int +) -> tuple[str, dict]: + """ + Eén of meer non-stream calls met jouw bestaande auto-continue logica. + Retourneert (full_text, last_response_json). + """ + resp = await llm_call_openai_compat( + messages, model=model, stream=False, + temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra=extra_payload if extra_payload else None + ) + content = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + finish_reason = (resp.get("choices",[{}])[0] or {}).get("finish_reason") + + continues = 0 + NEAR_MAX = int(os.getenv("LLM_AUTOCONT_NEAR_MAX", "48")) + def _near_cap(txt: str) -> bool: + try: + return approx_token_count(txt) >= max_tokens - NEAR_MAX + except Exception: + return False + while continues < max_autocont and ( + finish_reason in ("length","content_filter") + or content.endswith("…") + or _near_cap(content) + ): + follow = messages + [ + {"role":"assistant","content": content}, + {"role":"user","content": "Ga verder zonder herhaling; alleen het vervolg."} + ] + nxt = await llm_call_openai_compat( + follow, model=model, stream=False, + temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra=extra_payload if extra_payload else None + ) + more = (nxt.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + content = (content + ("\n" if content and more else "") + more).strip() + finish_reason = (nxt.get("choices",[{}])[0] or {}).get("finish_reason") + resp = nxt # laatst gezien resp teruggeven (usage etc.) + continues += 1 + return content, resp + +async def llm_call_autocont( + messages: list[dict], + *, + model: Optional[str] = None, + stream: bool = False, + temperature: float = 0.02, + top_p: float = 0.9, + max_tokens: int = 1024, + extra: Optional[dict] = None, + stop: Optional[Union[str, list[str]]] = None, + **kwargs +) -> dict | StreamingResponse: + """ + OpenAI-compatibele wrapper die non-stream antwoorden automatisch + doorcontinueert met jouw _complete_with_autocontinue(...). + - stream=True -> passthrough naar llm_call_openai_compat (ongewijzigd) + - stream=False -> retourneert één samengevoegd antwoord als OpenAI JSON + """ + mdl = (model or os.getenv("LLM_MODEL", "mistral-medium")).strip() + if stream: + # streaming blijft 1-op-1 zoals voorheen + return await llm_call_openai_compat( + messages, + model=mdl, + stream=True, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + extra=extra if extra else None, + stop=stop + ) + + max_autocont = int(os.getenv("LLM_AUTO_CONTINUES", "2")) + full_text, _last = await _complete_with_autocontinue( + messages, + model=mdl, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + extra_payload=extra if extra else None, + max_autocont=max_autocont + ) + # Bouw een standaard OpenAI-chat response met het samengevoegde antwoord + return _openai_chat_response(mdl, full_text, messages) + +def _normalize_tools_and_choice(body: dict) -> None: + tools = body.get("tools") or [] + tc = body.get("tool_choice") + + # 1) OWUI: tool_choice="required" -> maak het OpenAI-compat + if tc == "required": + names = [] + for t in tools: + fn = (t.get("function") or {}) if isinstance(t, dict) else {} + n = fn.get("name") + if n: + names.append(n) + + uniq = list(dict.fromkeys(names)) + if len(uniq) == 1: + # force die ene tool + body["tool_choice"] = {"type": "function", "function": {"name": uniq[0]}} + else: + # meerdere tools: upstream snapt "required" niet -> kies auto + body["tool_choice"] = "auto" + + # 2) Sommige clients sturen tools als [{"name":..., "parameters":...}] i.p.v. OpenAI {"type":"function","function":{...}} + if tools and isinstance(tools, list) and isinstance(tools[0], dict): + if "type" not in tools[0] and "name" in tools[0]: + body["tools"] = [{"type": "function", "function": t} for t in tools] + + # 3) (optioneel) backward compat: functions -> tools + if (not body.get("tools")) and body.get("functions"): + body["tools"] = [{"type": "function", "function": f} for f in body["functions"]] + body.pop("functions", None) + if body.get("function_call") and not body.get("tool_choice"): + # grof: map function_call->tool_choice + fc = body["function_call"] + if isinstance(fc, dict) and fc.get("name"): + body["tool_choice"] = {"type": "function", "function": {"name": fc["name"]}} + body.pop("function_call", None) + +@app.post("/v1/completions") +async def openai_completions(body: dict = Body(...), request: Request = None): + out = await openai_chat_completions(body,request) + return out + +@app.post("/v1/chat/completions") +async def openai_chat_completions(body: dict = Body(...), request: Request = None): + model = (body.get("model") or os.getenv("LLM_MODEL", "mistral-medium")).strip() + _normalize_tools_and_choice(body) + logging.info(str(body)) + logging.info(str(request)) + stream = bool(body.get("stream", False)) + raw_messages = body.get("messages") or [] + # normaliseer tool-berichten naar plain tekst voor het LLM + NORMALIZE_TOOL_MESSAGES = os.getenv("NORMALIZE_TOOL_MESSAGES", "0").lower() not in ("0","false","no") + if NORMALIZE_TOOL_MESSAGES: + norm_messages = [] + for m in raw_messages: + if m.get("role") == "tool": + nm = m.get("name") or "tool" + norm_messages.append({ + "role": "user", + "content": f"[{nm} RESULT]\n{m.get('content') or ''}" + }) + else: + norm_messages.append(m) + else: + norm_messages=raw_messages + + # --- minimal tool-calling glue (laat rest van je functie intact) --- + tools = body.get("tools") or [] + + RUN_BRIDGE = os.getenv("LLM_TOOL_RUNNER", "bridge").lower() == "bridge" + + # (optioneel maar vaak nodig) forceer jouw eigen tools i.p.v. wat OWUI meestuurt + if RUN_BRIDGE and os.getenv("FORCE_ALL_TOOLS", "1").lower() not in ("0","false","no"): + body["tools"] = _openai_tools_from_registry(_visible_registry(TOOLS_REGISTRY)) + tools = body["tools"] + + # bridge werkt het makkelijkst non-stream + if RUN_BRIDGE and stream and (body.get("tools") or []): + body["stream"] = False + stream = False + + # 'tool_choice' hier alleen lezen; later in de native branch wordt opnieuw naar body gekeken + tool_choice_req = body.get("tool_choice") # 'auto' | 'none' | 'required' | {...} + try: + logger.info("🧰 tools_count=%s, tool_choice=%s", len(tools), tool_choice_req) + except Exception: + pass + if RUN_BRIDGE and not stream: + + # OWUI stuurt vaak "required" als: "er MOET een tool worden gebruikt". + # Als er precies 1 tool is meegegeven, normaliseren we dat naar "force deze tool". + if tool_choice_req == "required" and tools: + names = [ (t.get("function") or {}).get("name") for t in tools if t.get("function") ] + names = [ n for n in names if n ] + # 1) exact 1 → force die + if len(set(names)) == 1: + tool_choice_req = {"type":"function","function":{"name": names[0]}} + else: + # 2) meerdere → kies op basis van user prompt (noem de toolnaam) + last_user = next((m for m in reversed(norm_messages) if m.get("role")=="user"), {}) + utext = (last_user.get("content") or "").lower() + mentioned = [n for n in names if n and n.lower() in utext] + if mentioned: + tool_choice_req = {"type":"function","function":{"name": mentioned[0]}} + logger.info("🔧 required->picked tool by mention: %s", mentioned[0]) + + + + # (1) Force: OWUI dwingt een specifieke tool af + if isinstance(tool_choice_req, dict) and (tool_choice_req.get("type") == "function"): + fname = (tool_choice_req.get("function") or {}).get("name") + if fname and fname not in TOOLS_REGISTRY: + # Onbekende tool → laat de LLM zelf native tool_calls teruggeven. + passthrough = dict(body) + passthrough["messages"] = norm_messages + passthrough["stream"] = False + client = app.state.HTTPX + r = await client.post(LLM_URL, json=passthrough) + try: + data = r.json() + data = _coerce_text_toolcalls_to_openai(data) + return JSONResponse(data, status_code=r.status_code) + except Exception: + return PlainTextResponse(r.text, status_code=r.status_code) + + if fname: + # Probeer lichte heuristiek voor bekende tools + last_user = next((m for m in reversed(norm_messages) if m.get("role")=="user"), {}) + utext = (last_user.get("content") or "") + args: dict = {} + + if fname == "rag_index_repo": + m = re.search(r'(https?://\S+)', utext) + if m: args["repo_url"] = m.group(1) + mb = re.search(r'\bbranch\s+([A-Za-z0-9._/-]+)', utext, re.I) + if mb: args["branch"] = mb.group(1) + elif fname == "rag_query": + args["query"] = utext.strip() + m = re.search(r"\bcollection(?:_name)?\s*[:=]\s*([A-Za-z0-9_.-]+)", utext, re.I) + if m: + args["collection_name"] = m.group(1) + elif fname == "summarize_text": + m = re.search(r':\s*(.+)$', utext, re.S) + args["text"] = (m.group(1).strip() if m else utext.strip())[:16000] + elif fname == "analyze_text": + m = re.search(r':\s*(.+)$', utext, re.S) + args["text"] = (m.group(1).strip() if m else utext.strip())[:20000] + elif fname == "improve_text": + m = re.search(r':\s*(.+)$', utext, re.S) + args["text"] = (m.group(1).strip() if m else utext.strip())[:20000] + elif fname == "validate_code_text": + code = re.search(r"```.*?\n(.*?)```", utext, re.S) + args["code"] = (code.group(1).strip() if code else utext.strip()) + elif fname == "improve_code_text": + code = re.search(r"```.*?\n(.*?)```", utext, re.S) + args["code"] = (code.group(1).strip() if code else utext.strip()) + elif fname == "vision_analyze": + m = re.search(r'(data:image\/[a-zA-Z]+;base64,[A-Za-z0-9+/=]+|https?://\S+)', utext) + if m: args["image_url"] = m.group(1) + + # Check verplichte velden; zo niet → native passthrough met alleen deze tool + required = (TOOLS_REGISTRY.get(fname, {}).get("parameters", {}) or {}).get("required", []) + if not all(k in args and args[k] for k in required): + passthrough = dict(body) + passthrough["messages"] = norm_messages + # Alleen deze tool meegeven + dwing deze tool af + only = [t for t in (body.get("tools") or []) if (t.get("function") or {}).get("name")==fname] + if only: passthrough["tools"] = only + passthrough["tool_choice"] = {"type":"function","function":{"name": fname}} + passthrough["stream"] = False + client = app.state.HTTPX + r = await client.post(LLM_URL, json=passthrough) + try: + data = r.json() + data = _coerce_text_toolcalls_to_openai(data) + return JSONResponse(data, status_code=r.status_code) + except Exception: + return PlainTextResponse(r.text, status_code=r.status_code) + + + # Heuristiek geslaagd → stuur tool_calls terug (compat met OWUI) + return { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [{ + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "tool_calls": [{ + "id": f"call_{uuid.uuid4().hex[:8]}", + "type": "function", + "function": {"name": fname, "arguments": json.dumps(args, ensure_ascii=False)} + }] + } + }] + } + + # Snelle escape: bij streaming en geen expliciete 'required' tool -> forceer directe streaming + if stream and tools and tool_choice_req in (None, "auto", "none") and \ + os.getenv("STREAM_PREFER_DIRECT", "0").lower() not in ("0","false","no"): + tools = [] # bypass tool glue zodat we rechtstreeks naar de echte streaming gaan + + # (2) Auto: vraag de LLM om 1+ function calls te produceren + if (tool_choice_req in (None, "auto")) and tools: + sys = _build_tools_system_prompt(tools) + ask = [{"role": "system", "content": sys}] + norm_messages + # jouw bestaande helper; hou 'm zoals je al gebruikt + resp = await llm_call_openai_compat(ask, stream=False, max_tokens=512) + txt = ((resp.get("choices") or [{}])[0].get("message") or {}).get("content", "") or "" + calls = detect_toolcalls_any(txt) #_extract_tool_calls_from_text(txt) + if calls: + return { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [{ + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "tool_calls": calls} + }] + } + # --- einde minimal tool-calling glue --- + + # Vision normalisatie (na tool->tekst normalisatie) + msgs, images_b64 = _normalize_openai_vision_messages(norm_messages) + messages = msgs if msgs else norm_messages + extra_payload = {"images": images_b64} if images_b64 else {} + + # Speciale modellen + if model == "repo-agent": + # === snelle bypass voor "unified diff" opdrachten met expliciete paden === + try: + fast = await _fast_unified_diff_task(messages) + except Exception as e: + fast = f"(Fast-diff pad mislukte: {e})" + if fast: + if stream: + async def _emit(): yield ("data: " + json.dumps({"id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "object":"chat.completion.chunk","created": int(time.time()),"model":"repo-agent","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":None}]}) + "\n\n").encode("utf-8"); yield ("data: " + json.dumps({"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk","created": int(time.time()),"model":"repo-agent","choices":[{"index":0,"delta":{"content": fast},"finish_reason":None}]}) + "\n\n").encode("utf-8"); yield b"data: [DONE]\n\n" + return StreamingResponse(_emit(), media_type="text/event-stream", headers={"Cache-Control":"no-cache, no-transform","X-Accel-Buffering":"no","Connection":"keep-alive"}) + return JSONResponse(_openai_chat_response("repo-agent", fast, messages)) + + if stream: + async def event_gen(): + import asyncio, time, json, contextlib + # Stuur meteen een role-delta om bytes te pushen + head = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk", + "created": int(time.time()),"model": model, + "choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason": None}]} + yield f"data: {json.dumps(head)}\n\n" + HEARTBEAT = float(os.getenv("SSE_HEARTBEAT_SEC","10")) + done = asyncio.Event() + result = {"text": ""} + # geef wat vroege status in de buitenste generator (hier mag je wel yielden) + for s in ("🔎 Indexeren/zoeken…", "🧠 Analyseren…", "🛠️ Voorstel genereren…"): + data = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk", + "created": int(time.time()),"model": model, + "choices":[{"index":0,"delta":{"content": s + "\n"},"finish_reason": None}]} + yield f"data: {json.dumps(data)}\n\n" + await asyncio.sleep(0) + + async def _work(): + try: + result["text"] = await handle_repo_agent(messages, request) + finally: + done.set() + worker = asyncio.create_task(_work()) + try: + while not done.is_set(): + try: + await asyncio.wait_for(done.wait(), timeout=HEARTBEAT) + except asyncio.TimeoutError: + yield ": ping\n\n" + # klaar → stuur content + data = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk", + "created": int(time.time()),"model": model, + "choices":[{"index":0,"delta":{"content": result['text']}, "finish_reason": None}]} + yield f"data: {json.dumps(data)}\n\n" + yield "data: [DONE]\n\n" + finally: + worker.cancel() + with contextlib.suppress(Exception): + await worker + return StreamingResponse( + event_gen(), + media_type="text/event-stream", + headers={"Cache-Control":"no-cache, no-transform","X-Accel-Buffering":"no","Connection":"keep-alive"} + ) + else: + text = await handle_repo_agent(messages, request) + return JSONResponse(_openai_chat_response(model, text, messages)) + + + if model == "repo-qa": + repo_hint, question, branch, n_ctx = _parse_repo_qa_from_messages(messages) + if not repo_hint: + friendly = ("Geef een repo-hint (URL of owner/repo).\nVoorbeeld:\n" + "repo: (repo URL eindigend op .git)\n" + "Vraag: (de vraag die je hebt over de code in de repo)") + return JSONResponse(_openai_chat_response("repo-qa", friendly, messages)) + try: + text = await repo_qa_answer(repo_hint, question, branch=branch, n_ctx=n_ctx) + except Exception as e: + logging.exception("repo-qa failed") + text = f"Repo-QA faalde: {e}" + if stream: + async def event_gen(): + data = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk","created": int(time.time()), + "model": "repo-qa", "choices":[{"index":0,"delta":{"content": text},"finish_reason": None}]} + yield f"data: {json.dumps(data)}\n\n"; yield "data: [DONE]\n\n" + return StreamingResponse(event_gen(), media_type="text/event-stream") + return JSONResponse(_openai_chat_response("repo-qa", text, messages)) + + # --- Tool-calling --- + tools_payload = body.get("tools") + tool_choice_req = body.get("tool_choice", "auto") + if tools_payload: + # native passthrough (llama.cpp voert tools NIET zelf uit, maar UI kan dit willen) + if LLM_FUNCTION_CALLING_MODE in ("native","auto") and stream: + passthrough = dict(body); passthrough["messages"]=messages + if images_b64: passthrough["images"]=images_b64 + STREAM_TOOLCALL_COERCE = os.getenv("STREAM_TOOLCALL_COERCE","1").lower() not in ("0","false","no") + async def _aiter(): + import asyncio, contextlib + client = app.state.HTTPX + async with client.stream("POST", LLM_URL, json=passthrough, timeout=None) as r: + r.raise_for_status() + HEARTBEAT = float(os.getenv("SSE_HEARTBEAT_SEC","10")) + q: asyncio.Queue[bytes] = asyncio.Queue(maxsize=100) + async def _reader(): + try: + async for chunk in r.aiter_bytes(): + if chunk: + await q.put(chunk) + except Exception: + pass + finally: + await q.put(b"__EOF__") + reader_task = asyncio.create_task(_reader()) + try: + buf = "" + acc = "" + suppress = False + while True: + try: + chunk = await asyncio.wait_for(q.get(), timeout=HEARTBEAT) + except asyncio.TimeoutError: + yield b": ping\n\n" + continue + if chunk == b"__EOF__": + break + if not STREAM_TOOLCALL_COERCE: + yield chunk + continue + try: + buf += chunk.decode("utf-8", errors="ignore") + except Exception: + yield chunk + continue + # SSE events zijn gescheiden door een lege regel + while "\n\n" in buf: + event, buf = buf.split("\n\n", 1) + if not event: + continue + if event.startswith(":"): + yield (event + "\n\n").encode("utf-8") + continue + lines = event.splitlines() + data_lines = [ln[5:].lstrip() for ln in lines if ln.startswith("data:")] + if not data_lines: + if not suppress: + yield (event + "\n\n").encode("utf-8") + continue + data_s = "\n".join(data_lines).strip() + if data_s == "[DONE]": + yield b"data: [DONE]\n\n" + return + try: + obj = json.loads(data_s) + except Exception: + if not suppress: + yield (event + "\n\n").encode("utf-8") + continue + try: + ch0 = (obj.get("choices") or [{}])[0] or {} + delta = ch0.get("delta") or {} + except Exception: + delta = {} + # Als upstream al echte tool_calls streamt: pass-through + if isinstance(delta, dict) and delta.get("tool_calls"): + if not suppress: + yield ("data: " + json.dumps(obj, ensure_ascii=False) + "\n\n").encode("utf-8") + continue + content = (delta.get("content") if isinstance(delta, dict) else None) + if isinstance(content, str) and content: + acc += content + if "[TOOL_CALLS" in acc: + suppress = True # onderdruk de text-tag stream + calls = detect_toolcalls_any(acc) or [] + if calls: + created = int(time.time()) + chunk_id = obj.get("id") or f"chatcmpl-{uuid.uuid4().hex[:24]}" + model_name = obj.get("model") or body.get("model") or "unknown" + tc_delta = [] + for i, tc in enumerate(calls): + tcc = dict(tc) + tcc["index"] = i + tc_delta.append(tcc) + first = { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model_name, + "choices": [{ + "index": 0, + "delta": {"role":"assistant", "tool_calls": tc_delta}, + "finish_reason": None + }] + } + second = { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": created, + "model": model_name, + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "tool_calls" + }] + } + yield ("data: " + json.dumps(first, ensure_ascii=False) + "\n\n").encode("utf-8") + yield ("data: " + json.dumps(second, ensure_ascii=False) + "\n\n").encode("utf-8") + yield b"data: [DONE]\n\n" + return + if not suppress: + yield ("data: " + json.dumps(obj, ensure_ascii=False) + "\n\n").encode("utf-8") + finally: + reader_task.cancel() + with contextlib.suppress(Exception): + await reader_task + return StreamingResponse( + _aiter(), + media_type="text/event-stream", + headers={"Cache-Control":"no-cache, no-transform","X-Accel-Buffering":"no","Connection":"keep-alive"} + ) + + if LLM_FUNCTION_CALLING_MODE in ("native","auto") and not stream: + # Relay-modus: laat LLM tools kiezen, bridge voert uit, daarna 2e run. + relay = os.getenv("LLM_TOOL_RUNNER", "bridge").lower() == "bridge" #passthrough + client = app.state.HTTPX + if not relay: + passthrough = dict(body); passthrough["messages"]=messages + if images_b64: passthrough["images"]=images_b64 + r = await client.post(LLM_URL, json=passthrough) + try: + data = r.json() + data = _coerce_text_toolcalls_to_openai(data) + return JSONResponse(data, status_code=r.status_code) + except Exception: + return PlainTextResponse(r.text, status_code=r.status_code) + + # Relay-modus: iteratief tools uitvoeren totdat de LLM stopt met tool_calls + max_rounds = int(os.getenv("LLM_TOOL_MAX_ROUNDS", "5")) + follow_messages = messages + last_status = None + for _round in range(max_rounds): + req_i = dict(body) + req_i["messages"] = follow_messages + req_i["stream"] = False + if images_b64: req_i["images"] = images_b64 + r_i = await client.post(LLM_URL, json=req_i) + last_status = r_i.status_code + try: + data_i = r_i.json() + except Exception: + return PlainTextResponse(r_i.text, status_code=r_i.status_code) + msg_i = ((data_i.get("choices") or [{}])[0].get("message") or {}) + tool_calls = msg_i.get("tool_calls") or [] + # Fallback: sommige backends gooien toolcalls als tekst (bv. [TOOL_CALLS]) + if not tool_calls: + txt = (msg_i.get("content") or "") + tool_calls = detect_toolcalls_any(txt) or [] + # Geen tool-calls? Geef direct door. + if not tool_calls: + data_i = _coerce_text_toolcalls_to_openai(data_i) + return JSONResponse(data_i, status_code=r_i.status_code) + + # Tools uitvoeren + tool_msgs = [] + for tc in tool_calls: + # Normaliseer tc structuur + tc_id = (tc or {}).get("id") or f"call_{uuid.uuid4().hex[:8]}" + fn = ((tc or {}).get("function") or {}) + tname = fn.get("name") + logger.info(f"Running tool: '{tname}'") + raw_args = fn.get("arguments") or "{}" + try: + args = json.loads(raw_args) if isinstance(raw_args, str) else (raw_args or {}) + except Exception: + args = {} + if not tname or tname not in TOOLS_REGISTRY: + out = {"error": f"Unknown tool '{tname}'"} + else: + try: + out = await _execute_tool(tname, args) + except Exception as e: + out = {"error": str(e)} + tool_msgs.append({ + "role": "tool", + "tool_call_id": tc_id, + "name": tname or "unknown", + "content": json.dumps(out, ensure_ascii=False) + }) + # Zorg dat assistant tool_calls een id heeft + if isinstance(tc, dict) and not tc.get("id"): + tc["id"] = tc_id + + follow_messages = follow_messages + [ + {"role": "assistant", "tool_calls": tool_calls}, + *tool_msgs + ] + + # Te veel tool-rondes → stop om loops te voorkomen + safe_msg = f"Te veel tool-rondes ({max_rounds}). Stop om loop te voorkomen." + return JSONResponse(_openai_chat_response(model, safe_msg, follow_messages), status_code=(last_status or 200)) + + # shim (non-stream) + if LLM_FUNCTION_CALLING_MODE == "shim" and not stream: + # 1) Laat LLM beslissen WELKE tool + args (strikte JSON) + tool_lines = [] + for tname, t in TOOLS_REGISTRY.items(): + tool_lines.append(f"- {tname}: {t['description']}\n schema: {json.dumps(t['parameters'])}") + sys = ("You can call tools.\n" + "If a tool is needed, reply with ONLY valid JSON:\n" + '{"call_tool":{"name":"","arguments":{...}}}\n' + "Otherwise reply with ONLY: {\"final_answer\":\"...\"}\n\nTools:\n" + "\n".join(tool_lines)) + decide = await llm_call_openai_compat( + [{"role":"system","content":sys}] + messages, + stream=False, temperature=float(body.get("temperature",0.02)), + top_p=float(body.get("top_p",0.9)), max_tokens=min(512, int(body.get("max_tokens",1024))), + extra=extra_payload if extra_payload else None + ) + raw = ((decide.get("choices",[{}])[0].get("message",{}) or {}).get("content","") or "").strip() + # haal evt. ```json fences weg + if raw.startswith("```"): + raw = extract_code_block(raw) + try: + obj = json.loads(raw) + except Exception: + # Model gaf geen JSON → behandel als final_answer + return JSONResponse(decide) + + if "final_answer" in obj: + return JSONResponse({ + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [{"index":0,"message":{"role":"assistant","content": obj["final_answer"]},"finish_reason":"stop"}], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + }) + + call = (obj.get("call_tool") or {}) + if isinstance(call, str): + call={"name": "Invalid_toolcall", "arguments": {}} + tname = call.get("name") + if tname not in TOOLS_REGISTRY: + return JSONResponse(_openai_chat_response(model, f"Onbekende tool: {tname}", messages)) + args = call.get("arguments") or {} + tool_out = await _execute_tool(tname, args) + + follow = messages + [ + {"role":"assistant","content": f"TOOL[{tname}] OUTPUT:\n{json.dumps(tool_out)[:8000]}"}, + {"role":"user","content": "Gebruik bovenstaande tool-output en geef nu het definitieve antwoord."} + ] + final = await llm_call_openai_compat( + follow, stream=False, + temperature=float(body.get("temperature",0.02)), + top_p=float(body.get("top_p",0.9)), + max_tokens=int(body.get("max_tokens",1024)), + extra=extra_payload if extra_payload else None + ) + return JSONResponse(final) + + # --- streaming passthrough (geen tools) --- + # --- streaming via onze wachtrij + upstreams --- + # --- streaming (zonder tools): nu mét windowing trim --- + # --- streaming (zonder tools): windowing + server-side auto-continue stream --- + simulate_stream=False + if simulate_stream: + if stream: + LLM_WINDOWING_ENABLE = os.getenv("LLM_WINDOWING_ENABLE", "1").lower() not in ("0","false","no") + MAX_CTX_TOKENS = int(os.getenv("LLM_CONTEXT_TOKENS", "13021")) + RESP_RESERVE = int(os.getenv("LLM_RESPONSE_RESERVE", "1024")) + temperature = float(body.get("temperature", 0.02)) + top_p = float(body.get("top_p", 0.9)) + # respecteer env-override voor default + _default_max = int(os.getenv("LLM_DEFAULT_MAX_TOKENS", "1024")) + max_tokens = int(body.get("max_tokens", _default_max)) + MAX_AUTOCONT = int(os.getenv("LLM_AUTO_CONTINUES", "3")) + + trimmed_stream_msgs = messages + try: + if LLM_WINDOWING_ENABLE and 'ConversationWindow' in globals(): + #thread_id = derive_thread_id({"model": model, "messages": messages}) if 'derive_thread_id' in globals() else uuid.uuid4().hex + thread_id = derive_thread_id(messages) if 'derive_thread_id' in globals() else uuid.uuid4().hex + running_summary = SUMMARY_STORE.get(thread_id, "") if isinstance(SUMMARY_STORE, dict) else (SUMMARY_STORE.get(thread_id) or "") + win = ConversationWindow(max_ctx_tokens=MAX_CTX_TOKENS, response_reserve=RESP_RESERVE, + tok_len=approx_token_count, running_summary=running_summary, + summary_header="Samenvatting tot nu toe") + for m in messages: + role, content = m.get("role","user"), m.get("content","") + if role in ("system","user","assistant"): + win.add(role, content) + async def _summarizer(old: str, chunk_msgs: list[dict]) -> str: + chunk_text = "" + for m in chunk_msgs: + chunk_text += f"\n[{m.get('role','user')}] {m.get('content','')}" + prompt = [ + {"role":"system","content":"Je bent een bondige notulist. Vat samen in max 10 bullets (feiten/besluiten/acties)."}, + {"role":"user","content": f"Vorige samenvatting:\n{old}\n\nNieuwe geschiedenis:\n{chunk_text}\n\nGeef geüpdatete samenvatting."} + ] + resp = await llm_call_openai_compat(prompt, stream=False, temperature=0.01, top_p=1.0, max_tokens=300) + return (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content", old or "") + trimmed_stream_msgs = await win.build_within_budget(system_prompt=None, summarizer=_summarizer) + new_summary = getattr(win, "running_summary", running_summary) + if isinstance(SUMMARY_STORE, dict): + if new_summary and new_summary != running_summary: + SUMMARY_STORE[thread_id] = new_summary + else: + try: + if new_summary and new_summary != running_summary: + SUMMARY_STORE.update(thread_id, new_summary) # type: ignore[attr-defined] + except Exception: + pass + except Exception: + trimmed_stream_msgs = messages + + # 1) haal volledige tekst op met non-stream + auto-continue + full_text, last_resp = await _complete_with_autocontinue( + trimmed_stream_msgs, + model=model, temperature=temperature, top_p=top_p, + max_tokens=max_tokens, extra_payload=extra_payload, + max_autocont=MAX_AUTOCONT + ) + + # 2) stream deze tekst als SSE in hapjes (simuleer live tokens) + async def _emit_sse(): + created = int(time.time()) + model_name = model + CHUNK = int(os.getenv("LLM_STREAM_CHUNK_CHARS", "800")) + # optioneel: stuur meteen role-delta (sommige UIs waarderen dat) + head = { + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion.chunk", + "created": created, + "model": model_name, + "choices": [{"index":0,"delta":{"role":"assistant"},"finish_reason": None}] + } + yield ("data: " + json.dumps(head, ensure_ascii=False) + "\n\n").encode("utf-8") + # content in blokken + for i in range(0, len(full_text), CHUNK): + piece = full_text[i:i+CHUNK] + data = { + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model_name, + "choices": [{"index":0,"delta":{"content": piece},"finish_reason": None}] + } + yield ("data: " + json.dumps(data, ensure_ascii=False) + "\n\n").encode("utf-8") + await asyncio.sleep(0) # laat event loop ademen + # afsluiten + done = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}","object":"chat.completion.chunk", + "created": int(time.time()),"model": model_name, + "choices": [{"index":0,"delta":{},"finish_reason":"stop"}]} + yield ("data: " + json.dumps(done, ensure_ascii=False) + "\n\n").encode("utf-8") + yield b"data: [DONE]\n\n" + + return StreamingResponse( + _emit_sse(), + media_type="text/event-stream", + headers={"Cache-Control":"no-cache, no-transform","X-Accel-Buffering":"no","Connection":"keep-alive"} + ) + else: + # --- ÉCHTE streaming (geen tools): direct passthrough met heartbeats --- + if stream: + temperature = float(body.get("temperature", 0.02)) + top_p = float(body.get("top_p", 0.9)) + _default_max = int(os.getenv("LLM_DEFAULT_MAX_TOKENS", "13021")) + max_tokens = int(body.get("max_tokens", _default_max)) + logger.info("UP tool_choice=%r tools0=%s", body.get("tool_choice"), (body.get("tools") or [{}])[0]) + return await llm_call_openai_compat( + messages, + model=model, + stream=True, + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + extra=extra_payload if extra_payload else None + ) + + + + # --- non-stream: windowing + auto-continue (zoals eerder gepatcht) --- + LLM_WINDOWING_ENABLE = os.getenv("LLM_WINDOWING_ENABLE", "1").lower() not in ("0","false","no") + MAX_CTX_TOKENS = int(os.getenv("LLM_CONTEXT_TOKENS", "42000")) + RESP_RESERVE = int(os.getenv("LLM_RESPONSE_RESERVE", "1024")) + MAX_AUTOCONT = int(os.getenv("LLM_AUTO_CONTINUES", "2")) + temperature = float(body.get("temperature", 0.02)) + top_p = float(body.get("top_p", 0.9)) + # Laat env de default bepalen, zodat OWUI niet hard op 1024 blijft hangen + _default_max = int(os.getenv("LLM_DEFAULT_MAX_TOKENS", "42000")) + max_tokens = int(body.get("max_tokens", _default_max)) + + trimmed = messages + try: + if LLM_WINDOWING_ENABLE and 'ConversationWindow' in globals(): + #thread_id = derive_thread_id({"model": model, "messages": messages}) if 'derive_thread_id' in globals() else uuid.uuid4().hex + thread_id = derive_thread_id(messages) if 'derive_thread_id' in globals() else uuid.uuid4().hex + running_summary = SUMMARY_STORE.get(thread_id, "") if isinstance(SUMMARY_STORE, dict) else (SUMMARY_STORE.get(thread_id) or "") + win = ConversationWindow(max_ctx_tokens=MAX_CTX_TOKENS, response_reserve=RESP_RESERVE, + tok_len=approx_token_count, running_summary=running_summary, + summary_header="Samenvatting tot nu toe") + for m in messages: + role, content = m.get("role","user"), m.get("content","") + if role in ("system","user","assistant"): + win.add(role, content) + async def _summarizer(old: str, chunk_msgs: list[dict]) -> str: + chunk_text = "" + for m in chunk_msgs: + chunk_text += f"\n[{m.get('role','user')}] {m.get('content','')}" + prompt = [ + {"role":"system","content":"Je bent een bondige notulist. Vat samen in max 10 bullets (feiten/besluiten/acties)."}, + {"role":"user","content": f"Vorige samenvatting:\n{old}\n\nNieuwe geschiedenis:\n{chunk_text}\n\nGeef geüpdatete samenvatting."} + ] + resp = await llm_call_openai_compat(prompt, stream=False, temperature=0.01, top_p=1.0, max_tokens=300) + return (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content", old or "") + trimmed = await win.build_within_budget(system_prompt=None, summarizer=_summarizer) + new_summary = getattr(win, "running_summary", running_summary) + if isinstance(SUMMARY_STORE, dict): + if new_summary and new_summary != running_summary: + SUMMARY_STORE[thread_id] = new_summary + else: + try: + if new_summary and new_summary != running_summary: + SUMMARY_STORE.update(thread_id, new_summary) # type: ignore[attr-defined] + except Exception: + pass + except Exception: + trimmed = messages + + resp = await llm_call_openai_compat(trimmed, model=model, stream=False, + temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra=extra_payload if extra_payload else None) + content = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + finish_reason = (resp.get("choices",[{}])[0] or {}).get("finish_reason") + + continues = 0 + # extra trigger: als we nagenoeg max_tokens geraakt hebben, ga door + NEAR_MAX = int(os.getenv("LLM_AUTOCONT_NEAR_MAX", "48")) + def _near_cap(txt: str) -> bool: + try: + return approx_token_count(txt) >= max_tokens - NEAR_MAX + except Exception: + return False + while continues < MAX_AUTOCONT and ( + finish_reason in ("length","content_filter") + or content.endswith("…") + or _near_cap(content) + ): + follow = trimmed + [{"role":"assistant","content": content}, + {"role":"user","content": "Ga verder zonder herhaling; alleen het vervolg."}] + nxt = await llm_call_openai_compat(follow, model=model, stream=False, + temperature=temperature, top_p=top_p, max_tokens=max_tokens, + extra=extra_payload if extra_payload else None) + more = (nxt.get("choices",[{}])[0].get("message",{}) or {}).get("content","") + content = (content + ("\n" if content and more else "") + more).strip() + finish_reason = (nxt.get("choices",[{}])[0] or {}).get("finish_reason") + continues += 1 + + if content: + resp["choices"][0]["message"]["content"] = content + resp["choices"][0]["finish_reason"] = finish_reason or resp["choices"][0].get("finish_reason", "stop") + try: + prompt_tokens = count_message_tokens(trimmed) if 'count_message_tokens' in globals() else approx_token_count(json.dumps(trimmed)) + except Exception: + prompt_tokens = approx_token_count(json.dumps(trimmed)) + completion_tokens = approx_token_count(content) + resp["usage"] = {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens} + return JSONResponse(resp) + + + +# ----------------------------------------------------------------------------- +# RAG Index (repo → Chroma) +# ----------------------------------------------------------------------------- + +_REPO_CACHE_PATH = os.path.join(CHROMA_PATH or "/rag_db", "repo_index_cache.json") +_SYMBOL_INDEX_PATH = os.path.join(CHROMA_PATH or "/rag_db", "symbol_index.json") + +def _load_symbol_index() -> dict: + try: + with open(_SYMBOL_INDEX_PATH, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return {} + +def _save_symbol_index(data: dict): + try: + _atomic_json_write(_SYMBOL_INDEX_PATH, data) + except Exception: + pass + +_SYMBOL_INDEX: dict[str, dict[str, list[dict]]] = _load_symbol_index() + +def _symkey(collection_effective: str, repo_name: str) -> str: + return f"{collection_effective}|{repo_name}" + + +def _load_repo_index_cache() -> dict: + try: + with open(_REPO_CACHE_PATH, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return {} + +def _save_repo_index_cache(data: dict): + try: + _atomic_json_write(_REPO_CACHE_PATH, data) + except Exception: + pass + +def _sha1_text(t: str) -> str: + return hashlib.sha1(t.encode("utf-8", errors="ignore")).hexdigest() + +def _repo_owner_repo_from_url(u: str) -> str: + try: + from urllib.parse import urlparse + p = urlparse(u).path.strip("/").split("/") + if len(p) >= 2: + name = p[-1] + if name.endswith(".git"): name = name[:-4] + return f"{p[-2]}/{name}" + except Exception: + pass + # fallback: basename + return os.path.basename(u).removesuffix(".git") + +def _summary_store_path(repo_url: str, branch: str) -> str: + key = _repo_owner_repo_from_url(repo_url).replace("/", "__") + return os.path.join(_SUMMARY_DIR, f"{key}__{branch}__{_EMBEDDER.slug}.json") + +def _load_summary_store(path: str) -> dict: + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + return {} + +def _save_summary_store(path: str, data: dict): + try: + _atomic_json_write(path, data) + except Exception: + pass + +async def _summarize_files_llm(items: list[tuple[str, str]]) -> dict[str, str]: + """ + items: list of (path, text) -> returns {path: one-line summary} + On-demand, tiny prompts; keep it cheap. + """ + out: dict[str, str] = {} + for path, text in items: + snippet = text[:2000] + prompt = [ + {"role":"system","content":"Geef één korte, functionele beschrijving (max 20 woorden) van dit bestand. Geen opsomming, geen code, 1 zin."}, + {"role":"user","content": f"Pad: {path}\n\nInhoud (ingekort):\n{snippet}\n\nAntwoord: "} + ] + try: + resp = await llm_call_openai_compat(prompt, stream=False, temperature=0.01, top_p=1.0, max_tokens=64) + summ = ((resp.get("choices") or [{}])[0].get("message") or {}).get("content","").strip() + except Exception: + summ = "" + out[path] = summ or "Bestand (korte beschrijving niet beschikbaar)" + return out + +async def _repo_summary_get_internal(repo_url: str, branch: str, paths: list[str]) -> dict[str, dict]: + """ + Returns {path: {"summary": str, "sha": str}}. Caches per file SHA, auto-invalidates when changed. + """ + repo_path = await _get_git_repo_async(repo_url, branch) + store_path = _summary_store_path(repo_url, branch) + store = _load_summary_store(store_path) # structure: {path: {"sha": "...", "summary": "..."}} + to_summarize: list[tuple[str, str]] = [] + for rel in paths: + p = Path(repo_path) / rel + if not p.exists(): + continue + text = _read_text_file(p) + fsha = _sha1_text(text) + rec = store.get(rel) or {} + if rec.get("sha") != fsha or not rec.get("summary"): + to_summarize.append((rel, text)) + if to_summarize: + new_summ = await _summarize_files_llm(to_summarize) + for rel, _ in to_summarize: + p = Path(repo_path) / rel + if not p.exists(): + continue + text = _read_text_file(p) + store[rel] = {"sha": _sha1_text(text), "summary": new_summ.get(rel, "")} + _save_summary_store(store_path, store) + # pack result + return {rel: {"summary": (store.get(rel) or {}).get("summary",""), "sha": (store.get(rel) or {}).get("sha","")} + for rel in paths} + +async def _grep_repo_paths(repo_url: str, branch: str, query: str, limit: int = 100) -> list[str]: + """ + Simpele fallback: lineaire scan op tekst-inhoud (alleen text-exts) -> unieke paden. + """ + repo_path = await _get_git_repo_async(repo_url, branch) + root = Path(repo_path) + qlow = (query or "").lower().strip() + hits: list[str] = [] + for p in root.rglob("*"): + if p.is_dir(): + continue + if set(p.parts) & PROFILE_EXCLUDE_DIRS: + continue + if p.suffix.lower() in BINARY_SKIP: + continue + try: + txt = _read_text_file(p) + except Exception: + continue + if not txt: + continue + if qlow in txt.lower(): + hits.append(str(p.relative_to(root))) + if len(hits) >= limit: + break + # uniq (preserve order) + seen=set(); out=[] + for h in hits: + if h in seen: continue + seen.add(h); out.append(h) + return out + +async def _meili_search_internal(query: str, *, repo_full: str | None, branch: str | None, limit: int = 50) -> list[dict]: + """ + Returns list of hits: {"path": str, "chunk_index": int|None, "score": float, "highlights": str} + """ + if not MEILI_ENABLED: + return [] + body = {"q": query, "limit": int(limit)} + filters = [] + if repo_full: + filters.append(f'repo_full = "{repo_full}"') + if branch: + filters.append(f'branch = "{branch}"') + if filters: + body["filter"] = " AND ".join(filters) + headers = {"Content-Type":"application/json"} + if MEILI_API_KEY: + headers["Authorization"] = f"Bearer {MEILI_API_KEY}" + # Meili ook vaak 'X-Meili-API-Key'; houd beide in ere: + headers["X-Meili-API-Key"] = MEILI_API_KEY + url = f"{MEILI_URL}/indexes/{MEILI_INDEX}/search" + client = app.state.HTTPX_PROXY if hasattr(app.state, "HTTPX_PROXY") else httpx.AsyncClient() + try: + r = await client.post(url, headers=headers, json=body, timeout=httpx.Timeout(30.0, connect=10.0)) + r.raise_for_status() + data = r.json() + except Exception: + return [] + hits = [] + for h in data.get("hits", []): + hits.append({ + "path": h.get("path"), + "chunk_index": h.get("chunk_index"), + "score": float(h.get("_rankingScore", h.get("_matchesPosition") or 0) or 0), + "highlights": h.get("_formatted") or {} + }) + # De-dup op path, hoogste score eerst + best: dict[str, dict] = {} + for h in sorted(hits, key=lambda x: x["score"], reverse=True): + p = h.get("path") + if p and p not in best: + best[p] = h + return list(best.values()) + +_MEILI_MEM_READY = False +_MEILI_MEM_LOCK = asyncio.Lock() + +def _meili_headers() -> dict: + headers = {"Content-Type": "application/json"} + if MEILI_API_KEY: + headers["Authorization"] = f"Bearer {MEILI_API_KEY}" + headers["X-Meili-API-Key"] = MEILI_API_KEY + return headers + +async def _meili_req(method: str, path: str, json_body=None, timeout: float = 15.0): + base = (MEILI_URL or "").rstrip("/") + url = f"{base}{path}" + client = app.state.HTTPX_PROXY if hasattr(app.state, "HTTPX_PROXY") else httpx.AsyncClient() + close_after = not hasattr(app.state, "HTTPX_PROXY") + try: + r = await client.request( + method, + url, + headers=_meili_headers(), + json=json_body, + timeout=httpx.Timeout(timeout, connect=min(5.0, timeout)), + ) + return r + finally: + if close_after: + await client.aclose() + +async def _ensure_meili_memory_index() -> bool: + """Zorg dat chat_memory index bestaat + goede settings heeft (filter/sort/searchable).""" + global _MEILI_MEM_READY + if not MEILI_URL: + return False + if _MEILI_MEM_READY: + return True + + async with _MEILI_MEM_LOCK: + if _MEILI_MEM_READY: + return True + + uid = MEILI_MEMORY_INDEX + + # 1) Bestaat index? + r = await _meili_req("GET", f"/indexes/{uid}", timeout=5.0) + if r.status_code == 404: + _ = await _meili_req("POST", "/indexes", json_body={"uid": uid, "primaryKey": "id"}, timeout=10.0) + + # 2) Settings (idempotent) + settings = { + "filterableAttributes": ["namespace", "role", "ts_unix", "source", "tags", "turn_id", "message_id"], + "sortableAttributes": ["ts_unix"], + "searchableAttributes": ["text"], + "displayedAttributes": ["*"], + } + _ = await _meili_req("PATCH", f"/indexes/{uid}/settings", json_body=settings, timeout=10.0) + + _MEILI_MEM_READY = True + return True + +async def _meili_memory_upsert(docs: list[dict]) -> None: + if not docs or not MEILI_URL: + return + ok = await _ensure_meili_memory_index() + if not ok: + return + + uid = MEILI_MEMORY_INDEX + path = f"/indexes/{uid}/documents?primaryKey=id" + CH = 500 + for i in range(0, len(docs), CH): + chunk = docs[i:i+CH] + r = await _meili_req("POST", path, json_body=chunk, timeout=30.0) + # geen hard fail; toolserver moet niet crashen door Meili + if r.status_code >= 400: + logger.warning("Meili memory upsert failed: %s %s", r.status_code, r.text[:400]) + +async def _meili_memory_search(namespace: str, query: str, limit: int = 40) -> list[dict]: + if not MEILI_URL: + return [] + ok = await _ensure_meili_memory_index() + if not ok: + return [] + + uid = MEILI_MEMORY_INDEX + # filter strikt op namespace + filt = f'namespace = "{namespace}"' + body = { + "q": query, + "limit": int(limit), + "filter": filt, + "showRankingScore": True, + } + r = await _meili_req("POST", f"/indexes/{uid}/search", json_body=body, timeout=20.0) + if r.status_code >= 400: + logger.warning("Meili memory search failed: %s %s", r.status_code, r.text[:400]) + return [] + + data = r.json() + out = [] + for h in data.get("hits", []) or []: + out.append({ + "id": h.get("id"), + "text": h.get("text") or "", + "score": float(h.get("_rankingScore") or 0.0), + "meta": { + "namespace": h.get("namespace"), + "role": h.get("role"), + "ts_unix": h.get("ts_unix"), + "source": h.get("source"), + "turn_id": h.get("turn_id"), + "tags": h.get("tags") or [], + "message_id": h.get("message_id"), + "chunk_index": h.get("chunk_index"), + } + }) + return out + + +async def _search_first_candidates(repo_url: str, branch: str, query: str, explicit_paths: list[str] | None = None, limit: int = 50) -> list[str]: + """ + Voorkeursvolgorde: expliciete paden → Meilisearch → grep → (laatste redmiddel) RAG op bestandsniveau + """ + if explicit_paths: + # valideer dat ze bestaan in de repo + repo_path = await _get_git_repo_async(repo_url, branch) + out=[] + for rel in explicit_paths: + if (Path(repo_path)/rel).exists(): + out.append(rel) + return out + repo_full = _repo_owner_repo_from_url(repo_url) + meili_hits = await _meili_search_internal(query, repo_full=repo_full, branch=branch, limit=limit) + if meili_hits: + return [h["path"] for h in meili_hits] + # fallback grep + return await _grep_repo_paths(repo_url, branch, query, limit=limit) + + +def _match_any(name: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(name, pat) for pat in patterns) + +def _looks_like_filename(q: str) -> bool: + q = (q or "").lower() + return any(ext in q for ext in (".php", ".py", ".js", ".ts", ".blade.php", ".vue", ".json", ".yaml", ".yml", ".txt", ".cpp", ".html", ".htm", ".xlsx", ".docx")) + + +def _rag_index_repo_sync( + *, + repo_url: str, + branch: str = "main", + profile: str = "auto", + include: str = "", + exclude_dirs: str = "", + chunk_chars: int = 3000, + overlap: int = 400, + collection_name: str = "code_docs", + force: bool = False +) -> dict: + repo_path = get_git_repo(repo_url, branch) + root = Path(repo_path) + repo = git.Repo(repo_path) + # HEAD sha + try: + head_sha = repo.head.commit.hexsha + except Exception: + head_sha = "" + # Cache key + cache = _load_repo_index_cache() + repo_key = f"{os.path.basename(repo_url)}|{branch}|{_collection_versioned(collection_name)}|{_EMBEDDER.slug}" + cached = cache.get(repo_key) + if not force and cached and cached.get("head_sha") == head_sha: + return { + "status": "skipped", + "reason": "head_unchanged", + "collection_effective": _collection_versioned(collection_name), + "detected_profile": profile if profile != "auto" else _detect_repo_profile(root), + "files_indexed": cached.get("files_indexed", 0), + "chunks_added": 0, + "note": f"Skip: HEAD {head_sha} al geïndexeerd (embedder={_EMBEDDER.slug})" + } + + # Profile → include patterns + if profile == "auto": + profile = _detect_repo_profile(root) + include_patterns = PROFILE_INCLUDES.get(profile, PROFILE_INCLUDES["generic"]) + if include.strip(): + include_patterns = [p.strip() for p in include.split(",") if p.strip()] + + exclude_set = set(PROFILE_EXCLUDE_DIRS) + if exclude_dirs.strip(): + exclude_set |= {d.strip() for d in exclude_dirs.split(",") if d.strip()} + + #collection = _get_collection(collection_name) + collection_name_eff = _collection_effective(collection_name) + collection = _get_collection(collection_name_eff) + + # --- Slim chunking toggles (env of via profile='smart') --- + use_smart_chunk = ( + (os.getenv("CHROMA_SMART_CHUNK","1").lower() not in ("0","false","no")) + or (str(profile).lower() == "smart") + ) + CH_TGT = int(os.getenv("CHUNK_TARGET_CHARS", "1800")) + CH_MAX = int(os.getenv("CHUNK_HARD_MAX", "2600")) + CH_MIN = int(os.getenv("CHUNK_MIN_CHARS", "800")) + + files_indexed = 0 + chunks_added = 0 + batch_documents: list[str] = [] + batch_metadatas: list[dict] = [] + batch_ids: list[str] = [] + BATCH_SIZE = 64 + + def flush(): + nonlocal chunks_added + if not batch_documents: + return + _collection_add(collection, batch_documents, batch_metadatas, batch_ids) + chunks_added += len(batch_documents) + batch_documents.clear(); batch_metadatas.clear(); batch_ids.clear() + docs_for_bm25: list[dict]=[] + docs_for_meili: list[dict]=[] + # tijdelijke symbol map voor deze run + # structuur: symbol_lower -> [{"path": str, "chunk_index": int}] + run_symbol_map: dict[str, list[dict]] = {} + + def _extract_symbol_hints(txt: str) -> list[str]: + hints = [] + for pat in [ + r"\bclass\s+([A-Za-z_][A-Za-z0-9_]*)\b", + r"\binterface\s+([A-Za-z_][A-Za-z0-9_]*)\b", + r"\btrait\s+([A-Za-z_][A-Za-z0-9_]*)\b", + r"\bdef\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"\bfunction\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"\bpublic\s+function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"\bprotected\s+function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + r"\bprivate\s+function\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(", + ]: + try: + hints += re.findall(pat, txt) + except Exception: + pass + # unique, cap + out = [] + for h in hints: + if h not in out: + out.append(h) + if len(out) >= 16: + break + return out + # precompute owner/repo once + repo_full_pre = _repo_owner_repo_from_url(repo_url) + if force: + try: + collection.delete(where={"repo_full": repo_full_pre, "branch": branch}) + except Exception: + pass + deleted_paths: set[str] = set() + for p in root.rglob("*"): + if p.is_dir(): + continue + if set(p.parts) & exclude_set: + continue + rel = str(p.relative_to(root)) + if not _match_any(rel, include_patterns): + continue + + #rel = str(p.relative_to(root)) + text = _read_text_file(p) + if not text: + continue + + files_indexed += 1 + # Verwijder bestaande chunks voor dit bestand (voorkomt bloat/duplicaten) + if rel not in deleted_paths: + try: + collection.delete(where={ + "repo_full": repo_full_pre, "branch": branch, "path": rel + }) + except Exception: + pass + deleted_paths.add(rel) + #chunks = _chunk_text(text, chunk_chars=int(chunk_chars), overlap=int(overlap)) + + # Kies slim chunking (taal/structuur-aware) of vaste chunks met overlap + if use_smart_chunk: + chunks = smart_chunk_text( + text, rel, + target_chars=CH_TGT, hard_max=CH_MAX, min_chunk=CH_MIN + ) + else: + chunks = _chunk_text(text, chunk_chars=int(chunk_chars), overlap=int(overlap)) + + for idx, ch in enumerate(chunks): + chunk_hash = _sha1_text(ch) + doc_id = f"{os.path.basename(repo_url)}:{branch}:{rel}:{idx}:{chunk_hash}" + # Zet zowel korte als lange repo-id neer + def _owner_repo_from_url(u: str) -> str: + try: + from urllib.parse import urlparse + p = urlparse(u).path.strip("/").split("/") + if len(p) >= 2: + name = p[-1] + if name.endswith(".git"): name = name[:-4] + return f"{p[-2]}/{name}" + except Exception: + pass + return os.path.basename(u).removesuffix(".git") + meta = { + "source": "git-repo", + "repo": os.path.basename(repo_url).removesuffix(".git"), + "repo_full": _owner_repo_from_url(repo_url), + "branch": branch, + "path": rel, + "chunk_index": idx, + "profile": profile, + "basename": os.path.basename(rel).lower(), + "path_lc": rel.lower(), + } + batch_documents.append(ch) + docs_for_bm25.append({"text": ch, "path": rel}) + batch_metadatas.append(meta) + batch_ids.append(doc_id) + if MEILI_ENABLED: + docs_for_meili.append({ + "id": hashlib.sha1(f"{doc_id}".encode("utf-8")).hexdigest(), + "repo": meta["repo"], + "repo_full": meta["repo_full"], + "branch": branch, + "path": rel, + "chunk_index": idx, + "text": ch[:4000], # houd het compact + "head_sha": head_sha, + "ts": int(time.time()) + }) + + # verzamel symbolen per chunk voor symbol-index + if os.getenv("RAG_SYMBOL_INDEX", "1").lower() not in ("0","false","no"): + for sym in _extract_symbol_hints(ch): + run_symbol_map.setdefault(sym.lower(), []).append({"path": rel, "chunk_index": idx}) + if len(batch_documents) >= BATCH_SIZE: + flush() + flush() + if BM25Okapi is not None and docs_for_bm25: + bm = BM25Okapi([_bm25_tok(d["text"]) for d in docs_for_bm25]) + repo_key = f"{os.path.basename(repo_url)}|{branch}|{_collection_versioned(collection_name)}" + _BM25_BY_REPO[repo_key] = (bm, docs_for_bm25) + + cache[repo_key] = {"head_sha": head_sha, "files_indexed": files_indexed, "embedder": _EMBEDDER.slug, "ts": int(time.time())} + _save_repo_index_cache(cache) + + # --- merge & persist symbol index --- + if os.getenv("RAG_SYMBOL_INDEX", "1").lower() not in ("0","false","no"): + col_eff = _collection_versioned(collection_name) + repo_name = os.path.basename(repo_url) + key = _symkey(col_eff, repo_name) + base = _SYMBOL_INDEX.get(key, {}) + # merge met dedupe en cap per symbol (om bloat te vermijden) + CAP = int(os.getenv("RAG_SYMBOL_CAP_PER_SYM", "200")) + for sym, entries in run_symbol_map.items(): + dst = base.get(sym, []) + seen = {(d["path"], d["chunk_index"]) for d in dst} + for e in entries: + tup = (e["path"], e["chunk_index"]) + if tup not in seen: + dst.append(e); seen.add(tup) + if len(dst) >= CAP: + break + base[sym] = dst + _SYMBOL_INDEX[key] = base + _save_symbol_index(_SYMBOL_INDEX) + + # === (optioneel) Meilisearch bulk upsert === + # NB: pas NA return plaatsen heeft geen effect; plaats dit blok vóór de return in jouw file. + if MEILI_ENABLED and docs_for_meili: + try: + headers = {"Content-Type":"application/json"} + if MEILI_API_KEY: + headers["Authorization"] = f"Bearer {MEILI_API_KEY}" + headers["X-Meili-API-Key"] = MEILI_API_KEY + url = f"{MEILI_URL}/indexes/{MEILI_INDEX}/documents?primaryKey=id" + # chunk upload om payloads klein te houden + CH = 500 + for i in range(0, len(docs_for_meili), CH): + chunk = docs_for_meili[i:i+CH] + requests.post(url, headers=headers, data=json.dumps(chunk), timeout=30) + except Exception as e: + logger.warning("Meili bulk upsert failed: %s", e) + + return { + "status": "ok", + "collection_effective": _collection_versioned(collection_name), + "detected_profile": profile, + "files_indexed": files_indexed, + "chunks_added": chunks_added, + "note": "Geïndexeerd met embedder=%s; smart_chunk=%s" % (_EMBEDDER.slug, bool(use_smart_chunk)) + } + + + + + +# Celery task +if celery_app: + @celery_app.task(name="task_rag_index_repo", bind=True, autoretry_for=(Exception,), retry_backoff=True, max_retries=5) + def task_rag_index_repo(self, args: dict): + return _rag_index_repo_sync(**args) + +# API endpoint: sync of async enqueue +@app.post("/rag/index-repo") +async def rag_index_repo( + repo_url: str = Form(...), + branch: str = Form("main"), + profile: str = Form("auto"), + include: str = Form(""), + exclude_dirs: str = Form(""), + chunk_chars: int = Form(3000), + overlap: int = Form(400), + collection_name: str = Form("code_docs"), + force: bool = Form(False), + async_enqueue: bool = Form(False) +): + args = dict( + repo_url=repo_url, branch=branch, profile=profile, + include=include, exclude_dirs=exclude_dirs, + chunk_chars=int(chunk_chars), overlap=int(overlap), + collection_name=collection_name, force=bool(force) + ) + if async_enqueue and celery_app: + task = task_rag_index_repo.delay(args) # type: ignore[name-defined] + return {"status": "enqueued", "task_id": task.id} + try: + return await run_in_threadpool(_rag_index_repo_sync, **args) + except Exception as e: + logger.exception("RAG index repo failed") + raise HTTPException(status_code=500, detail=str(e)) + +@app.get("/tasks/{task_id}") +def task_status(task_id: str): + if not celery_app: + raise HTTPException(status_code=400, detail="Celery niet geactiveerd") + try: + from celery.result import AsyncResult + res = AsyncResult(task_id, app=celery_app) + out = {"state": res.state} + if res.successful(): + out["result"] = res.result + elif res.failed(): + out["error"] = str(res.result) + return out + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + +# ----------------------------------------------------------------------------- +# RAG Query (Chroma) +# ----------------------------------------------------------------------------- +_LARAVEL_ROUTE_FILES = {"routes/web.php", "routes/api.php"} + +def _laravel_guess_view_paths_from_text(txt: str) -> list[str]: + out = [] + for m in re.finditer(r"return\s+view\(\s*['\"]([^'\"]+)['\"]", txt): + v = m.group(1).replace(".", "/") + out.append(f"resources/views/{v}.blade.php") + return out + +def _laravel_pairs_from_route_text(txt: str) -> list[tuple[str, str|None]]: + """Return [(controller_path, method_or_None), ...]""" + pairs = [] + # Route::get('x', [XController::class, 'show']) + for m in re.finditer(r"Route::(?:get|post|put|patch|delete)\([^;]+?\[(.*?)\]\s*\)", txt, re.S): + arr = m.group(1) + mm = re.search(r"([A-Za-z0-9_\\]+)::class\s*,\s*'([A-Za-z0-9_]+)'", arr) + if mm: + ctrl = mm.group(1).split("\\")[-1] + meth = mm.group(2) + pairs.append((f"app/Http/Controllers/{ctrl}.php", meth)) + # Route::resource('users', 'UserController') + for m in re.finditer(r"Route::resource\(\s*'([^']+)'\s*,\s*'([^']+)'\s*\)", txt): + res, ctrl = m.group(1), m.group(2).split("\\")[-1] + pairs.append((f"app/Http/Controllers/{ctrl}.php", None)) + return pairs + +async def rag_query_api( + *, + query: str, + n_results: int = 8, + collection_name: str = "code_docs", + repo: Optional[str] = None, + path_contains: Optional[str] = None, + profile: Optional[str] = None +) -> dict: + branch="main" + repo_full = None + repo_base = None + if repo: + if "://" in repo: # repo is URL + repo_full = _repo_owner_repo_from_url(repo) # -> "admin/cluster" + else: + repo_full = repo # al "owner/repo" + repo_base = repo_full.rsplit("/", 1)[-1] + repo=repo_full + #col = _get_collection(collection_name) + collection_name_eff = _collection_effective(collection_name) + col = _get_collection(collection_name_eff) + q_emb = _EMBEDDER.embed_query(query) + # Chroma: $and/$or moet >=2 where-expressies bevatten. + conds = [] + if repo: + conds.append({"repo_full": {"$eq": repo}}) + if branch: + conds.append({"branch": {"$eq": branch}}) + if profile: + conds.append({"profile": {"$eq": profile}}) + where = None + if len(conds) == 1: + where = conds[0] + elif len(conds) >= 2: + where = {"$and": conds} + + # ---- symbol hit set (repo-scoped) ---- + sym_hit_keys: set[str] = set() + sym_hit_weight = float(os.getenv("RAG_SYMBOL_HIT_W", "0.12")) + if sym_hit_weight > 0.0 and repo: + try: + # index werd opgeslagen onder basename(repo_url); hier dus normaliseren + repo_base = repo.rsplit("/", 1)[-1] + idx_key = _symkey(_collection_versioned(collection_name), repo_base) + symidx = _SYMBOL_INDEX.get(idx_key, {}) + # query-termen + terms = set(re.findall(r"[A-Za-z_][A-Za-z0-9_]*", query)) + # ook 'Foo\\BarController@show' splitsen + for m in re.finditer(r"([A-Za-z_][A-Za-z0-9_]*)(?:@|::|->)([A-Za-z_][A-Za-z0-9_]*)", query): + terms.add(m.group(1)); terms.add(m.group(2)) + for t in {x.lower() for x in terms}: + for e in symidx.get(t, []): + sym_hit_keys.add(f"{repo}::{e['path']}::{e['chunk_index']}") + except Exception: + pass + + # --- padhints uit query halen (expliciet genoemde bestanden/dirs) --- + # Herken ook gequote varianten en slimme quotes. + path_hints: set[str] = set() + PH_PATTERNS = [ + # 1) Relatief pad met directories + filename + (multi-dot) extensies + # Voorbeelden: src/foo/bar.py, app/Http/.../UserController.php, foo.tar.gz, index.blade.php + r'["“”\']?((?:[A-Za-z0-9_.-]+[\\/])+[A-Za-z0-9_.-]+(?:\.[A-Za-z0-9]{1,12})+)\b["“”\']?', + + # 2) Alleen filename + (multi-dot) extensies + # Voorbeelden: main.py, CMakeLists.txt (ook gedekt door specials), index.blade.php, foo.min.js + r'["“”\']?([A-Za-z0-9_.-]+(?:\.[A-Za-z0-9]{1,12})+)\b["“”\']?', + + # 3) Speciale bekende bestandsnamen zonder extensie + r'\b(Dockerfile(?:\.[A-Za-z0-9_.-]+)?)\b', # Dockerfile, Dockerfile.dev + r'\b(Makefile)\b', + r'\b(CMakeLists\.txt)\b', + r'\b(README(?:\.[A-Za-z0-9]{1,12})?)\b', # README, README.md + r'\b(LICENSE(?:\.[A-Za-z0-9]{1,12})?)\b', # LICENSE, LICENSE.txt + ] + + for pat in PH_PATTERNS: + for m in re.finditer(pat, query, flags=re.IGNORECASE): + path_hints.add(m.group(1).strip()) + clean_hints = set() + for h in path_hints: + h = h.strip().strip('"\''"“”") + if len(h) > 200: + continue + # skip dingen die op URL lijken + if "://" in h: + continue + clean_hints.add(h) + path_hints = clean_hints + + prefetch = [] + if path_hints and repo and _looks_like_filename(query): + repo_base = repo.rsplit("/", 1)[-1] + # jij indexeert meta["repo_full"] als owner/repo; bij rag_query kan repo "owner/repo" zijn. + repo_full = repo + + for hint in list(path_hints)[:10]: + h = hint.strip() + if not h: + continue + + # 1) exact pad (als er / in zit of het lijkt op resources/... etc.) + if "/" in h: + where = {"$and": [ + {"repo_full": {"$eq": repo_full}}, + {"branch": {"$eq": branch}}, + {"path": {"$eq": h}} + ] + } + got = col.get(where=where, include=["documents","metadatas"]) + else: + # 2) alleen bestandsnaam -> match op basename + where = {"$and": [ + {"repo_full": {"$eq": repo_full}}, + {"branch": {"$eq": branch}}, + {"basename": {"$eq": h.lower()}} + ] + } + got = col.get(where=where, include=["documents","metadatas"]) + + docs2 = got.get("documents") or [] + metas2 = got.get("metadatas") or [] + for d, m in zip(docs2, metas2): + prefetch.append({ + "document": d or "", + "metadata": m or {}, + "emb_sim": 1.0, + "distance": 0.0, + "score": 1.0, + }) + + res = col.query( + query_embeddings=[q_emb], + n_results=max(n_results, 30), # iets ruimer voor rerank + where=where or None, + include=["metadatas","documents","distances"] + ) + logger.info("RAG raw hits: %d (repo=%s)", len((res.get("documents") or [[]])[0]), repo or "-") + docs = (res.get("documents") or [[]])[0] + metas = (res.get("metadatas") or [[]])[0] + dists = (res.get("distances") or [[]])[0] + + # Filter path_contains en bouw kandidaten + cands = [] + # Voor frase-boost: haal een simpele “candidate phrase” uit query (>=2 woorden) + def _candidate_phrase(q: str) -> str | None: + q = q.strip().strip('"').strip("'") + words = re.findall(r"[A-Za-zÀ-ÿ0-9_+-]{2,}", q) + if len(words) >= 2: + return " ".join(words[:6]).lower() + return None + phrase = _candidate_phrase(query) + + for doc, meta, dist in zip(docs, metas, dists): + if path_contains and meta and path_contains.lower() not in (meta.get("path","").lower()): + continue + key = f"{(meta or {}).get('repo','')}::{(meta or {}).get('path','')}::{(meta or {}).get('chunk_index','')}" + emb_sim = 1.0 / (1.0 + float(dist or 0.0)) + # symbol-hit boost vóór verdere combinaties + if key in sym_hit_keys: + emb_sim = min(1.0, emb_sim + sym_hit_weight) + c = { + "document": doc or "", + "metadata": meta or {}, + "emb_sim": emb_sim, # afstand -> ~similarity (+symbol boost) + "distance": float(dist or 0.0) # bewaar ook de ruwe distance voor hybride retrieval + } + # --- extra boost: expliciet genoemde paths in query --- + p = (meta or {}).get("path","") + if path_hints: + for hint in path_hints: + if hint and (hint == p or hint.split("/")[-1] == p.split("/")[-1] or hint in p): + c["emb_sim"] = min(1.0, c["emb_sim"] + 0.12) + break + # --- extra boost: exacte frase in document (case-insensitive) --- + if phrase and phrase in (doc or "").lower(): + c["emb_sim"] = min(1.0, c["emb_sim"] + 0.10) + cands.append(c) + # --- Als er expliciete path_hints zijn, probeer ze "hard" toe te voegen bovenaan --- + # Dit helpt o.a. bij 'resources/views/log/edit.blade.php' exact noemen. + if path_hints and repo: + try: + # Probeer lokale repo te openen (snelle clone/update) en lees de bestanden in. + repo_base = repo.rsplit("/", 1)[-1] + # Zet een conservatieve guess in elkaar voor de URL: gebruiker werkt met Gitea + # en roept elders get_git_repo() aan — wij willen geen netwerk doen hier. + # We gebruiken in plaats daarvan de collectie-metadata om te bepalen + # of het pad al aanwezig is (via metas). + known_paths = { (m or {}).get("path","") for m in metas } + injects = [] + for hint in list(path_hints): + # Match ook alleen op bestandsnaam als volledige path niet gevonden is + matches = [p for p in known_paths if p == hint or p.endswith("/"+hint)] + for pth in matches: + # Zoek het eerste document dat bij dit path hoort en neem zijn tekst + for doc, meta in zip(docs, metas): + if (meta or {}).get("path","") == pth: + injects.append({"document": doc, "metadata": meta, "emb_sim": 1.0, "score": 1.0}) + break + if injects: + # Zet injects vooraan, zonder dubbelingen + seen_keys = set() + new_cands = [] + for j in injects + cands: + key = f"{(j.get('metadata') or {}).get('path','')}::{(j.get('metadata') or {}).get('chunk_index','')}" + if key in seen_keys: + continue + seen_keys.add(key) + new_cands.append(j) + cands = new_cands + except Exception: + pass + + if not cands: + fallback = [] + if BM25Okapi is not None and repo: + colname = _collection_versioned(collection_name) + repo_base = repo.rsplit("/", 1)[-1] + for k,(bm,docs) in _BM25_BY_REPO.items(): + # cache-key formaat: "||" + if (k.startswith(f"{repo_base}|") or k.startswith(f"{repo}|")) and k.endswith(colname): + scores = bm.get_scores(_bm25_tok(query)) + ranked = sorted(zip(docs, scores), key=lambda x: x[1], reverse=True)[:n_results] + for d,s in ranked: + fallback.append({ + "document": d["text"], + "metadata": {"repo": repo_base, "path": d["path"]}, + "score": float(s) + }) + break + if fallback: + return {"count": len(fallback), "results": fallback} + return {"count": 0, "results": []} + + + # BM25/Jaccard op kandidaten + def _tok(s: str) -> list[str]: + return re.findall(r"[A-Za-z0-9_]+", s.lower()) + bm25_scores = [] + if BM25Okapi: + bm = BM25Okapi([_tok(c["document"]) for c in cands]) + bm25_scores = bm.get_scores(_tok(query)) + else: + qset = set(_tok(query)) + for c in cands: + tset = set(_tok(c["document"])) + inter = len(qset & tset); uni = len(qset | tset) or 1 + bm25_scores.append(inter / uni) + + # --- Laravel cross-file boosts --- + # We hebben per kandidaat: {"document": ..., "metadata": {"repo": "...", "path": "..."}} + # Bouw een pad->index mapping + idx_by_path = {} + for i,c in enumerate(cands): + p = (c.get("metadata") or {}).get("path","") + idx_by_path.setdefault(p, []).append(i) + + # 1) Route files -> boost controllers/views + for c in cands: + meta = c.get("metadata") or {} + path = meta.get("path","") + if path not in _LARAVEL_ROUTE_FILES: + continue + txt = c.get("document","") + # controllers + for ctrl_path, _meth in _laravel_pairs_from_route_text(txt): + if ctrl_path in idx_by_path: + for j in idx_by_path[ctrl_path]: + cands[j]["emb_sim"] = min(1.0, cands[j]["emb_sim"] + 0.05) # kleine push + # views (heel grof: lookups in route-tekst komen zelden direct voor, skip hier) + + # 2) Controllers -> boost views die ze renderen + for c in cands: + meta = c.get("metadata") or {} + path = meta.get("path","") + if not path.startswith("app/Http/Controllers/") or not path.endswith(".php"): + continue + txt = c.get("document","") + for vpath in _laravel_guess_view_paths_from_text(txt): + if vpath in idx_by_path: + for j in idx_by_path[vpath]: + cands[j]["emb_sim"] = min(1.0, cands[j]["emb_sim"] + 0.05) + + # symbols-boost: kleine bonus als query-termen in symbols of bestandsnaam voorkomen + q_terms = set(re.findall(r"[A-Za-z0-9_]+", query.lower())) + for c in cands: + meta = (c.get("metadata") or {}) + # --- symbols (bugfix: syms_raw was undefined) --- + syms_raw = meta.get("symbols") + if isinstance(syms_raw, str): + syms = [s.strip() for s in syms_raw.split(",") if s.strip()] + elif isinstance(syms_raw, list): + syms = syms_raw + else: + syms = [] + if syms and (q_terms & {s.lower() for s in syms}): + c["emb_sim"] = min(1.0, c["emb_sim"] + 0.04) + + # --- filename exact-ish match --- + fname_terms = set(re.findall(r"[A-Za-z0-9_]+", meta.get("path","").split("/")[-1].lower())) + if fname_terms and (q_terms & fname_terms): + c["emb_sim"] = min(1.0, c["emb_sim"] + 0.02) + # lichte bonus als path exact één van de hints is + if path_hints and meta.get("path") in path_hints: + c["emb_sim"] = min(1.0, c["emb_sim"] + 0.06) + + + + # normaliseer + if len(bm25_scores) > 0: + mn, mx = min(bm25_scores), max(bm25_scores) + bm25_norm = [(s - mn) / (mx - mn) if mx > mn else 0.0 for s in bm25_scores] + else: + bm25_norm = [0.0] * len(cands) + + alpha = float(os.getenv("RAG_EMB_WEIGHT", "0.6")) + for c, b in zip(cands, bm25_norm): + c["score"] = alpha * c["emb_sim"] + (1.0 - alpha) * b + + ranked = sorted(cands, key=lambda x: x["score"], reverse=True)[:n_results] + ranked_full = sorted(cands, key=lambda x: x["score"], reverse=True) + if RAG_LLM_RERANK: + topK = ranked_full[:max(10, n_results)] + # bouw prompt + prompt = "Rerank the following code passages for the query. Return ONLY a JSON array of indices (0-based) in best-to-worst order.\n" + prompt += f"Query: {query}\n" + for i, r in enumerate(topK): + path = (r.get("metadata") or {}).get("path","") + snippet = (r.get("document") or "")[:600] + prompt += f"\n# {i} — {path}\n{snippet}\n" + resp = await llm_call_openai_compat( + [{"role":"system","content":"You are precise and return only valid JSON."}, + {"role":"user","content": prompt+"\n\nOnly JSON array."}], + stream=False, temperature=0.01, top_p=1.0, max_tokens=1024 + ) + try: + order = json.loads((resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","[]")) + reranked = [topK[i] for i in order if isinstance(i,int) and 0 <= i < len(topK)] + ranked = reranked[:n_results] if reranked else ranked_full[:n_results] + except Exception: + ranked = ranked_full[:n_results] + else: + ranked = ranked_full[:n_results] + if prefetch: + seen = set() + merged = [] + for r in prefetch + ranked: + meta = r.get("metadata") or {} + key = f"{meta.get('path','')}::{meta.get('chunk_index','')}" + if key in seen: + continue + seen.add(key) + merged.append(r) + ranked = merged[:n_results] + return { + "count": len(ranked), + "results": [{ + "document": r["document"], + "metadata": r["metadata"], + "file": (r["metadata"] or {}).get("path", ""), # <- expliciete bestandsnaam + "score": round(float(r["score"]), 4), + # houd distance beschikbaar voor hybrid_retrieve (embed-component) + "distance": float(r.get("distance", 0.0)) + } for r in ranked] + } + + +@app.post("/rag/query") +async def rag_query_endpoint( + query: str = Form(...), + n_results: int = Form(8), + collection_name: str = Form("code_docs"), + repo: str = Form(""), + path_contains: str = Form(""), + profile: str = Form("") +): + data = await rag_query_api( + query=query, n_results=n_results, collection_name=collection_name, + repo=(repo or None), path_contains=(path_contains or None), profile=(profile or None) + ) + return JSONResponse(data) + +# ----------------------------------------------------------------------------- +# Repo-agent endpoints +# ----------------------------------------------------------------------------- +@app.post("/agent/repo") +async def agent_repo(messages: ChatRequest, request: Request): + """ + Gespreks-interface met de repo-agent (stateful per sessie). + """ + # Convert naar list[dict] (zoals agent_repo verwacht) + msgs = [{"role": m.role, "content": m.content} for m in messages.messages] + text = await handle_repo_agent(msgs, request) + return PlainTextResponse(text) + +@app.post("/repo/qa") +async def repo_qa(req: RepoQARequest): + """ + Eén-shot vraag-antwoord over een repo. + """ + ans = await repo_qa_answer(req.repo_hint, req.question, branch=req.branch, n_ctx=req.n_ctx) + return PlainTextResponse(ans) + +# ----------------------------------------------------------------------------- +# Injecties voor agent_repo +# ----------------------------------------------------------------------------- +async def _rag_index_repo_internal(*, repo_url: str, branch: str, profile: str, + include: str, exclude_dirs: str, + chunk_chars: int, overlap: int, + collection_name: str): + # offload zware IO/CPU naar threadpool zodat de event-loop vrij blijft + return await run_in_threadpool( + _rag_index_repo_sync, + repo_url=repo_url, branch=branch, profile=profile, + include=include, exclude_dirs=exclude_dirs, + chunk_chars=chunk_chars, overlap=overlap, + collection_name=collection_name, force=False + ) + +async def _rag_query_internal(*, query: str, n_results: int, + collection_name: str, repo=None, path_contains=None, profile=None): + return await rag_query_api( + query=query, n_results=n_results, + collection_name=collection_name, + repo=repo, path_contains=path_contains, profile=profile + ) + +def _read_text_file_wrapper(p: Path | str) -> str: + return _read_text_file(Path(p) if not isinstance(p, Path) else p) + +async def _get_git_repo_async(repo_url: str, branch: str = "main") -> str: + # gitpython doet subprocess/IO → altijd in threadpool + return await run_in_threadpool(get_git_repo, repo_url, branch) + +# Registreer injecties +initialize_agent( + app=app, + get_git_repo_fn=_get_git_repo_async, + rag_index_repo_internal_fn=_rag_index_repo_internal, + rag_query_internal_fn=_rag_query_internal, + # Gebruik auto-continue wrapper zodat agent-antwoorden niet worden afgekapt + llm_call_fn=llm_call_autocont, + extract_code_block_fn=extract_code_block, + read_text_file_fn=_read_text_file_wrapper, + client_ip_fn=_client_ip, + profile_exclude_dirs=PROFILE_EXCLUDE_DIRS, + chroma_get_collection_fn=_get_collection, + embed_query_fn=_EMBEDDER.embed_query, + embed_documents_fn=_EMBEDDER.embed_documents, # ← nieuw: voor catalog-embeddings + # === nieuw: search-first + summaries + meili === + search_candidates_fn=_search_first_candidates, + repo_summary_get_fn=_repo_summary_get_internal, + meili_search_fn=_meili_search_internal, +) + +# ----------------------------------------------------------------------------- +# Health +# ----------------------------------------------------------------------------- +@app.get("/healthz") +def health(): + sem_val = getattr(app.state.LLM_SEM, "_value", 0) + ups = [{"url": u.url, "active": u.active, "ok": u.ok} for u in _UPS] + return { + "ok": True, + "embedder": _EMBEDDER.slug, + "chroma_mode": CHROMA_MODE, + "queue_len": len(app.state.LLM_QUEUE), + "permits_free": sem_val, + "upstreams": ups, + } + +@app.get("/metrics") +def metrics(): + ups = [{"url": u.url, "active": u.active, "ok": u.ok} for u in _UPS] + return { + "queue_len": len(app.state.LLM_QUEUE), + "sem_value": getattr(app.state.LLM_SEM, "_value", None), + "upstreams": ups + } + diff --git a/llm_client.py b/llm_client.py new file mode 100644 index 0000000..cd4ff41 --- /dev/null +++ b/llm_client.py @@ -0,0 +1,141 @@ +from __future__ import annotations +import os +import asyncio +import logging +from typing import List, Dict, Any, Optional + +import httpx + +from queue_helper import QueueManager + +logger = logging.getLogger(__name__) + +# ------------------------------------------------------------- +# Config voor onderliggende LLM-backend / proxy +# ------------------------------------------------------------- +# Je kunt één van deze zetten: +# - LLM_PROXY_URL: volledige URL naar OpenAI-compat endpoint (bv. http://host:8081/v1/completions of /v1/chat/completions) +# - LLM_API_BASE : base-url (fallback). Dan gebruiken we /v1/chat/completions +LLM_PROXY_URL = (os.getenv("LLM_PROXY_URL") or "").strip() +LLM_API_BASE = os.getenv("LLM_API_BASE", "").strip() or "http://127.0.0.1:11434" +LLM_DEFAULT_MODEL = os.getenv("LLM_MODEL", "gpt-4o-mini") +LLM_REQUEST_TIMEOUT = float(os.getenv("LLM_REQUEST_TIMEOUT", "180")) + +# Deze wordt in app.py gezet via init_llm_client(...) +LLM_QUEUE: QueueManager | None = None + + +def init_llm_client(queue: QueueManager) -> None: + global LLM_QUEUE + LLM_QUEUE = queue + logger.info("llm_client: LLM_QUEUE gekoppeld via init_llm_client.") + + +def _resolve_llm_url() -> str: + if LLM_PROXY_URL: + return LLM_PROXY_URL.rstrip("/") + # fallback: base -> chat completions + return f"{LLM_API_BASE.rstrip('/')}/v1/chat/completions" + + +def _messages_to_prompt(messages: List[Dict[str, Any]]) -> str: + # eenvoudige, robuuste prompt-serialisatie voor /v1/completions proxies + parts: list[str] = [] + for m in messages: + role = (m.get("role") or "user").upper() + content = m.get("content") or "" + parts.append(f"{role}: {content}") + parts.append("ASSISTANT:") + return "\n".join(parts) + + +def _chat_from_text(text: str) -> Dict[str, Any]: + return { + "object": "chat.completion", + "choices": [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": text}, + }], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +def _sync_model_infer(payload: Dict[str, Any]) -> Dict[str, Any]: + url = _resolve_llm_url() + try: + with httpx.Client(timeout=LLM_REQUEST_TIMEOUT) as client: + # detect endpoint type + is_chat = "/chat/" in url or url.endswith("/chat/completions") + if is_chat: + resp = client.post(url, json=payload) + resp.raise_for_status() + return resp.json() + # /v1/completions style + messages = payload.get("messages") or [] + prompt = payload.get("prompt") + if not prompt: + prompt = _messages_to_prompt(messages) + comp_payload: Dict[str, Any] = { + "model": payload.get("model") or LLM_DEFAULT_MODEL, + "prompt": prompt, + "max_tokens": payload.get("max_tokens", 2048), + "temperature": payload.get("temperature", 0.2), + "top_p": payload.get("top_p", 0.9), + "stream": False, + } + # pass-through extras if present + for k in ("stop", "presence_penalty", "frequency_penalty"): + if k in payload: + comp_payload[k] = payload[k] + resp = client.post(url, json=comp_payload) + resp.raise_for_status() + data = resp.json() + # normalize to chat.completion + try: + choice = (data.get("choices") or [{}])[0] + txt = choice.get("text") or choice.get("message", {}).get("content") or "" + return _chat_from_text(txt) + except Exception: + return _chat_from_text(str(data)[:2000]) + except Exception as exc: + logger.exception("LLM backend call failed: %s", exc) + return _chat_from_text(f"[LLM-fout] {exc}") + + +async def _llm_call( + messages: List[Dict[str, str]], + *, + stream: bool = False, + temperature: float = 0.2, + top_p: float = 0.9, + max_tokens: Optional[int] = None, + model: Optional[str] = None, + **extra: Any, +) -> Dict[str, Any]: + if stream: + raise NotImplementedError("_llm_call(stream=True) wordt momenteel niet ondersteund.") + + if LLM_QUEUE is None: + raise RuntimeError("LLM_QUEUE is niet geïnitialiseerd. Roep init_llm_client(...) aan in app.py") + + payload: Dict[str, Any] = { + "model": model or LLM_DEFAULT_MODEL, + "messages": messages, + "stream": False, + "temperature": float(temperature), + "top_p": float(top_p), + } + if max_tokens is not None: + payload["max_tokens"] = int(max_tokens) + payload.update(extra) + + loop = asyncio.get_running_loop() + try: + response: Dict[str, Any] = await loop.run_in_executor( + None, lambda: LLM_QUEUE.request_agent_sync(payload) + ) + return response + except Exception as exc: + logger.exception("_llm_call via agent-queue failed: %s", exc) + return _chat_from_text(f"[LLM-queue-fout] {exc}") diff --git a/queue_helper.py b/queue_helper.py new file mode 100644 index 0000000..f871f50 --- /dev/null +++ b/queue_helper.py @@ -0,0 +1,137 @@ +# ------------------------------------------------------------- +# queue_helper.py – minimalistisch thread‑based wachtrij‑manager +# ------------------------------------------------------------- +import threading, queue, uuid, time +from typing import Callable, Any, Dict + +# ------------------------------------------------------------------ +# Configuratie – pas eventueel aan +# ------------------------------------------------------------------ +USER_MAX_QUEUE = 20 # max. wachtende gebruikers +AGENT_MAX_QUEUE = 50 # max. wachtende agents (stil) +UPDATE_INTERVAL = 10.0 # sec tussen “position‑update” berichten +WORKER_TIMEOUT = 30.0 # max. tijd die een inference mag duren +# ------------------------------------------------------------------ + +class _Job: + __slots__ = ("job_id","payload","callback","created_at","event","result","error") + def __init__(self, payload: Dict, callback: Callable[[Dict], None]): + self.job_id = str(uuid.uuid4()) + self.payload = payload + self.callback = callback + self.created_at = time.time() + self.event = threading.Event() + self.result = None + self.error = None + def set_success(self, answer: Dict): + self.result = answer + self.event.set() + self.callback(answer) + def set_error(self, exc: Exception): + self.error = str(exc) + self.event.set() + self.callback({"error": self.error}) + +class QueueManager: + """Beheert één gedeelde queue + één worker‑thread.""" + def __init__(self, model_infer_fn: Callable[[Dict], Dict]): + self._infer_fn = model_infer_fn + self._user_q = queue.Queue(maxsize=USER_MAX_QUEUE) + self._agent_q = queue.Queue(maxsize=AGENT_MAX_QUEUE) + self._shutdown = threading.Event() + self._worker = threading.Thread(target=self._run_worker, + daemon=True, + name="LLM‑worker") + self._worker.start() + + # ---------- public API ---------- + def enqueue_user( + self, + payload: Dict, + progress_cb: Callable[[Dict], None], + *, + notify_position: bool = False, + ) -> tuple[str, int]: + job = _Job(payload, progress_cb) + try: self._user_q.put_nowait(job) + except queue.Full: raise RuntimeError(f"User‑queue vol (≥{USER_MAX_QUEUE})") + position = self._user_q.qsize() + if notify_position: + # start een aparte notifier-thread die periodiek de wachtrijpositie meldt + start_position_notifier(job, self._user_q) + return job.job_id, position + + def enqueue_agent(self, payload: Dict, progress_cb: Callable[[Dict], None]) -> str: + job = _Job(payload, progress_cb) + try: self._agent_q.put_nowait(job) + except queue.Full: raise RuntimeError(f"Agent‑queue vol (≥{AGENT_MAX_QUEUE})") + return job.job_id + + # ---------- sync helper voor agents/tools ---------- + def request_agent_sync(self, payload: Dict, timeout: float = WORKER_TIMEOUT) -> Dict: + """ + Gebruik dit voor interne calls (agents/tools). + - Job wordt in de agent-queue gezet (lagere prioriteit dan users). + - We wachten blokkerend tot de worker klaar is of tot timeout. + - Er worden GEEN wachtrij-meldingen ("U bent #...") verstuurd. + """ + result_box: Dict[str, Any] = {} + + def _cb(msg: Dict): + # alleen het eindresultaat is interessant voor tools/agents + result_box["answer"] = msg + + job = _Job(payload, _cb) + try: + self._agent_q.put_nowait(job) + except queue.Full: + raise RuntimeError(f"Agent-queue vol (≥{AGENT_MAX_QUEUE})") + + ok = job.event.wait(timeout) + if not ok: + raise TimeoutError(f"LLM-inference duurde langer dan {timeout} seconden.") + if job.error: + raise RuntimeError(job.error) + return result_box.get("answer") or {} + # ---------- worker ---------- + def _run_worker(self): + while not self._shutdown.is_set(): + job = self._pop_job(self._user_q) or self._pop_job(self._agent_q) + if not job: + time.sleep(0.1) + continue + try: + answer = self._infer_fn(job.payload) + job.set_success(answer) + except Exception as exc: + job.set_error(exc) + + def _pop_job(self, q: queue.Queue): + try: return q.get_nowait() + except queue.Empty: return None + + def stop(self): + self._shutdown.set() + self._worker.join(timeout=5) + +def start_position_notifier( + job: _Job, + queue_ref: queue.Queue, + interval: float = UPDATE_INTERVAL, + ): + """Stuurt elke `interval` seconden een bericht met de huidige positie.""" + def _notifier(): + # Stop zodra het job-event wordt gezet (success/fout/timeout upstream) + while not job.event.wait(interval): + # Neem een snapshot van de queue-inhoud op een thread-safe manier + with queue_ref.mutex: + snapshot = list(queue_ref.queue) + try: + pos = snapshot.index(job) + 1 # 1-based + except ValueError: + # Job staat niet meer in de wachtrij → geen updates meer nodig + break + job.callback({"info": f"U bent #{pos} in de wachtrij. Even geduld…" }) + t = threading.Thread(target=_notifier, daemon=True) + t.start() + return t diff --git a/rebuild.sh b/rebuild.sh new file mode 100755 index 0000000..2bb3dfb --- /dev/null +++ b/rebuild.sh @@ -0,0 +1,7 @@ +export HTTP_PROXY=http://192.168.100.2:8118 +export HTTPS_PROXY=http://192.168.100.2:8118 +export http_proxy=http://192.168.100.2:8118 +export https_proxy=http://192.168.100.2:8118 +docker stop toolserver +docker rm -f toolserver 2>/dev/null || true +docker build -t toolserver . --build-arg http_proxy=http://192.168.100.2:8118 --build-arg https_proxy=http://192.168.100.2:8118 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4f85179 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn[standard] +requests +python-docx +pypdf diff --git a/smart_rag.py b/smart_rag.py new file mode 100644 index 0000000..06d49f6 --- /dev/null +++ b/smart_rag.py @@ -0,0 +1,604 @@ +# smart_rag.py +# Kleine util-laag voor intent + hybride retrieval + context-assemblage. +from __future__ import annotations +import os, re, json, math, hashlib +from typing import List, Dict, Tuple, DefaultDict, Optional +from collections import defaultdict + + + +def _decamel(s: str) -> str: + s = re.sub(r"([a-z])([A-Z])", r"\1 \2", s) + s = s.replace("_", " ") + return re.sub(r"\s+", " ", s).strip() + +def _symbol_guess(q: str) -> list[str]: + # pak langste 'code-achtig' token als symboolkandidaat + toks = re.findall(r"[A-Za-z_][A-Za-z0-9_]{2,}", q) + toks.sort(key=len, reverse=True) + return toks[:2] + +def _simple_variants(q: str, max_k: int = 3) -> list[str]: + base = [q] + lo = q.lower().strip() + if lo and lo not in base: + base.append(lo) + dec = _decamel(q) + if dec and dec.lower() != lo and dec not in base: + base.append(dec) + syms = _symbol_guess(q) + for s in syms: + v = s.replace("_", " ") + if v not in base: + base.append(v) + v2 = s # raw symbool + if v2 not in base: + base.append(v2) + # cap + return base[: max(1, min(len(base), max_k))] + + +# --- Query routing + RRF fuse --- + +def _route_query_buckets(q: str) -> list[dict]: + """Hele lichte router: retourneert lijst subqueries met optionele path filters en boost.""" + lo = (q or "").lower() + buckets = [] + + # Queue/Jobs/Event pipeline (Laravel) + if any(w in lo for w in ["job", "queue", "listener", "event", "dispatch"]): + buckets.append({"q": q, "path_contains": "app/Jobs", "boost": 1.18}) + buckets.append({"q": q, "path_contains": "app/Listeners", "boost": 1.12}) + buckets.append({"q": q, "path_contains": "app/Events", "boost": 1.10}) + # Models / Migrations + if any(w in lo for w in ["model", "eloquent", "scope", "attribute"]): + buckets.append({"q": q, "path_contains": "app/Models", "boost": 1.12}) + if any(w in lo for w in ["migration", "schema", "table", "column"]): + buckets.append({"q": q, "path_contains": "database/migrations", "boost": 1.08}) + + # Laravel/Blade/UI + if any(w in lo for w in ["blade", "view", "template", "button", "placeholder", "label"]): + buckets.append({"q": q, "path_contains": "resources/views", "boost": 1.2}) + # Routes/controllers + if any(w in lo for w in ["route", "controller", "middleware", "api", "web.php", "controller@"]): + buckets.append({"q": q, "path_contains": "routes", "boost": 1.15}) + buckets.append({"q": q, "path_contains": "app/Http/Controllers", "boost": 1.2}) + # Config/ENV + if any(w in lo for w in ["env", "config", "database", "queue", "cache"]): + buckets.append({"q": q, "path_contains": "config", "boost": 1.15}) + buckets.append({"q": q, "path_contains": ".env", "boost": 1.1}) + # Docs/README + if any(w in lo for w in ["readme", "install", "setup", "document", "usage"]): + buckets.append({"q": q, "path_contains": "README", "boost": 1.05}) + buckets.append({"q": q, "path_contains": "docs", "boost": 1.05}) + + # Fallback: generiek + buckets.append({"q": q, "path_contains": None, "boost": 1.0}) + # dedup op (q, path_contains) + seen = set(); out = [] + for b in buckets: + key = (b["q"], b["path_contains"]) + if key in seen: continue + seen.add(key); out.append(b) + return out + +def rrf_fuse_ranked_lists(ranked_lists: list[list[dict]], k: int = 60) -> list[dict]: + """ + ranked_lists: bv. [[{key,score,item},...], ...] (elk al per kanaal/bucket gesorteerd) + Return: één samengevoegde lijst (dicts) met veld 'score_fused'. + """ + # bouw mapping + pos_maps: list[dict] = [] + for rl in ranked_lists or []: + pos = {} + for i, it in enumerate(rl, 1): + meta = it.get("metadata") or {} + key = f"{meta.get('repo','')}::{meta.get('path','')}::{meta.get('chunk_index','')}" + pos[key] = i + pos_maps.append(pos) + + fused: dict[str, float] = {} + ref_item: dict[str, dict] = {} + for idx, rl in enumerate(ranked_lists or []): + pos_map = pos_maps[idx] + for it in rl: + meta = it.get("metadata") or {} + key = f"{meta.get('repo','')}::{meta.get('path','')}::{meta.get('chunk_index','')}" + r = pos_map.get(key, 10**9) + fused[key] = fused.get(key, 0.0) + 1.0 / (k + r) + ref_item[key] = it + + out = [] + for key, f in fused.items(): + it = dict(ref_item[key]) + it["score_fused"] = f + out.append(it) + out.sort(key=lambda x: x.get("score_fused", 0.0), reverse=True) + return out + + +def _rrf_from_ranklists(ranklists: List[List[str]], k: int = int(os.getenv("RRF_K", "60"))) -> Dict[str, float]: + """ + Reciprocal Rank Fusion: neemt geordende lijsten (best eerst) en + geeft samengevoegde scores {key: rrf_score}. + """ + acc = defaultdict(float) + for lst in ranklists: + for i, key in enumerate(lst): + acc[key] += 1.0 / (k + i + 1) + return acc + +def _path_prior(path: str) -> float: + """ + Light-weight prior per pad. 0..1 schaal. Laravel paden krijgen bonus, + generieke code dirs ook een kleine bonus; binaire/test/asset minder. + """ + p = (path or "").replace("\\", "/").lower() + bonus = 0.0 + # Laravel priors + if p.startswith("routes/"): bonus += 0.35 + if p.startswith("app/http/controllers/"): bonus += 0.30 + if p.startswith("resources/views/"): bonus += 0.25 + if p.endswith(".blade.php"): bonus += 0.15 + # Generieke priors + if p.startswith(("src/", "app/", "lib/", "pages/", "components/")): bonus += 0.12 + if p.endswith((".php",".ts",".tsx",".js",".jsx",".py",".go",".rb",".java",".cs",".vue",".html",".md")): + bonus += 0.05 + # Demote obvious low-signal + if "/tests/" in p or p.startswith(("tests/", "test/")): bonus -= 0.10 + if p.endswith((".lock",".map",".min.js",".min.css")): bonus -= 0.10 + return max(0.0, min(1.0, bonus)) + + +def _safe_json_loads(s: str): + if not s: + return None + t = s.strip() + if t.startswith("```"): + t = re.sub(r"^```(?:json)?", "", t, count=1, flags=re.IGNORECASE).strip() + if t.endswith("```"): + t = t[:-3].strip() + try: + return json.loads(t) + except Exception: + return None + + +def _tok(s: str) -> List[str]: + return re.findall(r"[A-Za-z0-9_]+", s.lower()) + +def _jaccard(a: str, b: str) -> float: + A, B = set(_tok(a)), set(_tok(b)) + if not A or not B: return 0.0 + # heel kleine set-caps (noodrem tegen pathologische inputs) + if len(B) > 8000: + # reduceer B met stabiele (deterministische) sampling op basis van sha1 + def _stable_byte(tok: str) -> int: + return hashlib.sha1(tok.encode("utf-8")).digest()[0] + B = {t for t in B if _stable_byte(t) < 64} # ~25% sample + return len(A & B) / max(1, len(A | B)) + + +def _normalize(xs: List[float]) -> List[float]: + if not xs: return xs + lo, hi = min(xs), max(xs) + if hi <= lo: return [0.0]*len(xs) + return [(x - lo) / (hi - lo) for x in xs] + +async def enrich_intent(llm_call_fn, messages: List[Dict]) -> Dict: + """ + Zet ongestructureerde vraag om naar een compact plan. + Velden: task, constraints, file_hints, keywords, acceptance, ask(optional). + """ + user_text = "" + for m in reversed(messages): + if m.get("role") == "user": + user_text = m.get("content","").strip() + break + + sys = ("Je herstructureert een developer-vraag naar JSON. " + "Geef ALLEEN JSON, geen toelichting.") + usr = ( + "Zet de essentie van de vraag om naar dit schema:\n" + "{" + "\"task\": str, " + "\"constraints\": [str,...], " + "\"file_hints\": [str,...], " + "\"keywords\": [str,...], " + "\"acceptance\": [str,...], " + "\"ask\": str|null " + "}\n\n" + f"Vraag:\n{user_text}" + ) + try: + resp = await llm_call_fn( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.1, top_p=1.0, max_tokens=512 + ) + raw = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","{}") + spec = _safe_json_loads(raw) or {"task": user_text, "constraints": [], "file_hints": [], "keywords": [], "acceptance": [], "ask": None} #json.loads(raw.strip()) + except Exception: + # Veilige defaults + spec = { + "task": user_text, + "constraints": [], + "file_hints": [], + "keywords": [], + "acceptance": [], + "ask": None + } + # Minimalistische fallback sanity + for k in ("constraints","file_hints","keywords","acceptance"): + if not isinstance(spec.get(k), list): + spec[k] = [] + if not isinstance(spec.get("task"), str): + spec["task"] = user_text + if spec.get("ask") is not None and not isinstance(spec["ask"], str): + spec["ask"] = None + return spec + +async def expand_queries(llm_call_fn, q: str, k: int = 3) -> List[str]: + if str(os.getenv("RAG_EXPAND_QUERIES","1")).lower() in ("0","false"): + return [q] + sys = "Geef 3-4 korte NL/EN zoekvarianten als JSON array. Geen toelichting." + usr = f"Bronvraag:\n{q}\n\nAlleen JSON array." + try: + resp = await llm_call_fn( + [{"role":"system","content":sys},{"role":"user","content":usr}], + stream=False, temperature=0.2, top_p=0.9, max_tokens=240 + ) + raw = (resp.get("choices",[{}])[0].get("message",{}) or {}).get("content","[]") + arr = _safe_json_loads(raw) or [] + arr = [str(x).strip() for x in arr if str(x).strip()] + seen = {q.lower()} + base = [q] + for v in arr: + lv = v.lower() + if lv not in seen: + base.append(v); seen.add(lv) + return base[: max(1, min(len(base), k + 1))] + except Exception: + return [q] + +def _sim_from_chroma_distance(d: float|None) -> float: + """ + Converteer (Chroma) distance naar similarity in [0,1]; defensief tegen None/NaN/negatief. + """ + if d is None: + return 0.0 + try: + dv = float(d) + except Exception: + dv = 0.0 + if not math.isfinite(dv) or dv < 0: + return 0.0 + return 1.0 / (1.0 + dv) + + +async def hybrid_retrieve( + rag_query_internal_fn, + query: str, + *, + repo: str|None = None, + profile: str|None = None, + path_contains: str|None = None, + per_query_k: int = 30, + n_results: int = 8, + alpha: float = 0.6, + collection_name: str = "code_docs", + llm_call_fn=None, +) -> List[Dict]: + """ + Multi-variant retrieval met RRF-fusie + path-prior. + Return: lijst met dict(document, metadata, score) + """ + + # Optionele query-routing + RRF + use_route = str(os.getenv("RAG_ROUTE", "1")).lower() not in ("0", "false") + use_rrf = str(os.getenv("RAG_RRF", "1")).lower() not in ("0", "false") + # Optionele mini multi-query expansion (default aan) + use_expand = str(os.getenv("RAG_MULTI_EXPAND", "1")).lower() in ("1","true","yes") + k_variants = max(1, int(os.getenv("RAG_MULTI_K", "3"))) + per_query_k = max(1, int(per_query_k)) + n_results = max(1, int(n_results)) + if not (query or "").strip(): + return [] + # Multi-query variants: + if use_expand: + if llm_call_fn is not None: + variants = await expand_queries(llm_call_fn, query, k=k_variants) + else: + variants = _simple_variants(query, max_k=k_variants) + else: + variants = [query] + + ranked_lists = [] # voor RRF (alle varianten/buckets) + for qv in variants: + if use_route: + buckets = _route_query_buckets(qv) + for b in buckets: + # combineer globale path_contains-hint met bucket-specifieke filter + pc = b.get("path_contains") + if path_contains and not pc: + pc = path_contains + res = await rag_query_internal_fn( + query=b["q"], n_results=per_query_k, + collection_name=collection_name, + repo=repo, path_contains=pc, profile=profile + ) + lst = [] + for item in (res or {}).get("results", []): + # distance kan ontbreken bij oudere backends; defensieve cast + dist = item.get("distance", None) + try: dist = float(dist) if dist is not None else None + except Exception: dist = None + emb_sim = _sim_from_chroma_distance(dist) * float(b.get("boost",1.0)) + lst.append({**item, "emb_sim_routed": emb_sim}) + lst.sort(key=lambda x: x.get("emb_sim_routed",0.0), reverse=True) + # Laat RRF voldoende kandidaten zien (niet te vroeg afsnijden): + ranked_lists.append(lst[:per_query_k]) + else: + # geen routing: per variant direct query'en (consistent scoren/sorteren) + res = await rag_query_internal_fn( + query=qv, n_results=per_query_k, + collection_name=collection_name, + repo=repo, path_contains=path_contains, profile=profile + ) + lst = [] + for item in (res or {}).get("results", []): + dist = item.get("distance", None) + try: dist = float(dist) if dist is not None else None + except Exception: dist = None + emb_sim = _sim_from_chroma_distance(dist) + lst.append({**item, "emb_sim_routed": emb_sim}) + lst.sort(key=lambda x: x.get("emb_sim_routed", 0.0), reverse=True) + ranked_lists.append(lst[:per_query_k]) + + + + # Als RRF aanstaat: fuseer nu + items = rrf_fuse_ranked_lists(ranked_lists) if use_rrf else [x for rl in ranked_lists for x in rl] + + if not items: + return [] + + # Eenvoudige lexicale score (op samengevoegde set): + # neem het BESTE van alle varianten i.p.v. alleen de hoofdquery. + bm: List[float] = [] + if variants and len(variants) > 1: + for it in items: + doc = it.get("document", "") or "" + bm.append(max((_jaccard(v, doc) for v in variants), default=_jaccard(query, doc))) + else: + bm = [_jaccard(query, it.get("document","")) for it in items] + bm_norm = _normalize(bm) + + out = [] + for i, it in enumerate(items): + # Betere fallback: gebruik routed emb sim → plain emb_sim → distance + emb = ( + float(it.get("emb_sim_routed", 0.0)) + or float(it.get("emb_sim", 0.0)) + or _sim_from_chroma_distance(it.get("distance")) + ) + score = alpha * emb + (1.0 - alpha) * bm_norm[i] + meta = (it.get("metadata") or {}) + path = meta.get("path","") or "" + # — optioneel: path-prior + symbol-boost via env — + pp_w = float(os.getenv("RAG_PATH_PRIOR_W", "0.08")) + if pp_w > 0.0: + score += pp_w * _path_prior(path) + sym_w = float(os.getenv("RAG_SYM_BOOST", "0.04")) + if sym_w > 0.0: + syms_raw = meta.get("symbols") + if isinstance(syms_raw, str): + syms = [s.strip().lower() for s in syms_raw.split(",") if s.strip()] + elif isinstance(syms_raw, list): + syms = [str(s).strip().lower() for s in syms_raw if str(s).strip()] + else: + syms = [] + if syms: + q_terms = set(_tok(query)) + if q_terms & set(syms): + score += sym_w + out.append({**it, "score": float(score)}) + + out.sort(key=lambda x: x["score"], reverse=True) + return out[:int(n_results)] + +def assemble_context(chunks: List[Dict], *, max_chars: int = 24000) -> Tuple[str, float]: + """ + Budgeted stitching: + - groepeer per path + - per path: neem 1-3 fragmenten (op volgorde van chunk_index indien beschikbaar) + - verdeel char-budget over paden, zwaarder voor hogere scores + - behoud Laravel stitching + Retour: (context_text, top_score) + """ + if not chunks: + return "", 0.0 + + # 1) Groepeer per path en verzamel scores + (optioneel) chunk_index + by_path: Dict[str, List[Dict]] = {} + top_score = 0.0 + for r in chunks: + meta = (r.get("metadata") or {}) + path = meta.get("path","") or "" + r["_chunk_index"] = meta.get("chunk_index") + r["_score"] = float(r.get("score", 0.0) or 0.0) + top_score = max(top_score, r["_score"]) + by_path.setdefault(path, []).append(r) + + # 2) Per path: sorteer op chunk_index (indien beschikbaar) anders score; cap op N stukken + def _sort_key(x): + ci = x.get("_chunk_index") + return (0, int(ci)) if isinstance(ci, int) or (isinstance(ci, str) and str(ci).isdigit()) else (1, -x["_score"]) + + path_items = [] + max_pieces = int(os.getenv("CTX_PIECES_PER_PATH_CAP", "3")) + for p, lst in by_path.items(): + lst_sorted = sorted(lst, key=_sort_key) + path_items.append({ + "path": p, + "best_score": max(x["_score"] for x in lst_sorted), + "pieces": lst_sorted[:max(1, max_pieces)], # cap per bestand + }) + + # 3) Sorteer paden op best_score en bereken budgetverdeling (softmax-achtig, maar bounded) + path_items.sort(key=lambda t: t["best_score"], reverse=True) + # clamp scores naar [0,1] voor stabielere allocatie + scores = [min(1.0, max(0.0, t["best_score"])) for t in path_items] + # softmax-lite: exp(score*beta) normaliseren; beta iets lager om niet te scherp te verdelen + beta = float(os.getenv("CTX_ALLOC_BETA", "2.2")) + w = [math.exp(beta * s) for s in scores] + S = max(1e-9, sum(w)) + weights = [x / S for x in w] + + # 4) Bouw snelle lookup path->full body (voor Laravel stitching) + by_path_first_body: Dict[str, str] = {} + for t in path_items: + doc0 = (t["pieces"][0].get("document") or "").strip() + by_path_first_body[t["path"]] = doc0 + + # 5) Render met budget per pad + out = [] + used = 0 + for t, w_i in zip(path_items, weights): + p = t["path"] + # minimaal & maximaal budget per pad (chars) + min_chars = int(os.getenv("CTX_ALLOC_MIN_PER_PATH", "1200")) + max_chars_path = int(os.getenv("CTX_ALLOC_MAX_PER_PATH", "6000")) + alloc = min(max(min_chars, int(max_chars * w_i)), max_chars_path) + + # stitch 1..3 stukken van dit pad binnen alloc + header = f"### {p} (score={t['best_score']:.3f})" + block_buf = [header] + remaining = max(0, alloc - len(header) - 1) + + + for piece in t["pieces"]: + body = (piece.get("document") or "").strip() + # knip niet middenin een regel: neem tot remaining en rol terug tot laatste newline + if remaining <= 0: + break + if len(body) > remaining: + cut = body[:remaining] + nl = cut.rfind("\n") + if nl > 300: # laat niet té kort + body = cut[:nl] + "\n…" + else: + body = cut + "…" + block_buf.append(body) + remaining -= len(body) + if remaining <= 300: # hou wat over voor stitching + break + + block = "\n".join(block_buf) + + # --- Laravel mini-stitch zoals voorheen, maar budgetbewust + stitched = [] + if p in ("routes/web.php", "routes/api.php"): + for ctrl_path, _meth in _laravel_pairs_from_route_text(by_path_first_body.get(p,"")): + if ctrl_path in by_path_first_body and remaining > 400: + snippet = by_path_first_body[ctrl_path][:min(400, remaining)] + stitched.append(f"\n### {ctrl_path} (stitch)\n{snippet}") + remaining -= len(snippet) + if p.startswith("app/Http/Controllers/"): + for vpath in _laravel_guess_view_paths_from_text(by_path_first_body.get(p,"")): + if vpath in by_path_first_body and remaining > 400: + snippet = by_path_first_body[vpath][:min(400, remaining)] + stitched.append(f"\n### {vpath} (stitch)\n{snippet}") + remaining -= len(snippet) + + if stitched: + block += "\n" + "\n".join(stitched) + + # Past het volledige blok niet meer, knip netjes i.p.v. alles laten vallen + remaining_total = max_chars - used + if remaining_total <= 0: + break + if len(block) > remaining_total: + # Zorg dat we niet midden in markdown header afkappen + trimmed = block[:max(0, remaining_total - 1)] + block = trimmed + "…" + out.append(block) + used = max_chars + break + else: + out.append(block) + used += len(block) + + + # stop vroeg als we het budget bijna op hebben + if max_chars - used < 800: + break + + return ("\n\n".join(out), float(top_score)) + +# --- Laravel route/controller/view helpers (lightweight, cycle-safe) --- + +def _laravel_pairs_from_route_text(route_text: str): + """ + Parse routes/web.php|api.php tekst en yield (controller_path, method) guesses. + Ondersteunt: + - 'Controller@method' + - FQCN zoals App\\Http\\Controllers\\Foo\\BarController::class + """ + out = [] + + # 1) 'Controller@method' + for m in re.finditer(r"['\"]([A-Za-z0-9_\\]+)@([A-Za-z0-9_]+)['\"]", route_text): + fq = m.group(1) + method = m.group(2) + ctrl = fq.replace("\\\\","/").replace("\\","/") + name = ctrl.split("/")[-1] + guess = f"app/Http/Controllers/{ctrl}.php" + alt = f"app/Http/Controllers/{name}.php" + out.append((guess, method)) + out.append((alt, method)) + + # 2) FQCN ::class + for m in re.finditer(r"([A-Za-z_][A-Za-z0-9_\\]+)\s*::\s*class", route_text): + fq = m.group(1) + ctrl = fq.replace("\\\\","/").replace("\\","/") + name = ctrl.split("/")[-1] + guess = f"app/Http/Controllers/{ctrl}.php" + alt = f"app/Http/Controllers/{name}.php" + out.append((guess, None)) + out.append((alt, None)) + + # dedupe, behoud orde + seen = set(); dedup = [] + for p in out: + if p not in seen: + seen.add(p); dedup.append(p) + return dedup + + +def _laravel_guess_view_paths_from_text(controller_text: str): + """ + Parse simpele 'return view(\"foo.bar\")' patronen → resources/views/foo/bar.blade.php + """ + out = [] + for m in re.finditer(r"view\(\s*['\"]([A-Za-z0-9_.\/-]+)['\"]\s*\)", controller_text): + view = m.group(1).strip().strip(".") + # 'foo.bar' of 'foo/bar' + path = view.replace(".", "/") + out.append(f"resources/views/{path}.blade.php") + # dedupe + seen = set(); dedup = [] + for p in out: + if p not in seen: + seen.add(p); dedup.append(p) + return dedup + +# Public API surface +__all__ = [ + "enrich_intent", + "expand_queries", + "hybrid_retrieve", + "assemble_context", + "_laravel_pairs_from_route_text", + "_laravel_guess_view_paths_from_text", +] + diff --git a/toolserver.sh b/toolserver.sh new file mode 100755 index 0000000..99600f0 --- /dev/null +++ b/toolserver.sh @@ -0,0 +1 @@ +docker run -d --rm --name toolserver --network host -v /opt/SentenceTransformer:/opt/sentence-transformers -v /opt/piper/voices:/voices:ro -e LLM_TOOL_RUNNER=bridge -e LLM_UPSTREAMS="http://localhost:8000/v1/chat/completions,http://localhost:8001/v1/chat/completions" -e LLM_MAX_CONCURRENCY=2 -e REPO_AGENT_SMART=1 -e RAG_EXPAND_QUERIES=1 -e RAG_EXPAND_K=3 -e RAG_PER_QUERY_K=30 -e RAG_N_RESULT=8 -e RAG_EMB_WEIGHT=0.6 -e REPO_AGENT_CONTEXT_CHARS=24000 -e REPO_AGENT_ASK_CLARIFY=1 -e REPO_AGENT_ASK_THRESHOLD=0.35 -e PIPER_BIN=/usr/local/bin/piper -e PIPER_VOICE=/voices/nl_NL-mls-medium.onnx.gz -e LLM_WINDOWING_ENABLE=0 -e LLM_CONTEXT_TOKENS=42000 -e LLM_RESPONSE_RESERVE=1024 -e LLM_AUTO_CONTINUES=0 -e LLM_FUNCTION_CALLING_MODE=auto -e RAG_EMB_WEIGHT=0.6 -e LLM_URL="http://localhost:8000/v1/chat/completions" -e NO_PROXY="127.0.0.1,localhost,::1,host.docker.internal" -e RAG_TORCH_THREADS=6 -e OMP_NUM_THREADS=6 -e MKL_NUM_THREADS=6 -e OPENBLAS_NUM_THREADS=6 -e NUMEXPR_NUM_THREADS=6 -e LLM_READ_TIMEOUT=3600 -e NO_PROXY=localhost,127.0.0.1,::1,192.168.100.1,192.168.100.2 -e HTTP_PROXY=http://192.168.100.2:8118 -e HTTPS_PROXY=http://192.168.100.2:8118 -e MEILI_URL=http://localhost:7700 -e MEILI_KEY=0xipOmfgi_zMgdFplSdv7L8mlx0RPMQCNxVTNJc54lQ --gpus device=0 -e CUDA_VISIBLE_DEVICES=0 -e FORCE_ALL_TOOLS=0 -e AUTO_CONTINUE=0 -e LLM_PROXY_URL="http://192.168.100.1:8081/v1/chat/completions" -e ALLOWED_GIT_HOSTS="192.168.100.1,localhost,127.0.0.1,10.25.138.40" -e STREAM_PREFER_DIRECT=1 toolserver diff --git a/toolserver_toolset_v2.zip b/toolserver_toolset_v2.zip new file mode 100644 index 0000000..7d0b5de Binary files /dev/null and b/toolserver_toolset_v2.zip differ diff --git a/web_search.py b/web_search.py new file mode 100644 index 0000000..0095cf5 --- /dev/null +++ b/web_search.py @@ -0,0 +1,333 @@ +import os +import requests +from datetime import datetime +import json +import logging +from requests import get +from bs4 import BeautifulSoup +import concurrent.futures +from html.parser import HTMLParser +from urllib.parse import urlparse, urljoin +import re +import unicodedata +from pydantic import BaseModel, Field +import asyncio +from typing import Any, List, Optional +import httpx +logger = logging.getLogger("web_search") + +async def call_improve_web_query(text: str, max_chars: int = 20000, objective: str = None, style: str = None): + url = "http://localhost:8080/improve_web_query" + params = {"text": text, "max_chars": max_chars} + if objective: + params["objective"] = objective + if style: + params["style"] = style + + async with httpx.AsyncClient() as client: + response = await client.get(url, params=params) + return response.json() + +class HelpFunctions: + def __init__(self): + pass + + def get_base_url(self, url: str) -> str: + parsed_url = urlparse(url) + base_url = f"{parsed_url.scheme}://{parsed_url.netloc}" + return base_url + + def generate_excerpt(self, content: str, max_length: int = 200) -> str: + return content[:max_length] + "..." if len(content) > max_length else content + + def format_text(self, original_text: str) -> str: + soup = BeautifulSoup(original_text, "html.parser") + formatted_text = soup.get_text(separator=" ", strip=True) + formatted_text = unicodedata.normalize("NFKC", formatted_text) + formatted_text = re.sub(r"\s+", " ", formatted_text) + formatted_text = formatted_text.strip() + formatted_text = self.remove_emojis(formatted_text) + return formatted_text + + def remove_emojis(self, text: str) -> str: + return "".join(c for c in text if not unicodedata.category(c).startswith("So")) + + def process_search_result(self, result: dict, valves: Any) -> Optional[dict]: + title_site = self.remove_emojis(result["title"]) + url_site = result["url"] + snippet = result.get("content", "") + + # Check if the website is in the ignored list, but only if IGNORED_WEBSITES is not empty + if valves.IGNORED_WEBSITES: + base_url = self.get_base_url(url_site) + if any( + ignored_site.strip() in base_url + for ignored_site in valves.IGNORED_WEBSITES.split(",") + ): + return None + + try: + response_site = requests.get(url_site, timeout=20) + response_site.raise_for_status() + html_content = response_site.text + + soup = BeautifulSoup(html_content, "html.parser") + content_site = self.format_text(soup.get_text(separator=" ", strip=True)) + + truncated_content = self.truncate_to_n_words( + content_site, valves.PAGE_CONTENT_WORDS_LIMIT + ) + + return { + "title": title_site, + "url": url_site, + "content": truncated_content, + "snippet": self.remove_emojis(snippet), + } + + except requests.exceptions.RequestException as e: + return None + + def truncate_to_n_words(self, text: str, token_limit: int) -> str: + tokens = text.split() + truncated_tokens = tokens[:token_limit] + return " ".join(truncated_tokens) + +class EventEmitter: + def __init__(self, event_emitter: Optional[Any] = None): + self.event_emitter = event_emitter + + async def emit(self, description: str = "Unknown State", status: str = "in_progress", done: bool = False): + if self.event_emitter: + await self.event_emitter( + { + "type": "status", + "data": { + "status": status, + "description": description, + "done": done, + }, + } + ) + +class Tools: + class Valves(BaseModel): + SEARXNG_ENGINE_API_BASE_URL: str = Field( + default="http://192.168.100.1:8899/search", + description="The base URL for SearXNG Search Engine", + ) + IGNORED_WEBSITES: str = Field( + default="", + description="Comma-separated list of websites to ignore", + ) + RETURNED_SCRAPPED_PAGES_NO: int = Field( + default=3, + description="The number of Search Engine Results to Parse", + ) + SCRAPPED_PAGES_NO: int = Field( + default=5, + description="Total pages scapped. Ideally greater than one of the returned pages", + ) + PAGE_CONTENT_WORDS_LIMIT: int = Field( + default=5000, + description="Limit words content for each page.", + ) + CITATION_LINKS: bool = Field( + default=False, + description="If True, send custom citations with links", + ) + + def __init__(self): + self.valves = self.Valves() + self.headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3" + } + + async def search_web( + self, + query: str + ) -> str: + """ + Search the web using SearXNG and get the content of the relevant pages. + :param query: Web Query used in search engine. + :param __event_emitter__: Optional event emitter for status updates. + :return: The content of the pages in json format. + """ + logger.info(" in query: %s",query) + try: + #query=asyncio.run(call_improve_web_query(query)) + pass + except Exception as e: + logger.error("ERROR: %s",str(e)) + logger.info(" out query: %s",query) + __event_emitter__=None + functions = HelpFunctions() + emitter = EventEmitter(__event_emitter__) + + await emitter.emit(description=f"Initiating web search for: {query}") + + search_engine_url = self.valves.SEARXNG_ENGINE_API_BASE_URL + + # Ensure RETURNED_SCRAPPED_PAGES_NO does not exceed SCRAPPED_PAGES_NO + if self.valves.RETURNED_SCRAPPED_PAGES_NO > self.valves.SCRAPPED_PAGES_NO: + self.valves.RETURNED_SCRAPPED_PAGES_NO = self.valves.SCRAPPED_PAGES_NO + + params = { + "q": query, + "format": "json", + "number_of_results": self.valves.RETURNED_SCRAPPED_PAGES_NO, + } + + try: + await emitter.emit(description="Sending request to search engine") + resp = requests.get( + search_engine_url, params=params, headers=self.headers, timeout=120 + ) + logger.info("query : %s", query) + logger.info("REQUEST URL: %s", resp.url) + + resp.raise_for_status() + data = resp.json() + + results = data.get("results", []) + limited_results = results[: self.valves.SCRAPPED_PAGES_NO] + await emitter.emit(description=f"Retrieved {len(limited_results)} search results") + + except requests.exceptions.RequestException as e: + await emitter.emit( + status="error", + description=f"Error during search: {str(e)}", + done=True, + ) + return json.dumps({"error": str(e)}) + + results_json = [] + if limited_results: + await emitter.emit(description=f"Processing search results") + + with concurrent.futures.ThreadPoolExecutor() as executor: + futures = [ + executor.submit( + functions.process_search_result, result, self.valves + ) + for result in limited_results + ] + for future in concurrent.futures.as_completed(futures): + result_json = future.result() + if result_json: + try: + json.dumps(result_json) + results_json.append(result_json) + except (TypeError, ValueError): + continue + if len(results_json) >= self.valves.RETURNED_SCRAPPED_PAGES_NO: + break + + results_json = results_json[: self.valves.RETURNED_SCRAPPED_PAGES_NO] + + if self.valves.CITATION_LINKS and __event_emitter__: + for result in results_json: + await __event_emitter__( + { + "type": "citation", + "data": { + "document": [result["content"]], + "metadata": [{"source": result["url"]}], + "source": {"name": result["title"]}, + }, + } + ) + + await emitter.emit( + status="complete", + description=f"Web search completed. Retrieved content from {len(results_json)} pages", + done=True, + ) + + return json.dumps(results_json, ensure_ascii=False) + + async def get_website( + self, url: str + ) -> str: + """ + Web scrape the website provided and get the content of it. + :param url: The URL of the website. + :param __event_emitter__: Optional event emitter for status updates. + :return: The content of the website in json format. + """ + __event_emitter__=None + functions = HelpFunctions() + emitter = EventEmitter(__event_emitter__) + if 'http' not in url: + #assume that a https:// prepend is needed. + url='https://'+url + + await emitter.emit(description=f"Fetching content from URL: {url}") + + results_json = [] + + try: + response_site = requests.get(url, headers=self.headers, timeout=120) + response_site.raise_for_status() + html_content = response_site.text + + await emitter.emit(description="Parsing website content") + + soup = BeautifulSoup(html_content, "html.parser") + + page_title = soup.title.string if soup.title else "No title found" + page_title = unicodedata.normalize("NFKC", page_title.strip()) + page_title = functions.remove_emojis(page_title) + title_site = page_title + url_site = url + content_site = functions.format_text( + soup.get_text(separator=" ", strip=True) + ) + + truncated_content = functions.truncate_to_n_words( + content_site, self.valves.PAGE_CONTENT_WORDS_LIMIT + ) + + result_site = { + "title": title_site, + "url": url_site, + "content": truncated_content, + "excerpt": functions.generate_excerpt(content_site), + } + + results_json.append(result_site) + + if self.valves.CITATION_LINKS and __event_emitter__: + await __event_emitter__( + { + "type": "citation", + "data": { + "document": [truncated_content], + "metadata": [{"source": url_site}], + "source": {"name": title_site}, + }, + } + ) + + await emitter.emit( + status="complete", + description="Website content retrieved and processed successfully", + done=True, + ) + + except requests.exceptions.RequestException as e: + results_json.append( + { + "url": url, + "content": f"Failed to retrieve the page. Error: {str(e)}", + } + ) + + await emitter.emit( + status="error", + description=f"Error fetching website content: {str(e)}", + done=True, + ) + + return json.dumps(results_json, ensure_ascii=False) + diff --git a/windowing_utils.py b/windowing_utils.py new file mode 100644 index 0000000..3f8606b --- /dev/null +++ b/windowing_utils.py @@ -0,0 +1,172 @@ +# windowing_utils.py +from __future__ import annotations +from dataclasses import dataclass, field +from typing import List, Dict, Callable, Optional, Tuple, Awaitable +import hashlib +import os +import time + +# ---------- Token counting (vervang door echte tokenizer indien je wilt) +def approx_token_count(text: str) -> int: + # ~4 chars ≈ 1 token (ruwe maar stabiele vuistregel) + return max(1, len(text) // 4) + +def count_message_tokens(messages: List[Dict], tok_len: Callable[[str], int]) -> int: + total = 0 + for m in messages: + total += tok_len(m.get("content", "")) + return total + +# ---------- Thread ID + summary store +def derive_thread_id(body: Dict) -> str: + for key in ("conversation_id", "thread_id", "chat_id", "session_id", "room_id"): + if key in body and body[key]: + return str(body[key]) + parts = [str(body.get("model", ""))] + msgs = body.get("messages", [])[:2] + for m in msgs: + parts.append(m.get("role", "")) + parts.append(m.get("content", "")[:256]) + raw = "||".join(parts) + return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] + +class RunningSummaryStore: + def __init__(self): + self._mem: dict[str, str] = {} + def get(self, thread_id: str) -> str: + return self._mem.get(thread_id, "") + def update(self, thread_id: str, new_summary: str): + self._mem[thread_id] = new_summary + +SUMMARY_STORE = RunningSummaryStore() + +# ---------- Sliding window + running summary +@dataclass +class ConversationWindow: + max_ctx_tokens: int + response_reserve: int = 2048 + tok_len: Callable[[str], int] = approx_token_count + running_summary: str = "" + summary_header: str = "Samenvatting tot nu toe" + history: List[Dict] = field(default_factory=list) + + def add(self, role: str, content: str): + self.history.append({"role": role, "content": content}) + + def _base_messages(self, system_prompt: Optional[str]) -> List[Dict]: + msgs: List[Dict] = [] + if system_prompt: + msgs.append({"role": "system", "content": system_prompt}) + if self.running_summary: + msgs.append({"role": "system", "content": f"{self.summary_header}:\n{self.running_summary}"}) + return msgs + + async def build_within_budget( + self, + system_prompt: Optional[str], + summarizer: Optional[Callable[[str, List[Dict]], Awaitable[str]]] = None + ) -> List[Dict]: + budget = self.max_ctx_tokens - max(1, self.response_reserve) + working = self.history[:] + candidate = self._base_messages(system_prompt) + working + if count_message_tokens(candidate, self.tok_len) <= budget: + return candidate + + # 1) trim oudste turns + while working and count_message_tokens(self._base_messages(system_prompt) + working, self.tok_len) > budget: + working.pop(0) + candidate = self._base_messages(system_prompt) + working + if count_message_tokens(candidate, self.tok_len) <= budget: + self.history = working + return candidate + + # 2) samenvatten indien mogelijk + if summarizer is None: + while working and count_message_tokens(self._base_messages(system_prompt) + working, self.tok_len) > budget: + working.pop(0) + self.history = working + return self._base_messages(system_prompt) + working + + # samenvat in batches + working = self.history[:] + chunk_buf: List[Dict] = [] + + async def build_candidate(_summary: str, _working: List[Dict]) -> List[Dict]: + base = [] + if system_prompt: + base.append({"role": "system", "content": system_prompt}) + if _summary: + base.append({"role": "system", "content": f"{self.summary_header}:\n{_summary}"}) + return base + _working + + while working and count_message_tokens(await build_candidate(self.running_summary, working), self.tok_len) > budget: + chunk_buf.append(working.pop(0)) + # bij ~1500 tokens in buffer (ruw) samenvatten + if count_message_tokens([{"role":"system","content":str(chunk_buf)}], self.tok_len) > 1500 or not working: + self.running_summary = await summarizer(self.running_summary, chunk_buf) + chunk_buf = [] + + # verwerk eventuele overgebleven buffer zodat er geen turns verdwijnen + if chunk_buf: + self.running_summary = await summarizer(self.running_summary, chunk_buf) + chunk_buf = [] + + self.history = working + return await build_candidate(self.running_summary, working) + +# ---------- Repo chunking +from typing import Iterable +def split_text_tokens( + text: str, + tok_len: Callable[[str], int], + max_tokens: int, + overlap_tokens: int = 60 +) -> List[str]: + if tok_len(text) <= max_tokens: + return [text] + approx_ratio = max_tokens / max(1, tok_len(text)) + step = max(1000, int(len(text) * approx_ratio)) + chunks: List[str] = [] + i = 0 + while i < len(text): + ch = text[i:i+step] + while tok_len(ch) > max_tokens and len(ch) > 200: + ch = ch[:-200] + chunks.append(ch) + if overlap_tokens > 0: + ov_chars = max(100, overlap_tokens * 4) + i += max(1, len(ch) - ov_chars) + else: + i += len(ch) + return chunks + +def fit_context_under_budget( + items: List[Tuple[str,str]], tok_len: Callable[[str], int], budget_tokens: int +) -> List[Tuple[str,str]]: + res: List[Tuple[str,str]] = [] + used = 0 + for title, text in items: + t = tok_len(text) + if used + t <= budget_tokens: + res.append((title, text)) + used += t + else: + break + return res + +def build_repo_context( + files_ranked: List[Tuple[str, str, float]], + per_chunk_tokens: int = 2100, + overlap_tokens: int = 60, + ctx_budget_tokens: int = 5000, + tok_len: Callable[[str], int] = approx_token_count +) -> str: + expanded: List[Tuple[str,str]] = [] + for path, content, _ in files_ranked: + for i, ch in enumerate(split_text_tokens(content, tok_len, per_chunk_tokens, overlap_tokens)): + expanded.append((f"{path}#chunk{i+1}", ch)) + selected = fit_context_under_budget(expanded, tok_len, ctx_budget_tokens) + ctx = "" + for title, ch in selected: + ctx += f"\n\n=== {title} ===\n{ch}" + return ctx.strip()