diff --git a/blueprints/chat.py b/blueprints/chat.py index 6e5262d..d24d0c6 100644 --- a/blueprints/chat.py +++ b/blueprints/chat.py @@ -63,6 +63,36 @@ def ping_llm(): return jsonify({"status": "error", "error": str(e), "url": url, "model": model, "embed_model": embed_model}), 502 +@chat_bp.route("/debug-rag", methods=["GET"]) +@login_required +def debug_rag(): + """Diagnose RAG: show collection size, run a test query, return raw distances.""" + query = request.args.get("q", "test") + source_id = request.args.get("source_id", type=int) + source_type = request.args.get("source_type", "doc") + try: + collection = rag_service._get_collection() + total_chunks = collection.count() + source_ids = [source_id] if source_id else None + raw = collection.query( + query_texts=[query], + n_results=min(5, max(1, total_chunks)), + where=rag_service._build_where(current_user.id, source_ids, source_type if source_id else None), + include=["documents", "distances", "ids"], + ) + hits = [ + {"id": i, "distance": round(d, 4), "preview": t[:120]} + for i, d, t in zip( + (raw.get("ids") or [[]])[0], + (raw.get("distances") or [[]])[0], + (raw.get("documents") or [[]])[0], + ) + ] + return jsonify({"total_chunks": total_chunks, "query": query, "hits": hits}) + except Exception as e: + return jsonify({"error": str(e)}), 500 + + # ── Main chat ──────────────────────────────────────────────────────────────── @chat_bp.route("/sessions//ask", methods=["POST"]) @@ -108,6 +138,8 @@ def ask(session_id): except Exception as e: current_app.logger.warning(f"RAG lookup failed, continuing without context: {e}") + current_app.logger.info(f"RAG: found {len(chunks)} chunks for session {session_id}") + # Build history (last 10 messages for context window) history = [ {"role": m.role, "content": m.content} diff --git a/services/rag_service.py b/services/rag_service.py index 3f2c2d9..3e3cd0e 100644 --- a/services/rag_service.py +++ b/services/rag_service.py @@ -190,6 +190,16 @@ def _fetch_neighbor_docs(chunk_id: str, collection) -> list[tuple[str, str]]: return neighbors +def _build_where(user_id: int, source_ids=None, source_type=None) -> dict: + """Build a ChromaDB where-filter for user/source scoping.""" + conditions = [{"user_id": {"$eq": str(user_id)}}] + if source_ids is not None and len(source_ids) > 0: + conditions.append({"source_id": {"$in": [str(sid) for sid in source_ids]}}) + if source_type: + conditions.append({"source_type": {"$eq": source_type}}) + return {"$and": conditions} if len(conditions) > 1 else conditions[0] + + def similarity_search( query: str, user_id: int, @@ -199,13 +209,7 @@ def similarity_search( ) -> list[str]: """Multi-query search with neighbor expansion and reading-order sorting.""" collection = _get_collection() - - conditions = [{"user_id": {"$eq": str(user_id)}}] - if source_ids is not None and len(source_ids) > 0: - conditions.append({"source_id": {"$in": [str(sid) for sid in source_ids]}}) - if source_type: - conditions.append({"source_type": {"$eq": source_type}}) - where = {"$and": conditions} if len(conditions) > 1 else conditions[0] + where = _build_where(user_id, source_ids, source_type) # Generate multiple queries for broader recall queries = _expand_query(query) @@ -228,7 +232,7 @@ def similarity_search( dists = (results.get("distances") or [[]])[0] ids = (results.get("ids") or [[]])[0] for doc, dist, doc_id in zip(docs, dists, ids): - if doc_id not in seen_ids and dist < 0.65: + if doc_id not in seen_ids and dist < 0.80: seen_ids.add(doc_id) ranked.append((dist, doc_id, doc)) except Exception: