feat: Add knowledge base with document ingestion and file upload support

backend/api/ingestion/document_ingestion.py

@@ -0,0 +1,186 @@
+"""
+Document Ingestion Service
+
+Handles ingestion of various document types (PDF, DOCX, TXT, URL, raw_text)
+with metadata support and automatic type detection.
+"""
+
+import os
+import re
+import logging
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse
+import httpx
+
+logger = logging.getLogger("document_ingestion")
+
+
+def detect_source_type(content: str, filename: Optional[str] = None, url: Optional[str] = None) -> str:
+    """
+    Detect the source type from content, filename, or URL.
+    Returns: 'pdf', 'docx', 'txt', 'url', 'raw_text', 'markdown'
+    """
+    if url:
+        return "url"
+    
+    if filename:
+        ext = filename.lower().split('.')[-1] if '.' in filename else ''
+        if ext in ['pdf']:
+            return 'pdf'
+        elif ext in ['docx', 'doc']:
+            return 'docx'
+        elif ext in ['txt', 'text']:
+            return 'txt'
+        elif ext in ['md', 'markdown']:
+            return 'markdown'
+    
+    # Heuristic detection from content
+    content_lower = content.lower()
+    if 'http://' in content_lower or 'https://' in content_lower or 'www.' in content_lower:
+        return 'url'
+    
+    return 'raw_text'
+
+
+async def extract_text_from_url(url: str, timeout: int = 30) -> str:
+    """
+    Fetch and extract text content from a URL (async).
+    """
+    try:
+        async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+            response = await client.get(url)
+            response.raise_for_status()
+            
+            # Basic HTML stripping (for simple pages)
+            text = response.text
+            # Remove script and style tags
+            text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
+            text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL | re.IGNORECASE)
+            # Remove HTML tags
+            text = re.sub(r'<[^>]+>', ' ', text)
+            # Normalize whitespace
+            text = re.sub(r'\s+', ' ', text).strip()
+            
+            return text
+    except Exception as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        raise ValueError(f"Failed to fetch URL: {str(e)}")
+
+
+def normalize_text(text: str) -> str:
+    """
+    Sanitize and normalize text before ingestion.
+    """
+    # Remove excessive whitespace
+    text = re.sub(r'\s+', ' ', text)
+    # Remove control characters except newlines and tabs
+    text = re.sub(r'[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]', '', text)
+    # Strip leading/trailing whitespace
+    text = text.strip()
+    return text
+
+
+async def prepare_ingestion_payload(
+    tenant_id: str,
+    content: str,
+    source_type: Optional[str] = None,
+    filename: Optional[str] = None,
+    url: Optional[str] = None,
+    doc_id: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Prepare ingestion payload according to the system prompt specification.
+    
+    Returns:
+        {
+            "action": "ingest_document",
+            "tenant_id": "...",
+            "source_type": "pdf | docx | txt | url | raw_text",
+            "content": "...",
+            "metadata": {
+                "filename": "...",
+                "url": "...",
+                "doc_id": "..."
+            }
+        }
+    """
+    # Auto-detect source type if not provided
+    if not source_type:
+        source_type = detect_source_type(content, filename, url)
+    
+    # Handle URL: fetch content (async)
+    if source_type == "url" and url:
+        try:
+            content = await extract_text_from_url(url)
+        except Exception as e:
+            logger.warning(f"URL fetch failed, using provided content: {e}")
+    
+    # Normalize content
+    content = normalize_text(content)
+    
+    if not content:
+        raise ValueError("Content is empty after normalization")
+    
+    # Generate doc_id if not provided
+    if not doc_id:
+        if filename:
+            doc_id = filename
+        elif url:
+            parsed = urlparse(url)
+            doc_id = f"{parsed.netloc}{parsed.path}".replace('/', '_')[:100]
+        else:
+            import hashlib
+            doc_id = hashlib.md5(content.encode()).hexdigest()[:16]
+    
+    # Build metadata
+    ingestion_metadata = {
+        "doc_id": doc_id,
+        **(metadata or {})
+    }
+    
+    if filename:
+        ingestion_metadata["filename"] = filename
+    if url:
+        ingestion_metadata["url"] = url
+    
+    return {
+        "action": "ingest_document",
+        "tenant_id": tenant_id,
+        "source_type": source_type,
+        "content": content,
+        "metadata": ingestion_metadata
+    }
+
+
+async def process_ingestion(
+    payload: Dict[str, Any],
+    rag_client
+) -> Dict[str, Any]:
+    """
+    Process the ingestion payload by sending it to the RAG MCP server.
+    
+    Args:
+        payload: The ingestion payload from prepare_ingestion_payload
+        rag_client: RAGClient instance
+    
+    Returns:
+        Result from RAG ingestion
+    """
+    tenant_id = payload["tenant_id"]
+    content = payload["content"]
+    
+    # Send to RAG MCP server
+    result = await rag_client.ingest(content, tenant_id)
+    
+    # Enhance result with metadata
+    return {
+        "status": "ok",
+        "tenant_id": tenant_id,
+        "source_type": payload["source_type"],
+        "doc_id": payload["metadata"].get("doc_id"),
+        "chunks_stored": result.get("chunks_stored", 0),
+        "metadata": payload["metadata"],
+        **result
+    }
+

backend/api/routes/rag.py

@@ -1,15 +1,33 @@
-from fastapi import APIRouter, Header, HTTPException
+from fastapi import APIRouter, Header, HTTPException, UploadFile, File, Form
 from pydantic import BaseModel
+from typing import Optional, Dict, Any
 from api.mcp_clients.rag_client import RAGClient
+from api.services.document_ingestion import (
+    prepare_ingestion_payload,
+    process_ingestion,
+    detect_source_type,
+    normalize_text,
+    extract_text_from_file_bytes
+)
 
 router = APIRouter()
 rag_client = RAGClient()
 
 
 class IngestRequest(BaseModel):
+    """Legacy simple ingestion request"""
     content: str
 
 
+class DocumentIngestRequest(BaseModel):
+    """Enhanced ingestion request matching the system prompt specification"""
+    action: str = "ingest_document"
+    tenant_id: Optional[str] = None  # Can come from header
+    source_type: Optional[str] = None  # pdf | docx | txt | url | raw_text | markdown
+    content: str
+    metadata: Optional[Dict[str, Any]] = None
+
+
 class SearchRequest(BaseModel):
     query: str
 
@@ -43,6 +61,7 @@ async def rag_ingest(
     x_tenant_id: str = Header(None)
 ):
     """
+    Legacy ingestion endpoint - simple content ingestion.
     Ingest content into tenant knowledge base using the RAG MCP server.
     """
 
@@ -60,6 +79,129 @@ async def rag_ingest(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@router.post("/ingest-document")
+async def rag_ingest_document(
+    req: DocumentIngestRequest,
+    x_tenant_id: Optional[str] = Header(None)
+):
+    """
+    Enhanced document ingestion endpoint matching the system prompt specification.
+    
+    Supports:
+    - PDF, DOCX, TXT, Markdown files
+    - URLs (fetches content automatically)
+    - Raw text
+    - Metadata (filename, url, doc_id)
+    
+    Expected payload:
+    {
+        "action": "ingest_document",
+        "tenant_id": "...",
+        "source_type": "pdf | docx | txt | url | raw_text",
+        "content": "...",
+        "metadata": {
+            "filename": "...",
+            "url": "...",
+            "doc_id": "..."
+        }
+    }
+    """
+    # Use tenant_id from header if not in body (for backward compatibility)
+    tenant_id = req.tenant_id or x_tenant_id
+    if not tenant_id:
+        raise HTTPException(status_code=400, detail="Missing tenant ID")
+
+    try:
+        # Prepare ingestion payload (async for URL fetching)
+        payload = await prepare_ingestion_payload(
+            tenant_id=tenant_id,
+            content=req.content,
+            source_type=req.source_type,
+            filename=req.metadata.get("filename") if req.metadata else None,
+            url=req.metadata.get("url") if req.metadata else None,
+            doc_id=req.metadata.get("doc_id") if req.metadata else None,
+            metadata=req.metadata
+        )
+        
+        # Process ingestion
+        result = await process_ingestion(payload, rag_client)
+        
+        return {
+            "status": "ok",
+            "message": f"Document ingested successfully. {result.get('chunks_stored', 0)} chunk(s) stored.",
+            **result
+        }
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/ingest-file")
+async def rag_ingest_file(
+    file: UploadFile = File(...),
+    x_tenant_id: Optional[str] = Header(None),
+    tenant_id: Optional[str] = Form(None)
+):
+    """
+    File upload endpoint for binary files (PDF, DOCX, TXT, MD).
+    Extracts text server-side and ingests into knowledge base.
+    
+    Usage:
+        POST /rag/ingest-file
+        Headers:
+            x-tenant-id: <tenant_id>
+        Form Data:
+            file: <binary file>
+            tenant_id: <optional, can use header instead>
+    """
+    # Use tenant_id from form or header
+    tenant_id_value = tenant_id or x_tenant_id
+    if not tenant_id_value:
+        raise HTTPException(status_code=400, detail="Missing tenant ID")
+
+    try:
+        # Read file bytes
+        file_bytes = await file.read()
+        if not file_bytes:
+            raise HTTPException(status_code=400, detail="File is empty")
+
+        # Extract text from binary file
+        try:
+            extracted_text = extract_text_from_file_bytes(file_bytes, file.filename or "unknown")
+        except ValueError as e:
+            raise HTTPException(status_code=400, detail=str(e))
+
+        if not extracted_text or not extracted_text.strip():
+            raise HTTPException(status_code=400, detail="No text could be extracted from file")
+
+        # Prepare ingestion payload
+        payload = await prepare_ingestion_payload(
+            tenant_id=tenant_id_value,
+            content=extracted_text,
+            source_type=None,  # Auto-detect from filename
+            filename=file.filename,
+            url=None,
+            doc_id=None,
+            metadata=None
+        )
+        
+        # Process ingestion
+        result = await process_ingestion(payload, rag_client)
+        
+        return {
+            "status": "ok",
+            "message": f"File '{file.filename}' ingested successfully. {result.get('chunks_stored', 0)} chunk(s) stored.",
+            **result
+        }
+    except HTTPException:
+        raise
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
 @router.get("/list")
 async def rag_list(
     limit: int = 1000,

backend/api/services/INGESTION_SYSTEM.md

@@ -0,0 +1,154 @@
+# Document Ingestion System
+
+## Overview
+
+The backend now supports a comprehensive document ingestion system that matches the system prompt specification. This allows AI agents to automatically detect and ingest various document types (PDF, DOCX, TXT, URLs, raw text) with full metadata support.
+
+## Endpoints
+
+### 1. Legacy Endpoint (Backward Compatible)
+```
+POST /rag/ingest
+Headers:
+  x-tenant-id: <tenant_id>
+Body:
+  {
+    "content": "text content to ingest"
+  }
+```
+
+### 2. Enhanced Document Ingestion Endpoint
+```
+POST /rag/ingest-document
+Headers:
+  x-tenant-id: <tenant_id> (optional if in body)
+Body:
+  {
+    "action": "ingest_document",
+    "tenant_id": "<tenant_id>",  // Optional if in header
+    "source_type": "pdf | docx | txt | url | raw_text | markdown",  // Auto-detected if not provided
+    "content": "text content or URL",
+    "metadata": {
+      "filename": "document.pdf",
+      "url": "https://example.com/doc",
+      "doc_id": "unique-document-id"
+    }
+  }
+```
+
+## Features
+
+### Automatic Source Type Detection
+- **PDF**: Detected from `.pdf` extension or filename
+- **DOCX**: Detected from `.docx` or `.doc` extension
+- **TXT**: Detected from `.txt` or `.text` extension
+- **Markdown**: Detected from `.md` or `.markdown` extension
+- **URL**: Detected from URL in content or metadata
+- **Raw Text**: Default fallback for plain text
+
+### URL Processing
+- Automatically fetches content from URLs
+- Strips HTML tags and scripts
+- Normalizes whitespace
+- Handles redirects and timeouts
+
+### Text Normalization
+- Removes excessive whitespace
+- Strips control characters
+- Sanitizes input before ingestion
+
+### Metadata Support
+- `filename`: Original filename
+- `url`: Source URL
+- `doc_id`: Unique document identifier (auto-generated if not provided)
+- Custom metadata can be added to the metadata object
+
+## Usage Examples
+
+### Example 1: Ingest Raw Text
+```json
+{
+  "action": "ingest_document",
+  "tenant_id": "tenant123",
+  "source_type": "raw_text",
+  "content": "This is a company policy document...",
+  "metadata": {
+    "filename": "policy.txt",
+    "doc_id": "policy-2024-01"
+  }
+}
+```
+
+### Example 2: Ingest from URL
+```json
+{
+  "action": "ingest_document",
+  "tenant_id": "tenant123",
+  "source_type": "url",
+  "content": "https://example.com/documentation",
+  "metadata": {
+    "url": "https://example.com/documentation",
+    "doc_id": "docs-example-com"
+  }
+}
+```
+
+### Example 3: Ingest PDF (with extracted text)
+```json
+{
+  "action": "ingest_document",
+  "tenant_id": "tenant123",
+  "source_type": "pdf",
+  "content": "<extracted PDF text>",
+  "metadata": {
+    "filename": "manual.pdf",
+    "doc_id": "manual-2024"
+  }
+}
+```
+
+## Response Format
+
+```json
+{
+  "status": "ok",
+  "message": "Document ingested successfully. 5 chunk(s) stored.",
+  "tenant_id": "tenant123",
+  "source_type": "raw_text",
+  "doc_id": "policy-2024-01",
+  "chunks_stored": 5,
+  "metadata": {
+    "filename": "policy.txt",
+    "doc_id": "policy-2024-01"
+  }
+}
+```
+
+## Integration with AI Agents
+
+The system is designed to work with AI agents that follow the system prompt specification:
+
+1. **Agent detects** document/URL/pasted content
+2. **Agent prepares** ingestion payload with proper structure
+3. **Agent sends** to `POST /rag/ingest-document`
+4. **Backend processes**:
+   - Detects/validates source type
+   - Fetches URL content if needed
+   - Normalizes text
+   - Sends to RAG MCP server for chunking/embedding
+   - Stores in pgvector
+5. **Agent confirms** ingestion to user
+
+## Error Handling
+
+- **400 Bad Request**: Missing tenant_id, invalid payload, empty content
+- **500 Internal Server Error**: RAG MCP server error, database error, URL fetch failure
+
+## Notes
+
+- The legacy `/rag/ingest` endpoint remains for backward compatibility
+- Source type is auto-detected if not provided
+- URL fetching is async and handles timeouts gracefully
+- All content is normalized before ingestion
+- Metadata is preserved and stored with chunks
+

backend/api/services/agent_orchestrator.py

@@ -202,7 +202,18 @@ class AgentOrchestrator:
                 try:
                     fallback = await self.llm.simple_call(req.message, temperature=req.temperature)
                 except Exception as llm_error:
-                    fallback = f"I encountered an error while processing your request: {str(e)}. Additionally, the AI service is unavailable: {str(llm_error)}"
+                    error_msg = str(llm_error)
+                    if "Cannot connect" in error_msg or "Ollama" in error_msg:
+                        fallback = (
+                            f"I encountered an error while processing your request: {str(e)}\n\n"
+                            f"Additionally, the AI service (Ollama) is unavailable: {error_msg}\n\n"
+                            f"To fix:\n"
+                            f"1. Install Ollama from https://ollama.ai\n"
+                            f"2. Start: `ollama serve`\n"
+                            f"3. Pull model: `ollama pull {os.getenv('OLLAMA_MODEL', 'llama3.1:latest')}`"
+                        )
+                    else:
+                        fallback = f"I encountered an error while processing your request: {str(e)}. Additionally, the AI service is unavailable: {error_msg}"
                 return AgentResponse(
                     text=fallback,
                     decision=AgentDecision(action="respond", tool=None, tool_input=None, reason=f"tool_error_fallback: {e}"),
@@ -219,7 +230,19 @@ class AgentOrchestrator:
             llm_out = await self.llm.simple_call(req.message, temperature=req.temperature)
         except Exception as e:
             # If LLM fails, return a helpful error message
-            llm_out = f"I apologize, but I'm unable to process your request right now. The AI service is unavailable: {str(e)}"
+            error_msg = str(e)
+            if "Cannot connect" in error_msg or "Ollama" in error_msg:
+                llm_out = (
+                    f"I couldn't connect to the AI service (Ollama). "
+                    f"Error: {error_msg}\n\n"
+                    f"To fix this:\n"
+                    f"1. Install Ollama from https://ollama.ai\n"
+                    f"2. Start Ollama: `ollama serve`\n"
+                    f"3. Pull the model: `ollama pull {os.getenv('OLLAMA_MODEL', 'llama3.1:latest')}`\n"
+                    f"4. Or set OLLAMA_URL and OLLAMA_MODEL in your .env file"
+                )
+            else:
+                llm_out = f"I apologize, but I'm unable to process your request right now. The AI service is unavailable: {error_msg}"
             reasoning_trace.append({
                 "step": "error",
                 "tool": "llm",
@@ -377,7 +400,20 @@ class AgentOrchestrator:
             )
         except Exception as e:
             tool_traces.append({"tool": "llm", "error": str(e)})
-            fallback = f"I encountered an error while synthesizing the response: {str(e)}"
+            error_msg = str(e)
+            # Provide helpful error message
+            if "Cannot connect" in error_msg or "Ollama" in error_msg:
+                fallback = (
+                    f"I couldn't connect to the AI service (Ollama). "
+                    f"Error: {error_msg}\n\n"
+                    f"To fix this:\n"
+                    f"1. Install Ollama from https://ollama.ai\n"
+                    f"2. Start Ollama: `ollama serve`\n"
+                    f"3. Pull the model: `ollama pull {os.getenv('OLLAMA_MODEL', 'llama3.1:latest')}`\n"
+                    f"4. Or set OLLAMA_URL and OLLAMA_MODEL in your .env file"
+                )
+            else:
+                fallback = f"I encountered an error while synthesizing the response: {error_msg}"
             return AgentResponse(
                 text=fallback,
                 decision=AgentDecision(

backend/api/services/document_ingestion.py

@@ -0,0 +1,247 @@
+"""
+Document Ingestion Service
+
+Handles ingestion of various document types (PDF, DOCX, TXT, URL, raw_text)
+with metadata support and automatic type detection.
+"""
+
+import os
+import re
+import logging
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse
+import httpx
+from io import BytesIO
+
+logger = logging.getLogger("document_ingestion")
+
+
+def detect_source_type(content: str, filename: Optional[str] = None, url: Optional[str] = None) -> str:
+    """
+    Detect the source type from content, filename, or URL.
+    Returns: 'pdf', 'docx', 'txt', 'url', 'raw_text', 'markdown'
+    """
+    if url:
+        return "url"
+    
+    if filename:
+        ext = filename.lower().split('.')[-1] if '.' in filename else ''
+        if ext in ['pdf']:
+            return 'pdf'
+        elif ext in ['docx', 'doc']:
+            return 'docx'
+        elif ext in ['txt', 'text']:
+            return 'txt'
+        elif ext in ['md', 'markdown']:
+            return 'markdown'
+    
+    # Heuristic detection from content
+    content_lower = content.lower()
+    if 'http://' in content_lower or 'https://' in content_lower or 'www.' in content_lower:
+        return 'url'
+    
+    return 'raw_text'
+
+
+async def extract_text_from_url(url: str, timeout: int = 30) -> str:
+    """
+    Fetch and extract text content from a URL (async).
+    """
+    try:
+        async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+            response = await client.get(url)
+            response.raise_for_status()
+            
+            # Basic HTML stripping (for simple pages)
+            text = response.text
+            # Remove script and style tags
+            text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
+            text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL | re.IGNORECASE)
+            # Remove HTML tags
+            text = re.sub(r'<[^>]+>', ' ', text)
+            # Normalize whitespace
+            text = re.sub(r'\s+', ' ', text).strip()
+            
+            return text
+    except Exception as e:
+        logger.error(f"Failed to fetch URL {url}: {e}")
+        raise ValueError(f"Failed to fetch URL: {str(e)}")
+
+
+def extract_text_from_file_bytes(file_bytes: bytes, filename: str) -> str:
+    """
+    Extract text from binary file data (PDF, DOCX, etc.).
+    
+    Args:
+        file_bytes: Binary file content
+        filename: Original filename (for type detection)
+    
+    Returns:
+        Extracted text content
+    """
+    ext = filename.lower().split('.')[-1] if '.' in filename else ''
+    
+    # PDF extraction
+    if ext == 'pdf':
+        try:
+            import PyPDF2
+            pdf_file = BytesIO(file_bytes)
+            pdf_reader = PyPDF2.PdfReader(pdf_file)
+            text_parts = []
+            for page in pdf_reader.pages:
+                text_parts.append(page.extract_text())
+            return '\n'.join(text_parts)
+        except ImportError:
+            logger.warning("PyPDF2 not installed, cannot extract PDF text")
+            raise ValueError("PDF extraction requires PyPDF2. Install with: pip install PyPDF2")
+        except Exception as e:
+            logger.error(f"PDF extraction failed: {e}")
+            raise ValueError(f"Failed to extract text from PDF: {str(e)}")
+    
+    # DOCX extraction
+    elif ext in ['docx', 'doc']:
+        try:
+            from docx import Document
+            doc_file = BytesIO(file_bytes)
+            doc = Document(doc_file)
+            return '\n'.join(paragraph.text for paragraph in doc.paragraphs)
+        except ImportError:
+            logger.warning("python-docx not installed, cannot extract DOCX text")
+            raise ValueError("DOCX extraction requires python-docx. Install with: pip install python-docx")
+        except Exception as e:
+            logger.error(f"DOCX extraction failed: {e}")
+            raise ValueError(f"Failed to extract text from DOCX: {str(e)}")
+    
+    # Text files (TXT, MD)
+    elif ext in ['txt', 'md', 'markdown', 'text']:
+        try:
+            return file_bytes.decode('utf-8', errors='ignore')
+        except Exception as e:
+            logger.error(f"Text file decoding failed: {e}")
+            raise ValueError(f"Failed to decode text file: {str(e)}")
+    
+    else:
+        # Try to decode as UTF-8 text as fallback
+        try:
+            return file_bytes.decode('utf-8', errors='ignore')
+        except Exception:
+            raise ValueError(f"Unsupported file type: {ext}. Supported: pdf, docx, txt, md")
+
+
+def normalize_text(text: str) -> str:
+    """
+    Sanitize and normalize text before ingestion.
+    """
+    # Remove excessive whitespace
+    text = re.sub(r'\s+', ' ', text)
+    # Remove control characters except newlines and tabs
+    text = re.sub(r'[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]', '', text)
+    # Strip leading/trailing whitespace
+    text = text.strip()
+    return text
+
+
+async def prepare_ingestion_payload(
+    tenant_id: str,
+    content: str,
+    source_type: Optional[str] = None,
+    filename: Optional[str] = None,
+    url: Optional[str] = None,
+    doc_id: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    """
+    Prepare ingestion payload according to the system prompt specification.
+    
+    Returns:
+        {
+            "action": "ingest_document",
+            "tenant_id": "...",
+            "source_type": "pdf | docx | txt | url | raw_text",
+            "content": "...",
+            "metadata": {
+                "filename": "...",
+                "url": "...",
+                "doc_id": "..."
+            }
+        }
+    """
+    # Auto-detect source type if not provided
+    if not source_type:
+        source_type = detect_source_type(content, filename, url)
+    
+    # Handle URL: fetch content (async)
+    if source_type == "url" and url:
+        try:
+            content = await extract_text_from_url(url)
+        except Exception as e:
+            logger.warning(f"URL fetch failed, using provided content: {e}")
+    
+    # Normalize content
+    content = normalize_text(content)
+    
+    if not content:
+        raise ValueError("Content is empty after normalization")
+    
+    # Generate doc_id if not provided
+    if not doc_id:
+        if filename:
+            doc_id = filename
+        elif url:
+            parsed = urlparse(url)
+            doc_id = f"{parsed.netloc}{parsed.path}".replace('/', '_')[:100]
+        else:
+            import hashlib
+            doc_id = hashlib.md5(content.encode()).hexdigest()[:16]
+    
+    # Build metadata
+    ingestion_metadata = {
+        "doc_id": doc_id,
+        **(metadata or {})
+    }
+    
+    if filename:
+        ingestion_metadata["filename"] = filename
+    if url:
+        ingestion_metadata["url"] = url
+    
+    return {
+        "action": "ingest_document",
+        "tenant_id": tenant_id,
+        "source_type": source_type,
+        "content": content,
+        "metadata": ingestion_metadata
+    }
+
+
+async def process_ingestion(
+    payload: Dict[str, Any],
+    rag_client
+) -> Dict[str, Any]:
+    """
+    Process the ingestion payload by sending it to the RAG MCP server.
+    
+    Args:
+        payload: The ingestion payload from prepare_ingestion_payload
+        rag_client: RAGClient instance
+    
+    Returns:
+        Result from RAG ingestion
+    """
+    tenant_id = payload["tenant_id"]
+    content = payload["content"]
+    
+    # Send to RAG MCP server
+    result = await rag_client.ingest(content, tenant_id)
+    
+    # Enhance result with metadata
+    return {
+        "status": "ok",
+        "tenant_id": tenant_id,
+        "source_type": payload["source_type"],
+        "doc_id": payload["metadata"].get("doc_id"),
+        "chunks_stored": result.get("chunks_stored", 0),
+        "metadata": payload["metadata"],
+        **result
+    }
+

createingdummydata.py

@@ -0,0 +1,44 @@
+from docx import Document
+
+# Dummy data
+data = {
+    "Day": ["Day 1", "Day 2", "Day 3", "Day 4", "Day 5"],
+    "Breakfast": [
+        "Oatmeal with sliced bananas and honey",
+        "Scrambled eggs with toast and orange juice",
+        "Greek yogurt with granola and berries",
+        "Pancakes with maple syrup and strawberries",
+        "Smoothie (spinach, banana, yogurt, almond milk)"
+    ],
+    "Lunch": [
+        "Grilled chicken salad with mixed greens and vinaigrette",
+        "Turkey sandwich with lettuce, tomato, and chips",
+        "Vegetable soup with whole-grain roll",
+        "Tuna salad wrap with carrot sticks",
+        "Caesar salad with grilled shrimp"
+    ],
+    "Dinner": [
+        "Spaghetti with marinara sauce and garlic bread",
+        "Baked salmon with steamed broccoli and rice",
+        "Beef stir-fry with mixed vegetables and noodles",
+        "Chicken curry with basmati rice",
+        "Veggie pizza with side salad"
+    ]
+}
+
+# Create DOCX document
+doc = Document()
+doc.add_heading("5-Day Meal Plan", level=1)
+
+for i in range(5):
+    doc.add_heading(data["Day"][i], level=2)
+    doc.add_paragraph(f"Breakfast: {data['Breakfast'][i]}")
+    doc.add_paragraph(f"Lunch: {data['Lunch'][i]}")
+    doc.add_paragraph(f"Dinner: {data['Dinner'][i]}")
+    doc.add_paragraph("")
+
+# Save file
+path = "5_day_meal_plan.docx"
+doc.save(path)
+
+print(f"Saved DOCX file to: {path}")

frontend/components/knowledge-base-panel.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useState } from "react";
+import { useState, useRef } from "react";
 import Link from "next/link";
 
 type SearchResult = {
@@ -9,6 +9,8 @@ type SearchResult = {
   relevance?: number;
 };
 
+type SourceType = "raw_text" | "url" | "pdf" | "docx" | "txt" | "markdown";
+
 const API_BASE =
   process.env.NEXT_PUBLIC_API_URL?.replace(/\/$/, "") || "http://localhost:8000";
 
@@ -18,9 +20,13 @@ export function KnowledgeBasePanel() {
   const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
   const [isSearching, setIsSearching] = useState(false);
   const [ingestContent, setIngestContent] = useState("");
+  const [sourceType, setSourceType] = useState<SourceType>("raw_text");
+  const [filename, setFilename] = useState("");
+  const [url, setUrl] = useState("");
   const [isIngesting, setIsIngesting] = useState(false);
   const [ingestStatus, setIngestStatus] = useState<string | null>(null);
   const [searchError, setSearchError] = useState<string | null>(null);
+  const fileInputRef = useRef<HTMLInputElement>(null);
 
   async function handleSearch() {
     if (!searchQuery.trim() || isSearching) return;
@@ -56,30 +62,130 @@ export function KnowledgeBasePanel() {
     }
   }
 
+  async function handleFileUpload(event: React.ChangeEvent<HTMLInputElement>) {
+    const file = event.target.files?.[0];
+    if (!file) return;
+
+    // Detect file type from extension
+    const ext = file.name.split('.').pop()?.toLowerCase();
+    let detectedType: SourceType = "raw_text";
+    if (ext === "pdf") detectedType = "pdf";
+    else if (ext === "docx" || ext === "doc") detectedType = "docx";
+    else if (ext === "txt" || ext === "text") detectedType = "txt";
+    else if (ext === "md" || ext === "markdown") detectedType = "markdown";
+
+    setSourceType(detectedType);
+    setFilename(file.name);
+
+    // For binary files (PDF, DOCX), upload directly to server
+    if (detectedType === "pdf" || detectedType === "docx") {
+      await handleFileIngest(file);
+      return;
+    }
+
+    // For text files, read and show in textarea
+    const reader = new FileReader();
+    reader.onload = async (e) => {
+      const text = e.target?.result as string;
+      setIngestContent(text);
+    };
+    reader.readAsText(file);
+  }
+
+  async function handleFileIngest(file: File) {
+    setIsIngesting(true);
+    setIngestStatus(null);
+
+    try {
+      const formData = new FormData();
+      formData.append("file", file);
+
+      const response = await fetch(`${API_BASE}/rag/ingest-file`, {
+        method: "POST",
+        headers: {
+          "x-tenant-id": tenantId,
+        },
+        body: formData,
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json().catch(() => ({}));
+        throw new Error(
+          errorData.detail || `File ingestion failed: ${response.status}`,
+        );
+      }
+
+      const data = await response.json();
+      setIngestStatus(
+        `✅ ${data.message || `Successfully ingested ${data.chunks_stored || 0} chunk(s)`}`,
+      );
+      setFilename("");
+      if (fileInputRef.current) {
+        fileInputRef.current.value = "";
+      }
+    } catch (err) {
+      console.error(err);
+      setIngestStatus(
+        err instanceof Error
+          ? `❌ Error: ${err.message}`
+          : "Failed to ingest file. Is the RAG MCP server running?",
+      );
+    } finally {
+      setIsIngesting(false);
+    }
+  }
+
   async function handleIngest() {
     if (!ingestContent.trim() || isIngesting) return;
     setIsIngesting(true);
     setIngestStatus(null);
 
     try {
-      const response = await fetch(`${API_BASE}/rag/ingest`, {
+      // Prepare metadata
+      const metadata: Record<string, string> = {};
+      if (filename) metadata.filename = filename;
+      if (url || sourceType === "url") {
+        const ingestUrl = url || ingestContent.trim();
+        metadata.url = ingestUrl;
+      }
+      if (filename) {
+        // Generate doc_id from filename
+        metadata.doc_id = filename.replace(/[^a-zA-Z0-9]/g, "_").toLowerCase();
+      }
+
+      // Use the new enhanced endpoint
+      const response = await fetch(`${API_BASE}/rag/ingest-document`, {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
           "x-tenant-id": tenantId,
         },
-        body: JSON.stringify({ content: ingestContent }),
+        body: JSON.stringify({
+          action: "ingest_document",
+          tenant_id: tenantId,
+          source_type: sourceType,
+          content: sourceType === "url" ? (url || ingestContent.trim()) : ingestContent,
+          metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+        }),
       });
 
       if (!response.ok) {
-        throw new Error(`Ingestion failed: ${response.status}`);
+        const errorData = await response.json().catch(() => ({}));
+        throw new Error(
+          errorData.detail || `Ingestion failed: ${response.status}`,
+        );
       }
 
       const data = await response.json();
       setIngestStatus(
-        `✅ Successfully ingested ${data.chunks_stored || 0} chunk(s)`,
+        `✅ ${data.message || `Successfully ingested ${data.chunks_stored || 0} chunk(s)`}`,
       );
       setIngestContent("");
+      setFilename("");
+      setUrl("");
+      if (fileInputRef.current) {
+        fileInputRef.current.value = "";
+      }
     } catch (err) {
       console.error(err);
       setIngestStatus(
@@ -186,23 +292,95 @@ export function KnowledgeBasePanel() {
             Add to Knowledge Base
           </p>
           <p className="mt-2 text-sm text-slate-300">
-            Paste text content to ingest. It will be chunked, embedded, and
-            stored in your tenant's knowledge base.
+            Upload files (PDF, DOCX, TXT, MD), paste text, or provide URLs. Content will be chunked, embedded, and stored.
           </p>
+
+          {/* Source Type Selector */}
+          <div className="mt-4 flex flex-wrap gap-2">
+            {(["raw_text", "url", "pdf", "docx", "txt", "markdown"] as SourceType[]).map((type) => (
+              <button
+                key={type}
+                onClick={() => {
+                  setSourceType(type);
+                  if (type !== "url") setUrl("");
+                }}
+                className={`rounded-full px-4 py-2 text-xs font-semibold uppercase tracking-wider transition ${
+                  sourceType === type
+                    ? "bg-cyan-500 text-slate-950"
+                    : "bg-white/5 text-slate-300 hover:bg-white/10"
+                }`}
+              >
+                {type.replace("_", " ")}
+              </button>
+            ))}
+          </div>
+
+          {/* File Upload */}
+          <div className="mt-4">
+            <input
+              ref={fileInputRef}
+              type="file"
+              accept=".pdf,.docx,.doc,.txt,.md,.markdown"
+              onChange={handleFileUpload}
+              className="hidden"
+              id="file-upload"
+            />
+            <label
+              htmlFor="file-upload"
+              className="inline-flex cursor-pointer items-center gap-2 rounded-full border border-white/10 bg-white/5 px-4 py-2 text-sm text-slate-300 transition hover:bg-white/10"
+            >
+              📄 Upload File (PDF, DOCX, TXT, MD)
+            </label>
+            {filename && (
+              <span className="ml-3 text-sm text-cyan-300">{filename}</span>
+            )}
+          </div>
+
+          {/* URL Input (when source type is URL) */}
+          {sourceType === "url" && (
+            <input
+              type="url"
+              placeholder="Enter URL to fetch content from..."
+              value={url}
+              onChange={(e) => setUrl(e.target.value)}
+              className="mt-4 w-full rounded-2xl border border-white/10 bg-white/5 px-4 py-3 text-sm text-white outline-none focus:border-cyan-200/80"
+            />
+          )}
+
+          {/* Content Textarea */}
           <textarea
-            placeholder="Paste document content here (e.g., policy text, procedures, documentation)..."
+            placeholder={
+              sourceType === "url"
+                ? "Or paste URL here..."
+                : "Paste document content here (e.g., policy text, procedures, documentation, FAQs)..."
+            }
             value={ingestContent}
             onChange={(e) => setIngestContent(e.target.value)}
             className="mt-4 w-full rounded-2xl border border-white/10 bg-white/5 px-4 py-3 text-sm text-white outline-none focus:border-cyan-200/80"
             rows={6}
           />
+
+          {/* Filename Input (optional) */}
+          {sourceType !== "url" && (
+            <input
+              type="text"
+              placeholder="Filename (optional, e.g., policy.pdf)"
+              value={filename}
+              onChange={(e) => setFilename(e.target.value)}
+              className="mt-3 w-full rounded-2xl border border-white/10 bg-white/5 px-4 py-3 text-sm text-white outline-none focus:border-cyan-200/80"
+            />
+          )}
+
           <div className="mt-4 flex items-center gap-3">
             <button
               onClick={handleIngest}
-              disabled={isIngesting || !ingestContent.trim()}
+              disabled={
+                isIngesting ||
+                (!ingestContent.trim() && !url.trim() && sourceType === "url")
+              }
               className="rounded-2xl bg-gradient-to-r from-emerald-400 to-teal-500 px-6 py-2.5 font-semibold text-slate-950 shadow-lg shadow-emerald-500/30 transition hover:-translate-y-0.5 disabled:cursor-not-allowed disabled:opacity-60"
             >
-              {isIngesting ? "Ingesting…" : "Ingest Content"}
+              {isIngesting ? "Ingesting…" : `Ingest as ${sourceType.replace("_", " ")}`}
             </button>
             {ingestStatus && (
               <p className="text-sm text-slate-300">{ingestStatus}</p>

requirements.txt

@@ -8,4 +8,7 @@ supabase
 sentence-transformers
 pytest
 pytest-asyncio
-duckduckgo-search
+duckduckgo-search
+PyPDF2
+python-docx
+python-multipart