Integrate training data: Use actual training data instead of mock responses for intelligent AI responses

app.py

@@ -9,13 +9,15 @@ import json
 import logging
 from pathlib import Path
 from datetime import datetime
-from typing import Optional, Dict, Any
+from typing import Optional, Dict, Any, List
 from fastapi import FastAPI, HTTPException, Request, BackgroundTasks
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 import uvicorn
 import requests
+import re
+from difflib import SequenceMatcher
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -80,14 +82,76 @@ training_status = {
     "error": None
 }
 
+class TrainingDataLoader:
+    """Load and manage training data for intelligent responses"""
+    
+    def __init__(self, data_path: str = "data/textilindo_training_data.jsonl"):
+        self.data_path = data_path
+        self.training_data = []
+        self.load_data()
+    
+    def load_data(self):
+        """Load training data from JSONL file"""
+        try:
+            if os.path.exists(self.data_path):
+                with open(self.data_path, 'r', encoding='utf-8') as f:
+                    for line in f:
+                        line = line.strip()
+                        if line:
+                            try:
+                                data = json.loads(line)
+                                self.training_data.append(data)
+                            except json.JSONDecodeError:
+                                continue
+                logger.info(f"Loaded {len(self.training_data)} training samples")
+            else:
+                logger.warning(f"Training data file not found: {self.data_path}")
+        except Exception as e:
+            logger.error(f"Error loading training data: {e}")
+    
+    def find_best_match(self, user_input: str, threshold: float = 0.3) -> Optional[Dict]:
+        """Find the best matching training sample for user input"""
+        if not self.training_data:
+            return None
+        
+        user_input_lower = user_input.lower().strip()
+        best_match = None
+        best_score = 0
+        
+        for data in self.training_data:
+            instruction = data.get('instruction', '').lower().strip()
+            if not instruction:
+                continue
+            
+            # Calculate similarity score
+            score = SequenceMatcher(None, user_input_lower, instruction).ratio()
+            
+            # Also check for keyword matches
+            user_words = set(user_input_lower.split())
+            instruction_words = set(instruction.split())
+            keyword_score = len(user_words.intersection(instruction_words)) / max(len(user_words), 1)
+            
+            # Combine scores
+            combined_score = (score * 0.7) + (keyword_score * 0.3)
+            
+            if combined_score > best_score and combined_score >= threshold:
+                best_score = combined_score
+                best_match = data
+        
+        if best_match:
+            logger.info(f"Found match with score {best_score:.2f}: {best_match.get('instruction', '')[:50]}...")
+        
+        return best_match
+
 class TextilindoAI:
     """Textilindo AI Assistant using HuggingFace Inference API"""
     
     def __init__(self):
         self.api_key = os.getenv('HUGGINGFACE_API_KEY')
         # Use a model available on free HuggingFace Inference API
-        self.model = os.getenv('DEFAULT_MODEL', 'microsoft/DialoGPT-small')  # Try DialoGPT-small
+        self.model = os.getenv('DEFAULT_MODEL', 'gpt2')  # Use GPT-2 which is available
         self.system_prompt = self.load_system_prompt()
+        self.data_loader = TrainingDataLoader()
         
         if not self.api_key:
             logger.warning("HUGGINGFACE_API_KEY not found. Using mock responses.")
@@ -138,13 +202,21 @@ The company uses yards for sales.
 Minimum purchase is 1 roll (67-70 yards)."""
     
     def generate_response(self, user_message: str) -> str:
-        """Generate response using HuggingFace Inference API"""
+        """Generate response using training data and HuggingFace Inference API"""
+        
+        # First, try to find a match in training data
+        training_match = self.data_loader.find_best_match(user_message)
+        if training_match:
+            logger.info("Using training data response")
+            return training_match.get('output', '')
+        
+        # If no training data match, try HuggingFace API if available
         if not self.client:
-            logger.warning("No HuggingFace client available, using mock response")
-            return self.get_mock_response(user_message)
+            logger.warning("No HuggingFace client available, using fallback response")
+            return self.get_fallback_response(user_message)
         
         try:
-            # Use DialoGPT conversation format
+            # Use GPT-2 conversation format
             prompt = f"User: {user_message}\nAssistant:"
             
             logger.info(f"Using model: {self.model}")
@@ -152,45 +224,55 @@ Minimum purchase is 1 roll (67-70 yards)."""
             
             logger.info(f"Generating response for prompt: {prompt[:100]}...")
             
-            # Generate response with DialoGPT parameters
+            # Generate response with GPT-2 parameters
             response = self.client.text_generation(
                 prompt,
-                max_new_tokens=200,
-                temperature=0.7,
+                max_new_tokens=150,
+                temperature=0.8,
                 top_p=0.9,
-                top_k=40,
-                repetition_penalty=1.1,
-                stop_sequences=["User:", "Assistant:"]
+                top_k=50,
+                repetition_penalty=1.2,
+                do_sample=True,
+                stop_sequences=["User:", "Assistant:", "\n\n"]
             )
             
             logger.info(f"Raw AI response: {response[:200]}...")
             
-            # Clean up the response for DialoGPT
+            # Clean up the response for GPT-2
             if "Assistant:" in response:
                 assistant_response = response.split("Assistant:")[-1].strip()
             else:
                 assistant_response = response.strip()
             
-            # Remove any remaining special tokens
+            # Remove any remaining special tokens and clean up
             assistant_response = assistant_response.replace("<|end|>", "").replace("<|user|>", "").strip()
             
+            # Remove any incomplete sentences or cut-off text
+            if assistant_response.endswith(('.', '!', '?')):
+                pass  # Complete sentence
+            elif '.' in assistant_response:
+                # Take only the first complete sentence
+                assistant_response = assistant_response.split('.')[0] + '.'
+            else:
+                # If no complete sentence, take first 100 characters
+                assistant_response = assistant_response[:100]
+            
             logger.info(f"Cleaned AI response: {assistant_response[:100]}...")
             
-            # If response is too short or generic, use mock response
+            # If response is too short or generic, use fallback
             if len(assistant_response) < 10 or "I don't know" in assistant_response.lower():
-                logger.warning("AI response too short, using mock response")
-                return self.get_mock_response(user_message)
-            
-            # For testing: if it's a non-Textilindo question, return the AI response directly
-            if not any(keyword in user_message.lower() for keyword in ['textilindo', 'lokasi', 'jam', 'katalog', 'produk', 'sample', 'pembelian', 'pembayaran', 'ongkir']):
-                logger.info("Non-Textilindo question detected, returning AI response directly")
-                return assistant_response
+                logger.warning("AI response too short, using fallback response")
+                return self.get_fallback_response(user_message)
                 
             return assistant_response
                 
         except Exception as e:
             logger.error(f"Error generating response: {e}")
-            return self.get_mock_response(user_message)
+            return self.get_fallback_response(user_message)
+    
+    def get_fallback_response(self, user_message: str) -> str:
+        """Fallback response when no training data match and no API available"""
+        return f"Halo! Saya adalah asisten AI Textilindo. Saya bisa membantu Anda dengan pertanyaan tentang produk dan layanan kami, atau sekadar mengobrol! Bagaimana saya bisa membantu Anda hari ini? 😊"
     
     def get_mock_response(self, user_message: str) -> str:
         """Enhanced mock responses with better context awareness"""