Update app.py

app.py

@@ -326,32 +326,23 @@ def search_budget_proposals(query: str) -> str:
         context_parts = []
         language_specific_matches = []
         
-        # Filter matches by language - prioritize documents in the user's language
+        # Filter matches to only include English documents
+        english_matches = []
         for match in matches:
             metadata = match.get('metadata', {})
             file_path = metadata.get('file_path', '')
             
-            # Check if this document has content in the detected language
-            is_language_match = False
-            if detected_language == 'en':
-                # For English, prefer documents without language suffixes
-                is_language_match = not any(lang in file_path.lower() for lang in ['_sin_', '_tam_', '-sin', '-tam', 'sinhala', 'tamil'])
-            elif detected_language == 'si':
-                # For Sinhala, prefer documents with Sinhala indicators
-                is_language_match = any(indicator in file_path.lower() for indicator in ['_sin_', '-sin', 'sinhala', 'si/', '- sinhala'])
-            elif detected_language == 'ta':
-                # For Tamil, prefer documents with Tamil indicators
-                is_language_match = any(indicator in file_path.lower() for indicator in ['_tam_', '-tam', 'tamil', 'ta/'])
+            # Only include English documents (no language suffixes)
+            is_english_document = not any(lang in file_path.lower() for lang in ['_sin_', '_tam_', '-sin', '-tam', 'sinhala', 'tamil', 'si/', 'ta/'])
             
-            if is_language_match:
-                language_specific_matches.append(match)
+            if is_english_document:
+                english_matches.append(match)
         
-        # If no language-specific matches found, use all matches (fallback)
-        if not language_specific_matches:
-            language_specific_matches = matches[:1]  # Take only the most relevant fallback
+        # Use English matches only, or fallback to top match if no English documents found
+        if english_matches:
+            language_specific_matches = english_matches[:1]  # Take only the most relevant English document
         else:
-            # Always return only the most relevant document in the user's language
-            language_specific_matches = language_specific_matches[:1]
+            language_specific_matches = matches[:1]  # Fallback to any document if no English found
         
         logger.info(f"Returning {len(language_specific_matches)} most relevant document(s) for {detected_language}")
         
@@ -670,47 +661,1314 @@ def extract_sources_from_search_context(search_context: str, user_language: str
             if pdf in search_context:
                 found_files.add(pdf)
     
-    # Filter by user language preference and prioritize by relevance
+    # Filter to return sources in the user's language, but prioritize by relevance
     language_filtered_files = []
     
-    # First, collect all language-appropriate documents
+    # First, try to find documents in the user's language
     for pdf in found_files:
-        # Determine document language from filename
         doc_language = get_document_language(pdf)
         
-        # Language matching logic
+        # Language matching logic - return sources in user's language
         should_include = False
         if user_language == 'en' or user_language == 'singlish':
-            # English users can see all documents, but prefer English versions
-            if doc_language in ['en', 'english'] or user_language == 'singlish':
+            # English users get English documents
+            if doc_language in ['en', 'english']:
                 should_include = True
-            elif not any(f for f in found_files if get_document_language(f) in ['en', 'english']):
-                # If no English version available, show other languages
+        elif user_language == 'si' or user_language == 'sinhala':
+            # Sinhala users get Sinhala documents
+            if doc_language in ['si', 'sinhala']:
+                should_include = True
+        elif user_language == 'ta' or user_language == 'tamil':
+            # Tamil users get Tamil documents
+            if doc_language in ['ta', 'tamil']:
+                should_include = True
+        else:
+            # Default: show English documents
+            if doc_language in ['en', 'english']:
+                should_include = True
+            
+        if should_include:
+            language_filtered_files.append(pdf)
+    
+    # If no language-specific documents found, fallback to English
+    if not language_filtered_files:
+        for pdf in found_files:
+            doc_language = get_document_language(pdf)
+            if doc_language in ['en', 'english']:
+                language_filtered_files.append(pdf)
+    
+    # If still no documents, use any available document
+    if not language_filtered_files and found_files:
+        language_filtered_files = [list(found_files)[0]]
+    
+    # Return only the most relevant document in the user's language
+    if language_filtered_files:
+        language_filtered_files = [language_filtered_files[0]]
+    
+    # Convert to list with short names and correct URLs
+    for pdf in language_filtered_files:
+        sources.append({
+            "filename": pdf,
+            "short_name": get_short_document_name(pdf),
+            "pdf_url": get_correct_pdf_url(pdf)
+        })
+    
+    return sources
+
+def get_document_language(filename: str) -> str:
+    """Determine the language of a document from its filename"""
+    filename_lower = filename.lower()
+    
+    if any(indicator in filename_lower for indicator in ['_sin_', '-sin', 'sinhala', 'si/', '- sinhala']):
+        return 'si'
+    elif any(indicator in filename_lower for indicator in ['_tam_', '-tam', 'tamil', 'ta/']):
+        return 'ta'
+    elif '_raj_' in filename_lower:
+        return 'en'  # Treat Raj as English/default
+    elif '_en_' in filename_lower or '_english_' in filename_lower:
+        return 'en'
+    else:
+        # Default to English if no language indicator found
+        return 'en'
+
+def get_correct_pdf_url(filename: str) -> str:
+    """Get the correct PDF URL based on document language"""
+    doc_language = get_document_language(filename)
+    
+    # Map language to directory
+    if doc_language == 'si':
+        return f"../si/assets/pdfs/{filename}"
+    elif doc_language == 'ta':
+        return f"../ta/assets/pdfs/{filename}"
+    else:
+        # English documents
+        return f"assets/pdfs/{filename}"
+
+def extract_sources_from_response(response: str) -> List[Dict[str, str]]:
+    """Extract source documents mentioned in the response with short names (fallback method)"""
+    sources = []
+    
+    # Get dynamically available PDF files
+    available_pdfs = get_available_pdfs()
+    
+    # Look for source patterns like "(Source: filename.pdf)" or "(Sources: file1.pdf, file2.pdf)"
+    # Also look for partial matches for the new budget proposal files
+    found_files = set()
+    for pdf in available_pdfs:
+        if pdf in response:
+            found_files.add(pdf)
+        # Also check for partial matches (e.g., "MaternityLeaveBenefit" matches the full filename)
+        elif any(keyword in response for keyword in pdf.split('_') if len(keyword) > 5):
+            found_files.add(pdf)
+    
+    # Convert to list with short names and correct URLs
+    for pdf in found_files:
+        sources.append({
+            "filename": pdf,
+            "short_name": get_short_document_name(pdf),
+            "pdf_url": get_correct_pdf_url(pdf)
+        })
+    
+    return sources
+
+def generate_response_with_rag(user_message: str, session_id: str) -> Dict[str, Any]:
+    """Generate response using RAG with memory and multilingual support"""
+    try:
+        # Process multilingual input
+        processed_message, original_language, needs_translation, transliteration_used, ai_detection_used, confidence = simple_process_input(user_message)
+        logger.info(f"Input processing: original='{user_message}', processed='{processed_message}', lang='{original_language}', transliteration='{transliteration_used}', ai_detection='{ai_detection_used}', confidence='{confidence:.2f}'")
+        
+        # Get or create memory for this session
+        memory = get_or_create_memory(session_id)
+        
+        # Let Gemini handle both specific and general questions intelligently
+        # Always search with the user's actual query - Gemini will handle vague questions
+        search_context = search_budget_proposals(processed_message)
+        
+        # Get conversation history for context
+        chat_history = memory.chat_memory.messages
+        conversation_context = ""
+        if chat_history:
+            # Get last few messages for context
+            recent_messages = chat_history[-6:]  # Last 3 exchanges
+            conversation_parts = []
+            for msg in recent_messages:
+                if isinstance(msg, HumanMessage):
+                    conversation_parts.append(f"User: {msg.content}")
+                elif isinstance(msg, AIMessage):
+                    conversation_parts.append(f"Assistant: {msg.content}")
+            conversation_context = "\n".join(conversation_parts)
+        
+        # Create a prompt with conversation history and retrieved context
+        language_instruction = ""
+        if original_language == 'si':
+            language_instruction = "\n\nIMPORTANT: The user asked in Sinhala. Please respond in the same language (Sinhala) using proper Sinhala script and formal language appropriate for policy discussions. The question was: '{}'".format(user_message)
+        elif original_language == 'ta':
+            language_instruction = "\n\nIMPORTANT: The user asked in Tamil. Please respond in the same language (Tamil) using proper Tamil script and formal language appropriate for policy discussions. Use Sri Lankan Tamil terminology and context. The question was: '{}'".format(user_message)
+        elif original_language == 'singlish':
+            language_instruction = "\n\nIMPORTANT: The user asked in Singlish (Romanized Sinhala - Sinhala words written in English letters). Please respond in proper Sinhala script using formal language appropriate for policy discussions. Translate their question and provide a comprehensive answer in Sinhala. The original question was: '{}'".format(user_message)
+        elif original_language == 'romanized_tamil':
+            language_instruction = "\n\nIMPORTANT: The user asked in Romanized Tamil (Tamil words written in English letters). Please respond in proper Tamil script using formal language appropriate for policy discussions. Use Sri Lankan Tamil terminology and context. Translate their question and provide a comprehensive answer in Tamil. The original question was: '{}'".format(user_message)
+        
+        prompt = f"""You are a helpful assistant for budget proposals in Sri Lanka. You can communicate in English, Sinhala, Tamil (Sri Lankan Tamil), and understand Singlish and Romanized Tamil.
+
+FORMATTING RULES:
+- DO NOT use asterisks (*) for formatting or emphasis
+- DO NOT use markdown formatting like **bold** or *italic*
+- Use plain text without any special formatting characters
+- Keep responses clean and readable without formatting symbols
+
+IMPORTANT: This website contains various budget proposals for Sri Lanka including:
+- Maternity leave benefits proposals (multiple language versions)
+- EPF (Employee Provident Fund) taxation removal proposals
+- Industrial land expansion proposals
+- Cigarette tax reform proposals  
+- Electricity tariff reforms
+- Tax policy changes
+- Economic growth initiatives
+- Social protection measures
+- Budget 2025 and 2026 proposals
+
+Based on the following information from the budget proposals database:
+
+{search_context}
+
+{conversation_context}
+
+Current user question: {processed_message}
+Original user input: {user_message}
+{language_instruction}
+
+Guidelines:
+- For general questions like "monada meh" (what is this), "help", or vague inquiries, provide a helpful overview of available budget proposals
+- Never say "I couldn't process your request" - always provide useful information about budget proposals  
+- Be professional but approachable in any language
+- Include specific details from the retrieved information when available
+- When mentioning proposals, refer to them by topic (e.g., "maternity leave benefits proposal", "EPF tax removal proposal") - DO NOT include long document filenames
+- If the search doesn't return relevant results, provide an overview of available proposals with examples
+- For vague questions, proactively explain what's available and guide users to specific topics (EPF, electricity, maternity leave, cigarette taxes, etc.)
+- Keep responses clear and informative
+- Reference previous conversation context when relevant
+- Maintain conversation continuity
+- Be culturally sensitive when discussing Sri Lankan policies
+- When responding in Sinhala, use appropriate formal language for policy discussions
+- When responding in Tamil, use Sri Lankan Tamil dialect and formal language appropriate for policy discussions
+- Always be helpful - turn any question into an opportunity to inform about budget proposals
+
+Please provide a helpful response:"""
+
+        # Generate response using the LLM directly
+        response = llm.invoke(prompt)
+        response_text = response.content.strip()
+        
+        # No need to translate response - Gemini handles language matching automatically
+        
+        # Extract sources from search context (where the actual filenames are)
+        sources = extract_sources_from_search_context(search_context, original_language)
+        
+        # Add messages to memory (store original user message for context)
+        memory.chat_memory.add_user_message(user_message)
+        memory.chat_memory.add_ai_message(response_text)
+        
+        # Get updated conversation history for context
+        chat_history = memory.chat_memory.messages
+        
+        return {
+            "response": response_text,
+            "confidence": "high",
+            "session_id": session_id,
+            "conversation_length": len(chat_history),
+            "memory_used": True,
+            "rag_used": True,
+            "sources": sources,
+            "language_detected": original_language,
+            "translation_used": needs_translation,
+            "transliteration_used": transliteration_used,
+            "ai_detection_used": ai_detection_used,
+            "detection_confidence": confidence
+        }
+        
+    except Exception as e:
+        logger.error(f"Error generating response with RAG: {e}")
+        # Provide error message in appropriate language
+        error_message = "I'm sorry, I'm having trouble processing your request right now. Please try again later."
+        
+        return {
+            "response": error_message,
+            "confidence": "error",
+            "session_id": session_id,
+            "memory_used": False,
+            "rag_used": False,
+            "sources": [],
+            "language_detected": original_language if 'original_language' in locals() else 'en',
+            "translation_used": False,
+            "transliteration_used": False,
+            "ai_detection_used": False,
+            "detection_confidence": 0.0
+        }
+
+def clear_session_memory(session_id: str) -> bool:
+    """Clear memory for a specific session"""
+    try:
+        if session_id in conversation_memories:
+            del conversation_memories[session_id]
+            logger.info(f"Cleared memory for session: {session_id}")
+            return True
+        return False
+    except Exception as e:
+        logger.error(f"Error clearing memory: {e}")
+        return False
+
+@app.route('/api/chat', methods=['POST'])
+def chat():
+    """Enhanced chat endpoint with memory"""
+    try:
+        data = request.get_json()
+        user_message = data.get('message', '').strip()
+        session_id = data.get('session_id', 'default')
+        
+        if not user_message:
+            return jsonify({
+                "error": "Message is required"
+            }), 400
+        
+        # Generate response with memory
+        result = generate_response_with_rag(user_message, session_id)
+        
+        return jsonify({
+            "response": result["response"],
+            "confidence": result["confidence"],
+            "session_id": session_id,
+            "conversation_length": result.get("conversation_length", 0),
+            "memory_used": result.get("memory_used", False),
+            "rag_used": result.get("rag_used", False),
+            "sources": result.get("sources", []),
+            "user_message": user_message,
+            "language_detected": result.get("language_detected", "en"),
+            "translation_used": result.get("translation_used", False),
+            "transliteration_used": result.get("transliteration_used", False),
+            "ai_detection_used": result.get("ai_detection_used", False),
+            "detection_confidence": result.get("detection_confidence", 0.0)
+        })
+    
+    except Exception as e:
+        logger.error(f"Chat API error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/clear', methods=['POST'])
+def clear_chat():
+    """Clear chat memory for a session"""
+    try:
+        data = request.get_json()
+        session_id = data.get('session_id', 'default')
+        
+        success = clear_session_memory(session_id)
+        
+        return jsonify({
+            "success": success,
+            "session_id": session_id,
+            "message": "Chat memory cleared successfully" if success else "Session not found"
+        })
+    
+    except Exception as e:
+        logger.error(f"Clear chat error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/sessions', methods=['GET'])
+def list_sessions():
+    """List all active chat sessions"""
+    try:
+        sessions = []
+        for session_id, memory in conversation_memories.items():
+            messages = memory.chat_memory.messages
+            sessions.append({
+                "session_id": session_id,
+                "message_count": len(messages),
+                "last_activity": datetime.now().isoformat()  # Simplified for now
+            })
+        
+        return jsonify({
+            "sessions": sessions,
+            "total_sessions": len(sessions)
+        })
+    
+    except Exception as e:
+        logger.error(f"List sessions error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/history/<session_id>', methods=['GET'])
+def get_chat_history(session_id: str):
+    """Get chat history for a specific session"""
+    try:
+        if session_id not in conversation_memories:
+            return jsonify({
+                "session_id": session_id,
+                "history": [],
+                "message_count": 0
+            })
+        
+        memory = conversation_memories[session_id]
+        messages = memory.chat_memory.messages
+        
+        history = []
+        for msg in messages:
+            if isinstance(msg, HumanMessage):
+                history.append({
+                    "type": "human",
+                    "content": msg.content,
+                    "timestamp": datetime.now().isoformat()
+                })
+            elif isinstance(msg, AIMessage):
+                history.append({
+                    "type": "ai",
+                    "content": msg.content,
+                    "timestamp": datetime.now().isoformat()
+                })
+        
+        return jsonify({
+            "session_id": session_id,
+            "history": history,
+            "message_count": len(history)
+        })
+    
+    except Exception as e:
+        logger.error(f"Get chat history error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/health', methods=['GET'])
+def chat_health():
+    """Health check for the enhanced chatbot"""
+    try:
+        # Test LangChain connection and vector database
+        test_agent = create_agent("health_check")
+        test_response = test_agent.invoke({"input": "Hello"})
+        
+        # Test vector database connection
+        pc_index = get_pinecone_index()
+        vector_db_status = "connected" if pc_index else "disconnected"
+        
+        return jsonify({
+            "status": "healthy",
+            "message": "Enhanced budget proposals chatbot with RAG is running",
+            "langchain_status": "connected" if test_response else "disconnected",
+            "vector_db_status": vector_db_status,
+            "rag_enabled": True,
+            "active_sessions": len(conversation_memories),
+            "memory_enabled": True
+        })
+    except Exception as e:
+        return jsonify({
+            "status": "unhealthy",
+            "message": f"Error: {str(e)}"
+        }), 500
+
+@app.route('/api/chat/debug/<session_id>', methods=['GET'])
+def debug_session(session_id: str):
+    """Debug endpoint to check session memory"""
+    try:
+        memory_exists = session_id in conversation_memories
+        memory_info = {
+            "session_id": session_id,
+            "memory_exists": memory_exists,
+            "total_sessions": len(conversation_memories),
+            "session_keys": list(conversation_memories.keys())
+        }
+        
+        if memory_exists:
+            memory = conversation_memories[session_id]
+            messages = memory.chat_memory.messages
+            memory_info.update({
+                "message_count": len(messages),
+                "messages": [
+                    {
+                        "type": getattr(msg, 'type', 'unknown'),
+                        "content": getattr(msg, 'content', '')[:100] + "..." if len(getattr(msg, 'content', '')) > 100 else getattr(msg, 'content', '')
+                    }
+                    for msg in messages
+                ]
+            })
+        
+        return jsonify(memory_info)
+    
+    except Exception as e:
+        logger.error(f"Debug session error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/suggestions', methods=['GET'])
+def get_chat_suggestions():
+    """Get suggested questions for the chatbot with multilingual support"""
+    suggestions = [
+        "What are the maternity leave benefits proposed? 🤱",
+        "What are the industrial land expansion proposals? 🏭",
+        "How do the cigarette tax proposals work? 💰",
+        "What changes are proposed for electricity tariffs? ⚡",
+        "Tell me about the EPF taxation removal proposals 💰",
+        "What tax reforms are being suggested? 🏛️",
+        "How will these proposals affect the economy? 📈",
+        "What is the cost of implementing these proposals? 💵",
+        "Can you compare the costs of different proposals? ⚖️",
+        "What are the main benefits of these proposals? ✨",
+        "Budget proposals gana kiyanna 📋",
+        "EPF eka gana mokadda thiyenne? 💰",
+        "Industrial land expansion kiyannako 🏭",
+        "Electricity bill eka wenas wenawada? ⚡",
+        "Maternity leave benefits kiyannako 🤱",
+        "මේ budget proposals වල cost එක කීයද? 💵",
+        "රජයේ ආර්థික ප්‍රතිපත්ති ගැන කියන්න 🏛️"
+    ]
+    
+    return jsonify({
+        "suggestions": suggestions,
+        "supported_languages": ["English", "Sinhala", "Singlish"]
+    })
+
+@app.route('/api/chat/available-pdfs', methods=['GET'])
+def get_available_pdfs_endpoint():
+    """Get list of available PDF files with short names for UI display"""
+    try:
+        available_pdfs = get_available_pdfs()
+        
+        # Create list with both full names and short names
+        pdf_list = []
+        short_names = []
+        for pdf in available_pdfs:
+            short_name = get_short_document_name(pdf)
+            pdf_list.append({
+                "filename": pdf,
+                "short_name": short_name,
+                "type": "PDF" if pdf.endswith('.pdf') else "DOCX"
+            })
+            short_names.append(short_name)
+        
+        return jsonify({
+            "available_pdfs": available_pdfs,
+            "pdf_list": pdf_list,
+            "short_names": short_names,  # Simple array for easy frontend use
+            "count": len(available_pdfs),
+            "pdf_directory": "Budget_Proposals copy-2/assets/pdfs"
+        })
+    except Exception as e:
+        logger.error(f"Error getting available PDFs: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/document-names', methods=['GET'])
+def get_document_names():
+    """Get document names with short names for UI display"""
+    try:
+        available_pdfs = get_available_pdfs()
+        
+        # Create mapping of full names to short names
+        document_mapping = {}
+        for pdf in available_pdfs:
+            document_mapping[pdf] = get_short_document_name(pdf)
+        
+        return jsonify({
+            "document_mapping": document_mapping,
+            "count": len(available_pdfs)
+        })
+    except Exception as e:
+        logger.error(f"Error getting document names: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/short-document-names', methods=['GET'])
+def get_short_document_names():
+    """Get just the short document names as a simple array for frontend display"""
+    try:
+        available_pdfs = get_available_pdfs()
+        
+        # Create simple array of short names
+        short_names = []
+        for pdf in available_pdfs:
+            short_names.append(get_short_document_name(pdf))
+        
+        return jsonify({
+            "short_names": short_names,
+            "count": len(short_names)
+        })
+    except Exception as e:
+        logger.error(f"Error getting short document names: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/document-buttons', methods=['GET'])
+def get_document_buttons():
+    """Get document names formatted specifically for UI buttons (simple strings only)"""
+    try:
+        available_pdfs = get_available_pdfs()
+        
+        # Create simple array of just the short names as strings
+        button_names = []
+        for pdf in available_pdfs:
+            button_names.append(get_short_document_name(pdf))
+        
+        # Return just the array of strings - no objects
+        return jsonify(button_names)
+    except Exception as e:
+        logger.error(f"Error getting document buttons: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/api/chat/detect-language', methods=['POST'])
+def detect_language():
+    """Test language detection functionality"""
+    try:
+        data = request.get_json()
+        text = data.get('text', '').strip()
+        
+        if not text:
+            return jsonify({
+                "error": "Text is required"
+            }), 400
+        
+        processed_message, original_language, needs_translation, transliteration_used, ai_detection_used, confidence = simple_process_input(text)
+        
+        return jsonify({
+            "original_text": text,
+            "processed_text": processed_message,
+            "language_detected": original_language,
+            "translation_needed": needs_translation,
+            "transliteration_used": transliteration_used,
+            "ai_detection_used": ai_detection_used,
+            "detection_confidence": confidence,
+            "contains_sinhala": detect_sinhala_content(text),
+            "is_singlish": detect_singlish(text)
+        })
+    
+    except Exception as e:
+        logger.error(f"Language detection error: {e}")
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/', methods=['GET'])
+def home():
+    """Home endpoint with API documentation"""
+    return jsonify({
+        "message": "Multilingual Budget Proposals Chatbot API with Swabhasha Pipeline",
+        "version": "2.1.0",
+        "supported_languages": ["English", "Sinhala", "Tamil (Sri Lankan)", "Romanized Sinhala (Singlish)", "Romanized Tamil"],
+        "features": ["RAG", "Memory", "Swabhasha Transliteration", "Google Translation", "FAISS Vector Store"],
+        "pipeline": "Romanized Sinhala → Swabhasha → Sinhala Script → Google Translate → English → LLM → Response",
+        "endpoints": {
+            "POST /api/chat": "Chat with memory, RAG, and multilingual support",
+            "POST /api/chat/clear": "Clear chat memory",
+            "GET /api/chat/sessions": "List active sessions",
+            "GET /api/chat/history/<session_id>": "Get chat history",
+            "GET /api/chat/health": "Health check",
+            "GET /api/chat/suggestions": "Get suggested questions (multilingual)",
+            "GET /api/chat/available-pdfs": "Get available PDF files with short names",
+            "GET /api/chat/document-names": "Get document name mapping (full to short names)",
+            "GET /api/chat/short-document-names": "Get simple array of short document names",
+            "GET /api/chat/document-buttons": "Get document names as simple string array for UI buttons",
+            "POST /api/chat/detect-language": "Test language detection"
+        },
+        "status": "running"
+    })
+
+if __name__ == '__main__':
+    app.run(debug=False, host='0.0.0.0', port=7860)
+#!/usr/bin/env python3
+"""
+Enhanced Budget Proposals Chatbot API using LangChain with Memory and Agentic RAG
+"""
+
+from flask import Flask, request, jsonify
+from flask_cors import CORS
+import os
+import logging
+import json
+from datetime import datetime
+from typing import Dict, List, Any
+
+# LangChain imports
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.schema import HumanMessage, AIMessage
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.chains import LLMChain
+from langchain_community.chat_message_histories import RedisChatMessageHistory
+from langchain.tools import Tool
+from langchain.agents import AgentExecutor, create_openai_functions_agent
+from langchain.agents.openai_functions_agent.base import OpenAIFunctionsAgent
+from langchain.schema import BaseMessage
+
+# Vector database imports
+from pinecone import Pinecone
+from sentence_transformers import SentenceTransformer
+
+# Language detection imports
+import re
+import requests
+import json
+
+app = Flask(__name__)
+CORS(app)
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Configure Gemini
+GEMINI_API_KEY = os.getenv('GEMINI_API_KEY')
+if not GEMINI_API_KEY:
+    logger.error("GEMINI_API_KEY not found in environment variables")
+    raise ValueError("Please set GEMINI_API_KEY in your .env file")
+
+# Configure Pinecone
+PINECONE_API_KEY = os.getenv('PINECONE_API_KEY')
+if not PINECONE_API_KEY:
+    logger.error("PINECONE_API_KEY not found in environment variables")
+    raise ValueError("Please set PINECONE_API_KEY in your .env file")
+
+# Configure Hugging Face (optional - needed for some models)
+HF_TOKEN = os.getenv('HUGGINGFACE_TOKEN')
+if HF_TOKEN:
+    logger.info("Hugging Face token found - will use for model downloads")
+else:
+    logger.warning("HUGGINGFACE_TOKEN not found - some models may not work")
+
+# Initialize Pinecone and embedding model - Using all-MiniLM model only
+pc = Pinecone(api_key=PINECONE_API_KEY)
+BUDGET_INDEX_NAME = "budget-proposals-optimized"  # Index for all-MiniLM model
+
+# Initialize all-MiniLM model (no HF token needed)
+embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+logger.info("✅ all-MiniLM model loaded successfully")
+
+# Initialize LangChain components
+llm = ChatGoogleGenerativeAI(
+    model="gemini-2.5-flash",
+    google_api_key=GEMINI_API_KEY,
+    temperature=0.7,
+    max_tokens=2000  # Increased for longer Sinhala responses
+)
+
+# Simplified initialization - Let Gemini handle everything
+logger.info("Using Gemini for all language processing (transliteration, translation, responses)")
+
+def detect_sinhala_content(text: str) -> bool:
+    """Detect if text contains Sinhala characters"""
+    # Sinhala Unicode range: U+0D80 to U+0DFF
+    sinhala_pattern = re.compile(r'[\u0D80-\u0DFF]')
+    return bool(sinhala_pattern.search(text))
+
+def detect_tamil_content(text: str) -> bool:
+    """Detect if text contains Tamil characters"""
+    # Tamil Unicode range: U+0B80 to U+0BFF
+    tamil_pattern = re.compile(r'[\u0B80-\u0BFF]')
+    return bool(tamil_pattern.search(text))
+
+def simple_detect_language(text: str) -> Dict[str, Any]:
+    """Simplified language detection with Tamil support - let Gemini handle the complexity"""
+    try:
+        # Check for Sinhala Unicode first (most reliable)
+        has_sinhala_unicode = detect_sinhala_content(text)
+        if has_sinhala_unicode:
+            return {
+                'language': 'si',
+                'confidence': 0.95,
+                'is_sinhala_unicode': True,
+                'is_tamil_unicode': False,
+                'is_romanized_sinhala': False,
+                'is_english': False,
+                'detection_method': 'unicode_detection'
+            }
+        
+        # Check for Tamil Unicode
+        has_tamil_unicode = detect_tamil_content(text)
+        if has_tamil_unicode:
+            return {
+                'language': 'ta',
+                'confidence': 0.95,
+                'is_sinhala_unicode': False,
+                'is_tamil_unicode': True,
+                'is_romanized_sinhala': False,
+                'is_english': False,
+                'detection_method': 'unicode_detection'
+            }
+        
+        # Use enhanced rule-based detection for Singlish
+        return enhanced_rule_based_detection(text)
+        
+    except Exception as e:
+        logger.error(f"Language detection failed: {e}")
+        return rule_based_language_detection(text)
+
+def enhanced_rule_based_detection(text: str) -> Dict[str, Any]:
+    """Enhanced rule-based detection with Singlish and Romanized Tamil recognition"""
+    has_sinhala_unicode = detect_sinhala_content(text)
+    has_tamil_unicode = detect_tamil_content(text)
+    is_romanized_sinhala = detect_singlish(text) and not has_sinhala_unicode and not has_tamil_unicode
+    is_romanized_tamil = detect_romanized_tamil(text) and not has_sinhala_unicode and not has_tamil_unicode and not is_romanized_sinhala
+    
+    # More sophisticated Singlish detection
+    if not has_sinhala_unicode and not is_romanized_sinhala:
+        # Check for common Sinhala sentence patterns in English letters
+        sinhala_patterns = [
+            r'\b(mokadda|kohomada|api|oya|mama)\b',
+            r'\b(eka|meka|thiyenne|kiyala)\b',
+            r'\b(gana|genna|danna|karanna)\b',
+            r'\b(budget|proposal).*\b(gana|eka)\b'
+        ]
+        
+        text_lower = text.lower()
+        pattern_matches = sum(1 for pattern in sinhala_patterns if re.search(pattern, text_lower))
+        
+        if pattern_matches >= 2:  # More conservative threshold to avoid false positives
+            is_romanized_sinhala = True
+    
+    if has_sinhala_unicode:
+        language_code = 'si'
+        confidence = 0.9
+    elif has_tamil_unicode:
+        language_code = 'ta'
+        confidence = 0.9
+    elif is_romanized_sinhala:
+        language_code = 'singlish'
+        confidence = 0.8
+    elif is_romanized_tamil:
+        language_code = 'romanized_tamil'
+        confidence = 0.8
+    else:
+        language_code = 'en'
+        confidence = 0.7
+    
+    return {
+        'language': language_code,
+        'confidence': confidence,
+        'is_sinhala_unicode': has_sinhala_unicode,
+        'is_tamil_unicode': has_tamil_unicode,
+        'is_romanized_sinhala': is_romanized_sinhala,
+        'is_romanized_tamil': is_romanized_tamil,
+        'is_english': language_code == 'en',
+        'detection_method': 'enhanced_rule_based'
+    }
+
+def rule_based_language_detection(text: str) -> Dict[str, Any]:
+    """Fallback rule-based language detection with Tamil and Romanized Tamil support"""
+    has_sinhala_unicode = detect_sinhala_content(text)
+    has_tamil_unicode = detect_tamil_content(text)
+    is_romanized_sinhala = detect_singlish(text) and not has_sinhala_unicode and not has_tamil_unicode
+    is_romanized_tamil = detect_romanized_tamil(text) and not has_sinhala_unicode and not has_tamil_unicode and not is_romanized_sinhala
+    is_english = not has_sinhala_unicode and not has_tamil_unicode and not is_romanized_sinhala and not is_romanized_tamil
+    
+    if has_sinhala_unicode:
+        language_code = 'si'
+    elif has_tamil_unicode:
+        language_code = 'ta'
+    elif is_romanized_sinhala:
+        language_code = 'singlish'
+    elif is_romanized_tamil:
+        language_code = 'romanized_tamil'
+    else:
+        language_code = 'en'
+    
+    return {
+        'language': language_code,
+        'confidence': 0.8,  # Default confidence for rule-based
+        'is_sinhala_unicode': has_sinhala_unicode,
+        'is_tamil_unicode': has_tamil_unicode,
+        'is_romanized_sinhala': is_romanized_sinhala,
+        'is_romanized_tamil': is_romanized_tamil,
+        'is_english': is_english,
+        'detection_method': 'rule_based'
+    }
+
+def detect_singlish(text: str) -> bool:
+    """Detect common Singlish patterns and words"""
+    singlish_words = [
+        'mokadda', 'kohomada', 'api', 'oya', 'mama', 'eka', 'meka', 'oya', 'dan', 'kiyala',
+        'karan', 'karanna', 'gana', 'genna', 'danna', 'ahala', 'denna',
+        'mata', 'ape', 'wage', 'wenas', 'thiyenne', 'kiyanawa', 'balanawa', 'pennanna',
+        'sampura', 'mudal', 'pasal', 'vyaparayak', 'rajaye', 'arthikaya', 'sammandala',
+        'kara', 'karanna', 'giya', 'yanawa', 'enawa', 'gihin', 'awe', 'nane', 'inne',
+        'danna', 'kiyanna', 'balanna', 'ganna', 'denna', 'yanna', 'enna'
+    ]
+    
+    # Convert to lowercase and check for common Singlish words
+    text_lower = text.lower()
+    singlish_word_count = sum(1 for word in singlish_words if word in text_lower)
+    
+    # Consider it Singlish if it has 3 or more Singlish words (more conservative)
+    return singlish_word_count >= 3
+
+def detect_romanized_tamil(text: str) -> bool:
+    """Detect common Romanized Tamil patterns and words (Tamil written in English letters)"""
+    romanized_tamil_words = [
+        # Common Tamil words in Roman script
+        'enna', 'epdi', 'enga', 'yaar', 'naa', 'nee', 'avar', 'ivan', 'ival', 'ithu', 'athu',
+        'vandhu', 'ponga', 'vanga', 'sollu', 'kelu', 'paaru', 'irukku', 'irukkanga', 'irundhu',
+        'seiya', 'panna', 'mudiyum', 'mudiyathu', 'venum', 'vendam', 'puriyuthu', 'puriyala',
+        'nalla', 'ketta', 'romba', 'konjam', 'neraya', 'kammi', 'adhikam', 'thaan', 'daan',
+        # Budget/government related Tamil terms (excluding common English words)
+        'sarkar', 'arasaangam', 'vyavasai', 'panam', 'kaasu', 'thogai',
+        'nilai', 'mari', 'maatram', 'thiruththam', 'yojana', 'thittam', 'mudhal', 'selavu',
+        'varumanam', 'aayam', 'viduli'
+    ]
+    
+    # Convert to lowercase and check for common Romanized Tamil words
+    text_lower = text.lower()
+    tamil_word_count = sum(1 for word in romanized_tamil_words if word in text_lower)
+    
+    # Consider it Romanized Tamil if it has 3 or more Tamil words (more conservative)
+    return tamil_word_count >= 3
+
+# Removed: AI transliteration and Google Translate functions
+# Gemini will handle all transliteration and translation needs
+
+def simple_process_input(user_message: str) -> tuple:
+    """
+    Simplified input processing - let Gemini handle everything
+    """
+    # Step 1: Simple language detection
+    language_info = simple_detect_language(user_message)
+    original_language = language_info['language']
+    confidence = language_info['confidence']
+    detection_method = language_info['detection_method']
+    
+    logger.info(f"Language detection: {original_language} (confidence: {confidence:.2f}, method: {detection_method})")
+    
+    # Use original message for all processing - Gemini will handle the rest
+    processed_message = user_message
+    needs_translation = False  # Gemini handles translation internally
+    transliteration_used = False  # Gemini handles transliteration internally
+    ai_detection_used = detection_method == 'ai'
+    
+    logger.info(f"Input processing: keeping original '{user_message}' for Gemini to handle")
+    
+    return processed_message, original_language, needs_translation, transliteration_used, ai_detection_used, confidence
+
+# Removed: translate_response_if_needed function
+# Gemini handles all language responses automatically
+
+def get_pinecone_index():
+    """Get the Pinecone index - single index for all languages"""
+    try:
+        return pc.Index(BUDGET_INDEX_NAME)
+    except Exception as e:
+        logger.error(f"Error accessing Pinecone index: {e}")
+        return None
+
+def get_embedding_model():
+    """Get the embedding model - single model for all languages"""
+    return embed_model
+
+def search_budget_proposals(query: str) -> str:
+    """Search budget proposals using all-MiniLM model for all languages"""
+    try:
+        # Detect language for logging and result filtering
+        language_info = simple_detect_language(query)
+        detected_language = language_info.get('language', 'en')
+        
+        logger.info(f"Detected language: {detected_language} for query: {query[:50]}...")
+        
+        # Get index and model (single model for all languages)
+        index = get_pinecone_index()
+        if not index:
+            return "Error: Could not access vector database."
+        
+        # Use all-MiniLM model for all languages
+        model = get_embedding_model()
+        query_embedding = model.encode(query).tolist()
+        
+        logger.info(f"Using all-MiniLM model for {detected_language}")
+        
+        # Query the vector database directly
+        search_results = index.query(
+            vector=query_embedding,
+            top_k=5,
+            include_metadata=True
+        )
+        
+        matches = search_results.get('matches', [])
+        
+        # Debug: Log what we're getting from the vector database
+        logger.info(f"Vector DB returned {len(matches)} results")
+        if matches:
+            sample_match = matches[0]
+            logger.info(f"Sample match metadata keys: {list(sample_match.get('metadata', {}).keys())}")
+        
+        if not matches:
+            return "No relevant budget proposals found in the database."
+        
+        # Build context from vector database results, filtering by language
+        context_parts = []
+        language_specific_matches = []
+        
+        # Filter matches to only include English documents
+        english_matches = []
+        for match in matches:
+            metadata = match.get('metadata', {})
+            file_path = metadata.get('file_path', '')
+            
+            # Only include English documents (no language suffixes)
+            is_english_document = not any(lang in file_path.lower() for lang in ['_sin_', '_tam_', '-sin', '-tam', 'sinhala', 'tamil', 'si/', 'ta/'])
+            
+            if is_english_document:
+                english_matches.append(match)
+        
+        # Use English matches only, or fallback to top match if no English documents found
+        if english_matches:
+            language_specific_matches = english_matches[:1]  # Take only the most relevant English document
+        else:
+            language_specific_matches = matches[:1]  # Fallback to any document if no English found
+        
+        logger.info(f"Returning {len(language_specific_matches)} most relevant document(s) for {detected_language}")
+        
+        for match in language_specific_matches:
+            metadata = match.get('metadata', {})
+            score = match.get('score', 0)
+            
+            file_path = metadata.get('file_path', '')
+            category = metadata.get('category', '')
+            title = metadata.get('title', '')
+            content = metadata.get('content', '')
+            summary = metadata.get('summary', '')
+            cost = metadata.get('costLKR', '')
+            
+            # Include relevance score for debugging
+            context_parts.append(f"From {file_path} ({category}) [Relevance: {score:.3f}]: {title}")
+            
+            # Prioritize content over summary, but include both if available
+            if content and len(content.strip()) > 50:  # Only use substantial content
+                context_parts.append(f"Content: {content}")
+            elif summary:
+                context_parts.append(f"Summary: {summary}")
+            
+            # Always include cost information if available
+            if cost and cost != "No Costing Available":
+                context_parts.append(f"Cost: {cost}")
+            
+            # Add any additional relevant fields from metadata
+            if metadata.get('implementation_period'):
+                context_parts.append(f"Implementation Period: {metadata.get('implementation_period')}")
+            if metadata.get('beneficiaries'):
+                context_parts.append(f"Beneficiaries: {metadata.get('beneficiaries')}")
+            if metadata.get('revenue_impact'):
+                context_parts.append(f"Revenue Impact: {metadata.get('revenue_impact')}")
+            if metadata.get('proposal_type'):
+                context_parts.append(f"Proposal Type: {metadata.get('proposal_type')}")
+            if metadata.get('sector'):
+                context_parts.append(f"Sector: {metadata.get('sector')}")
+        
+        return "\n\n".join(context_parts)
+        
+    except Exception as e:
+        logger.error(f"Error searching vector database: {e}")
+        return f"Error searching database: {str(e)}"
+
+# Create the RAG tool
+search_tool = Tool(
+    name="search_budget_proposals",
+    description="Search for relevant budget proposals in the vector database. Use this when you need specific information about budget proposals, costs, policies, or implementation details.",
+    func=search_budget_proposals
+)
+
+# Create the prompt template for the agent
+agent_prompt = ChatPromptTemplate.from_messages([
+    ("system", """You are a helpful assistant for budget proposals in Sri Lanka. You have access to a vector database containing detailed information about various budget proposals. You can communicate in English, Sinhala, and understand Singlish (Sinhala written in English letters).
+
+When a user asks about budget proposals, you should:
+1. Use the search_budget_proposals tool to find relevant information
+2. Provide accurate, detailed responses based on the retrieved information
+3. Reference proposals by their content/topic, not by filename
+4. Be professional but approachable in any language
+5. If the search doesn't return relevant results, acknowledge this and provide general guidance
+6. Respond in the same language or style as the user's question when possible
+
+Guidelines:
+- Always use the search tool for specific questions about budget proposals
+- When mentioning proposals, refer to them by topic (e.g., "maternity leave benefits proposal", "EPF tax removal proposal") rather than document filenames
+- Keep responses clear and informative in any language
+- Use a balanced tone - helpful but not overly casual
+- If asked about topics not covered, redirect to relevant topics professionally
+- Be culturally sensitive when discussing Sri Lankan policies and economic matters
+- When responding in Sinhala, use appropriate formal language for policy discussions
+- DO NOT include long document filenames in your responses - refer to proposals by their topic instead"""),
+    MessagesPlaceholder(variable_name="chat_history"),
+    ("human", "{input}"),
+    MessagesPlaceholder(variable_name="agent_scratchpad")
+])
+
+# Store conversation memories for different sessions
+conversation_memories: Dict[str, ConversationBufferWindowMemory] = {}
+
+def get_or_create_memory(session_id: str) -> ConversationBufferWindowMemory:
+    """Get or create a memory instance for a session"""
+    if session_id not in conversation_memories:
+        # Create new memory with window of 10 messages (5 exchanges)
+        conversation_memories[session_id] = ConversationBufferWindowMemory(
+            k=10,  # Remember last 10 messages
+            return_messages=True,
+            memory_key="chat_history"
+        )
+        logger.info(f"Created new memory for session: {session_id}")
+    
+    return conversation_memories[session_id]
+
+def create_agent(session_id: str) -> AgentExecutor:
+    """Create a LangChain agent with memory and RAG capabilities"""
+    memory = get_or_create_memory(session_id)
+    
+    # Create the agent
+    agent = create_openai_functions_agent(
+        llm=llm,
+        tools=[search_tool],
+        prompt=agent_prompt
+    )
+    
+    # Create agent executor with memory
+    agent_executor = AgentExecutor(
+        agent=agent,
+        tools=[search_tool],
+        memory=memory,
+        verbose=False,
+        handle_parsing_errors=True
+    )
+    
+    return agent_executor
+
+def get_short_document_name(filename: str) -> str:
+    """
+    Convert long document names to shorter, user-friendly names automatically
+    
+    SHORT NAME GENERATION GUIDE:
+    ===========================
+    
+    1. MANUAL MAPPING (Priority 1):
+       - Add entries to the 'short_names' dictionary for specific files
+       - Format: 'full_filename_without_extension': 'Short Display Name'
+       - Example: '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01': 'Maternity Leave Benefits'
+    
+    2. AUTOMATIC PATTERN MATCHING (Priority 2):
+       - System automatically detects proposal types and languages
+       - Proposal Types Detected:
+         * MaternityLeaveBenefit/MaternityLeave → "Maternity Leave Benefits"
+         * RemovalOfTaxationOnEPF/EPF → "EPF Tax Removal" 
+         * ExpandingIndustrialLand/IndustrialLand → "Industrial Land Expansion"
+         * Budget2025/Budget2026 → "Budget 2025/2026 Proposals"
+         * Template → "Budget Template"
+         * OnePagers → "Budget YYYY One-Pagers"
+       
+       - Language Detection:
+         * _Sin_/_Sinhala_ → "(Sinhala)"
+         * _Tam_/_Tamil_ ��� "(Tamil)"
+         * _En_/_English_ → "(EN)"
+         * _Raj_ → No language suffix (treated as default/English)
+         * No language indicator → No language suffix
+    
+    3. GENERIC FALLBACK (Priority 3):
+       - Removes date prefixes: 20250813_ → ""
+       - Removes language suffixes: _Sin_, _Tam_, _Raj_, _En_, _F_, _Final_, _D01
+       - Removes budget prefixes: Budget2026Proposal_ → ""
+       - Converts underscores to spaces: _ → " "
+       - Capitalizes words: "maternity leave" → "Maternity Leave"
+       - Limits length: Truncates to 37 chars + "..." if longer than 40
+    
+    EXAMPLES:
+    =========
+    Input: "20250813_Budget2026Proposal_MaternityLeaveBenefit_Sin_F.pdf"
+    Output: "Maternity Leave Benefits (Sinhala)"
+    
+    Input: "20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Tam_F.pdf"  
+    Output: "EPF Tax Removal (Tamil)"
+    
+    Input: "20250813_Budget2026_Proposal_ExpandingIndustrialLand_En_F.pdf"
+    Output: "Industrial Land Expansion (EN)"
+    
+    Input: "20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01.pdf"
+    Output: "Maternity Leave Benefits" (no language suffix)
+    
+    HOW TO ADD NEW DOCUMENTS:
+    =========================
+    1. Drop the PDF/DOCX file in the assets/pdfs/ folder
+    2. The system will automatically generate a short name using pattern matching
+    3. If you want a custom name, add it to the 'short_names' dictionary
+    4. No code changes needed for automatic naming!
+    """
+    # Remove file extension
+    name = filename.replace('.pdf', '').replace('.docx', '')
+    
+    # Create mapping for common document types (can be updated manually for special cases)
+    short_names = {
+        '20241211_Econ_VRProposals_Budget2025_OnePagers': 'Budget 2025 One-Pagers',
+        '20250813_Budget2026_Proposal_ExpandingIndustrialLand_En_F': 'Industrial Land Expansion (EN)',
+        '20250813_Budget2026Proposal_ExpandingIndustrialLand_F': 'Industrial Land Expansion (English)',
+        '20250813_Budget2026Proposal_ExpandingIndustrialLand_F - Sinhala': 'Industrial Land Expansion (Sinhala)',
+        '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01': 'Maternity Leave Benefits',
+        '20250813_Budget2026Proposal_RemovalOfTaxationOnEPF_Raj_F': 'EPF Tax Removal',
+        '20250825_Budget2026Proposal_MaternityLeaveBenefit_Sin_F': 'Maternity Leave Benefits (Sinhala)',
+        '20250825_Budget2026Proposal_MaternityLeaveBenefit_Tam_F': 'Maternity Leave Benefits (Tamil)',
+        '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Sin_Final': 'EPF Tax Removal (Sinhala)',
+        '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Tam_F': 'EPF Tax Removal (Tamil)',
+        '20250908_Budget2026Proposal_Template': 'Budget 2026 Template'
+    }
+    
+    # Return short name if found in manual mapping
+    if name in short_names:
+        return short_names[name]
+    
+    # Automatic pattern-based naming (works for new files without manual updates)
+    # Extract year
+    year_match = re.search(r'20\d{2}', name)
+    year = year_match.group() if year_match else ''
+    
+    # Extract language indicators
+    language = ''
+    if '_Sin_' in name or '_Sinhala_' in name:
+        language = ' (Sinhala)'
+    elif '_Tam_' in name or '_Tamil_' in name:
+        language = ' (Tamil)'
+    elif '_Raj_' in name:
+        language = ' (Raj)'
+    elif '_En_' in name or '_English_' in name:
+        language = ' (EN)'
+    
+    # Extract proposal type
+    if 'MaternityLeaveBenefit' in name or 'MaternityLeave' in name:
+        return f'Maternity Leave Benefits{language}'
+    elif 'RemovalOfTaxationOnEPF' in name or 'EPF' in name:
+        return f'EPF Tax Removal{language}'
+    elif 'ExpandingIndustrialLand' in name or 'IndustrialLand' in name:
+        return f'Industrial Land Expansion{language}'
+    elif 'Budget' in name and year:
+        return f'Budget {year} Proposals{language}'
+    elif 'Template' in name:
+        return f'Budget Template{language}'
+    elif 'OnePagers' in name:
+        return f'Budget {year} One-Pagers'
+    else:
+        # Generic fallback - clean up the name
+        # Remove date prefixes and common suffixes
+        clean_name = re.sub(r'^\d{8}_', '', name)  # Remove date prefix
+        clean_name = re.sub(r'_(En|Sin|Tam|Raj|F|Final|D01)$', '', clean_name)  # Remove language/version suffixes
+        clean_name = re.sub(r'Budget\d{4}Proposal_?', '', clean_name)  # Remove budget proposal prefix
+        clean_name = re.sub(r'_', ' ', clean_name)  # Replace underscores with spaces
+        
+        # Capitalize words
+        clean_name = ' '.join(word.capitalize() for word in clean_name.split())
+        
+        # Limit length
+        if len(clean_name) > 40:
+            clean_name = clean_name[:37] + '...'
+        
+        return clean_name + language
+
+def get_available_pdfs() -> List[str]:
+    """Dynamically get list of available PDF files from all language directories"""
+    try:
+        import os
+        import glob
+        
+        # Search in all language directories
+        pdf_dirs = [
+            "Budget_Proposals copy-2/en/assets/pdfs/",
+            "Budget_Proposals copy-2/si/assets/pdfs/",
+            "Budget_Proposals copy-2/ta/assets/pdfs/",
+            "Budget_Proposals copy-2/assets/pdfs/"
+        ]
+        
+        pdf_files = set()
+        for pdf_dir in pdf_dirs:
+            if os.path.exists(pdf_dir):
+                files = [f for f in os.listdir(pdf_dir) if f.lower().endswith(('.pdf', '.docx'))]
+                pdf_files.update(files)
+        
+        if pdf_files:
+            return list(pdf_files)
+        else:
+            # Fallback to known PDFs if no directories exist
+            return [
+                '20250813_Budget2026Proposal_ExpandingIndustrialLand_F.pdf',
+                '20250813_Budget2026Proposal_ExpandingIndustrialLand_F - Sinhala.pdf',
+                '20250813_Budget2026Proposal_ExpandingIndustrialLand_F - TamilReviewed.pdf',
+                '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01.pdf',
+                '20250813_Budget2026Proposal_RemovalOfTaxationOnEPF_Raj_F.pdf',
+                '20250825_Budget2026Proposal_MaternityLeaveBenefit_Sin_F.pdf',
+                '20250825_Budget2026Proposal_MaternityLeaveBenefit_Tam_F.pdf',
+                '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Sin_Final.pdf',
+                '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Tam_F.pdf'
+            ]
+    except Exception as e:
+        logger.error(f"Error getting available PDFs: {e}")
+        # Fallback to known PDFs
+        return [
+            '20250813_Budget2026Proposal_ExpandingIndustrialLand_F.pdf',
+            '20250813_Budget2026Proposal_ExpandingIndustrialLand_F - Sinhala.pdf',
+            '20250813_Budget2026Proposal_ExpandingIndustrialLand_F - TamilReviewed.pdf',
+            '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01.pdf',
+            '20250813_Budget2026Proposal_RemovalOfTaxationOnEPF_Raj_F.pdf',
+            '20250825_Budget2026Proposal_MaternityLeaveBenefit_Sin_F.pdf',
+            '20250825_Budget2026Proposal_MaternityLeaveBenefit_Tam_F.pdf',
+            '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Sin_Final.pdf',
+            '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Tam_F.pdf'
+        ]
+
+def extract_sources_from_search_context(search_context: str, user_language: str = 'en') -> List[Dict[str, str]]:
+    """Extract source documents from search context with short names, filtered by user language"""
+    sources = []
+    
+    # Get dynamically available PDF files
+    available_pdfs = get_available_pdfs()
+    
+    # Look for the specific pattern "From {filename} ({category}):" in search context
+    import re
+    found_files = set()
+    
+    # Pattern to match "From filename.pdf (category):" or "From filename.docx (category):"
+    # Updated to handle assets/pdfs/ prefix and empty parentheses, and stop at the colon
+    from_pattern = r'From\s+assets/pdfs/([^:]+\.(?:pdf|docx))\s*\([^)]*\)'
+    matches = re.findall(from_pattern, search_context)
+    
+    for match in matches:
+        if match in available_pdfs:
+            found_files.add(match)
+    
+    # Fallback: if no "From" pattern found, look for direct filename mentions
+    if not found_files:
+        for pdf in available_pdfs:
+            if pdf in search_context:
+                found_files.add(pdf)
+    
+    # Filter to return sources in the user's language, but prioritize by relevance
+    language_filtered_files = []
+    
+    # First, try to find documents in the user's language
+    for pdf in found_files:
+        doc_language = get_document_language(pdf)
+        
+        # Language matching logic - return sources in user's language
+        should_include = False
+        if user_language == 'en' or user_language == 'singlish':
+            # English users get English documents
+            if doc_language in ['en', 'english']:
                 should_include = True
         elif user_language == 'si' or user_language == 'sinhala':
-            # Sinhala users prefer Sinhala documents
+            # Sinhala users get Sinhala documents
             if doc_language in ['si', 'sinhala']:
                 should_include = True
-            elif not any(f for f in found_files if get_document_language(f) in ['si', 'sinhala']):
-                # If no Sinhala version available, show other languages
-                should_include = True
         elif user_language == 'ta' or user_language == 'tamil':
-            # Tamil users prefer Tamil documents
+            # Tamil users get Tamil documents
             if doc_language in ['ta', 'tamil']:
                 should_include = True
-            elif not any(f for f in found_files if get_document_language(f) in ['ta', 'tamil']):
-                # If no Tamil version available, show other languages
-                should_include = True
         else:
-            # Default: show all documents
-            should_include = True
+            # Default: show English documents
+            if doc_language in ['en', 'english']:
+                should_include = True
             
         if should_include:
             language_filtered_files.append(pdf)
     
-        # Always return only the most relevant document in the user's language
-        if language_filtered_files:
-            language_filtered_files = [language_filtered_files[0]]  # Single most relevant document
+    # If no language-specific documents found, fallback to English
+    if not language_filtered_files:
+        for pdf in found_files:
+            doc_language = get_document_language(pdf)
+            if doc_language in ['en', 'english']:
+                language_filtered_files.append(pdf)
+    
+    # If still no documents, use any available document
+    if not language_filtered_files and found_files:
+        language_filtered_files = [list(found_files)[0]]
+    
+    # Return only the most relevant document in the user's language
+    if language_filtered_files:
+        language_filtered_files = [language_filtered_files[0]]
     
     # Convert to list with short names and correct URLs
     for pdf in language_filtered_files: