Update app.py

app.py

@@ -387,7 +387,63 @@ def create_agent(session_id: str) -> AgentExecutor:
     return agent_executor
 
 def get_short_document_name(filename: str) -> str:
-    """Convert long document names to shorter, user-friendly names automatically"""
+    """
+    Convert long document names to shorter, user-friendly names automatically
+    
+    SHORT NAME GENERATION GUIDE:
+    ===========================
+    
+    1. MANUAL MAPPING (Priority 1):
+       - Add entries to the 'short_names' dictionary for specific files
+       - Format: 'full_filename_without_extension': 'Short Display Name'
+       - Example: '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01': 'Maternity Leave Benefits'
+    
+    2. AUTOMATIC PATTERN MATCHING (Priority 2):
+       - System automatically detects proposal types and languages
+       - Proposal Types Detected:
+         * MaternityLeaveBenefit/MaternityLeave → "Maternity Leave Benefits"
+         * RemovalOfTaxationOnEPF/EPF → "EPF Tax Removal" 
+         * ExpandingIndustrialLand/IndustrialLand → "Industrial Land Expansion"
+         * Budget2025/Budget2026 → "Budget 2025/2026 Proposals"
+         * Template → "Budget Template"
+         * OnePagers → "Budget YYYY One-Pagers"
+       
+       - Language Detection:
+         * _Sin_/_Sinhala_ → "(Sinhala)"
+         * _Tam_/_Tamil_ → "(Tamil)"
+         * _En_/_English_ → "(EN)"
+         * _Raj_ → No language suffix (treated as default/English)
+         * No language indicator → No language suffix
+    
+    3. GENERIC FALLBACK (Priority 3):
+       - Removes date prefixes: 20250813_ → ""
+       - Removes language suffixes: _Sin_, _Tam_, _Raj_, _En_, _F_, _Final_, _D01
+       - Removes budget prefixes: Budget2026Proposal_ → ""
+       - Converts underscores to spaces: _ → " "
+       - Capitalizes words: "maternity leave" → "Maternity Leave"
+       - Limits length: Truncates to 37 chars + "..." if longer than 40
+    
+    EXAMPLES:
+    =========
+    Input: "20250813_Budget2026Proposal_MaternityLeaveBenefit_Sin_F.pdf"
+    Output: "Maternity Leave Benefits (Sinhala)"
+    
+    Input: "20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Tam_F.pdf"  
+    Output: "EPF Tax Removal (Tamil)"
+    
+    Input: "20250813_Budget2026_Proposal_ExpandingIndustrialLand_En_F.pdf"
+    Output: "Industrial Land Expansion (EN)"
+    
+    Input: "20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01.pdf"
+    Output: "Maternity Leave Benefits" (no language suffix)
+    
+    HOW TO ADD NEW DOCUMENTS:
+    =========================
+    1. Drop the PDF/DOCX file in the assets/pdfs/ folder
+    2. The system will automatically generate a short name using pattern matching
+    3. If you want a custom name, add it to the 'short_names' dictionary
+    4. No code changes needed for automatic naming!
+    """
     # Remove file extension
     name = filename.replace('.pdf', '').replace('.docx', '')
     
@@ -395,8 +451,8 @@ def get_short_document_name(filename: str) -> str:
     short_names = {
         '20241211_Econ_VRProposals_Budget2025_OnePagers': 'Budget 2025 One-Pagers',
         '20250813_Budget2026_Proposal_ExpandingIndustrialLand_En_F': 'Industrial Land Expansion (EN)',
-        '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01': 'Maternity Leave Benefits (Raj)',
-        '20250813_Budget2026Proposal_RemovalOfTaxationOnEPF_Raj_F': 'EPF Tax Removal (Raj)',
+        '20250813_Budget2026Proposal_MaternityLeaveBenefit_Raj_D01': 'Maternity Leave Benefits',
+        '20250813_Budget2026Proposal_RemovalOfTaxationOnEPF_Raj_F': 'EPF Tax Removal',
         '20250825_Budget2026Proposal_MaternityLeaveBenefit_Sin_F': 'Maternity Leave Benefits (Sinhala)',
         '20250825_Budget2026Proposal_MaternityLeaveBenefit_Tam_F': 'Maternity Leave Benefits (Tamil)',
         '20250825_Budget2026Proposal_RemovalOfTaxationOnEPF_Sin_Final': 'EPF Tax Removal (Sinhala)',
@@ -490,8 +546,8 @@ def get_available_pdfs() -> List[str]:
             '20250908_Budget2026Proposal_Template.pdf'
         ]
 
-def extract_sources_from_search_context(search_context: str) -> List[Dict[str, str]]:
-    """Extract source documents from search context with short names"""
+def extract_sources_from_search_context(search_context: str, user_language: str = 'en') -> List[Dict[str, str]]:
+    """Extract source documents from search context with short names, filtered by user language"""
     sources = []
     
     # Get dynamically available PDF files
@@ -515,8 +571,41 @@ def extract_sources_from_search_context(search_context: str) -> List[Dict[str, s
             if pdf in search_context:
                 found_files.add(pdf)
     
-    # Convert to list with short names
+    # Filter by user language preference
+    language_filtered_files = set()
+    
     for pdf in found_files:
+        # Determine document language from filename
+        doc_language = get_document_language(pdf)
+        
+        # Language matching logic
+        if user_language == 'en' or user_language == 'singlish':
+            # English users can see all documents, but prefer English versions
+            if doc_language in ['en', 'english'] or user_language == 'singlish':
+                language_filtered_files.add(pdf)
+            elif not any(f for f in found_files if get_document_language(f) in ['en', 'english']):
+                # If no English version available, show other languages
+                language_filtered_files.add(pdf)
+        elif user_language == 'si' or user_language == 'sinhala':
+            # Sinhala users prefer Sinhala documents
+            if doc_language in ['si', 'sinhala']:
+                language_filtered_files.add(pdf)
+            elif not any(f for f in found_files if get_document_language(f) in ['si', 'sinhala']):
+                # If no Sinhala version available, show other languages
+                language_filtered_files.add(pdf)
+        elif user_language == 'ta' or user_language == 'tamil':
+            # Tamil users prefer Tamil documents
+            if doc_language in ['ta', 'tamil']:
+                language_filtered_files.add(pdf)
+            elif not any(f for f in found_files if get_document_language(f) in ['ta', 'tamil']):
+                # If no Tamil version available, show other languages
+                language_filtered_files.add(pdf)
+        else:
+            # Default: show all documents
+            language_filtered_files.add(pdf)
+    
+    # Convert to list with short names
+    for pdf in language_filtered_files:
         sources.append({
             "filename": pdf,
             "short_name": get_short_document_name(pdf)
@@ -524,6 +613,22 @@ def extract_sources_from_search_context(search_context: str) -> List[Dict[str, s
     
     return sources
 
+def get_document_language(filename: str) -> str:
+    """Determine the language of a document from its filename"""
+    filename_lower = filename.lower()
+    
+    if '_sin_' in filename_lower or '_sinhala_' in filename_lower:
+        return 'si'
+    elif '_tam_' in filename_lower or '_tamil_' in filename_lower:
+        return 'ta'
+    elif '_raj_' in filename_lower:
+        return 'en'  # Treat Raj as English/default
+    elif '_en_' in filename_lower or '_english_' in filename_lower:
+        return 'en'
+    else:
+        # Default to English if no language indicator found
+        return 'en'
+
 def extract_sources_from_response(response: str) -> List[Dict[str, str]]:
     """Extract source documents mentioned in the response with short names (fallback method)"""
     sources = []
@@ -643,7 +748,7 @@ Please provide a helpful response:"""
         # No need to translate response - Gemini handles language matching automatically
         
         # Extract sources from search context (where the actual filenames are)
-        sources = extract_sources_from_search_context(search_context)
+        sources = extract_sources_from_search_context(search_context, original_language)
         
         # Add messages to memory (store original user message for context)
         memory.chat_memory.add_user_message(user_message)