fix

config.py

@@ -17,7 +17,7 @@ CORE_DATA_FOLDER = os.path.join(BASE_DIR, "data", "core")
 # PENDING_UPLOADS_FOLDER = os.path.join(BASE_DIR, "data", "pending_uploads")
 # PROCESSED_FILES_FOLDER = os.path.join(BASE_DIR, "data", "processed_files")
 # FAILED_FILES_FOLDER = os.path.join(BASE_DIR, "data", "failed_files")
-PROCESSED_HASH_LOG = os.path.join(BASE_DIR, "data", "processed_hashes.log")
+# PROCESSED_HASH_LOG = os.path.join(BASE_DIR, "data", "processed_hashes.log")
 PENDING_UPLOADS_FOLDER = '/tmp/pending_uploads'
 LEGAL_DIC_FOLDER = os.path.join(BASE_DIR, "data", "dictionary")
 

db/mongoDB.py

@@ -20,6 +20,7 @@ class MongoDatabase:
     users: Optional[AsyncIOMotorCollection] = None
     token_blacklist: Optional[AsyncIOMotorCollection] = None
     conversations: Optional[AsyncIOMotorCollection] = None
+    processed_documents: Optional[AsyncIOMotorCollection] = None
 
 # Tạo một instance duy nhất để import và sử dụng trong toàn bộ ứng dụng
 mongo_db = MongoDatabase()
@@ -54,6 +55,8 @@ async def connect_to_mongo():
         mongo_db.users = mongo_db.db["users"]
         mongo_db.token_blacklist = mongo_db.db["token_blacklist"]
         mongo_db.conversations = mongo_db.db["conversations"]
+        mongo_db.processed_documents = mongo_db.db["processed_documents"]
+
 
         # 4. Tạo TTL index một cách an toàn
         # Lấy danh sách index hiện có

services/document_service.py

@@ -38,7 +38,7 @@ def convert_to_text_content(source_path: str) -> str:
     logger.info(f"✅ Successfully extracted content from {source_file.name}.")
     return content
 
-def full_process_and_ingest_pipeline(filepath: str, file_hash: str, embedding_model):
+async def full_process_and_ingest_pipeline(filepath: str, file_hash: str, embedding_model):
     filename = os.path.basename(filepath)
     logger.info(f"BACKGROUND TASK: Starting full pipeline for: {filename} (Hash: {file_hash[:10]}...)")
     weaviate_client = None
@@ -66,7 +66,7 @@ def full_process_and_ingest_pipeline(filepath: str, file_hash: str, embedding_mo
 
         ingest_chunks_with_native_batching(weaviate_client, collection_name, processed_chunks, embeddings_model)
 
-        utils.log_processed_hash(file_hash)
+        await utils.log_processed_hash(file_hash, filename)
         logger.info(f"✅ Successfully ingested '{filename}'.")
         # shutil.move(filepath, os.path.join(config.PROCESSED_FILES_FOLDER, filename))
         logger.info(f"Moved '{filename}' to processed folder.")

utils/utils.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from schemas.chat import  Message
 from redis.asyncio import Redis
 import bcrypt
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from jose import jwt
 from config import SECRET_KEY, ALGORITHM, ACCESS_TOKEN_EXPIRE_MINUTES
 from typing import List, Dict,  Optional
@@ -455,20 +455,33 @@ def calculate_file_hash(filepath: str) -> str:
             sha256_hash.update(byte_block)
     return sha256_hash.hexdigest()
 
-def check_if_hash_exists(file_hash: str) -> bool:
-    if not os.path.exists(config.PROCESSED_HASH_LOG):
-        return False
-    try:
-        with open(config.PROCESSED_HASH_LOG, "r") as f:
-            processed_hashes = {line.strip() for line in f}
-            return file_hash in processed_hashes
-    except IOError as e:
-        logger.error(f"Could not read hash log file: {e}")
-        return False
-
-def log_processed_hash(file_hash: str):
+# def check_if_hash_exists(file_hash: str) -> bool:
+#     if not os.path.exists(config.PROCESSED_HASH_LOG):
+#         return False
+#     try:
+#         with open(config.PROCESSED_HASH_LOG, "r") as f:
+#             processed_hashes = {line.strip() for line in f}
+#             return file_hash in processed_hashes
+#     except IOError as e:
+#         logger.error(f"Could not read hash log file: {e}")
+#         return False
+
+async def check_if_hash_exists(file_hash: str) -> bool:
+    # Đếm số document có hash tương ứng
+    count = await mongo_db.processed_documents.count_documents({"file_hash": file_hash})
+    return count > 0
+
+async def log_processed_hash(file_hash: str, filename: str):
     try:
-        with open(config.PROCESSED_HASH_LOG, "a") as f:
-            f.write(file_hash + "\n")
+        document_record = {
+            "file_hash": file_hash,          # Hash của file
+            "original_filename": filename,   # Tên file gốc
+            "processed_at": datetime.now(timezone.utc), # Thời gian xử lý
+            "status": "SUCCESS",
+            # Thêm các thông tin khác nếu cần, ví dụ:
+            # "source_url": "https://url_cua_file_tren_s3_hoac_cloudinary",
+            # "user_uploader": user_email
+        }
+        await mongo_db.processed_documents.insert_one(document_record)
     except IOError as e:
         logger.error(f"Could not write to hash log file: {e}")