Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -26,7 +26,7 @@ We will load our environment variables here.
|
|
| 26 |
"""
|
| 27 |
HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
|
| 28 |
HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
|
| 29 |
-
HF_TOKEN = os.environ["
|
| 30 |
|
| 31 |
# ---- GLOBAL DECLARATIONS ---- #
|
| 32 |
|
|
@@ -37,7 +37,7 @@ HF_TOKEN = os.environ["HF_TOKEN"]
|
|
| 37 |
3. Load HuggingFace Embeddings (remember to use the URL we set above)
|
| 38 |
4. Index Files if they do not exist, otherwise load the vectorstore
|
| 39 |
"""
|
| 40 |
-
document_loader = TextLoader("
|
| 41 |
documents = document_loader.load()
|
| 42 |
|
| 43 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
|
|
@@ -49,9 +49,9 @@ hf_embeddings = HuggingFaceEndpointEmbeddings(
|
|
| 49 |
huggingfacehub_api_token=HF_TOKEN,
|
| 50 |
)
|
| 51 |
|
| 52 |
-
if os.path.exists("
|
| 53 |
vectorstore = FAISS.load_local(
|
| 54 |
-
"
|
| 55 |
hf_embeddings,
|
| 56 |
allow_dangerous_deserialization=True, # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
| 57 |
)
|
|
@@ -59,7 +59,7 @@ if os.path.exists("./data/vectorstore"):
|
|
| 59 |
print("Loaded Vectorstore")
|
| 60 |
else:
|
| 61 |
print("Indexing Files")
|
| 62 |
-
os.makedirs("
|
| 63 |
for i in range(0, len(split_documents), 32):
|
| 64 |
if i == 0:
|
| 65 |
vectorstore = FAISS.from_documents(
|
|
@@ -67,7 +67,7 @@ else:
|
|
| 67 |
)
|
| 68 |
continue
|
| 69 |
vectorstore.add_documents(split_documents[i : i + 32])
|
| 70 |
-
vectorstore.save_local("
|
| 71 |
|
| 72 |
hf_retriever = vectorstore.as_retriever()
|
| 73 |
|
|
|
|
| 26 |
"""
|
| 27 |
HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
|
| 28 |
HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
|
| 29 |
+
HF_TOKEN = os.environ["HF_API_KEY"]
|
| 30 |
|
| 31 |
# ---- GLOBAL DECLARATIONS ---- #
|
| 32 |
|
|
|
|
| 37 |
3. Load HuggingFace Embeddings (remember to use the URL we set above)
|
| 38 |
4. Index Files if they do not exist, otherwise load the vectorstore
|
| 39 |
"""
|
| 40 |
+
document_loader = TextLoader("paul_graham_essays.txt")
|
| 41 |
documents = document_loader.load()
|
| 42 |
|
| 43 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
|
|
|
|
| 49 |
huggingfacehub_api_token=HF_TOKEN,
|
| 50 |
)
|
| 51 |
|
| 52 |
+
if os.path.exists("vectorstore"):
|
| 53 |
vectorstore = FAISS.load_local(
|
| 54 |
+
"vectorstore",
|
| 55 |
hf_embeddings,
|
| 56 |
allow_dangerous_deserialization=True, # this is necessary to load the vectorstore from disk as it's stored as a `.pkl` file.
|
| 57 |
)
|
|
|
|
| 59 |
print("Loaded Vectorstore")
|
| 60 |
else:
|
| 61 |
print("Indexing Files")
|
| 62 |
+
os.makedirs("vectorstore", exist_ok=True)
|
| 63 |
for i in range(0, len(split_documents), 32):
|
| 64 |
if i == 0:
|
| 65 |
vectorstore = FAISS.from_documents(
|
|
|
|
| 67 |
)
|
| 68 |
continue
|
| 69 |
vectorstore.add_documents(split_documents[i : i + 32])
|
| 70 |
+
vectorstore.save_local("vectorstore")
|
| 71 |
|
| 72 |
hf_retriever = vectorstore.as_retriever()
|
| 73 |
|