Update app.py

app.py

@@ -1,69 +1,91 @@
-#refer llama recipes for more info https://github.com/huggingface/huggingface-llama-recipes/blob/main/inference-api.ipynb
-#huggingface-llama-recipes : https://github.com/huggingface/huggingface-llama-recipes/tree/main
-import gradio as gr
-from openai import OpenAI
-import os
+# #refer llama recipes for more info https://github.com/huggingface/huggingface-llama-recipes/blob/main/inference-api.ipynb
+# #huggingface-llama-recipes : https://github.com/huggingface/huggingface-llama-recipes/tree/main
+# import gradio as gr
+# from openai import OpenAI
+# import os
 
-ACCESS_TOKEN = os.getenv("HF_TOKEN")
+# ACCESS_TOKEN = os.getenv("HF_TOKEN")
 
-client = OpenAI(
-    base_url="https://integrate.api.nvidia.com/v1",
-    api_key=ACCESS_TOKEN,
-)
+# client = OpenAI(
+#     base_url="https://integrate.api.nvidia.com/v1",
+#     api_key=ACCESS_TOKEN,
+# )
 
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
+# def respond(
+#     message,
+#     history: list[tuple[str, str]],
+#     system_message,
+#     max_tokens,
+#     temperature,
+#     top_p,
+# ):
+#     messages = [{"role": "system", "content": system_message}]
 
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
+#     for val in history:
+#         if val[0]:
+#             messages.append({"role": "user", "content": val[0]})
+#         if val[1]:
+#             messages.append({"role": "assistant", "content": val[1]})
 
-    messages.append({"role": "user", "content": message})
+#     messages.append({"role": "user", "content": message})
 
-    response = ""
+#     response = ""
     
-    for message in  client.chat.completions.create(
-        model="nvidia/llama-3.1-nemotron-70b-instruct",
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-        messages=messages,
-    ):
-        token = message.choices[0].delta.content
+#     for message in  client.chat.completions.create(
+#         model="nvidia/llama-3.1-nemotron-70b-instruct",
+#         max_tokens=max_tokens,
+#         stream=True,
+#         temperature=temperature,
+#         top_p=top_p,
+#         messages=messages,
+#     ):
+#         token = message.choices[0].delta.content
         
-        response += token
-        yield response
+#         response += token
+#         yield response
         
-chatbot = gr.Chatbot(height=600)
+# chatbot = gr.Chatbot(height=600)
 
-service = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="", label="Системный промпт"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Максимальная длина ответа"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Температура"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="top_p",
-        ),
+# service = gr.ChatInterface(
+#     respond,
+#     additional_inputs=[
+#         gr.Textbox(value="", label="Системный промпт"),
+#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Максимальная длина ответа"),
+#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Температура"),
+#         gr.Slider(
+#             minimum=0.1,
+#             maximum=1.0,
+#             value=0.95,
+#             step=0.05,
+#             label="top_p",
+#         ),
         
-    ],
-    fill_height=True,
-    chatbot=chatbot,
-    theme=gr.themes.Soft(),
+#     ],
+#     fill_height=True,
+#     chatbot=chatbot,
+#     theme=gr.themes.Soft(),
+# )
+# if __name__ == "__main__":
+#     service.launch()
+
+
+from openai import OpenAI
+
+client = OpenAI(
+  base_url = "https://integrate.api.nvidia.com/v1",
+  api_key = "$API_KEY_REQUIRED_IF_EXECUTING_OUTSIDE_NGC"
+)
+
+completion = client.chat.completions.create(
+  model="nvidia/llama-3.1-nemotron-70b-instruct",
+  messages=[{"role":"user","content":"Write a limerick about the wonders of GPU computing."}],
+  temperature=0.5,
+  top_p=1,
+  max_tokens=1024,
+  stream=True
 )
-if __name__ == "__main__":
-    service.launch()
+
+for chunk in completion:
+  if chunk.choices[0].delta.content is not None:
+    print(chunk.choices[0].delta.content, end="")
+