inference provider ok

front/package-lock.json

@@ -10,6 +10,7 @@
       "dependencies": {
         "@gradio/client": "^1.12.0",
         "@huggingface/hub": "^1.0.1",
+        "@huggingface/inference": "^3.3.4",
         "@sec-ant/readable-stream": "^0.6.0",
         "autoprefixer": "^10.4.20",
         "base64-arraybuffer": "^1.0.2",
@@ -967,6 +968,18 @@
         "node": ">=18"
       }
     },
+    "node_modules/@huggingface/inference": {
+      "version": "3.3.4",
+      "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-3.3.4.tgz",
+      "integrity": "sha512-IMTaZelduC6xywmm124NgpcnFZ1jPjdUNZgEUuigGneOiwnzWVBhrFt35Gz/oOSqHiQB1tTZG29v7oCJNR2Fog==",
+      "license": "MIT",
+      "dependencies": {
+        "@huggingface/tasks": "^0.15.9"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@huggingface/tasks": {
       "version": "0.15.9",
       "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.15.9.tgz",

front/package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "@gradio/client": "^1.12.0",
     "@huggingface/hub": "^1.0.1",
+    "@huggingface/inference": "^3.3.4",
     "@sec-ant/readable-stream": "^0.6.0",
     "autoprefixer": "^10.4.20",
     "base64-arraybuffer": "^1.0.2",

front/src/App.tsx

@@ -36,6 +36,7 @@ function App() {
               setScript={setGeneratedScript}
               setBusy={setBusy}
               busy={busy}
+              hfToken={hfToken}
             />
 
             <PodcastGenerator

front/src/components/AuthCard.tsx

@@ -28,7 +28,7 @@ export const AuthCard = ({
       console.log('oauthHandleRedirectIfPresent', res);
       if (res) {
         try {
-          const myself = whoAmI({ accessToken: res.accessToken });
+          const myself = await whoAmI({ accessToken: res.accessToken });
           console.log('myself', myself);
         } catch (e) {
           console.log(e);

front/src/components/ScriptMaker.tsx

@@ -1,8 +1,9 @@
 import { useEffect, useState } from 'react';
 import { CONFIG } from '../config';
 import { getPromptGeneratePodcastScript } from '../utils/prompts';
-import { getSSEStreamAsync } from '../utils/utils';
+//import { getSSEStreamAsync } from '../utils/utils';
 import { EXAMPLES } from '../examples';
+import { HfInference } from '@huggingface/inference';
 
 interface SplitContent {
   thought: string;
@@ -30,11 +31,19 @@ export const ScriptMaker = ({
   setScript,
   setBusy,
   busy,
+  hfToken,
 }: {
   setScript: (script: string) => void;
   setBusy: (busy: boolean) => void;
   busy: boolean;
+  hfToken: string;
 }) => {
+  const [model, setModel] = useState<string>(CONFIG.inferenceProviderModels[0]);
+  const [customModel, setCustomModel] = useState<string>(
+    CONFIG.inferenceProviderModels[0]
+  );
+  const usingModel = model === 'custom' ? customModel : model;
+
   const [input, setInput] = useState<string>('');
   const [note, setNote] = useState<string>('');
   const [thought, setThought] = useState<string>('');
@@ -49,12 +58,15 @@ export const ScriptMaker = ({
     setThought('');
     try {
       let responseContent = '';
+      /*
       const fetchResponse = await fetch(CONFIG.llmEndpoint, {
         method: 'POST',
         headers: {
           'Content-Type': 'application/json',
+          'Authorization': `Bearer ${hfToken}`,
         },
         body: JSON.stringify({
+          model: usingModel,
           messages: [
             {
               role: 'user',
@@ -63,18 +75,33 @@ export const ScriptMaker = ({
           ],
           temperature: 0.3,
           stream: true,
+          provider: CONFIG.inferenceProvider,
         }),
       });
       if (fetchResponse.status !== 200) {
         const body = await fetchResponse.json();
-        throw new Error(body?.error?.message || 'Unknown error');
+        throw new Error(body?.error?.message || body?.error || 'Unknown error');
       }
       const chunks = getSSEStreamAsync(fetchResponse);
+      */
+      const client = new HfInference(hfToken);
+      const chunks = client.chatCompletionStream({
+        model: usingModel,
+        messages: [
+          {
+            role: 'user',
+            content: getPromptGeneratePodcastScript(input, note),
+          },
+        ],
+        temperature: 0.3,
+        stream: true,
+        provider: CONFIG.inferenceProvider,
+      });
       for await (const chunk of chunks) {
         // const stop = chunk.stop;
-        if (chunk.error) {
-          throw new Error(chunk.error?.message || 'Unknown error');
-        }
+        //if (chunk.error) {
+        //  throw new Error(chunk.error?.message || 'Unknown error');
+        //}
         const addedContent = chunk.choices[0].delta.content;
         responseContent += addedContent;
         const { thought, codeBlock } = splitContent(responseContent);
@@ -85,7 +112,7 @@ export const ScriptMaker = ({
       }
     } catch (error) {
       console.error(error);
-      alert('Failed to generate the script. Please try again.');
+      alert(`ERROR: ${error}`);
     }
     setIsGenerating(false);
   };
@@ -124,6 +151,7 @@ export const ScriptMaker = ({
           onChange={(e) => setInput(e.target.value)}
           disabled={isGenerating || busy}
         ></textarea>
+
         <textarea
           className="textarea textarea-bordered w-full h-24 p-2"
           placeholder="Optional note (the theme, tone, etc)..."
@@ -132,6 +160,29 @@ export const ScriptMaker = ({
           disabled={isGenerating || busy}
         ></textarea>
 
+        <select
+          className="select select-bordered"
+          value={model}
+          onChange={(e) => setModel(e.target.value)}
+        >
+          {CONFIG.inferenceProviderModels.map((s) => (
+            <option key={s} value={s}>
+              {s}
+            </option>
+          ))}
+          <option value="custom">Custom</option>
+        </select>
+
+        {model === 'custom' && (
+          <input
+            type="text"
+            placeholder="Use a custom model from HF Hub (must be supported by Inference Providers)"
+            className="input input-bordered w-full"
+            value={customModel}
+            onChange={(e) => setCustomModel(e.target.value)}
+          />
+        )}
+
         {thought.length > 0 && (
           <>
             <p>Thought process:</p>

front/src/config.ts

@@ -7,8 +7,16 @@ const LLM_ENDPOINT = window.huggingface?.variables?.LLM_ENDPOINT;
 export const CONFIG = {
   llmEndpoint:
     LLM_ENDPOINT ||
-    'https://gnb1thady6h3noiz.us-east-1.aws.endpoints.huggingface.cloud/v1/chat/completions',
+    'https://router.huggingface.co/hf-inference/v1/chat/completions',
   ttsSpaceId: TTS_SPACE_ID || 'ngxson/kokoro-podcast-backend',
+  inferenceProviderModels: [
+    //'deepseek-ai/DeepSeek-R1-Distill-Qwen-14B',
+    //'deepseek-ai/DeepSeek-R1-Distill-Llama-8B',
+    //'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B',
+    'deepseek-ai/DeepSeek-R1',
+    'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B',
+  ],
+  inferenceProvider: 'together',
 };
 
 console.log({ CONFIG });