Spaces:
Running
Running
Merge pull request #6 from all-in-aigc/feature/support-openai
Browse files- .gitignore +2 -0
- README.md +16 -3
- package.json +1 -0
- src/app/queries/predict.ts +46 -1
.gitignore
CHANGED
@@ -33,3 +33,5 @@ yarn-error.log*
|
|
33 |
# typescript
|
34 |
*.tsbuildinfo
|
35 |
next-env.d.ts
|
|
|
|
|
|
33 |
# typescript
|
34 |
*.tsbuildinfo
|
35 |
next-env.d.ts
|
36 |
+
|
37 |
+
pnpm-lock.yaml
|
README.md
CHANGED
@@ -81,14 +81,27 @@ HF_INFERENCE_ENDPOINT_URL="path to your inference endpoint url"
|
|
81 |
|
82 |
To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
|
83 |
|
84 |
-
### Option 3:
|
85 |
|
86 |
-
|
87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
|
89 |
### Notes
|
90 |
|
91 |
-
It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for
|
92 |
|
93 |
## The Rendering API
|
94 |
|
|
|
81 |
|
82 |
To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
|
83 |
|
84 |
+
### Option 3: Use an OpenAI API Key
|
85 |
|
86 |
+
This is a new option added recently, where you can use OpenAI API with an OpenAI API Key.
|
87 |
|
88 |
+
To activate it, create a `.env.local` configuration file:
|
89 |
+
|
90 |
+
```bash
|
91 |
+
LLM_ENGINE="OPENAI"
|
92 |
+
# default openai api base url is: https://api.openai.com/v1
|
93 |
+
OPENAI_API_BASE_URL="Your OpenAI API Base URL"
|
94 |
+
OPENAI_API_KEY="Your OpenAI API Key"
|
95 |
+
OPENAI_API_MODEL="gpt-3.5-turbo"
|
96 |
+
```
|
97 |
+
|
98 |
+
### Option 4: Fork and modify the code to use a different LLM system
|
99 |
+
|
100 |
+
Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. Claude, Replicate), or a human-generated story instead (by returning mock or static data).
|
101 |
|
102 |
### Notes
|
103 |
|
104 |
+
It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for Claude or Replicate)
|
105 |
|
106 |
## The Rendering API
|
107 |
|
package.json
CHANGED
@@ -43,6 +43,7 @@
|
|
43 |
"html2canvas": "^1.4.1",
|
44 |
"lucide-react": "^0.260.0",
|
45 |
"next": "13.4.10",
|
|
|
46 |
"pick": "^0.0.1",
|
47 |
"postcss": "8.4.26",
|
48 |
"react": "18.2.0",
|
|
|
43 |
"html2canvas": "^1.4.1",
|
44 |
"lucide-react": "^0.260.0",
|
45 |
"next": "13.4.10",
|
46 |
+
"openai": "^4.10.0",
|
47 |
"pick": "^0.0.1",
|
48 |
"postcss": "8.4.26",
|
49 |
"react": "18.2.0",
|
src/app/queries/predict.ts
CHANGED
@@ -1,8 +1,11 @@
|
|
1 |
"use server"
|
2 |
|
3 |
-
import { LLMEngine } from "@/types"
|
4 |
import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
|
5 |
|
|
|
|
|
|
|
|
|
6 |
const hf = new HfInference(process.env.HF_API_TOKEN)
|
7 |
|
8 |
|
@@ -10,6 +13,7 @@ const hf = new HfInference(process.env.HF_API_TOKEN)
|
|
10 |
const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
|
11 |
const inferenceEndpoint = `${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`
|
12 |
const inferenceModel = `${process.env.HF_INFERENCE_API_MODEL || ""}`
|
|
|
13 |
|
14 |
let hfie: HfInferenceEndpoint
|
15 |
|
@@ -34,6 +38,16 @@ switch (llmEngine) {
|
|
34 |
throw new Error(error)
|
35 |
}
|
36 |
break;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
|
38 |
default:
|
39 |
const error = "No Inference Endpoint URL or Inference API Model defined"
|
@@ -45,6 +59,10 @@ export async function predict(inputs: string) {
|
|
45 |
|
46 |
console.log(`predict: `, inputs)
|
47 |
|
|
|
|
|
|
|
|
|
48 |
const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
|
49 |
|
50 |
let instructions = ""
|
@@ -92,4 +110,31 @@ export async function predict(inputs: string) {
|
|
92 |
.replaceAll("<|assistant|>", "")
|
93 |
.replaceAll('""', '"')
|
94 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
95 |
}
|
|
|
1 |
"use server"
|
2 |
|
|
|
3 |
import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
|
4 |
|
5 |
+
import type { ChatCompletionMessage } from "openai/resources/chat"
|
6 |
+
import { LLMEngine } from "@/types"
|
7 |
+
import OpenAI from "openai"
|
8 |
+
|
9 |
const hf = new HfInference(process.env.HF_API_TOKEN)
|
10 |
|
11 |
|
|
|
13 |
const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
|
14 |
const inferenceEndpoint = `${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`
|
15 |
const inferenceModel = `${process.env.HF_INFERENCE_API_MODEL || ""}`
|
16 |
+
const openaiApiKey = `${process.env.OPENAI_API_KEY || ""}`
|
17 |
|
18 |
let hfie: HfInferenceEndpoint
|
19 |
|
|
|
38 |
throw new Error(error)
|
39 |
}
|
40 |
break;
|
41 |
+
|
42 |
+
case "OPENAI":
|
43 |
+
if (openaiApiKey) {
|
44 |
+
console.log("Using an OpenAI API Key")
|
45 |
+
} else {
|
46 |
+
const error = "No OpenAI API key defined"
|
47 |
+
console.error(error)
|
48 |
+
throw new Error(error)
|
49 |
+
}
|
50 |
+
break;
|
51 |
|
52 |
default:
|
53 |
const error = "No Inference Endpoint URL or Inference API Model defined"
|
|
|
59 |
|
60 |
console.log(`predict: `, inputs)
|
61 |
|
62 |
+
if (llmEngine==="OPENAI") {
|
63 |
+
return predictWithOpenAI(inputs)
|
64 |
+
}
|
65 |
+
|
66 |
const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
|
67 |
|
68 |
let instructions = ""
|
|
|
110 |
.replaceAll("<|assistant|>", "")
|
111 |
.replaceAll('""', '"')
|
112 |
)
|
113 |
+
}
|
114 |
+
|
115 |
+
async function predictWithOpenAI(inputs: string) {
|
116 |
+
const openaiApiBaseUrl = `${process.env.OPENAI_API_BASE_URL || "https://api.openai.com/v1"}`
|
117 |
+
const openaiApiModel = `${process.env.OPENAI_API_MODEL || "gpt-3.5-turbo"}`
|
118 |
+
|
119 |
+
const openai = new OpenAI({
|
120 |
+
apiKey: openaiApiKey,
|
121 |
+
baseURL: openaiApiBaseUrl,
|
122 |
+
})
|
123 |
+
|
124 |
+
const messages: ChatCompletionMessage[] = [
|
125 |
+
{ role: "system", content: inputs },
|
126 |
+
]
|
127 |
+
|
128 |
+
try {
|
129 |
+
const res = await openai.chat.completions.create({
|
130 |
+
messages: messages,
|
131 |
+
stream: false,
|
132 |
+
model: openaiApiModel,
|
133 |
+
temperature: 0.8
|
134 |
+
})
|
135 |
+
|
136 |
+
return res.choices[0].message.content
|
137 |
+
} catch (err) {
|
138 |
+
console.error(`error during generation: ${err}`)
|
139 |
+
}
|
140 |
}
|