me@idoubi.cc commited on
Commit
287a603
1 Parent(s): 166428d

support openai api

Browse files
Files changed (4) hide show
  1. .gitignore +2 -0
  2. README.md +16 -3
  3. package.json +1 -0
  4. src/app/queries/predict.ts +46 -1
.gitignore CHANGED
@@ -33,3 +33,5 @@ yarn-error.log*
33
  # typescript
34
  *.tsbuildinfo
35
  next-env.d.ts
 
 
 
33
  # typescript
34
  *.tsbuildinfo
35
  next-env.d.ts
36
+
37
+ pnpm-lock.yaml
README.md CHANGED
@@ -81,14 +81,27 @@ HF_INFERENCE_ENDPOINT_URL="path to your inference endpoint url"
81
 
82
  To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
83
 
84
- ### Option 3: Fork and modify the code to use a different LLM system
85
 
86
- Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. OpenAI, Replicate), or a human-generated story instead (by returning mock or static data).
87
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
  ### Notes
90
 
91
- It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for OpenAI or Replicate)
92
 
93
  ## The Rendering API
94
 
 
81
 
82
  To run this kind of LLM locally, you can use [TGI](https://github.com/huggingface/text-generation-inference) (Please read [this post](https://github.com/huggingface/text-generation-inference/issues/726) for more information about the licensing).
83
 
84
+ ### Option 3: Use an OpenAI API Key
85
 
86
+ This is a new option added recently, where you can use OpenAI API with an OpenAI API Key.
87
 
88
+ To activate it, create a `.env.local` configuration file:
89
+
90
+ ```bash
91
+ LLM_ENGINE="OPENAI"
92
+ # default openai api base url is: https://api.openai.com/v1
93
+ OPENAI_API_BASE_URL="Your OpenAI API Base URL"
94
+ OPENAI_API_KEY="Your OpenAI API Key"
95
+ OPENAI_API_MODEL="gpt-3.5-turbo"
96
+ ```
97
+
98
+ ### Option 4: Fork and modify the code to use a different LLM system
99
+
100
+ Another option could be to disable the LLM completely and replace it with another LLM protocol and/or provider (eg. Claude, Replicate), or a human-generated story instead (by returning mock or static data).
101
 
102
  ### Notes
103
 
104
+ It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for Claude or Replicate)
105
 
106
  ## The Rendering API
107
 
package.json CHANGED
@@ -43,6 +43,7 @@
43
  "html2canvas": "^1.4.1",
44
  "lucide-react": "^0.260.0",
45
  "next": "13.4.10",
 
46
  "pick": "^0.0.1",
47
  "postcss": "8.4.26",
48
  "react": "18.2.0",
 
43
  "html2canvas": "^1.4.1",
44
  "lucide-react": "^0.260.0",
45
  "next": "13.4.10",
46
+ "openai": "^4.10.0",
47
  "pick": "^0.0.1",
48
  "postcss": "8.4.26",
49
  "react": "18.2.0",
src/app/queries/predict.ts CHANGED
@@ -1,8 +1,11 @@
1
  "use server"
2
 
3
- import { LLMEngine } from "@/types"
4
  import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
5
 
 
 
 
 
6
  const hf = new HfInference(process.env.HF_API_TOKEN)
7
 
8
 
@@ -10,6 +13,7 @@ const hf = new HfInference(process.env.HF_API_TOKEN)
10
  const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
11
  const inferenceEndpoint = `${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`
12
  const inferenceModel = `${process.env.HF_INFERENCE_API_MODEL || ""}`
 
13
 
14
  let hfie: HfInferenceEndpoint
15
 
@@ -34,6 +38,16 @@ switch (llmEngine) {
34
  throw new Error(error)
35
  }
36
  break;
 
 
 
 
 
 
 
 
 
 
37
 
38
  default:
39
  const error = "No Inference Endpoint URL or Inference API Model defined"
@@ -45,6 +59,10 @@ export async function predict(inputs: string) {
45
 
46
  console.log(`predict: `, inputs)
47
 
 
 
 
 
48
  const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
49
 
50
  let instructions = ""
@@ -92,4 +110,31 @@ export async function predict(inputs: string) {
92
  .replaceAll("<|assistant|>", "")
93
  .replaceAll('""', '"')
94
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  }
 
1
  "use server"
2
 
 
3
  import { HfInference, HfInferenceEndpoint } from "@huggingface/inference"
4
 
5
+ import type { ChatCompletionMessage } from "openai/resources/chat"
6
+ import { LLMEngine } from "@/types"
7
+ import OpenAI from "openai"
8
+
9
  const hf = new HfInference(process.env.HF_API_TOKEN)
10
 
11
 
 
13
  const llmEngine = `${process.env.LLM_ENGINE || ""}` as LLMEngine
14
  const inferenceEndpoint = `${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`
15
  const inferenceModel = `${process.env.HF_INFERENCE_API_MODEL || ""}`
16
+ const openaiApiKey = `${process.env.OPENAI_API_KEY || ""}`
17
 
18
  let hfie: HfInferenceEndpoint
19
 
 
38
  throw new Error(error)
39
  }
40
  break;
41
+
42
+ case "OPENAI":
43
+ if (openaiApiKey) {
44
+ console.log("Using an OpenAI API Key")
45
+ } else {
46
+ const error = "No OpenAI API key defined"
47
+ console.error(error)
48
+ throw new Error(error)
49
+ }
50
+ break;
51
 
52
  default:
53
  const error = "No Inference Endpoint URL or Inference API Model defined"
 
59
 
60
  console.log(`predict: `, inputs)
61
 
62
+ if (llmEngine==="OPENAI") {
63
+ return predictWithOpenAI(inputs)
64
+ }
65
+
66
  const api = llmEngine ==="INFERENCE_ENDPOINT" ? hfie : hf
67
 
68
  let instructions = ""
 
110
  .replaceAll("<|assistant|>", "")
111
  .replaceAll('""', '"')
112
  )
113
+ }
114
+
115
+ async function predictWithOpenAI(inputs: string) {
116
+ const openaiApiBaseUrl = `${process.env.OPENAI_API_BASE_URL || "https://api.openai.com/v1"}`
117
+ const openaiApiModel = `${process.env.OPENAI_API_MODEL || "gpt-3.5-turbo"}`
118
+
119
+ const openai = new OpenAI({
120
+ apiKey: openaiApiKey,
121
+ baseURL: openaiApiBaseUrl,
122
+ })
123
+
124
+ const messages: ChatCompletionMessage[] = [
125
+ { role: "system", content: inputs },
126
+ ]
127
+
128
+ try {
129
+ const res = await openai.chat.completions.create({
130
+ messages: messages,
131
+ stream: false,
132
+ model: openaiApiModel,
133
+ temperature: 0.8
134
+ })
135
+
136
+ return res.choices[0].message.content
137
+ } catch (err) {
138
+ console.error(`error during generation: ${err}`)
139
+ }
140
  }