jbilcke-hf HF staff commited on
Commit
cdb6f86
1 Parent(s): 4043a8e
Files changed (5) hide show
  1. .env +5 -3
  2. README.md +22 -7
  3. package-lock.json +12 -0
  4. package.json +1 -0
  5. src/app/engine/render.ts +162 -68
.env CHANGED
@@ -1,13 +1,15 @@
1
  # ------------- IMAGE API CONFIG --------------
2
  # Supported values:
3
  # - VIDEOCHAIN
4
- RENDERING_ENGINE="VIDEOCHAIN"
 
5
 
6
  VIDEOCHAIN_API_URL="http://localhost:7860"
7
  VIDEOCHAIN_API_TOKEN=
8
 
9
- # Not supported yet
10
- REPLICATE_TOKEN=
 
11
 
12
  # ------------- LLM API CONFIG ----------------
13
  # Supported values:
 
1
  # ------------- IMAGE API CONFIG --------------
2
  # Supported values:
3
  # - VIDEOCHAIN
4
+ # - REPLICATE
5
+ RENDERING_ENGINE="REPLICATE"
6
 
7
  VIDEOCHAIN_API_URL="http://localhost:7860"
8
  VIDEOCHAIN_API_TOKEN=
9
 
10
+ REPLICATE_API_TOKEN=
11
+ REPLICATE_API_MODEL="stabilityai/sdxl"
12
+ REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
13
 
14
  # ------------- LLM API CONFIG ----------------
15
  # Supported values:
README.md CHANGED
@@ -17,17 +17,19 @@ First, I would like to highlight that everything is open-source (see [here](http
17
  However the project isn't a monolithic Space that can be duplicated and ran immediately:
18
  it requires various components to run for the frontend, backend, LLM, SDXL etc.
19
 
20
- If you try to duplicate the project, you will see it requires some variables:
21
 
22
  - `LLM_ENGINE`: can be either "INFERENCE_API" or "INFERENCE_ENDPOINT"
23
  - `HF_API_TOKEN`: necessary if you decide to use an inference api model or a custom inference endpoint
24
  - `HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
25
- - `RENDERING_ENGINE`: can only be "VIDEOCHAIN" for now, unless you code your custom solution
26
  - `VIDEOCHAIN_API_URL`: url to the VideoChain API server
27
  - `VIDEOCHAIN_API_TOKEN`: secret token to access the VideoChain API server
 
 
 
28
 
29
  In addition, there are some community sharing variables that you can just ignore.
30
-
31
  Those variables are not required to run the AI Comic Factory on your own website or computer
32
  (they are meant to create a connection with the Hugging Face community,
33
  and thus only make sense for official Hugging Face apps):
@@ -104,10 +106,23 @@ You will have to [clone](https://huggingface.co/spaces/jbilcke-hf/VideoChain-API
104
  Unfortunately, I haven't had the time to write the documentation for VideoChain yet.
105
  (When I do I will update this document to point to the VideoChain's README)
106
 
107
- ### Option 2: Use another SDXL API
108
 
109
- If you fork the project you will be able to modify the code to use the Stable Diffusion technology of your choice (local, open-source, your custom HF Space etc)
110
 
111
- ### Notes
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
 
113
- It is possible that I modify the AI Comic Factory to make it easier in the future (eg. add support for Replicate)
 
17
  However the project isn't a monolithic Space that can be duplicated and ran immediately:
18
  it requires various components to run for the frontend, backend, LLM, SDXL etc.
19
 
20
+ If you try to duplicate the project and open the `.env` you will see it requires some variables:
21
 
22
  - `LLM_ENGINE`: can be either "INFERENCE_API" or "INFERENCE_ENDPOINT"
23
  - `HF_API_TOKEN`: necessary if you decide to use an inference api model or a custom inference endpoint
24
  - `HF_INFERENCE_ENDPOINT_URL`: necessary if you decide to use a custom inference endpoint
25
+ - `RENDERING_ENGINE`: can only be "VIDEOCHAIN" or "REPLICATE" for now, unless you code your custom solution
26
  - `VIDEOCHAIN_API_URL`: url to the VideoChain API server
27
  - `VIDEOCHAIN_API_TOKEN`: secret token to access the VideoChain API server
28
+ - `REPLICATE_API_TOKEN`: in case you want to use Replicate.com
29
+ - `REPLICATE_API_MODEL`: optional, defaults to "stabilityai/sdxl"
30
+ - `REPLICATE_API_MODEL_VERSION`: optional, in case you want to change the version
31
 
32
  In addition, there are some community sharing variables that you can just ignore.
 
33
  Those variables are not required to run the AI Comic Factory on your own website or computer
34
  (they are meant to create a connection with the Hugging Face community,
35
  and thus only make sense for official Hugging Face apps):
 
106
  Unfortunately, I haven't had the time to write the documentation for VideoChain yet.
107
  (When I do I will update this document to point to the VideoChain's README)
108
 
 
109
 
110
+ ### Option 2: Use Replicate
111
 
112
+ To use Replicate, create a `.env.local` configuration file:
113
+
114
+ ```bash
115
+ RENDERING_ENGINE="REPLICATE"
116
+
117
+ REPLICATE_API_TOKEN="Your Replicate token"
118
+
119
+ REPLICATE_API_MODEL="stabilityai/sdxl"
120
+
121
+ REPLICATE_API_MODEL_VERSION="da77bc59ee60423279fd632efb4795ab731d9e3ca9705ef3341091fb989b7eaf"
122
+ ```
123
+
124
+ ### Option 3: Use another SDXL API
125
+
126
+ If you fork the project you will be able to modify the code to use the Stable Diffusion technology of your choice (local, open-source, proprietary, your custom HF Space etc).
127
 
128
+ It would even be something else, such as Dall-E.
package-lock.json CHANGED
@@ -48,6 +48,7 @@
48
  "react-circular-progressbar": "^2.1.0",
49
  "react-dom": "18.2.0",
50
  "react-virtualized-auto-sizer": "^1.0.20",
 
51
  "sbd": "^1.0.19",
52
  "sharp": "^0.32.5",
53
  "styled-components": "^6.0.7",
@@ -7900,6 +7901,17 @@
7900
  "jsesc": "bin/jsesc"
7901
  }
7902
  },
 
 
 
 
 
 
 
 
 
 
 
7903
  "node_modules/resolve": {
7904
  "version": "1.22.4",
7905
  "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz",
 
48
  "react-circular-progressbar": "^2.1.0",
49
  "react-dom": "18.2.0",
50
  "react-virtualized-auto-sizer": "^1.0.20",
51
+ "replicate": "^0.17.0",
52
  "sbd": "^1.0.19",
53
  "sharp": "^0.32.5",
54
  "styled-components": "^6.0.7",
 
7901
  "jsesc": "bin/jsesc"
7902
  }
7903
  },
7904
+ "node_modules/replicate": {
7905
+ "version": "0.17.0",
7906
+ "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.17.0.tgz",
7907
+ "integrity": "sha512-m/ZpGpHpYeUeExvgOm5a+dG/48q5ard2C7xJiOES1TDn6+0IlDOmTp75m/dPQU8xjd3g164ZwMMzJjqAZmNaRw==",
7908
+ "engines": {
7909
+ "git": ">=2.11.0",
7910
+ "node": ">=18.0.0",
7911
+ "npm": ">=7.19.0",
7912
+ "yarn": ">=1.7.0"
7913
+ }
7914
+ },
7915
  "node_modules/resolve": {
7916
  "version": "1.22.4",
7917
  "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.4.tgz",
package.json CHANGED
@@ -49,6 +49,7 @@
49
  "react-circular-progressbar": "^2.1.0",
50
  "react-dom": "18.2.0",
51
  "react-virtualized-auto-sizer": "^1.0.20",
 
52
  "sbd": "^1.0.19",
53
  "sharp": "^0.32.5",
54
  "styled-components": "^6.0.7",
 
49
  "react-circular-progressbar": "^2.1.0",
50
  "react-dom": "18.2.0",
51
  "react-virtualized-auto-sizer": "^1.0.20",
52
+ "replicate": "^0.17.0",
53
  "sbd": "^1.0.19",
54
  "sharp": "^0.32.5",
55
  "styled-components": "^6.0.7",
src/app/engine/render.ts CHANGED
@@ -1,9 +1,17 @@
1
  "use server"
2
 
 
 
3
  import { RenderRequest, RenderedScene, RenderingEngine } from "@/types"
 
 
4
 
5
  const renderingEngine = `${process.env.RENDERING_ENGINE || ""}` as RenderingEngine
6
 
 
 
 
 
7
  // note: there is no / at the end in the variable
8
  // so we have to add it ourselves if needed
9
  const apiUrl = process.env.VIDEOCHAIN_API_URL
@@ -37,54 +45,89 @@ export async function newRender({
37
 
38
 
39
  try {
40
- // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
 
 
 
 
 
 
 
 
 
 
41
 
42
- const res = await fetch(`${apiUrl}/render`, {
43
- method: "POST",
44
- headers: {
45
- Accept: "application/json",
46
- "Content-Type": "application/json",
47
- Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
48
- },
49
- body: JSON.stringify({
50
- prompt,
51
- // negativePrompt, unused for now
52
- nbFrames: 1,
53
- nbSteps: 25, // 20 = fast, 30 = better, 50 = best
54
- actionnables: [], // ["text block"],
55
- segmentation: "disabled", // "firstframe", // one day we will remove this param, to make it automatic
56
- width,
57
- height,
58
-
59
- // no need to upscale right now as we generate tiny panels
60
- // maybe later we can provide an "export" button to PDF
61
- // unfortunately there are too many requests for upscaling,
62
- // the server is always down
63
- upscalingFactor: 1, // 2,
64
-
65
- // analyzing doesn't work yet, it seems..
66
- analyze: false, // analyze: true,
67
-
68
- cache: "ignore"
69
- } as Partial<RenderRequest>),
70
- cache: 'no-store',
71
- // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
72
- // next: { revalidate: 1 }
73
- })
74
 
75
- // console.log("res:", res)
76
- // The return value is *not* serialized
77
- // You can return Date, Map, Set, etc.
78
-
79
- // Recommendation: handle errors
80
- if (res.status !== 200) {
81
- // This will activate the closest `error.js` Error Boundary
82
- throw new Error('Failed to fetch data')
83
- }
84
-
85
- const response = (await res.json()) as RenderedScene
86
 
87
- return response
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  } catch (err) {
89
  console.error(err)
90
  return defaulResult
@@ -108,32 +151,83 @@ export async function getRender(renderId: string) {
108
  }
109
 
110
  try {
111
- // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
112
- const res = await fetch(`${apiUrl}/render/${renderId}`, {
113
- method: "GET",
114
- headers: {
115
- Accept: "application/json",
116
- "Content-Type": "application/json",
117
- Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
118
- },
119
- cache: 'no-store',
120
- // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
121
- // next: { revalidate: 1 }
122
- })
123
 
124
- // console.log("res:", res)
125
- // The return value is *not* serialized
126
- // You can return Date, Map, Set, etc.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
 
128
- // Recommendation: handle errors
129
- if (res.status !== 200) {
130
- // This will activate the closest `error.js` Error Boundary
131
- throw new Error('Failed to fetch data')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  }
133
-
134
- const response = (await res.json()) as RenderedScene
135
- // console.log("response:", response)
136
- return response
137
  } catch (err) {
138
  console.error(err)
139
  defaulResult.status = "error"
 
1
  "use server"
2
 
3
+ import Replicate, { Prediction } from "replicate"
4
+
5
  import { RenderRequest, RenderedScene, RenderingEngine } from "@/types"
6
+ import { generateSeed } from "@/lib/generateSeed"
7
+ import { sleep } from "@/lib/sleep"
8
 
9
  const renderingEngine = `${process.env.RENDERING_ENGINE || ""}` as RenderingEngine
10
 
11
+ const replicateToken = `${process.env.REPLICATE_API_TOKEN || ""}`
12
+ const replicateModel = `${process.env.REPLICATE_API_MODEL || ""}`
13
+ const replicateModelVersion = `${process.env.REPLICATE_API_MODEL_VERSION || ""}`
14
+
15
  // note: there is no / at the end in the variable
16
  // so we have to add it ourselves if needed
17
  const apiUrl = process.env.VIDEOCHAIN_API_URL
 
45
 
46
 
47
  try {
48
+ if (renderingEngine === "REPLICATE") {
49
+ if (!replicateToken) {
50
+ throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
51
+ }
52
+ if (!replicateModel) {
53
+ throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
54
+ }
55
+ if (!replicateModelVersion) {
56
+ throw new Error(`you need to configure your REPLICATE_API_MODEL_VERSION in order to use the REPLICATE rendering engine`)
57
+ }
58
+ const replicate = new Replicate({ auth: replicateToken })
59
 
60
+ // console.log("Calling replicate..")
61
+ const seed = generateSeed()
62
+ const prediction = await replicate.predictions.create({
63
+ version: replicateModelVersion,
64
+ input: { prompt, seed }
65
+ })
66
+
67
+ // console.log("prediction:", prediction)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
 
69
+ // no need to reply straight away: good things take time
70
+ // also our friends at Replicate won't like it if we spam them with requests
71
+ await sleep(4000)
 
 
 
 
 
 
 
 
72
 
73
+ return {
74
+ renderId: prediction.id,
75
+ status: "pending",
76
+ assetUrl: "",
77
+ alt: prompt,
78
+ error: prediction.error,
79
+ maskUrl: "",
80
+ segments: []
81
+ } as RenderedScene
82
+ } else {
83
+ // console.log(`calling POST ${apiUrl}/render with prompt: ${prompt}`)
84
+ const res = await fetch(`${apiUrl}/render`, {
85
+ method: "POST",
86
+ headers: {
87
+ Accept: "application/json",
88
+ "Content-Type": "application/json",
89
+ Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
90
+ },
91
+ body: JSON.stringify({
92
+ prompt,
93
+ // negativePrompt, unused for now
94
+ nbFrames: 1,
95
+ nbSteps: 25, // 20 = fast, 30 = better, 50 = best
96
+ actionnables: [], // ["text block"],
97
+ segmentation: "disabled", // "firstframe", // one day we will remove this param, to make it automatic
98
+ width,
99
+ height,
100
+
101
+ // no need to upscale right now as we generate tiny panels
102
+ // maybe later we can provide an "export" button to PDF
103
+ // unfortunately there are too many requests for upscaling,
104
+ // the server is always down
105
+ upscalingFactor: 1, // 2,
106
+
107
+ // analyzing doesn't work yet, it seems..
108
+ analyze: false, // analyze: true,
109
+
110
+ cache: "ignore"
111
+ } as Partial<RenderRequest>),
112
+ cache: 'no-store',
113
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
114
+ // next: { revalidate: 1 }
115
+ })
116
+
117
+
118
+ // console.log("res:", res)
119
+ // The return value is *not* serialized
120
+ // You can return Date, Map, Set, etc.
121
+
122
+ // Recommendation: handle errors
123
+ if (res.status !== 200) {
124
+ // This will activate the closest `error.js` Error Boundary
125
+ throw new Error('Failed to fetch data')
126
+ }
127
+
128
+ const response = (await res.json()) as RenderedScene
129
+ return response
130
+ }
131
  } catch (err) {
132
  console.error(err)
133
  return defaulResult
 
151
  }
152
 
153
  try {
154
+ if (renderingEngine === "REPLICATE") {
155
+ if (!replicateToken) {
156
+ throw new Error(`you need to configure your REPLICATE_API_TOKEN in order to use the REPLICATE rendering engine`)
157
+ }
158
+ if (!replicateModel) {
159
+ throw new Error(`you need to configure your REPLICATE_API_MODEL in order to use the REPLICATE rendering engine`)
160
+ }
 
 
 
 
 
161
 
162
+ // const replicate = new Replicate({ auth: replicateToken })
163
+
164
+ // console.log("Calling replicate..")
165
+ // const prediction = await replicate.predictions.get(renderId)
166
+ // console.log("Prediction:", prediction)
167
+
168
+ // console.log(`calling GET https://api.replicate.com/v1/predictions/${renderId}`)
169
+ const res = await fetch(`https://api.replicate.com/v1/predictions/${renderId}`, {
170
+ method: "GET",
171
+ headers: {
172
+ // Accept: "application/json",
173
+ // "Content-Type": "application/json",
174
+ Authorization: `Token ${replicateToken}`,
175
+ },
176
+ cache: 'no-store',
177
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
178
+ // next: { revalidate: 1 }
179
+ })
180
 
181
+ // console.log("res:", res)
182
+ // The return value is *not* serialized
183
+ // You can return Date, Map, Set, etc.
184
+
185
+ // Recommendation: handle errors
186
+ if (res.status !== 200) {
187
+ // This will activate the closest `error.js` Error Boundary
188
+ throw new Error('Failed to fetch data')
189
+ }
190
+
191
+ const response = (await res.json()) as any
192
+ // console.log("response:", response)
193
+
194
+ return {
195
+ renderId,
196
+ status: response?.error ? "error" : response?.status === "succeeded" ? "completed" : "pending",
197
+ assetUrl: `${response?.output || ""}`,
198
+ alt: `${response?.input?.prompt || ""}`,
199
+ error: `${response?.error || ""}`,
200
+ maskUrl: "",
201
+ segments: []
202
+ } as RenderedScene
203
+ } else {
204
+ // console.log(`calling GET ${apiUrl}/render with renderId: ${renderId}`)
205
+ const res = await fetch(`${apiUrl}/render/${renderId}`, {
206
+ method: "GET",
207
+ headers: {
208
+ Accept: "application/json",
209
+ "Content-Type": "application/json",
210
+ Authorization: `Bearer ${process.env.VIDEOCHAIN_API_TOKEN}`,
211
+ },
212
+ cache: 'no-store',
213
+ // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
214
+ // next: { revalidate: 1 }
215
+ })
216
+
217
+ // console.log("res:", res)
218
+ // The return value is *not* serialized
219
+ // You can return Date, Map, Set, etc.
220
+
221
+ // Recommendation: handle errors
222
+ if (res.status !== 200) {
223
+ // This will activate the closest `error.js` Error Boundary
224
+ throw new Error('Failed to fetch data')
225
+ }
226
+
227
+ const response = (await res.json()) as RenderedScene
228
+ // console.log("response:", response)
229
+ return response
230
  }
 
 
 
 
231
  } catch (err) {
232
  console.error(err)
233
  defaulResult.status = "error"