Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Commit
•
bb4f86a
1
Parent(s):
a9f8c5a
afternoon release
Browse files- package-lock.json +14 -14
- package.json +3 -3
- src/app/api/assistant/askAnyAssistant.ts +2 -3
- src/app/api/resolve/providers/comfy/getComfyWorkflow.ts +0 -5
- src/app/api/resolve/providers/huggingface/index.ts +0 -1
- src/app/api/resolve/providers/replicate/index.ts +0 -2
- src/app/api/resolve/providers/stabilityai/index.ts +0 -1
- src/app/api/resolve/providers/stabilityai/performRequest.ts +0 -1
- src/app/main.tsx +1 -1
- src/components/forms/FormSwitch.tsx +0 -1
- src/components/monitor/DynamicPlayer/VideoClipBuffer.tsx +0 -3
- src/components/toolbars/top-menu/ActivitySpinner/index.tsx +24 -0
- src/components/toolbars/top-menu/IsBusy/index.tsx +20 -0
- src/components/toolbars/top-menu/TopMenuLogo/index.tsx +100 -0
- src/components/toolbars/top-menu/image/index.tsx +6 -1
- src/components/toolbars/top-menu/index.tsx +9 -14
- src/components/toolbars/top-menu/music/index.tsx +4 -1
- src/components/toolbars/top-menu/sound/index.tsx +4 -1
- src/components/toolbars/top-menu/video/index.tsx +4 -1
- src/components/toolbars/top-menu/voice/index.tsx +4 -1
- src/controllers/io/parseFileIntoSegments.ts +38 -23
- src/controllers/io/parseFilesIntoSegments.ts +1 -7
- src/controllers/io/useIO.ts +8 -15
- src/controllers/monitor/useMonitor.ts +0 -1
- src/controllers/renderer/useRenderLoop.ts +0 -1
- src/controllers/renderer/useRenderer.ts +1 -3
- src/controllers/resolver/getDefaultResolverState.ts +17 -8
- src/controllers/resolver/types.ts +23 -7
- src/controllers/resolver/useResolver.ts +82 -47
- src/lib/core/constants.ts +1 -1
- src/lib/hooks/useTitle.ts +16 -0
package-lock.json
CHANGED
@@ -8,9 +8,9 @@
|
|
8 |
"name": "@aitube/clapper",
|
9 |
"version": "0.0.1",
|
10 |
"dependencies": {
|
11 |
-
"@aitube/clap": "0.0.
|
12 |
-
"@aitube/engine": "0.0.
|
13 |
-
"@aitube/timeline": "0.0.
|
14 |
"@fal-ai/serverless-client": "^0.10.3",
|
15 |
"@huggingface/hub": "^0.15.1",
|
16 |
"@huggingface/inference": "^2.7.0",
|
@@ -106,9 +106,9 @@
|
|
106 |
}
|
107 |
},
|
108 |
"node_modules/@aitube/clap": {
|
109 |
-
"version": "0.0.
|
110 |
-
"resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.
|
111 |
-
"integrity": "sha512-
|
112 |
"dependencies": {
|
113 |
"pure-uuid": "^1.8.1"
|
114 |
},
|
@@ -117,23 +117,23 @@
|
|
117 |
}
|
118 |
},
|
119 |
"node_modules/@aitube/engine": {
|
120 |
-
"version": "0.0.
|
121 |
-
"resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.
|
122 |
-
"integrity": "sha512-
|
123 |
"peerDependencies": {
|
124 |
-
"@aitube/clap": "0.0.
|
125 |
}
|
126 |
},
|
127 |
"node_modules/@aitube/timeline": {
|
128 |
-
"version": "0.0.
|
129 |
-
"resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.
|
130 |
-
"integrity": "sha512-
|
131 |
"dependencies": {
|
132 |
"date-fns": "^3.6.0",
|
133 |
"react-virtualized-auto-sizer": "^1.0.24"
|
134 |
},
|
135 |
"peerDependencies": {
|
136 |
-
"@aitube/clap": "0.0.
|
137 |
"@radix-ui/react-slider": "^1.1.2",
|
138 |
"@react-spring/three": "^9.7.3",
|
139 |
"@react-spring/types": "^9.7.3",
|
|
|
8 |
"name": "@aitube/clapper",
|
9 |
"version": "0.0.1",
|
10 |
"dependencies": {
|
11 |
+
"@aitube/clap": "0.0.27",
|
12 |
+
"@aitube/engine": "0.0.21",
|
13 |
+
"@aitube/timeline": "0.0.26",
|
14 |
"@fal-ai/serverless-client": "^0.10.3",
|
15 |
"@huggingface/hub": "^0.15.1",
|
16 |
"@huggingface/inference": "^2.7.0",
|
|
|
106 |
}
|
107 |
},
|
108 |
"node_modules/@aitube/clap": {
|
109 |
+
"version": "0.0.27",
|
110 |
+
"resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.27.tgz",
|
111 |
+
"integrity": "sha512-3mPciwLyw4FkMI1c2EnumldlCfbVucOTTUZvqcyZVrNUfHIvjy9MJwpg55rCrER3rdfJpduHkK1muk5orA2c0Q==",
|
112 |
"dependencies": {
|
113 |
"pure-uuid": "^1.8.1"
|
114 |
},
|
|
|
117 |
}
|
118 |
},
|
119 |
"node_modules/@aitube/engine": {
|
120 |
+
"version": "0.0.21",
|
121 |
+
"resolved": "https://registry.npmjs.org/@aitube/engine/-/engine-0.0.21.tgz",
|
122 |
+
"integrity": "sha512-seVvSnzQgUTsb/Vj9Ri6lhZN9LS6Td1D9ytcq9VQgZYSBWRW6zENZdZzOKn0QB8x4MCe+Zdr1NZlZfpukaJa7A==",
|
123 |
"peerDependencies": {
|
124 |
+
"@aitube/clap": "0.0.27"
|
125 |
}
|
126 |
},
|
127 |
"node_modules/@aitube/timeline": {
|
128 |
+
"version": "0.0.26",
|
129 |
+
"resolved": "https://registry.npmjs.org/@aitube/timeline/-/timeline-0.0.26.tgz",
|
130 |
+
"integrity": "sha512-uFC1oF86g2/kWL6eG5bEy1Z1zJUp27t/zkx35o9cnJ0zgNGYMq5XgvZJ91SYdC8TBvKBRosbZroIOx+7EraCRw==",
|
131 |
"dependencies": {
|
132 |
"date-fns": "^3.6.0",
|
133 |
"react-virtualized-auto-sizer": "^1.0.24"
|
134 |
},
|
135 |
"peerDependencies": {
|
136 |
+
"@aitube/clap": "0.0.27",
|
137 |
"@radix-ui/react-slider": "^1.1.2",
|
138 |
"@react-spring/three": "^9.7.3",
|
139 |
"@react-spring/types": "^9.7.3",
|
package.json
CHANGED
@@ -10,9 +10,9 @@
|
|
10 |
"lint": "next lint"
|
11 |
},
|
12 |
"dependencies": {
|
13 |
-
"@aitube/clap": "0.0.
|
14 |
-
"@aitube/engine": "0.0.
|
15 |
-
"@aitube/timeline": "0.0.
|
16 |
"@fal-ai/serverless-client": "^0.10.3",
|
17 |
"@huggingface/hub": "^0.15.1",
|
18 |
"@huggingface/inference": "^2.7.0",
|
|
|
10 |
"lint": "next lint"
|
11 |
},
|
12 |
"dependencies": {
|
13 |
+
"@aitube/clap": "0.0.27",
|
14 |
+
"@aitube/engine": "0.0.21",
|
15 |
+
"@aitube/timeline": "0.0.26",
|
16 |
"@fal-ai/serverless-client": "^0.10.3",
|
17 |
"@huggingface/hub": "^0.15.1",
|
18 |
"@huggingface/inference": "^2.7.0",
|
src/app/api/assistant/askAnyAssistant.ts
CHANGED
@@ -112,7 +112,7 @@ export async function askAnyAssistant({
|
|
112 |
category: segment.category,
|
113 |
} as SimplifiedSegmentData))
|
114 |
|
115 |
-
console.log("INPUT:", JSON.stringify(inputData, null, 2))
|
116 |
|
117 |
const chain = chatPrompt.pipe(coerceable).pipe(parser)
|
118 |
|
@@ -143,7 +143,7 @@ export async function askAnyAssistant({
|
|
143 |
inputData: JSON.stringify(inputData),
|
144 |
})
|
145 |
|
146 |
-
console.log("OUTPUT:", JSON.stringify(result, null, 2))
|
147 |
|
148 |
/*
|
149 |
this whole code doesn't work well actually..
|
@@ -175,7 +175,6 @@ export async function askAnyAssistant({
|
|
175 |
const errObj = err1 as any
|
176 |
try {
|
177 |
const keys = Object.keys(errObj)
|
178 |
-
// console.log("keys:", keys)
|
179 |
if (errObj.llmOutput) {
|
180 |
return {
|
181 |
prompt: "",
|
|
|
112 |
category: segment.category,
|
113 |
} as SimplifiedSegmentData))
|
114 |
|
115 |
+
// console.log("INPUT:", JSON.stringify(inputData, null, 2))
|
116 |
|
117 |
const chain = chatPrompt.pipe(coerceable).pipe(parser)
|
118 |
|
|
|
143 |
inputData: JSON.stringify(inputData),
|
144 |
})
|
145 |
|
146 |
+
// console.log("OUTPUT:", JSON.stringify(result, null, 2))
|
147 |
|
148 |
/*
|
149 |
this whole code doesn't work well actually..
|
|
|
175 |
const errObj = err1 as any
|
176 |
try {
|
177 |
const keys = Object.keys(errObj)
|
|
|
178 |
if (errObj.llmOutput) {
|
179 |
return {
|
180 |
prompt: "",
|
src/app/api/resolve/providers/comfy/getComfyWorkflow.ts
CHANGED
@@ -32,11 +32,6 @@ export function getComfyWorkflow(request: ResolveRequest) {
|
|
32 |
}
|
33 |
output[`${i}`] = node
|
34 |
})
|
35 |
-
|
36 |
-
console.log("DEBUG:", {
|
37 |
-
nodes,
|
38 |
-
output
|
39 |
-
})
|
40 |
|
41 |
return JSON.stringify(output)
|
42 |
}
|
|
|
32 |
}
|
33 |
output[`${i}`] = node
|
34 |
})
|
|
|
|
|
|
|
|
|
|
|
35 |
|
36 |
return JSON.stringify(output)
|
37 |
}
|
src/app/api/resolve/providers/huggingface/index.ts
CHANGED
@@ -13,7 +13,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
|
|
13 |
throw new Error(`Missing API key for "Hugging Face"`)
|
14 |
}
|
15 |
|
16 |
-
console.log(`key: ${request.settings.huggingFaceApiKey}`)
|
17 |
const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey)
|
18 |
|
19 |
if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
|
|
|
13 |
throw new Error(`Missing API key for "Hugging Face"`)
|
14 |
}
|
15 |
|
|
|
16 |
const hf: HfInferenceEndpoint = new HfInference(request.settings.huggingFaceApiKey)
|
17 |
|
18 |
if (request.segment.category !== ClapSegmentCategory.STORYBOARD) {
|
src/app/api/resolve/providers/replicate/index.ts
CHANGED
@@ -28,8 +28,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
|
|
28 |
}
|
29 |
})
|
30 |
|
31 |
-
console.log(`Replicate replied: `, output)
|
32 |
-
|
33 |
segment.assetUrl = await decodeOutput(output)
|
34 |
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
|
35 |
} catch (err) {
|
|
|
28 |
}
|
29 |
})
|
30 |
|
|
|
|
|
31 |
segment.assetUrl = await decodeOutput(output)
|
32 |
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
|
33 |
} catch (err) {
|
src/app/api/resolve/providers/stabilityai/index.ts
CHANGED
@@ -62,7 +62,6 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
|
|
62 |
}
|
63 |
|
64 |
segment.assetUrl = await decodeOutput(content)
|
65 |
-
// console.log(`segment.assetUrl = ${segment.assetUrl.slice(0, 80)}..`)
|
66 |
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
|
67 |
} catch (err) {
|
68 |
console.error(`failed to call Stability.ai: `, err)
|
|
|
62 |
}
|
63 |
|
64 |
segment.assetUrl = await decodeOutput(content)
|
|
|
65 |
segment.assetSourceType = getClapAssetSourceType(segment.assetUrl)
|
66 |
} catch (err) {
|
67 |
console.error(`failed to call Stability.ai: `, err)
|
src/app/api/resolve/providers/stabilityai/performRequest.ts
CHANGED
@@ -54,7 +54,6 @@ export async function performRequest({
|
|
54 |
)
|
55 |
|
56 |
if (response.status === 200) {
|
57 |
-
console.log("response.data: ", response.data)
|
58 |
const buffer = Buffer.from(response.data)
|
59 |
const rawAssetUrl = `data:image/${payload.output_format};base64,${buffer.toString('base64')}`
|
60 |
const assetUrl = await decodeOutput(rawAssetUrl)
|
|
|
54 |
)
|
55 |
|
56 |
if (response.status === 200) {
|
|
|
57 |
const buffer = Buffer.from(response.data)
|
58 |
const rawAssetUrl = `data:image/${payload.output_format};base64,${buffer.toString('base64')}`
|
59 |
const assetUrl = await decodeOutput(rawAssetUrl)
|
src/app/main.tsx
CHANGED
@@ -39,7 +39,7 @@ function MainContent() {
|
|
39 |
NativeTypes.FILE,
|
40 |
],
|
41 |
drop: (item: DroppableThing): void => {
|
42 |
-
console.log("DROP", item)
|
43 |
openFiles(item.files)
|
44 |
},
|
45 |
collect: (monitor) => ({
|
|
|
39 |
NativeTypes.FILE,
|
40 |
],
|
41 |
drop: (item: DroppableThing): void => {
|
42 |
+
// console.log("DROP", item)
|
43 |
openFiles(item.files)
|
44 |
},
|
45 |
collect: (monitor) => ({
|
src/components/forms/FormSwitch.tsx
CHANGED
@@ -20,7 +20,6 @@ export function FormSwitch({ label, className, checked, onCheckedChange, horizon
|
|
20 |
<Switch
|
21 |
checked={checked}
|
22 |
onCheckedChange={(checked) => {
|
23 |
-
console.log("onCheckedChange: " + checked)
|
24 |
onCheckedChange(!checked)
|
25 |
}}
|
26 |
/>
|
|
|
20 |
<Switch
|
21 |
checked={checked}
|
22 |
onCheckedChange={(checked) => {
|
|
|
23 |
onCheckedChange(!checked)
|
24 |
}}
|
25 |
/>
|
src/components/monitor/DynamicPlayer/VideoClipBuffer.tsx
CHANGED
@@ -18,12 +18,9 @@ export function VideoClipBuffer({
|
|
18 |
const togglePlayVideo = (play: boolean) => {
|
19 |
const player = ref.current
|
20 |
if (!player) { return }
|
21 |
-
// console.log(`togglePlayVideo(${play}) (current status = ${player.paused ? "paused" : "playing"})`)
|
22 |
if (play && player.paused) {
|
23 |
-
// console.log("playing video..")
|
24 |
player.play()
|
25 |
} else if (!play && !player.paused) {
|
26 |
-
// console.log("pausing video..")
|
27 |
player.pause()
|
28 |
}
|
29 |
}
|
|
|
18 |
const togglePlayVideo = (play: boolean) => {
|
19 |
const player = ref.current
|
20 |
if (!player) { return }
|
|
|
21 |
if (play && player.paused) {
|
|
|
22 |
player.play()
|
23 |
} else if (!play && !player.paused) {
|
|
|
24 |
player.pause()
|
25 |
}
|
26 |
}
|
src/components/toolbars/top-menu/ActivitySpinner/index.tsx
ADDED
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
import { CgSpinnerTwoAlt } from "react-icons/cg"
|
3 |
+
|
4 |
+
import { cn } from "@/lib/utils"
|
5 |
+
|
6 |
+
export function ActivitySpinner({
|
7 |
+
className = "",
|
8 |
+
isBusy = false,
|
9 |
+
}: {
|
10 |
+
className?: string
|
11 |
+
isBusy?: boolean
|
12 |
+
}) {
|
13 |
+
return (
|
14 |
+
<CgSpinnerTwoAlt
|
15 |
+
className={cn(
|
16 |
+
`w-2 h-2 text-stone-200`,
|
17 |
+
className,
|
18 |
+
isBusy
|
19 |
+
? 'opacity-100 animate-spin'
|
20 |
+
: 'opacity-0',
|
21 |
+
)}
|
22 |
+
/>
|
23 |
+
)
|
24 |
+
}
|
src/components/toolbars/top-menu/IsBusy/index.tsx
ADDED
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { GoDotFill } from "react-icons/go"
|
2 |
+
|
3 |
+
import { cn } from "@/lib/utils"
|
4 |
+
|
5 |
+
export function IsBusy({
|
6 |
+
nbPendingTasks = 0,
|
7 |
+
}: {
|
8 |
+
nbPendingTasks?: number
|
9 |
+
}) {
|
10 |
+
return (
|
11 |
+
<GoDotFill
|
12 |
+
className={cn(
|
13 |
+
`ml-[1px] -mt-[9px] w-1.5 h-1.5 text-yellow-400`,
|
14 |
+
nbPendingTasks > 0
|
15 |
+
? 'opacity-100 animate-pulse'
|
16 |
+
: 'opacity-0'
|
17 |
+
)}
|
18 |
+
/>
|
19 |
+
)
|
20 |
+
}
|
src/components/toolbars/top-menu/TopMenuLogo/index.tsx
ADDED
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { MdMovieFilter, MdOutlineMovieFilter } from "react-icons/md"
|
2 |
+
import { PiPauseFill, PiPlayFill } from "react-icons/pi"
|
3 |
+
import { BiSolidMoviePlay } from "react-icons/bi"
|
4 |
+
// import Image from "next/image"
|
5 |
+
|
6 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
7 |
+
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"
|
8 |
+
// import logo from "../../../app/logo-v2.png"
|
9 |
+
|
10 |
+
import { ActivitySpinner } from "../ActivitySpinner"
|
11 |
+
import { cn } from "@aitube/timeline"
|
12 |
+
|
13 |
+
export function TopMenuLogo() {
|
14 |
+
const isPaused = useResolver(s => s.isPaused)
|
15 |
+
const togglePause = useResolver(s => s.togglePause)
|
16 |
+
const isBusyResolving = useResolver(s => s.isBusyResolving)
|
17 |
+
|
18 |
+
// we could display this in the modal, but it is not very informative
|
19 |
+
// it would be more useful to display things like:
|
20 |
+
// nb remaining assets to process per category (10 images, 40 videos..)
|
21 |
+
// estimated total time
|
22 |
+
// a little timeline of the production rate (a tiny chart)
|
23 |
+
// rendering rate (frames per second etc, kbps etc)
|
24 |
+
const nbRequestsRunningInParallel = useResolver(s => s.nbRequestsRunningInParallel)
|
25 |
+
|
26 |
+
// it doesn't look great when minified like this
|
27 |
+
// <Image src={logo} height={32} alt="Clapper" /> */}
|
28 |
+
|
29 |
+
// we have to disable the button when we are in an intermediary state,
|
30 |
+
// such has a paused resolver with pending tasks
|
31 |
+
// otherwise users get confused, thinking it didn't work
|
32 |
+
// and they might click multiple times
|
33 |
+
const isDisabled = isBusyResolving && isPaused
|
34 |
+
|
35 |
+
return (
|
36 |
+
<Tooltip>
|
37 |
+
<TooltipTrigger className="flex items-center justify-center">
|
38 |
+
<div
|
39 |
+
className={cn(`
|
40 |
+
flex items-center justify-center
|
41 |
+
group
|
42 |
+
`, isDisabled ? "" : "pointer-cursor")}
|
43 |
+
onClick={() => {
|
44 |
+
if (isDisabled) { return }
|
45 |
+
togglePause()
|
46 |
+
}}
|
47 |
+
>
|
48 |
+
{isBusyResolving
|
49 |
+
? <div
|
50 |
+
className="
|
51 |
+
flex items-center justify-center
|
52 |
+
w-4 h-4
|
53 |
+
">
|
54 |
+
<ActivitySpinner
|
55 |
+
isBusy={isBusyResolving}
|
56 |
+
className="absolute opacity-100 group-hover:opacity-0 w-4 h-4 text-yellow-400"
|
57 |
+
/>
|
58 |
+
{isPaused
|
59 |
+
|
60 |
+
// imho using the original "logo" (sort of) is a bit more elegant than a simple play button
|
61 |
+
// ? <PiPlayFill className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
|
62 |
+
? <BiSolidMoviePlay className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
|
63 |
+
|
64 |
+
: <PiPauseFill className="absolute opacity-0 group-hover:opacity-100 text-yellow-400" />
|
65 |
+
|
66 |
+
}
|
67 |
+
</div>
|
68 |
+
: <BiSolidMoviePlay
|
69 |
+
className=" text-yellow-400"
|
70 |
+
/>}
|
71 |
+
<span className="
|
72 |
+
pr-2
|
73 |
+
scale-[88%]
|
74 |
+
text-yellow-400
|
75 |
+
text-lg font-bold tracking-[-0.03em]
|
76 |
+
">Clapper</span>
|
77 |
+
</div>
|
78 |
+
</TooltipTrigger>
|
79 |
+
<TooltipContent>
|
80 |
+
<div className="text-xs text-yellow-300">
|
81 |
+
{/*
|
82 |
+
<p>{
|
83 |
+
nbRequestsRunningInParallel || 0
|
84 |
+
} remaining task{
|
85 |
+
nbRequestsRunningInParallel > 1 ? 's' : ''
|
86 |
+
}</p>
|
87 |
+
*/}
|
88 |
+
{isBusyResolving
|
89 |
+
? <span>Some tasks are still pending.</span>
|
90 |
+
: <span>No pending task in progress.</span>
|
91 |
+
}
|
92 |
+
<br/>
|
93 |
+
{isPaused
|
94 |
+
? <span>Click to RESUME generation.</span>
|
95 |
+
: <span>Click to PAUSE generation.</span>}
|
96 |
+
</div>
|
97 |
+
</TooltipContent>
|
98 |
+
</Tooltip>
|
99 |
+
)
|
100 |
+
}
|
src/components/toolbars/top-menu/image/index.tsx
CHANGED
@@ -19,9 +19,14 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
|
|
19 |
import { availableComputeProvidersForImages } from "@/components/settings/constants"
|
20 |
import { ImageGenerationModelList } from "../lists/ImageGenerationModelList"
|
21 |
import { ImageUpscalingModelList } from "../lists/ImageUpscalingModelList"
|
|
|
|
|
22 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
|
|
|
|
23 |
|
24 |
export function TopMenuImage() {
|
|
|
25 |
const setShowSettings = useUI(s => s.setShowSettings)
|
26 |
const imageProvider = useSettings(s => s.imageProvider)
|
27 |
const setImageProvider = useSettings(s => s.setImageProvider)
|
@@ -33,7 +38,7 @@ export function TopMenuImage() {
|
|
33 |
const setImageRenderingStrategy = useSettings((s) => s.setImageRenderingStrategy)
|
34 |
return (
|
35 |
<MenubarMenu>
|
36 |
-
<MenubarTrigger>Image</MenubarTrigger>
|
37 |
<MenubarContent>
|
38 |
<MenubarSub>
|
39 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.IMAGE) }}>Show advanced settings</MenubarItem>
|
|
|
19 |
import { availableComputeProvidersForImages } from "@/components/settings/constants"
|
20 |
import { ImageGenerationModelList } from "../lists/ImageGenerationModelList"
|
21 |
import { ImageUpscalingModelList } from "../lists/ImageUpscalingModelList"
|
22 |
+
import { IsBusy } from "../IsBusy"
|
23 |
+
|
24 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
25 |
+
import { useRenderer } from "@/controllers/renderer"
|
26 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
27 |
|
28 |
export function TopMenuImage() {
|
29 |
+
const nbPendingRequestsForImage = useResolver(s => s.nbPendingRequestsForImage)
|
30 |
const setShowSettings = useUI(s => s.setShowSettings)
|
31 |
const imageProvider = useSettings(s => s.imageProvider)
|
32 |
const setImageProvider = useSettings(s => s.setImageProvider)
|
|
|
38 |
const setImageRenderingStrategy = useSettings((s) => s.setImageRenderingStrategy)
|
39 |
return (
|
40 |
<MenubarMenu>
|
41 |
+
<MenubarTrigger><span>Image</span><IsBusy nbPendingTasks={nbPendingRequestsForImage} /></MenubarTrigger>
|
42 |
<MenubarContent>
|
43 |
<MenubarSub>
|
44 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.IMAGE) }}>Show advanced settings</MenubarItem>
|
src/components/toolbars/top-menu/index.tsx
CHANGED
@@ -1,30 +1,25 @@
|
|
|
|
|
|
1 |
import { Menubar } from "@/components/ui/menubar"
|
|
|
|
|
2 |
|
3 |
import { TopMenuFile } from "./file"
|
4 |
-
|
5 |
import { TopMenuImage } from "./image"
|
6 |
import { TopMenuVideo } from "./video"
|
7 |
import { TopMenuVoice } from "./voice"
|
8 |
import { TopMenuSound } from "./sound"
|
9 |
import { TopMenuMusic } from "./music"
|
10 |
import { TopMenuAssistant } from "./assistant"
|
11 |
-
|
12 |
import { TopMenuView } from "./view"
|
13 |
-
import {
|
14 |
-
import { APP_REVISION } from "@/lib/core/constants"
|
15 |
-
import Image from "next/image"
|
16 |
-
import logo from "../../../app/logo-v2.png"
|
17 |
|
18 |
export function TopMenu() {
|
|
|
|
|
19 |
return (
|
20 |
-
<Menubar className="w-full">
|
21 |
-
|
22 |
-
it doesn't look great when minified like this
|
23 |
-
<Image src={logo} height={32} alt="Clapper" /> */}
|
24 |
-
<span
|
25 |
-
className="scale-[88%] text-yellow-400 pl-2 text-lg font-bold tracking-[-0.03em] mr-2"
|
26 |
-
><span>Clapper</span><span className="text-yellow-100">.</span><span
|
27 |
-
className="hidden absolute text-3xs text-red-500/100 tracking-[0.05em] uppercase mt-1.5 -ml-0 -rotate-6 ">ai</span></span>
|
28 |
<TopMenuFile />
|
29 |
<TopMenuImage />
|
30 |
<TopMenuVideo />
|
|
|
1 |
+
import { cn } from "@aitube/timeline"
|
2 |
+
|
3 |
import { Menubar } from "@/components/ui/menubar"
|
4 |
+
import { APP_REVISION } from "@/lib/core/constants"
|
5 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
6 |
|
7 |
import { TopMenuFile } from "./file"
|
|
|
8 |
import { TopMenuImage } from "./image"
|
9 |
import { TopMenuVideo } from "./video"
|
10 |
import { TopMenuVoice } from "./voice"
|
11 |
import { TopMenuSound } from "./sound"
|
12 |
import { TopMenuMusic } from "./music"
|
13 |
import { TopMenuAssistant } from "./assistant"
|
|
|
14 |
import { TopMenuView } from "./view"
|
15 |
+
import { TopMenuLogo } from "./TopMenuLogo"
|
|
|
|
|
|
|
16 |
|
17 |
export function TopMenu() {
|
18 |
+
const isBusyResolving = useResolver(s => s.isBusyResolving)
|
19 |
+
|
20 |
return (
|
21 |
+
<Menubar className="w-full ml-1">
|
22 |
+
<TopMenuLogo />
|
|
|
|
|
|
|
|
|
|
|
|
|
23 |
<TopMenuFile />
|
24 |
<TopMenuImage />
|
25 |
<TopMenuVideo />
|
src/components/toolbars/top-menu/music/index.tsx
CHANGED
@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
|
|
19 |
import { availableComputeProvidersForMusic } from "@/components/settings/constants"
|
20 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
21 |
import { MusicGenerationModelList } from "../lists/MusicGenerationModelList"
|
|
|
|
|
22 |
|
23 |
export function TopMenuMusic() {
|
|
|
24 |
const setShowSettings = useUI(s => s.setShowSettings)
|
25 |
const musicProvider = useSettings(s => s.musicProvider)
|
26 |
const setMusicProvider = useSettings(s => s.setMusicProvider)
|
@@ -31,7 +34,7 @@ export function TopMenuMusic() {
|
|
31 |
|
32 |
return (
|
33 |
<MenubarMenu>
|
34 |
-
<MenubarTrigger>Music</MenubarTrigger>
|
35 |
<MenubarContent>
|
36 |
<MenubarSub>
|
37 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.MUSIC) }}>Show advanced settings</MenubarItem>
|
|
|
19 |
import { availableComputeProvidersForMusic } from "@/components/settings/constants"
|
20 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
21 |
import { MusicGenerationModelList } from "../lists/MusicGenerationModelList"
|
22 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
23 |
+
import { IsBusy } from "../IsBusy"
|
24 |
|
25 |
export function TopMenuMusic() {
|
26 |
+
const nbPendingRequestsForMusic = useResolver(s => s.nbPendingRequestsForMusic)
|
27 |
const setShowSettings = useUI(s => s.setShowSettings)
|
28 |
const musicProvider = useSettings(s => s.musicProvider)
|
29 |
const setMusicProvider = useSettings(s => s.setMusicProvider)
|
|
|
34 |
|
35 |
return (
|
36 |
<MenubarMenu>
|
37 |
+
<MenubarTrigger><span>Music</span><IsBusy nbPendingTasks={nbPendingRequestsForMusic} /></MenubarTrigger>
|
38 |
<MenubarContent>
|
39 |
<MenubarSub>
|
40 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.MUSIC) }}>Show advanced settings</MenubarItem>
|
src/components/toolbars/top-menu/sound/index.tsx
CHANGED
@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
|
|
19 |
import { availableComputeProvidersForSound } from "@/components/settings/constants"
|
20 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
21 |
import { SoundGenerationModelList } from "../lists/SoundGenerationModelList"
|
|
|
|
|
22 |
|
23 |
export function TopMenuSound() {
|
|
|
24 |
const setShowSettings = useUI(s => s.setShowSettings)
|
25 |
const soundProvider = useSettings(s => s.soundProvider)
|
26 |
const setSoundProvider = useSettings(s => s.setSoundProvider)
|
@@ -30,7 +33,7 @@ export function TopMenuSound() {
|
|
30 |
const setSoundRenderingStrategy = useSettings((s) => s.setSoundRenderingStrategy)
|
31 |
return (
|
32 |
<MenubarMenu>
|
33 |
-
<MenubarTrigger>Sound</MenubarTrigger>
|
34 |
<MenubarContent>
|
35 |
<MenubarSub>
|
36 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.SOUND) }}>Show advanced settings</MenubarItem>
|
|
|
19 |
import { availableComputeProvidersForSound } from "@/components/settings/constants"
|
20 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
21 |
import { SoundGenerationModelList } from "../lists/SoundGenerationModelList"
|
22 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
23 |
+
import { IsBusy } from "../IsBusy"
|
24 |
|
25 |
export function TopMenuSound() {
|
26 |
+
const nbPendingRequestsForSound = useResolver(s => s.nbPendingRequestsForSound)
|
27 |
const setShowSettings = useUI(s => s.setShowSettings)
|
28 |
const soundProvider = useSettings(s => s.soundProvider)
|
29 |
const setSoundProvider = useSettings(s => s.setSoundProvider)
|
|
|
33 |
const setSoundRenderingStrategy = useSettings((s) => s.setSoundRenderingStrategy)
|
34 |
return (
|
35 |
<MenubarMenu>
|
36 |
+
<MenubarTrigger><span>Sound</span><IsBusy nbPendingTasks={nbPendingRequestsForSound} /></MenubarTrigger>
|
37 |
<MenubarContent>
|
38 |
<MenubarSub>
|
39 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.SOUND) }}>Show advanced settings</MenubarItem>
|
src/components/toolbars/top-menu/video/index.tsx
CHANGED
@@ -20,8 +20,11 @@ import { availableComputeProvidersForVideos } from "@/components/settings/consta
|
|
20 |
import { VideoGenerationModelList } from "../lists/VideoGenerationModelList"
|
21 |
import { VideoUpscalingModelList } from "../lists/VideoUpscalingModelList"
|
22 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
|
|
|
|
23 |
|
24 |
export function TopMenuVideo() {
|
|
|
25 |
const setShowSettings = useUI(s => s.setShowSettings)
|
26 |
const videoProvider = useSettings(s => s.videoProvider)
|
27 |
const setVideoProvider = useSettings(s => s.setVideoProvider)
|
@@ -33,7 +36,7 @@ export function TopMenuVideo() {
|
|
33 |
const setVideoRenderingStrategy = useSettings((s) => s.setVideoRenderingStrategy)
|
34 |
return (
|
35 |
<MenubarMenu>
|
36 |
-
<MenubarTrigger>Video</MenubarTrigger>
|
37 |
<MenubarContent>
|
38 |
<MenubarSub>
|
39 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.VIDEO) }}>Show advanced settings</MenubarItem>
|
|
|
20 |
import { VideoGenerationModelList } from "../lists/VideoGenerationModelList"
|
21 |
import { VideoUpscalingModelList } from "../lists/VideoUpscalingModelList"
|
22 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
23 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
24 |
+
import { IsBusy } from "../IsBusy"
|
25 |
|
26 |
export function TopMenuVideo() {
|
27 |
+
const nbPendingRequestsForVideo = useResolver(s => s.nbPendingRequestsForVideo)
|
28 |
const setShowSettings = useUI(s => s.setShowSettings)
|
29 |
const videoProvider = useSettings(s => s.videoProvider)
|
30 |
const setVideoProvider = useSettings(s => s.setVideoProvider)
|
|
|
36 |
const setVideoRenderingStrategy = useSettings((s) => s.setVideoRenderingStrategy)
|
37 |
return (
|
38 |
<MenubarMenu>
|
39 |
+
<MenubarTrigger><span>Video</span><IsBusy nbPendingTasks={nbPendingRequestsForVideo} /></MenubarTrigger>
|
40 |
<MenubarContent>
|
41 |
<MenubarSub>
|
42 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.VIDEO) }}>Show advanced settings</MenubarItem>
|
src/components/toolbars/top-menu/voice/index.tsx
CHANGED
@@ -19,8 +19,11 @@ import { RenderingStrategyList } from "../lists/RenderingStrategyList"
|
|
19 |
import { availableComputeProvidersForVoice } from "@/components/settings/constants"
|
20 |
import { VoiceGenerationModelList } from "../lists/VoiceGenerationModelList"
|
21 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
|
|
|
|
22 |
|
23 |
export function TopMenuVoice() {
|
|
|
24 |
const setShowSettings = useUI(s => s.setShowSettings)
|
25 |
const voiceProvider = useSettings(s => s.voiceProvider)
|
26 |
const setVoiceProvider = useSettings(s => s.setVoiceProvider)
|
@@ -30,7 +33,7 @@ export function TopMenuVoice() {
|
|
30 |
const setVoiceRenderingStrategy = useSettings((s) => s.setVoiceRenderingStrategy)
|
31 |
return (
|
32 |
<MenubarMenu>
|
33 |
-
<MenubarTrigger>Voice</MenubarTrigger>
|
34 |
<MenubarContent>
|
35 |
<MenubarSub>
|
36 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.VOICE) }}>Show advanced settings</MenubarItem>
|
|
|
19 |
import { availableComputeProvidersForVoice } from "@/components/settings/constants"
|
20 |
import { VoiceGenerationModelList } from "../lists/VoiceGenerationModelList"
|
21 |
import { ComputeProvider, SettingsCategory } from "@/types"
|
22 |
+
import { useResolver } from "@/controllers/resolver/useResolver"
|
23 |
+
import { IsBusy } from "../IsBusy"
|
24 |
|
25 |
export function TopMenuVoice() {
|
26 |
+
const nbPendingRequestsForVoice = useResolver(s => s.nbPendingRequestsForVoice)
|
27 |
const setShowSettings = useUI(s => s.setShowSettings)
|
28 |
const voiceProvider = useSettings(s => s.voiceProvider)
|
29 |
const setVoiceProvider = useSettings(s => s.setVoiceProvider)
|
|
|
33 |
const setVoiceRenderingStrategy = useSettings((s) => s.setVoiceRenderingStrategy)
|
34 |
return (
|
35 |
<MenubarMenu>
|
36 |
+
<MenubarTrigger><span>Voice</span><IsBusy nbPendingTasks={nbPendingRequestsForVoice} /></MenubarTrigger>
|
37 |
<MenubarContent>
|
38 |
<MenubarSub>
|
39 |
<MenubarItem onClick={() => { setShowSettings(SettingsCategory.VOICE) }}>Show advanced settings</MenubarItem>
|
src/controllers/io/parseFileIntoSegments.ts
CHANGED
@@ -1,53 +1,50 @@
|
|
1 |
"use client"
|
2 |
|
3 |
-
import { ClapOutputType, ClapSegment, ClapSegmentCategory, generateSeed, newSegment, UUID } from "@aitube/clap"
|
4 |
import { findFreeTrack } from "@aitube/timeline"
|
5 |
|
6 |
import { RuntimeSegment } from "@/types"
|
7 |
|
8 |
import { analyzeAudio } from "../audio/analyzeAudio"
|
9 |
import { ResourceCategory, ResourceType } from "./types"
|
|
|
10 |
|
11 |
-
export async function parseFileIntoSegments({ file
|
12 |
/**
|
13 |
* The file to import
|
14 |
*/
|
15 |
file: File
|
16 |
-
|
17 |
-
|
18 |
-
/**
|
19 |
-
* all existing segments
|
20 |
-
*/
|
21 |
-
segments: ClapSegment[]
|
22 |
}): Promise<ClapSegment[]> {
|
23 |
-
console.log(`
|
24 |
-
console.log(`file size
|
25 |
-
console.log(`file type
|
26 |
|
27 |
const extension = file.name.split(".").pop()?.toLowerCase()
|
28 |
|
29 |
console.log("TODO: open a popup to ask if this is a voice character sample, dialogue, music etc")
|
30 |
|
31 |
let type: ResourceType = "misc"
|
32 |
-
let
|
33 |
|
34 |
const newSegments: ClapSegment[] = []
|
35 |
|
36 |
switch (file.type) {
|
37 |
case "image/webp":
|
38 |
type = "image"
|
39 |
-
|
40 |
break;
|
41 |
|
42 |
-
case "audio/mpeg":
|
|
|
43 |
case "audio/wav":
|
44 |
case "audio/mp4":
|
|
|
45 |
case "audio/m4a": // shouldn't exist
|
46 |
-
case "audio/x-m4a": // should be rare,
|
47 |
case "audio/webm":
|
48 |
// for background track, or as an inspiration track, or a voice etc
|
49 |
type = "audio"
|
50 |
-
|
51 |
|
52 |
// TODO: add caption analysis
|
53 |
const { durationInMs,durationInSteps, bpm, audioBuffer } = await analyzeAudio(file)
|
@@ -62,17 +59,35 @@ export async function parseFileIntoSegments({ file, segments }: {
|
|
62 |
const endTimeInSteps = durationInSteps
|
63 |
const endTimeInMs = startTimeInMs + durationInMs
|
64 |
|
65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
66 |
prompt: "audio track",
|
67 |
startTimeInMs, // start time of the segment
|
68 |
endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
|
69 |
-
|
|
|
70 |
label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined)
|
71 |
-
category
|
72 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
73 |
|
74 |
const audioSegment: RuntimeSegment = {
|
75 |
...clapSegment,
|
|
|
76 |
outputType: ClapOutputType.AUDIO,
|
77 |
outputGain: 1,
|
78 |
audioBuffer,
|
@@ -81,13 +96,13 @@ export async function parseFileIntoSegments({ file, segments }: {
|
|
81 |
// console.log("newSegment:", audioSegment)
|
82 |
|
83 |
// poof! type disappears.. it's magic
|
84 |
-
newSegments.push(audioSegment
|
85 |
break;
|
86 |
|
87 |
case "text/plain":
|
88 |
// for dialogue, prompts..
|
89 |
type = "text"
|
90 |
-
|
91 |
break;
|
92 |
|
93 |
default:
|
@@ -116,6 +131,6 @@ export async function parseFileIntoSegments({ file, segments }: {
|
|
116 |
|
117 |
// Note: uploading is optional, some file type don't need it (eg. text prompt)
|
118 |
|
119 |
-
return
|
120 |
}
|
121 |
|
|
|
1 |
"use client"
|
2 |
|
3 |
+
import { ClapAssetSource, ClapOutputType, ClapSegment, ClapSegmentCategory, ClapSegmentStatus, generateSeed, newSegment, UUID } from "@aitube/clap"
|
4 |
import { findFreeTrack } from "@aitube/timeline"
|
5 |
|
6 |
import { RuntimeSegment } from "@/types"
|
7 |
|
8 |
import { analyzeAudio } from "../audio/analyzeAudio"
|
9 |
import { ResourceCategory, ResourceType } from "./types"
|
10 |
+
import { blobToBase64DataUri } from "@/lib/utils/blobToBase64DataUri"
|
11 |
|
12 |
+
export async function parseFileIntoSegments({ file }: {
|
13 |
/**
|
14 |
* The file to import
|
15 |
*/
|
16 |
file: File
|
|
|
|
|
|
|
|
|
|
|
|
|
17 |
}): Promise<ClapSegment[]> {
|
18 |
+
// console.log(`parseFileIntoSegments(): filename = ${file.name}`)
|
19 |
+
// console.log(`parseFileIntoSegments(): file size = ${file.size} bytes`)
|
20 |
+
// console.log(`parseFileIntoSegments(): file type = ${file.type}`)
|
21 |
|
22 |
const extension = file.name.split(".").pop()?.toLowerCase()
|
23 |
|
24 |
console.log("TODO: open a popup to ask if this is a voice character sample, dialogue, music etc")
|
25 |
|
26 |
let type: ResourceType = "misc"
|
27 |
+
let resourceCategory: ResourceCategory = "misc"
|
28 |
|
29 |
const newSegments: ClapSegment[] = []
|
30 |
|
31 |
switch (file.type) {
|
32 |
case "image/webp":
|
33 |
type = "image"
|
34 |
+
resourceCategory = "control_image"
|
35 |
break;
|
36 |
|
37 |
+
case "audio/mpeg": // this is the "official" one
|
38 |
+
case "audio/mp3": // this is just an alias
|
39 |
case "audio/wav":
|
40 |
case "audio/mp4":
|
41 |
+
case "audio/x-mp4": // should be rare, normally is is audio/mp4
|
42 |
case "audio/m4a": // shouldn't exist
|
43 |
+
case "audio/x-m4a": // should be rare, normally is is audio/mp4
|
44 |
case "audio/webm":
|
45 |
// for background track, or as an inspiration track, or a voice etc
|
46 |
type = "audio"
|
47 |
+
resourceCategory = "background_music"
|
48 |
|
49 |
// TODO: add caption analysis
|
50 |
const { durationInMs,durationInSteps, bpm, audioBuffer } = await analyzeAudio(file)
|
|
|
59 |
const endTimeInSteps = durationInSteps
|
60 |
const endTimeInMs = startTimeInMs + durationInMs
|
61 |
|
62 |
+
// ok let's stop for a minute there:
|
63 |
+
// if someone drops a .mp3, and assuming we don't yet have the UI to select the category,
|
64 |
+
// do you think it should be a SOUND, a VOICE or a MUSIC by default?
|
65 |
+
// I expect people will use AI service providers for sound and voice,
|
66 |
+
// maybe in some case music too, but there are also many people
|
67 |
+
// who will want to use their own track eg. to create a music video
|
68 |
+
const category = ClapSegmentCategory.MUSIC
|
69 |
+
|
70 |
+
const assetUrl = await blobToBase64DataUri(file)
|
71 |
+
|
72 |
+
const newSegmentData: Partial<ClapSegment> = {
|
73 |
prompt: "audio track",
|
74 |
startTimeInMs, // start time of the segment
|
75 |
endTimeInMs, // end time of the segment (startTimeInMs + durationInMs)
|
76 |
+
status: ClapSegmentStatus.COMPLETED,
|
77 |
+
// track: findFreeTrack({ segments, startTimeInMs, endTimeInMs }), // track row index
|
78 |
label: `${file.name} (${Math.round(durationInMs / 1000)}s @ ${Math.round(bpm * 100) / 100} BPM)`, // a short label to name the segment (optional, can be human or LLM-defined)
|
79 |
+
category,
|
80 |
+
assetUrl,
|
81 |
+
assetDurationInMs: endTimeInMs,
|
82 |
+
assetSourceType: ClapAssetSource.DATA,
|
83 |
+
assetFileFormat: `${file.type}`,
|
84 |
+
}
|
85 |
+
|
86 |
+
const clapSegment = newSegment(newSegmentData)
|
87 |
|
88 |
const audioSegment: RuntimeSegment = {
|
89 |
...clapSegment,
|
90 |
+
|
91 |
outputType: ClapOutputType.AUDIO,
|
92 |
outputGain: 1,
|
93 |
audioBuffer,
|
|
|
96 |
// console.log("newSegment:", audioSegment)
|
97 |
|
98 |
// poof! type disappears.. it's magic
|
99 |
+
newSegments.push(audioSegment)
|
100 |
break;
|
101 |
|
102 |
case "text/plain":
|
103 |
// for dialogue, prompts..
|
104 |
type = "text"
|
105 |
+
resourceCategory = "text_prompt"
|
106 |
break;
|
107 |
|
108 |
default:
|
|
|
131 |
|
132 |
// Note: uploading is optional, some file type don't need it (eg. text prompt)
|
133 |
|
134 |
+
return newSegments
|
135 |
}
|
136 |
|
src/controllers/io/parseFilesIntoSegments.ts
CHANGED
@@ -2,21 +2,15 @@
|
|
2 |
import { ClapSegment } from "@aitube/clap"
|
3 |
import { parseFileIntoSegments } from "./parseFileIntoSegments"
|
4 |
|
5 |
-
export async function parseFilesIntoSegments({ files
|
6 |
/**
|
7 |
* The files to import
|
8 |
*/
|
9 |
files: File[]
|
10 |
-
|
11 |
-
/**
|
12 |
-
* all existing segments
|
13 |
-
*/
|
14 |
-
segments: ClapSegment[]
|
15 |
}): Promise<ClapSegment[]> {
|
16 |
return (
|
17 |
await Promise.all(files.map(file => parseFileIntoSegments({
|
18 |
file,
|
19 |
-
segments,
|
20 |
})))
|
21 |
).reduce((acc, segments) => [...acc, ...segments], [])
|
22 |
}
|
|
|
2 |
import { ClapSegment } from "@aitube/clap"
|
3 |
import { parseFileIntoSegments } from "./parseFileIntoSegments"
|
4 |
|
5 |
+
export async function parseFilesIntoSegments({ files }: {
|
6 |
/**
|
7 |
* The files to import
|
8 |
*/
|
9 |
files: File[]
|
|
|
|
|
|
|
|
|
|
|
10 |
}): Promise<ClapSegment[]> {
|
11 |
return (
|
12 |
await Promise.all(files.map(file => parseFileIntoSegments({
|
13 |
file,
|
|
|
14 |
})))
|
15 |
).reduce((acc, segments) => [...acc, ...segments], [])
|
16 |
}
|
src/controllers/io/useIO.ts
CHANGED
@@ -31,7 +31,8 @@ export const useIO = create<IOStore>((set, get) => ({
|
|
31 |
},
|
32 |
openFiles: async (files: File[]) => {
|
33 |
const { openClapBlob, openScreenplay } = get()
|
34 |
-
const
|
|
|
35 |
|
36 |
if (Array.isArray(files)) {
|
37 |
console.log("user tried to drop some files:", files)
|
@@ -66,25 +67,17 @@ export const useIO = create<IOStore>((set, get) => ({
|
|
66 |
// screenplay files: -> analyze (if there is existing data, show a modal asking to save or not)
|
67 |
// mp3 file: ->
|
68 |
if (isAudioFile) {
|
69 |
-
const newSegments = await parseFileIntoSegments({
|
70 |
-
|
71 |
-
|
|
|
|
|
72 |
})
|
|
|
73 |
}
|
74 |
|
75 |
const isVideoFile = fileType.startsWith("video/")
|
76 |
|
77 |
-
|
78 |
-
// for the moment let's not care of the coordinates at all
|
79 |
-
/*
|
80 |
-
parseFilesIntoSegments({
|
81 |
-
files,
|
82 |
-
columnIndex,
|
83 |
-
rowIndex,
|
84 |
-
segments,
|
85 |
-
segment
|
86 |
-
})
|
87 |
-
*/
|
88 |
}
|
89 |
},
|
90 |
openScreenplay: async (projectName: string, fileName: string, fileContent: string | Blob): Promise<void> => {
|
|
|
31 |
},
|
32 |
openFiles: async (files: File[]) => {
|
33 |
const { openClapBlob, openScreenplay } = get()
|
34 |
+
const timeline: TimelineStore = useTimeline.getState()
|
35 |
+
const { segments, addSegments } = timeline
|
36 |
|
37 |
if (Array.isArray(files)) {
|
38 |
console.log("user tried to drop some files:", files)
|
|
|
67 |
// screenplay files: -> analyze (if there is existing data, show a modal asking to save or not)
|
68 |
// mp3 file: ->
|
69 |
if (isAudioFile) {
|
70 |
+
const newSegments = await parseFileIntoSegments({ file })
|
71 |
+
|
72 |
+
console.log("calling timeline.addSegments with:", newSegments)
|
73 |
+
await timeline.addSegments({
|
74 |
+
segments: newSegments
|
75 |
})
|
76 |
+
return
|
77 |
}
|
78 |
|
79 |
const isVideoFile = fileType.startsWith("video/")
|
80 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
81 |
}
|
82 |
},
|
83 |
openScreenplay: async (projectName: string, fileName: string, fileContent: string | Blob): Promise<void> => {
|
src/controllers/monitor/useMonitor.ts
CHANGED
@@ -131,7 +131,6 @@ export const useMonitor = create<MonitorStore>((set, get) => ({
|
|
131 |
// we force a full state recompute
|
132 |
// and we also pass jumpedSomewhere=true to indicate that we
|
133 |
// need a buffer transition
|
134 |
-
// console.log(`forcing a state update`)
|
135 |
renderLoop(true)
|
136 |
}
|
137 |
},
|
|
|
131 |
// we force a full state recompute
|
132 |
// and we also pass jumpedSomewhere=true to indicate that we
|
133 |
// need a buffer transition
|
|
|
134 |
renderLoop(true)
|
135 |
}
|
136 |
},
|
src/controllers/renderer/useRenderLoop.ts
CHANGED
@@ -45,7 +45,6 @@ export function useRenderLoop(): void {
|
|
45 |
|
46 |
const newTimelineUpdateAtInMs = performance.now()
|
47 |
const elapsedTimeInMs = newTimelineUpdateAtInMs - lastTimelineUpdateAtInMs
|
48 |
-
// console.log(`TODO: move the timeline cursor according to the elapsed time`)
|
49 |
setCursorTimestampAtInMs(cursorTimestampAtInMs + elapsedTimeInMs)
|
50 |
setLastTimelineUpdateAtInMs(newTimelineUpdateAtInMs)
|
51 |
|
|
|
45 |
|
46 |
const newTimelineUpdateAtInMs = performance.now()
|
47 |
const elapsedTimeInMs = newTimelineUpdateAtInMs - lastTimelineUpdateAtInMs
|
|
|
48 |
setCursorTimestampAtInMs(cursorTimestampAtInMs + elapsedTimeInMs)
|
49 |
setLastTimelineUpdateAtInMs(newTimelineUpdateAtInMs)
|
50 |
|
src/controllers/renderer/useRenderer.ts
CHANGED
@@ -206,9 +206,7 @@ export const useRenderer = create<RendererStore>((set, get) => ({
|
|
206 |
}
|
207 |
|
208 |
}
|
209 |
-
|
210 |
-
// console.log("useRenderer: computeBufferedSegments() returning:", results)
|
211 |
-
|
212 |
return results
|
213 |
},
|
214 |
|
|
|
206 |
}
|
207 |
|
208 |
}
|
209 |
+
|
|
|
|
|
210 |
return results
|
211 |
},
|
212 |
|
src/controllers/resolver/getDefaultResolverState.ts
CHANGED
@@ -3,6 +3,7 @@ import { ResolverState } from "./types"
|
|
3 |
export function getDefaultResolverState(): ResolverState {
|
4 |
const state: ResolverState = {
|
5 |
isRunning: false,
|
|
|
6 |
|
7 |
defaultParallelismQuotas: {
|
8 |
video: 1,
|
@@ -12,15 +13,23 @@ export function getDefaultResolverState(): ResolverState {
|
|
12 |
music: 1,
|
13 |
},
|
14 |
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
},
|
22 |
|
23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
}
|
25 |
return state
|
26 |
}
|
|
|
3 |
export function getDefaultResolverState(): ResolverState {
|
4 |
const state: ResolverState = {
|
5 |
isRunning: false,
|
6 |
+
isPaused: false,
|
7 |
|
8 |
defaultParallelismQuotas: {
|
9 |
video: 1,
|
|
|
13 |
music: 1,
|
14 |
},
|
15 |
|
16 |
+
// used for UI display, show some metrics
|
17 |
+
currentParallelismQuotaForVideo: 1,
|
18 |
+
currentParallelismQuotaForImage: 1,
|
19 |
+
currentParallelismQuotaForVoice: 1,
|
20 |
+
currentParallelismQuotaForSound: 1,
|
21 |
+
currentParallelismQuotaForMusic: 1,
|
|
|
22 |
|
23 |
+
// just some aliases for convenience
|
24 |
+
nbPendingRequestsForVideo: 0,
|
25 |
+
nbPendingRequestsForImage: 0,
|
26 |
+
nbPendingRequestsForVoice: 0,
|
27 |
+
nbPendingRequestsForSound: 0,
|
28 |
+
nbPendingRequestsForMusic: 0,
|
29 |
+
|
30 |
+
nbRequestsRunningInParallel: 0,
|
31 |
+
|
32 |
+
isBusyResolving: false,
|
33 |
}
|
34 |
return state
|
35 |
}
|
src/controllers/resolver/types.ts
CHANGED
@@ -1,8 +1,17 @@
|
|
1 |
import { ClapSegment } from "@aitube/clap"
|
2 |
|
3 |
export type ResolverState = {
|
|
|
|
|
|
|
4 |
isRunning: boolean
|
5 |
|
|
|
|
|
|
|
|
|
|
|
|
|
6 |
defaultParallelismQuotas: {
|
7 |
video: number
|
8 |
image: number
|
@@ -12,22 +21,29 @@ export type ResolverState = {
|
|
12 |
}
|
13 |
|
14 |
// used for UI display, show some metrics
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
music: number
|
21 |
-
}
|
22 |
|
|
|
23 |
// used for UI display, show some metrics
|
|
|
|
|
|
|
|
|
|
|
24 |
nbRequestsRunningInParallel: number
|
|
|
25 |
}
|
26 |
|
27 |
export type ResolverControls = {
|
28 |
startLoop: () => void
|
29 |
runLoop: () => Promise<void>
|
30 |
|
|
|
|
|
31 |
/**
|
32 |
* This resolve a segment
|
33 |
*
|
|
|
1 |
import { ClapSegment } from "@aitube/clap"
|
2 |
|
3 |
export type ResolverState = {
|
4 |
+
// this indicate the status of the loop
|
5 |
+
// normally once it is running it is never stopped,
|
6 |
+
// since this is used to check if we have pending tasks
|
7 |
isRunning: boolean
|
8 |
|
9 |
+
// used to "pause" the resolution
|
10 |
+
// request have have already be sent to the API providers
|
11 |
+
// will still be honored, which is why the number of pending
|
12 |
+
// requests won't drop to 0 immediately
|
13 |
+
isPaused: boolean
|
14 |
+
|
15 |
defaultParallelismQuotas: {
|
16 |
video: number
|
17 |
image: number
|
|
|
21 |
}
|
22 |
|
23 |
// used for UI display, show some metrics
|
24 |
+
currentParallelismQuotaForVideo: number
|
25 |
+
currentParallelismQuotaForImage: number
|
26 |
+
currentParallelismQuotaForVoice: number
|
27 |
+
currentParallelismQuotaForSound: number
|
28 |
+
currentParallelismQuotaForMusic: number
|
|
|
|
|
29 |
|
30 |
+
// just some aliases for convenience,
|
31 |
// used for UI display, show some metrics
|
32 |
+
nbPendingRequestsForVideo: number
|
33 |
+
nbPendingRequestsForImage: number
|
34 |
+
nbPendingRequestsForVoice: number
|
35 |
+
nbPendingRequestsForSound: number
|
36 |
+
nbPendingRequestsForMusic: number
|
37 |
nbRequestsRunningInParallel: number
|
38 |
+
isBusyResolving: boolean
|
39 |
}
|
40 |
|
41 |
export type ResolverControls = {
|
42 |
startLoop: () => void
|
43 |
runLoop: () => Promise<void>
|
44 |
|
45 |
+
togglePause: (isPaused?: boolean) => boolean
|
46 |
+
|
47 |
/**
|
48 |
* This resolve a segment
|
49 |
*
|
src/controllers/resolver/useResolver.ts
CHANGED
@@ -55,24 +55,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
55 |
}, waitTimeIfNothingToDoInMs)
|
56 |
}
|
57 |
|
58 |
-
//
|
59 |
-
|
60 |
-
const allStrategiesAreOnDemand =
|
61 |
-
imageRenderingStrategy === RenderingStrategy.ON_DEMAND &&
|
62 |
-
videoRenderingStrategy === RenderingStrategy.ON_DEMAND &&
|
63 |
-
soundRenderingStrategy === RenderingStrategy.ON_DEMAND &&
|
64 |
-
voiceRenderingStrategy === RenderingStrategy.ON_DEMAND &&
|
65 |
-
musicRenderingStrategy
|
66 |
-
|
67 |
-
// nothing to do
|
68 |
-
if (allStrategiesAreOnDemand) {
|
69 |
-
// console.log(`useResolver.runLoop(): all strategies are on-demand only`)
|
70 |
-
return runLoopAgain()
|
71 |
-
}
|
72 |
-
|
73 |
-
// ---------- end of the very trivial case ----------------
|
74 |
-
|
75 |
-
|
76 |
|
77 |
// console.log(`useResolver.runLoop()`)
|
78 |
const timelineState: TimelineStore = useTimeline.getState()
|
@@ -99,13 +83,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
99 |
//
|
100 |
// -------------------------------------------------------------------------
|
101 |
|
102 |
-
const { defaultParallelismQuotas } = get()
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
-
// note that we to create a copy here, that way we can modify it
|
105 |
-
const parallelismQuotas = {
|
106 |
-
...defaultParallelismQuotas,
|
107 |
-
}
|
108 |
-
|
109 |
// console.log(`useResolver.runLoop() parallelismQuotas = `, parallelismQuotas)
|
110 |
|
111 |
// we do not need ot get currentParallelismQuotas,
|
@@ -127,11 +112,13 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
127 |
// it is thus vital to deduct it from the parallelism quota,
|
128 |
// to avoir triggering quota limit on the providers side
|
129 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
130 |
-
|
131 |
}
|
132 |
continue
|
133 |
}
|
134 |
|
|
|
|
|
135 |
if (videoRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
136 |
continue
|
137 |
}
|
@@ -150,8 +137,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
150 |
continue
|
151 |
}
|
152 |
|
153 |
-
if (
|
154 |
-
|
155 |
segmentsToRender.push(s)
|
156 |
}
|
157 |
} else if (s.category === ClapSegmentCategory.STORYBOARD) {
|
@@ -165,12 +152,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
165 |
// it is thus vital to deduct it from the parallelism quota,
|
166 |
// to avoir triggering quoote limit on the providers side
|
167 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
168 |
-
|
169 |
}
|
170 |
|
171 |
continue
|
172 |
}
|
173 |
// console.log(`useResolver.runLoop(): found a storyboard segment that has to be generated`)
|
|
|
|
|
174 |
|
175 |
if (imageRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
176 |
continue
|
@@ -192,9 +181,9 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
192 |
|
193 |
// console.log(`useResolver.runLoop(): strategy is good to go`)
|
194 |
|
195 |
-
if (
|
196 |
// console.log(`useResolver.runLoop(): quota is good to go`)
|
197 |
-
|
198 |
segmentsToRender.push(s)
|
199 |
}
|
200 |
} else if (s.category === ClapSegmentCategory.DIALOGUE) {
|
@@ -205,12 +194,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
205 |
// it is thus vital to deduct it from the parallelism quota,
|
206 |
// to avoir triggering quoote limit on the providers side
|
207 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
208 |
-
|
209 |
}
|
210 |
|
211 |
continue
|
212 |
}
|
213 |
|
|
|
|
|
214 |
if (voiceRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
215 |
continue
|
216 |
}
|
@@ -229,8 +220,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
229 |
continue
|
230 |
}
|
231 |
|
232 |
-
if (
|
233 |
-
|
234 |
segmentsToRender.push(s)
|
235 |
}
|
236 |
} else if (s.category === ClapSegmentCategory.SOUND) {
|
@@ -241,11 +232,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
241 |
// it is thus vital to deduct it from the parallelism quota,
|
242 |
// to avoir triggering quoote limit on the providers side
|
243 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
244 |
-
|
245 |
}
|
246 |
|
247 |
continue
|
248 |
}
|
|
|
|
|
|
|
249 |
if (soundRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
250 |
continue
|
251 |
}
|
@@ -264,8 +258,8 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
264 |
continue
|
265 |
}
|
266 |
|
267 |
-
if (
|
268 |
-
|
269 |
segmentsToRender.push(s)
|
270 |
}
|
271 |
} else if (s.category === ClapSegmentCategory.MUSIC) {
|
@@ -276,12 +270,14 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
276 |
// it is thus vital to deduct it from the parallelism quota,
|
277 |
// to avoir triggering quoote limit on the providers side
|
278 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
279 |
-
|
280 |
}
|
281 |
|
282 |
continue
|
283 |
}
|
284 |
|
|
|
|
|
285 |
if (musicRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
286 |
continue
|
287 |
}
|
@@ -300,21 +296,13 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
300 |
continue
|
301 |
}
|
302 |
|
303 |
-
if (
|
304 |
-
|
305 |
segmentsToRender.push(s)
|
306 |
}
|
307 |
} // else continue
|
308 |
}
|
309 |
|
310 |
-
if (!segmentsToRender.length) {
|
311 |
-
// nothing to do - this will be the most common case
|
312 |
-
return runLoopAgain()
|
313 |
-
}
|
314 |
-
|
315 |
-
// console.log(`useResolver.runLoop(): firing and forgetting ${segmentsToRender.length} new resolveSegment promises`)
|
316 |
-
// we fire and forget
|
317 |
-
segmentsToRender.forEach(segment => resolveSegment(segment))
|
318 |
|
319 |
// we don't want to do something like this:
|
320 |
// await Promise.allSettled(segmentsRenderingPromises)
|
@@ -323,9 +311,56 @@ export const useResolver = create<ResolverStore>((set, get) => ({
|
|
323 |
// the idea here is that we don't want to wait for all segments
|
324 |
// to finish before starting new ones.
|
325 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
326 |
return runLoopAgain()
|
327 |
},
|
328 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
329 |
/**
|
330 |
* This resolve a segment
|
331 |
*
|
|
|
55 |
}, waitTimeIfNothingToDoInMs)
|
56 |
}
|
57 |
|
58 |
+
// note: do not create a return condition in case all strategies are "on demand"
|
59 |
+
// otherwise we won't be able to get the status of current tasks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
60 |
|
61 |
// console.log(`useResolver.runLoop()`)
|
62 |
const timelineState: TimelineStore = useTimeline.getState()
|
|
|
83 |
//
|
84 |
// -------------------------------------------------------------------------
|
85 |
|
86 |
+
const { defaultParallelismQuotas, isPaused } = get()
|
87 |
+
|
88 |
+
let currentParallelismQuotaForVideo = defaultParallelismQuotas.video
|
89 |
+
let currentParallelismQuotaForImage = defaultParallelismQuotas.image
|
90 |
+
let currentParallelismQuotaForVoice = defaultParallelismQuotas.voice
|
91 |
+
let currentParallelismQuotaForSound = defaultParallelismQuotas.sound
|
92 |
+
let currentParallelismQuotaForMusic = defaultParallelismQuotas.music
|
93 |
|
|
|
|
|
|
|
|
|
|
|
94 |
// console.log(`useResolver.runLoop() parallelismQuotas = `, parallelismQuotas)
|
95 |
|
96 |
// we do not need ot get currentParallelismQuotas,
|
|
|
112 |
// it is thus vital to deduct it from the parallelism quota,
|
113 |
// to avoir triggering quota limit on the providers side
|
114 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
115 |
+
currentParallelismQuotaForVideo = Math.max(0, currentParallelismQuotaForVideo - 1)
|
116 |
}
|
117 |
continue
|
118 |
}
|
119 |
|
120 |
+
if (isPaused) { continue }
|
121 |
+
|
122 |
if (videoRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
123 |
continue
|
124 |
}
|
|
|
137 |
continue
|
138 |
}
|
139 |
|
140 |
+
if (currentParallelismQuotaForVideo > 0) {
|
141 |
+
currentParallelismQuotaForVideo = Math.max(0, currentParallelismQuotaForVideo - 1)
|
142 |
segmentsToRender.push(s)
|
143 |
}
|
144 |
} else if (s.category === ClapSegmentCategory.STORYBOARD) {
|
|
|
152 |
// it is thus vital to deduct it from the parallelism quota,
|
153 |
// to avoir triggering quoote limit on the providers side
|
154 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
155 |
+
currentParallelismQuotaForImage = Math.max(0, currentParallelismQuotaForImage - 1)
|
156 |
}
|
157 |
|
158 |
continue
|
159 |
}
|
160 |
// console.log(`useResolver.runLoop(): found a storyboard segment that has to be generated`)
|
161 |
+
|
162 |
+
if (isPaused) { continue }
|
163 |
|
164 |
if (imageRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
165 |
continue
|
|
|
181 |
|
182 |
// console.log(`useResolver.runLoop(): strategy is good to go`)
|
183 |
|
184 |
+
if (currentParallelismQuotaForImage > 0) {
|
185 |
// console.log(`useResolver.runLoop(): quota is good to go`)
|
186 |
+
currentParallelismQuotaForImage = Math.max(0, currentParallelismQuotaForImage - 1)
|
187 |
segmentsToRender.push(s)
|
188 |
}
|
189 |
} else if (s.category === ClapSegmentCategory.DIALOGUE) {
|
|
|
194 |
// it is thus vital to deduct it from the parallelism quota,
|
195 |
// to avoir triggering quoote limit on the providers side
|
196 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
197 |
+
currentParallelismQuotaForVoice = Math.max(0, currentParallelismQuotaForVoice - 1)
|
198 |
}
|
199 |
|
200 |
continue
|
201 |
}
|
202 |
|
203 |
+
if (isPaused) { continue }
|
204 |
+
|
205 |
if (voiceRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
206 |
continue
|
207 |
}
|
|
|
220 |
continue
|
221 |
}
|
222 |
|
223 |
+
if (currentParallelismQuotaForVoice > 0) {
|
224 |
+
currentParallelismQuotaForVoice = Math.max(0, currentParallelismQuotaForVoice - 1)
|
225 |
segmentsToRender.push(s)
|
226 |
}
|
227 |
} else if (s.category === ClapSegmentCategory.SOUND) {
|
|
|
232 |
// it is thus vital to deduct it from the parallelism quota,
|
233 |
// to avoir triggering quoote limit on the providers side
|
234 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
235 |
+
currentParallelismQuotaForSound = Math.max(0, currentParallelismQuotaForSound - 1)
|
236 |
}
|
237 |
|
238 |
continue
|
239 |
}
|
240 |
+
|
241 |
+
if (isPaused) { continue }
|
242 |
+
|
243 |
if (soundRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
244 |
continue
|
245 |
}
|
|
|
258 |
continue
|
259 |
}
|
260 |
|
261 |
+
if (currentParallelismQuotaForSound > 0) {
|
262 |
+
currentParallelismQuotaForSound = Math.max(0, currentParallelismQuotaForSound - 1)
|
263 |
segmentsToRender.push(s)
|
264 |
}
|
265 |
} else if (s.category === ClapSegmentCategory.MUSIC) {
|
|
|
270 |
// it is thus vital to deduct it from the parallelism quota,
|
271 |
// to avoir triggering quoote limit on the providers side
|
272 |
if (s.status === ClapSegmentStatus.IN_PROGRESS) {
|
273 |
+
currentParallelismQuotaForMusic = Math.max(0, currentParallelismQuotaForMusic - 1)
|
274 |
}
|
275 |
|
276 |
continue
|
277 |
}
|
278 |
|
279 |
+
if (isPaused) { continue }
|
280 |
+
|
281 |
if (musicRenderingStrategy === RenderingStrategy.ON_DEMAND) {
|
282 |
continue
|
283 |
}
|
|
|
296 |
continue
|
297 |
}
|
298 |
|
299 |
+
if (currentParallelismQuotaForMusic > 0) {
|
300 |
+
currentParallelismQuotaForMusic = Math.max(0, currentParallelismQuotaForMusic - 1)
|
301 |
segmentsToRender.push(s)
|
302 |
}
|
303 |
} // else continue
|
304 |
}
|
305 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
306 |
|
307 |
// we don't want to do something like this:
|
308 |
// await Promise.allSettled(segmentsRenderingPromises)
|
|
|
311 |
// the idea here is that we don't want to wait for all segments
|
312 |
// to finish before starting new ones.
|
313 |
|
314 |
+
const nbPendingRequestsForVideo = defaultParallelismQuotas.video - currentParallelismQuotaForVideo
|
315 |
+
const nbPendingRequestsForImage = defaultParallelismQuotas.image - currentParallelismQuotaForImage
|
316 |
+
const nbPendingRequestsForVoice = defaultParallelismQuotas.voice - currentParallelismQuotaForVoice
|
317 |
+
const nbPendingRequestsForSound = defaultParallelismQuotas.sound - currentParallelismQuotaForSound
|
318 |
+
const nbPendingRequestsForMusic = defaultParallelismQuotas.music - currentParallelismQuotaForMusic
|
319 |
+
|
320 |
+
const nbRequestsRunningInParallel =
|
321 |
+
nbPendingRequestsForVideo
|
322 |
+
+ nbPendingRequestsForImage
|
323 |
+
+ nbPendingRequestsForVoice
|
324 |
+
+ nbPendingRequestsForSound
|
325 |
+
+ nbPendingRequestsForMusic
|
326 |
+
|
327 |
+
const isBusyResolving = nbRequestsRunningInParallel > 0
|
328 |
+
|
329 |
+
set({
|
330 |
+
currentParallelismQuotaForVideo,
|
331 |
+
currentParallelismQuotaForImage,
|
332 |
+
currentParallelismQuotaForVoice,
|
333 |
+
currentParallelismQuotaForSound,
|
334 |
+
currentParallelismQuotaForMusic,
|
335 |
+
// just some aliases for convenience
|
336 |
+
nbPendingRequestsForVideo,
|
337 |
+
nbPendingRequestsForImage,
|
338 |
+
nbPendingRequestsForVoice,
|
339 |
+
nbPendingRequestsForSound,
|
340 |
+
nbPendingRequestsForMusic,
|
341 |
+
nbRequestsRunningInParallel,
|
342 |
+
isBusyResolving
|
343 |
+
})
|
344 |
+
|
345 |
+
// console.log(`useResolver.runLoop(): firing and forgetting ${segmentsToRender.length} new resolveSegment promises`)
|
346 |
+
// we fire and forget
|
347 |
+
segmentsToRender.forEach(segment => resolveSegment(segment))
|
348 |
+
|
349 |
return runLoopAgain()
|
350 |
},
|
351 |
|
352 |
+
|
353 |
+
togglePause: (isPaused?: boolean): boolean => {
|
354 |
+
const { isPaused: previouslyPaused } = get()
|
355 |
+
if (typeof isPaused === "boolean") {
|
356 |
+
set({ isPaused })
|
357 |
+
return isPaused
|
358 |
+
} else {
|
359 |
+
set({ isPaused: !previouslyPaused })
|
360 |
+
return !previouslyPaused
|
361 |
+
}
|
362 |
+
},
|
363 |
+
|
364 |
/**
|
365 |
* This resolve a segment
|
366 |
*
|
src/lib/core/constants.ts
CHANGED
@@ -4,7 +4,7 @@
|
|
4 |
export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
|
5 |
|
6 |
export const APP_NAME = "Clapper AI"
|
7 |
-
export const APP_REVISION = "r20240616-
|
8 |
|
9 |
export const APP_DOMAIN = "Clapper.app"
|
10 |
export const APP_LINK = "https://clapper.app"
|
|
|
4 |
export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
|
5 |
|
6 |
export const APP_NAME = "Clapper AI"
|
7 |
+
export const APP_REVISION = "r20240616-1825"
|
8 |
|
9 |
export const APP_DOMAIN = "Clapper.app"
|
10 |
export const APP_LINK = "https://clapper.app"
|
src/lib/hooks/useTitle.ts
ADDED
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import { useState, useEffect } from 'react'
|
2 |
+
|
3 |
+
export const useTitle = (): {
|
4 |
+
title: string
|
5 |
+
changeTitle: (newTitle: string) => void
|
6 |
+
} => {
|
7 |
+
const [title, setTitle] = useState<string>(document.title)
|
8 |
+
|
9 |
+
useEffect(() => {
|
10 |
+
document.title = title
|
11 |
+
}, [title])
|
12 |
+
|
13 |
+
const changeTitle = (newTitle: string) => setTitle(newTitle)
|
14 |
+
|
15 |
+
return { title, changeTitle }
|
16 |
+
}
|