working on audio generation

Dockerfile

@@ -10,6 +10,19 @@ RUN apk add git git-lfs procps htop vim nano
 
 RUN apk add alpine-sdk pkgconfig
 
+# For FFMPEG and gl concat
+RUN apk add curl python3 python3-dev libx11-dev libsm-dev libxrender libxext-dev mesa-dev xvfb libxi-dev glew-dev
+
+# For fonts, emojis etc
+RUN apk add font-terminus font-noto font-noto-cjk font-noto-extra font-noto-emoji
+RUN apk add font-arabic-misc font-inconsolata font-dejavu font-awesome 
+RUN apk add ttf-opensans
+
+# For Puppeteer
+RUN apk add build-base gcompat udev chromium
+
+RUN apk add --no-cache ffmpeg
+
 # Set up a new user named "user" with user ID 1000
 RUN adduser --disabled-password --uid 1001 user
 
@@ -30,11 +43,15 @@ COPY --chown=user package*.json /app
 # make sure the .env is copied as well
 COPY --chown=user .env /app
 
+RUN ffmpeg -version
+
 # Copy the current directory contents into the container at /app setting the owner to the user
 COPY --chown=user . /app
 
 RUN npm ci --force
 
+# expose our secrets during the static build
+# otherwise Upstash becomes unhappy
 RUN npm run build
 
 EXPOSE 3000

package-lock.json

@@ -58,6 +58,8 @@
         "cmdk": "^0.2.1",
         "eslint": "8.57.0",
         "eslint-config-next": "14.1.0",
+        "fluent-ffmpeg": "^2.1.3",
+        "fs-extra": "^11.2.0",
         "lucide-react": "^0.334.0",
         "mlt-xml": "^2.0.2",
         "monaco-editor": "^0.49.0",
@@ -76,7 +78,7 @@
         "react-hotkeys-hook": "^4.5.0",
         "react-icons": "^5.2.1",
         "react-reflex": "^4.2.6",
-        "replicate": "^0.30.1",
+        "replicate": "^0.30.2",
         "sharp": "^0.33.4",
         "sonner": "^1.4.41",
         "tailwind-merge": "^2.3.0",
@@ -92,6 +94,9 @@
         "yaml": "^2.4.2",
         "zustand": "^4.5.2",
         "zx": "^8.1.2"
+      },
+      "devDependencies": {
+        "@types/fluent-ffmpeg": "^2.1.24"
       }
     },
     "node_modules/@aitube/clap": {
@@ -1446,9 +1451,9 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz",
-      "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA=="
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.4.tgz",
+      "integrity": "sha512-3EtkY5VDkuV2+lNmKlbkibIJxcO4oIHEhBWne6PaAp+76J9KoSsGvNikp6ivzAT8dhhBMYrm6op2pS1ApG0Hzg=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "14.1.0",
@@ -1459,9 +1464,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz",
-      "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.4.tgz",
+      "integrity": "sha512-AH3mO4JlFUqsYcwFUHb1wAKlebHU/Hv2u2kb1pAuRanDZ7pD/A/KPD98RHZmwsJpdHQwfEc/06mgpSzwrJYnNg==",
       "cpu": [
         "arm64"
       ],
@@ -1474,9 +1479,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz",
-      "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.4.tgz",
+      "integrity": "sha512-QVadW73sWIO6E2VroyUjuAxhWLZWEpiFqHdZdoQ/AMpN9YWGuHV8t2rChr0ahy+irKX5mlDU7OY68k3n4tAZTg==",
       "cpu": [
         "x64"
       ],
@@ -1489,9 +1494,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz",
-      "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.4.tgz",
+      "integrity": "sha512-KT6GUrb3oyCfcfJ+WliXuJnD6pCpZiosx2X3k66HLR+DMoilRb76LpWPGb4tZprawTtcnyrv75ElD6VncVamUQ==",
       "cpu": [
         "arm64"
       ],
@@ -1504,9 +1509,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz",
-      "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.4.tgz",
+      "integrity": "sha512-Alv8/XGSs/ytwQcbCHwze1HmiIkIVhDHYLjczSVrf0Wi2MvKn/blt7+S6FJitj3yTlMwMxII1gIJ9WepI4aZ/A==",
       "cpu": [
         "arm64"
       ],
@@ -1519,9 +1524,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz",
-      "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.4.tgz",
+      "integrity": "sha512-ze0ShQDBPCqxLImzw4sCdfnB3lRmN3qGMB2GWDRlq5Wqy4G36pxtNOo2usu/Nm9+V2Rh/QQnrRc2l94kYFXO6Q==",
       "cpu": [
         "x64"
       ],
@@ -1534,9 +1539,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz",
-      "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.4.tgz",
+      "integrity": "sha512-8dwC0UJoc6fC7PX70csdaznVMNr16hQrTDAMPvLPloazlcaWfdPogq+UpZX6Drqb1OBlwowz8iG7WR0Tzk/diQ==",
       "cpu": [
         "x64"
       ],
@@ -1549,9 +1554,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz",
-      "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.4.tgz",
+      "integrity": "sha512-jxyg67NbEWkDyvM+O8UDbPAyYRZqGLQDTPwvrBBeOSyVWW/jFQkQKQ70JDqDSYg1ZDdl+E3nkbFbq8xM8E9x8A==",
       "cpu": [
         "arm64"
       ],
@@ -1564,9 +1569,9 @@
       }
     },
     "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
-      "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.4.tgz",
+      "integrity": "sha512-twrmN753hjXRdcrZmZttb/m5xaCBFa48Dt3FbeEItpJArxriYDunWxJn+QFXdJ3hPkm4u7CKxncVvnmgQMY1ag==",
       "cpu": [
         "ia32"
       ],
@@ -1579,9 +1584,9 @@
       }
     },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz",
-      "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.4.tgz",
+      "integrity": "sha512-tkLrjBzqFTP8DVrAAQmZelEahfR9OxWpFR++vAI9FBhCiIxtwHwBHC23SBHCTURBtwB4kc/x44imVOnkKGNVGg==",
       "cpu": [
         "x64"
       ],
@@ -3122,6 +3127,15 @@
       "resolved": "https://registry.npmjs.org/@types/draco3d/-/draco3d-1.4.10.tgz",
       "integrity": "sha512-AX22jp8Y7wwaBgAixaSvkoG4M/+PlAcm3Qs4OW8yT9DM4xUpWKeFhLueTAyZF39pviAdcDdeJoACapiAceqNcw=="
     },
+    "node_modules/@types/fluent-ffmpeg": {
+      "version": "2.1.24",
+      "resolved": "https://registry.npmjs.org/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.24.tgz",
+      "integrity": "sha512-g5oQO8Jgi2kFS3tTub7wLvfLztr1s8tdXmRd8PiL/hLMLzTIAyMR2sANkTggM/rdEDAg3d63nYRRVepwBiCw5A==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/fs-extra": {
       "version": "11.0.4",
       "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz",
@@ -3694,6 +3708,11 @@
       "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz",
       "integrity": "sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ=="
     },
+    "node_modules/async": {
+      "version": "0.2.10",
+      "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
+      "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ=="
+    },
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -4844,9 +4863,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.798",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.798.tgz",
-      "integrity": "sha512-by9J2CiM9KPGj9qfp5U4FcPSbXJG7FNzqnYaY4WLzX+v2PHieVGmnsA4dxfpGE3QEC7JofpPZmn7Vn1B9NR2+Q=="
+      "version": "1.4.799",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.799.tgz",
+      "integrity": "sha512-3D3DwWkRTzrdEpntY0hMLYwj7SeBk1138CkPE8sBDSj3WzrzOiG2rHm3luw8jucpf+WiyLBCZyU9lMHyQI9M9Q=="
     },
     "node_modules/emoji-regex": {
       "version": "9.2.2",
@@ -5680,6 +5699,29 @@
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz",
       "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw=="
     },
+    "node_modules/fluent-ffmpeg": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.3.tgz",
+      "integrity": "sha512-Be3narBNt2s6bsaqP6Jzq91heDgOEaDCJAXcE3qcma/EJBSy5FB4cvO31XBInuAuKBx8Kptf8dkhjK0IOru39Q==",
+      "dependencies": {
+        "async": "^0.2.9",
+        "which": "^1.1.1"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/fluent-ffmpeg/node_modules/which": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
+      "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==",
+      "dependencies": {
+        "isexe": "^2.0.0"
+      },
+      "bin": {
+        "which": "bin/which"
+      }
+    },
     "node_modules/for-each": {
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz",
@@ -5753,6 +5795,19 @@
         "url": "https://github.com/sponsors/rawify"
       }
     },
+    "node_modules/fs-extra": {
+      "version": "11.2.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz",
+      "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==",
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.14"
+      }
+    },
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -6747,6 +6802,17 @@
         "node": ">=6"
       }
     },
+    "node_modules/jsonfile": {
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
+      "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "node_modules/jsx-ast-utils": {
       "version": "3.3.5",
       "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz",
@@ -7156,11 +7222,11 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="
     },
     "node_modules/next": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz",
-      "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==",
+      "version": "14.2.4",
+      "resolved": "https://registry.npmjs.org/next/-/next-14.2.4.tgz",
+      "integrity": "sha512-R8/V7vugY+822rsQGQCjoLhMuC9oFj9SOi4Cl4b2wjDrseD0LRZ10W7R6Czo4w9ZznVSshKjuIomsRjvm9EKJQ==",
       "dependencies": {
-        "@next/env": "14.2.3",
+        "@next/env": "14.2.4",
         "@swc/helpers": "0.5.5",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
@@ -7175,15 +7241,15 @@
         "node": ">=18.17.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.2.3",
-        "@next/swc-darwin-x64": "14.2.3",
-        "@next/swc-linux-arm64-gnu": "14.2.3",
-        "@next/swc-linux-arm64-musl": "14.2.3",
-        "@next/swc-linux-x64-gnu": "14.2.3",
-        "@next/swc-linux-x64-musl": "14.2.3",
-        "@next/swc-win32-arm64-msvc": "14.2.3",
-        "@next/swc-win32-ia32-msvc": "14.2.3",
-        "@next/swc-win32-x64-msvc": "14.2.3"
+        "@next/swc-darwin-arm64": "14.2.4",
+        "@next/swc-darwin-x64": "14.2.4",
+        "@next/swc-linux-arm64-gnu": "14.2.4",
+        "@next/swc-linux-arm64-musl": "14.2.4",
+        "@next/swc-linux-x64-gnu": "14.2.4",
+        "@next/swc-linux-x64-musl": "14.2.4",
+        "@next/swc-win32-arm64-msvc": "14.2.4",
+        "@next/swc-win32-ia32-msvc": "14.2.4",
+        "@next/swc-win32-x64-msvc": "14.2.4"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
@@ -9648,6 +9714,14 @@
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
       "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
     },
+    "node_modules/universalify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/update-browserslist-db": {
       "version": "1.0.16",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz",

package.json

@@ -60,6 +60,8 @@
     "cmdk": "^0.2.1",
     "eslint": "8.57.0",
     "eslint-config-next": "14.1.0",
+    "fluent-ffmpeg": "^2.1.3",
+    "fs-extra": "^11.2.0",
     "lucide-react": "^0.334.0",
     "mlt-xml": "^2.0.2",
     "monaco-editor": "^0.49.0",
@@ -78,7 +80,7 @@
     "react-hotkeys-hook": "^4.5.0",
     "react-icons": "^5.2.1",
     "react-reflex": "^4.2.6",
-    "replicate": "^0.30.1",
+    "replicate": "^0.30.2",
     "sharp": "^0.33.4",
     "sonner": "^1.4.41",
     "tailwind-merge": "^2.3.0",
@@ -94,5 +96,8 @@
     "yaml": "^2.4.2",
     "zustand": "^4.5.2",
     "zx": "^8.1.2"
+  },
+  "devDependencies": {
+    "@types/fluent-ffmpeg": "^2.1.24"
   }
 }

src/app/api/resolve/providers/falai/index.ts

@@ -131,7 +131,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
     ) {
       const result = await fal.run(request.settings.falAiModelForSound, {
         input: {
-          // note how we use the *segment* prompt for music
+          // note how we use the *segment* prompt for music or sound
           prompt: request.segment.prompt,
 
           sync_mode: true,
@@ -139,7 +139,7 @@ export async function resolveSegment(request: ResolveRequest): Promise<ClapSegme
         },
       }) as FalAiAudioResponse
 
-      content = result?.audio_file?.url
+      content = await decodeOutput(result?.audio_file?.url)
     } else if (
       request.segment.category === ClapSegmentCategory.DIALOGUE
     ) {

src/app/api/resolve/providers/replicate/index.ts

@@ -2,8 +2,6 @@ import Replicate from 'replicate'
 
 import { ResolveRequest } from "@/types"
 import { ClapSegment, ClapSegmentCategory, ClapSegmentStatus, getClapAssetSourceType } from "@aitube/clap"
-import { getVideoPrompt } from "@aitube/engine"
-import { fetchContentToBase64 } from '@/lib/utils/fetchContentToBase64'
 import { getResolveRequestPrompts } from '@/lib/utils/getResolveRequestPrompts'
 import { decodeOutput } from '@/lib/utils/decodeOutput'
 

src/components/monitor/DynamicPlayer/index.tsx

@@ -33,14 +33,6 @@ export const DynamicPlayer = ({
     upcomingStoryboardSegment
   } = useRenderer(s => s.bufferedSegments)
 
-  console.log(`DynamicPlayer:`, {
-    activeVideoSegment,
-    upcomingVideoSegment,
-    activeStoryboardSegment,
-    upcomingStoryboardSegment
-  })
-
-
   const currentSegment =
     activeVideoSegment?.assetUrl
     ? activeVideoSegment
@@ -123,10 +115,10 @@ export const DynamicPlayer = ({
     return () => {
       clearTimeout(timeoutRef.current)
     }
-}, [
-  currentSegmentKey,
-  preloadSegmentKey
-])
+  }, [
+    currentSegmentKey,
+    preloadSegmentKey
+  ])
 
   return (
     <div className={cn(`@container flex flex-col items-center flex-grow w-full`, className)}>

src/lib/core/constants.ts

@@ -4,7 +4,7 @@
 export const HARD_LIMIT_NB_MAX_ASSETS_TO_GENERATE_IN_PARALLEL = 32
 
 export const APP_NAME = "Clapper AI"
-export const APP_REVISION = "r20240611-2327"
+export const APP_REVISION = "r20240612-0135"
 
 export const APP_DOMAIN = "Clapper.app"
 export const APP_LINK = "https://clapper.app"

src/lib/ffmpeg/convertAudioToMp3.ts

@@ -0,0 +1,50 @@
+import ffmpeg from 'fluent-ffmpeg';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+// Define a map from media type to file extension
+const EXTENSIONS: { [type: string]: string } = {
+  'audio/x-wav': 'wav',
+  'audio/wav': 'wav',
+  'audio/mp3': 'mp3',
+  'audio/mpeg': 'mp3',
+  'audio/webm': 'webm',
+  // add more if needed
+};
+
+export async function convertAudioToMp3(dataUri: string): Promise<string> {
+  const match = dataUri.match(/^data:(.*?);base64,(.*)$/);
+  if (!match) throw new Error('Invalid data URI');
+
+  const mediaType = match[1];
+  const base64Audio = match[2];
+
+  const format = EXTENSIONS[mediaType];
+  if (!format) throw new Error('Unsupported media type: ' + mediaType);
+
+  if (format === 'mp3') return dataUri;  // If it's already a mp3, then do nothing
+
+  const randomName = crypto.randomBytes(15).toString('hex');
+  const inputPath = path.join(__dirname, `${randomName}.${format}`);
+  const outputPath = path.join(__dirname, `${randomName}.mp3`);
+
+  // Decode base64 and write to an audio file
+  const audioBuffer = Buffer.from(base64Audio, 'base64');
+  await fs.promises.writeFile(inputPath, audioBuffer);
+
+  return new Promise<string>((resolve, reject) => {
+    ffmpeg(inputPath)
+      .output(outputPath)
+      .on('end', async () => {
+        const mp3File = await fs.promises.readFile(outputPath);
+        await fs.promises.unlink(inputPath);
+        await fs.promises.unlink(outputPath);
+        resolve('data:audio/mp3;base64,' + mp3File.toString('base64'));
+      })
+      .on('error', async (err) => {
+        await fs.promises.unlink(inputPath);
+        reject(err);
+      }).run();
+  });
+}

src/lib/utils/decodeOutput.ts

@@ -1,3 +1,5 @@
+import { convertAudioToMp3 } from '@/lib/ffmpeg/convertAudioToMp3'
+
 import { fetchContentToBase64 } from "./fetchContentToBase64"
 import { convertToJpeg } from "./convertToJpeg"
 
@@ -19,9 +21,16 @@ export async function decodeOutput(input: any): Promise<string> {
     //
     // if you really want to have a pro, Hollywood-grade storyboard storage,
     // this isn't impossible but then you need to use either file paths or remote URL paths
+     // and if you want some lossless like this, we should add a parameter to support that
     const jpegImageAsBase64 = await convertToJpeg(base64Url)
 
     return jpegImageAsBase64
+  } else if (base64Url.startsWith("data:audio/wav") || base64Url.startsWith("data:audio/x-wav")) {
+    // same logic as for images here: we convert to a more compact format
+    // and if you want some lossless, we should add a parameter to support that
+    const mp4AudioAsBase64 = await convertAudioToMp3(base64Url)
+
+    return mp4AudioAsBase64
   }
 
   return base64Url