ready for the update

database.json

@@ -2,6 +2,93 @@
   "version": 1,
   "startAtShotId": "",
   "sequences": [
+    {
+      "sequenceId": "da6329e1-c863-45e0-aa18-5161cf5eeacd",
+      "skip": false,
+      "lastGenerationAt": "",
+      "videoPrompt": "a photorealistic of a space opera movie",
+      "audioPrompt": "intense and fast-paced electronic and orchestra music for a space opera with synthetisers, electronic drum machine",
+      "tags": [
+        "trailer",
+        "comedy",
+        "action",
+        "adventure",
+        "science-fiction"
+      ],
+      "channel": "main",
+      "shots": [
+        {
+          "shotId": "63462eef-7fa0-46b6-855a-c70dcebfb414",
+          "index": 0,
+          "lastGenerationAt": "",
+          "videoPrompt": "Photorealistic movie footage of a detailed NASA-inspired, pyramid-shaped spaceship soaring towards a strange planet laden with a colossal castle, glinting enchantingly under the golden hour sunlight, viewed from the perspective of an ultra wide-angle cinema camera, a spectacular, cinematic, photorealistic, movie quality shot.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "5d4b89f0-ba68-499c-be41-61ca447720c7",
+          "index": 1,
+          "lastGenerationAt": "",
+          "videoPrompt": "Photorealistic film of a raccoon pilot in an expansive spaceship cockpit, wearing an airman's leather jacket and goggles, illuminated by ambient lighting from the spacecraft's controls, shot with high details, digitally remastered, award-winning.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "c1d2e49d-abb2-42c8-8ed7-d9eadcbfc152",
+          "index": 2,
+          "lastGenerationAt": "",
+          "videoPrompt": "Ultra-wide angle photorealistic shot of huge, freeze-breathing dragons, their scales glistening unnervingly under the sunlight, amid massive octopus creatures upturning alien houses, shot with a mix of ambient lighting and backlighting in superb photorealistic, cinematic, movie quality.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "268fdad6-07db-426b-812d-dc03c30ffe49",
+          "index": 3,
+          "lastGenerationAt": "",
+          "videoPrompt": "Captivating photorealistic footage of our raccoon pilot passionately manipulating the spaceship's controls amidst the tumultuous situation outdoors, captured in soft lighting from a Canon EOS cinema-grade camera, showcasing splendid, award-winning composition and detail",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "c431acda-a572-4fc0-ba4a-179d98219f48",
+          "index": 4,
+          "lastGenerationAt": "",
+          "videoPrompt": "Space opera drama unfolding in a photorealistic footage of phenomenal, giant icy dragons sweeping over expansive, medieval-style castle, casting long, dramatic shadows under the slowly setting sun, captured in sublime movie quality.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "62971863-1448-4044-a50c-242af4de953e",
+          "index": 5,
+          "lastGenerationAt": "",
+          "videoPrompt": "Photorealistic film shot of the ecstatic raccoon pilot reacting to the chaos outside, speeding towards the planet in his spaceship, the merriment in his beady eyes reflected in the surreal and breathtaking high-quality, movie lighting, captured with stunning detail.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "4aa96204-c78f-47b3-b6ca-a9e1faf5b7a0",
+          "index": 6,
+          "lastGenerationAt": "",
+          "videoPrompt": "Highly detailed, photorealistic footage of ominous eight-legged giant octopuses briskly roaming a wild, otherworldly landscape, crushing alien houses in their path, as our raccoon hero watches in awe, their movement fluid like in an epic space opera movie.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "bd89e7ce-01e9-449a-b468-550e43956e77",
+          "index": 7,
+          "lastGenerationAt": "",
+          "videoPrompt": "Photorealistic movie shot of intricate details of the spaceship cockpit with the dedicated and absorbed raccoon pilot urgently plotting a course for escape, shadows and lights from the control panels play on his fur, showcasing stunning detail.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "4eba4b78-1d2f-4795-82b0-f56e6004e8fe",
+          "index": 8,
+          "lastGenerationAt": "",
+          "videoPrompt": "Ultra-wide angle, photorealistic shot of the planet's dawn, with the iridescent glow revealing the frozen wreckage caused by the dragons, taking on an ethereal quality amidst the chaos, shot in a cinematic manner, maintaining high details to procure an award-winning shot.",
+          "audioPrompt": ""
+        },
+        {
+          "shotId": "c3f5a185-cace-46bf-a74c-27f7114e75ce",
+          "index": 9,
+          "lastGenerationAt": "",
+          "videoPrompt": "Cinematic, photorealistic video of our raccoon hero in his spaceship, glancing back at the strange planet, a mix of pride and melancholy in his gaze as his spaceship disappears into the starry void, lighting from the control panel casting a soft glow on his figure, award-winning filmography.",
+          "audioPrompt": ""
+        }
+      ]
+    },
     {
       "sequenceId": "3b7fb664-f50a-4e2d-b63b-e1fedb22e501",
       "skip": false,
@@ -382,7 +469,7 @@
     },
     {
       "sequenceId": "a8ea6aac-c857-436c-877a-28ebe3420209",
-      "skip": false,
+      "skip": true,
       "lastGenerationAt": "",
       "videoPrompt": "a photorealistic movie, experimental horror movie showing creepy places, pool of blood, a creepy mansion, a woman in white near the road in the forest, mist and foggy woods, a cemetery, a creepy animal or deer-like monster creeping around. The movie is esoteric, there are some dreamy places.",
       "audioPrompt": "ambient electronic music from the 50s kraftwerk inspiration",

scripts/README.md

@@ -0,0 +1,9 @@
+
+
+## Scripts
+
+### interpolate.sh
+
+This is a script meant to be run on an existing video database.
+This can help making quick trials of interpolation algorithms,
+without modifying the video creating pipeline.

scripts/init.sh

@@ -5,10 +5,6 @@ echo "creating the storage folders for channel 1.."
 mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_1
 mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_1
 
-bash scripts/download_fresh_music.sh
-
-bash scripts/censorship.sh
-
 echo "creating the playlists for channel 1.." 
 echo "ffconcat version 1.0" > channel_1_video.txt
 echo "ffconcat version 1.0" > channel_1_audio.txt
@@ -21,3 +17,29 @@ mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_2
 echo "creating the playlists for channel 2.." 
 echo "ffconcat version 1.0" > channel_2_video.txt
 echo "ffconcat version 1.0" > channel_2_video.txt
+
+
+# ------------- CHANNEL 3 --------------
+echo "creating the storage folders for channel 3.."
+mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3
+mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_3
+
+echo "creating the playlists for channel 3.." 
+echo "ffconcat version 1.0" > channel_3_video.txt
+echo "ffconcat version 1.0" > channel_3_video.txt
+
+
+# ------------- CHANNEL 4 --------------
+echo "creating the storage folders for channel 4.."
+mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4
+mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_4
+
+echo "creating the playlists for channel 4.." 
+echo "ffconcat version 1.0" > channel_4_video.txt
+echo "ffconcat version 1.0" > channel_4_video.txt
+
+# ------------ UPDATE MUSIC ------------
+bash scripts/download_fresh_music.sh
+
+# ---------- CONTENT ALIGNMENT ---------
+bash scripts/censorship.sh

scripts/interpolate.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Make sure to provide your directories paths
+INPUT_DIR=$WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2
+OUTPUT_DIR=$WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3
+TEMP_DIR=$(mktemp -d)
+
+# Loop through all .mp4 files in input directory
+for FILE_PATH in $INPUT_DIR*.mp4; do
+    # get the base file name
+    FILE_NAME=$(basename "$FILE_PATH")
+    
+    # Create a temp file path
+    TEMP_FILE_PATH=$TEMP_DIR"/"$FILE_NAME
+
+    # Check if file does not exist in output directory
+    if [ ! -f "$OUTPUT_DIR$FILE_NAME" ]; then
+        echo "Processing $FILE_NAME, output will be saved in $OUTPUT_DIR$FILE_NAME"
+        
+        # Run your node command
+        if ! npm run postprod:interpolate "$FILE_PATH" "$TEMP_FILE_PATH"
+        then
+            # The first attempt has failed, retrying
+            echo "Attempt 1 failed for $FILE_NAME, retrying..."
+
+            if ! npm run postprod:interpolate "$FILE_PATH" "$TEMP_FILE_PATH"
+            then
+                # Both attempts have failed, skipping the file
+                echo "Both attempts failed for $FILE_NAME, skipping the file..."
+                continue
+            fi
+        fi
+
+        # Running ffmpeg to resize and apply unsharp mask
+        # Also we speed things up, and re-add the noise
+        if ! ffmpeg -hide_banner -v fatal -nostats -loglevel 0 -i "$TEMP_FILE_PATH" -vf "setpts=0.35*PTS,scale=-1:576:lanczos,unsharp=5:5:0.2:5:5:0.2,noise=c0s=10:c0f=t+u" -r 24 "$OUTPUT_DIR$FILE_NAME"
+        then
+            # ffmpeg command failed
+            echo "ffmpeg command failed for $FILE_NAME, skipping the file..."
+        fi
+        
+    else
+        echo "File $FILE_NAME already exists in the output directory, skipping."
+    fi
+done
+
+# Remove temp folder
+rm -r "$TEMP_DIR"

scripts/stream3.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+COUNT=0
+
+echo "Starting FFMPEG live stream for channel 3"
+while true; do
+    # Note: for now we also use channel 1 for audio!
+    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_3_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/smooth
+done

scripts/stream4.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+COUNT=0
+
+echo "Starting FFMPEG live stream for channel 4"
+while true; do
+    # Note: for now we also use channel 1 for audio!
+    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_4_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/interactive
+done

scripts/video3.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+echo "Starting the video collection stream for channel 3.."
+echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3*.mp4"
+current_count=0
+
+while true; do
+    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3*.mp4 2> /dev/null | wc -l)
+
+    if [ $new_count -ne $current_count ]; then
+        echo "there are $new_count videos files for channel 3"
+
+        echo "Updating playliss for channel 3.."
+        current_count=$new_count
+        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2*.mp4)
+
+        echo "ffconcat version 1.0" > channel_3_video_tmp.txt
+        for (( i=0; i<${#files[@]}; i++ )); do
+            echo "file '${files[$i]}'" >> channel_3_video_tmp.txt
+        done
+        mv channel_3_video_tmp.txt channel_3_video.txt
+    fi
+
+    # the new playlist will only be updated after the current playlist ended
+    # so there is no emergency here
+    sleep 60
+done

scripts/video4.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "Starting the video collection stream for channel 4.."
+echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4"
+current_count=0
+
+while true; do
+    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4 2> /dev/null | wc -l)
+
+    if [ $new_count -ne $current_count ]; then
+        echo "there are $new_count videos files for channel 4"
+
+        echo "Updating playliss for channel 4.."
+        current_count=$new_count
+        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4)
+
+        echo "ffconcat version 1.0" > channel_4_video_tmp.txt
+        for (( i=0; i<${#files[@]}; i++ )); do
+            echo "file '${files[$i]}'" >> channel_4_video_tmp.txt
+        done
+        mv channel_4_video_tmp.txt channel_4_video.txt
+    fi
+
+    # channel 4 is an interactive channel, so we have tighter delays
+    sleep 20
+done

src/index.mts

@@ -8,6 +8,7 @@ import { callZeroscope } from './callZeroscope.mts'
 import { downloadVideo } from './downloadVideo.mts'
 import { getDatabase } from './getDatabase.mts'
 import { callMusicgen } from './callMusicgen.mts'
+import { interpolateVideo } from './interpolateVideo.mts'
 
 let hasReachedStartingPoint = false
 
@@ -125,6 +126,9 @@ ${sequence.videoPrompt}
           await upscaleVideo(shotFileName, shot.videoPrompt)
         }
 
+        console.log('- enhancing shot..')
+        // await interpolateVideo(shotFileName)
+
         console.log('- enhancing shot..')
         await enhanceVideo(shotFileName)
 

src/interpolate/TODO_USE_ME.mts

@@ -0,0 +1,27 @@
+import { promises as fs } from "node:fs"
+import path from "node:path"
+
+import { client } from "@gradio/client"
+import tmpDir from "temp-dir"
+
+const spaceUrl = "https://jbilcke-hf-video-frame-interpolation.hf.space"
+
+export const interpolateVideo = (fileName: string) {
+
+  const filePath = path.join(tmpDir, fileName)
+ 
+  const app = await client(spaceUrl)
+
+  const video = await fs.readFile(filePath)
+  
+  const result = await app.predict(1, [
+    video.toString('base64'), 	// blob in 'parameter_5' Video component		
+    1, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
+    "24", // string  in 'FPS output' Radio component
+  ])
+
+  const data = (result as any).data
+
+  console.log(data)
+
+}

src/interpolateVideo.mts

@@ -1,27 +1,65 @@
-import { promises as fs } from "node:fs"
-import path from "node:path"
+import path from 'node:path'
+import fs from 'node:fs'
 
-import { client } from "@gradio/client"
-import tmpDir from "temp-dir"
+import tmpDir from 'temp-dir'
+import puppeteer from 'puppeteer'
+import { downloadVideo } from './downloadVideo.mts'
 
-const spaceUrl = "https://jbilcke-hf-video-frame-interpolation.hf.space"
+// TODO we should use an inference endpoint instead
+export async function interpolateVideo(fileName: string) {
+  
+  const browser = await puppeteer.launch({
+    // headless: true,
+    protocolTimeout: 300000,
+  })
 
-export const interpolateVideo = (fileName: string) {
+  const spaceUrl = process.env.WEBTV_INTERPOLATION_SPACE_URL
 
-  const filePath = path.join(tmpDir, fileName)
- 
-  const app = await client(spaceUrl)
+  const page = await browser.newPage()
 
-  const video = await fs.readFile(filePath)
+  await page.goto(spaceUrl, {
+    waitUntil: 'networkidle2',
+  })
+
+  const inputFilePath = path.join(tmpDir, fileName)
+  // console.log(`local file to upscale: ${inputFilePath}`)
   
-  const result = await app.predict(1, [
-    video.toString('base64'), 	// blob in 'parameter_5' Video component		
-    1, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
-    "24", // string  in 'FPS output' Radio component
-  ])
+  await new Promise(r => setTimeout(r, 3000))
+
+  const fileField = await page.$('input[type=file]')
+
+  // console.log(`uploading file..`)
+  await fileField.uploadFile(inputFilePath)
+
+  // console.log('looking for the button to submit')
+  const submitButton = await page.$('button.lg')
+
+  // console.log('clicking on the button')
+  await submitButton.click()
 
-  const data = (result as any).data
 
-  console.log(data)
+  await page.waitForSelector('a[download="interpolated_result.mp4"]', {
+    timeout: 300000, // need to be large enough in case someone else attemps to use our space
+  })
+
+  const interpolatedFileUrl = await page.$$eval('a[download="interpolated_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
+
+  const tmpFileName = `${fileName}_xl`
+
+  // console.log('downloading file from space..')
+  console.log(`- downloading ${fileName} from ${interpolatedFileUrl}`)
+
+  await downloadVideo(interpolatedFileUrl, tmpFileName)
+
+  const tmpFilePath = path.join(tmpDir, tmpFileName)
+  const filePath = path.join(tmpDir, fileName)
+
+  await fs.promises.copyFile(tmpFilePath, filePath)
+  try {
+    await fs.promises.unlink(tmpFilePath)
+  } catch (err) {
+    console.log('failed to cleanup (no big deal..)')
+  }
 
+  return fileName
 }

src/postprod/interpolate.mts

@@ -1,26 +1,12 @@
-import { promises as fs } from "node:fs"
-import { client } from "@gradio/client"
+import { interpolateVideo } from "./interpolateVideoForScript.mts"
 
 // to test this file:
-//  npm run postprod:interpolate sandbox/video/1688471841394.mp4
+//  npm run postprod:interpolate sandbox/video/1688471841394.mp4 sandbox/video/output.mp4
 
-const spaceUrl = "https://jbilcke-hf-video-frame-interpolation.hf.space"
-const filePath = process.argv[2]
+const inputFilePath = process.argv[2]
+const outputFilePath = process.argv[3]
 
-console.log('filePath:', filePath)
-const vid = await fs.readFile(filePath)
- 
-const b64 = vid.toString('base64')
+console.log('input file path:', inputFilePath)
+console.log('output file path:', outputFilePath)
 
-console.log(b64.slice(0, 20))
-
-const app = await client(spaceUrl)
-
-/*
-const result = await app.predict(1, [
-	exampleVideo, 	// blob in 'parameter_5' Video component		
-	1, // number (numeric value between 1 and 4) in 'Interpolation Steps' Slider component		
-	"8", // string  in 'FPS output' Radio component
-]);
-console.log(result?.data)
-*/
+await interpolateVideo(inputFilePath, outputFilePath)

src/postprod/interpolateVideoForScript.mts

@@ -0,0 +1,53 @@
+import fs from 'node:fs'
+
+import puppeteer from 'puppeteer'
+
+// TODO we should use an inference endpoint instead
+export async function interpolateVideo(inputFilePath: string, outputFilePath: string) {
+  
+  const browser = await puppeteer.launch({
+    headless: true,
+    protocolTimeout: 400000,
+  })
+
+  const spaceUrl = process.env.WEBTV_INTERPOLATION_SPACE_URL
+
+  const page = await browser.newPage()
+  await page.goto(spaceUrl, { waitUntil: 'networkidle2' })
+  
+  await new Promise(r => setTimeout(r, 3000))
+
+  const fileField = await page.$('input[type=file]')
+
+  // console.log(`uploading file..`)
+  await fileField.uploadFile(inputFilePath)
+
+  // console.log('looking for the button to submit')
+  const submitButton = await page.$('button.lg')
+
+  // console.log('clicking on the button')
+  await submitButton.click()
+
+  await page.waitForSelector('a[download="interpolated_result.mp4"]', {
+    timeout: 400000, // need to be large enough in case someone else attemps to use our space
+  })
+
+  const interpolatedFileUrl = await page.$$eval('a[download="interpolated_result.mp4"]', el => el.map(x => x.getAttribute("href"))[0])
+
+  // console.log('downloading file from space..')
+  console.log(`- downloading from ${interpolatedFileUrl}`)
+
+
+  // download the video
+  const response = await fetch(interpolatedFileUrl)
+  // write it to the disk
+  const arrayBuffer = await response.arrayBuffer()
+  
+  await fs.promises.writeFile(
+    outputFilePath,
+    Buffer.from(arrayBuffer)
+  )
+  console.log('success! wrote to:', outputFilePath)
+
+  process.exit()
+}

test.sh

@@ -0,0 +1,2 @@
+echo "$FOO$BAR"
+