working on new structure

database.json

@@ -158,7 +158,7 @@
 },
     {
       "sequenceId": "46e94a7b-dc27-4949-819e-bb9bce60c460",
-      "skip": false,
+      "skip": true,
       "lastGenerationAt": "",
       "videoPrompt": "a photorealistic compilation of various computer renderings and simulation featuring classical 3D animation of torus, cube, phong, shading, cast shadow in a minimal textured setting, a teapot, a monkey head",
       "audioPrompt": "electronic goal music from the 90s",

package.json

@@ -5,9 +5,9 @@
   "main": "src/index.mts",
   "scripts": {
     "start": "node --loader ts-node/esm src/index.mts",
-    "test": "node --loader ts-node/esm src/test.mts",
-    "postprod:interpolate": "node --loader ts-node/esm src/postprod/interpolate_legacy.mts",
-    "postprod:interpolate_legacy": "node --loader ts-node/esm src/postprod/interpolate_legacy.mts",
+    "batch:updatePlaylists": "node --loader ts-node/esm src/batch/updatePlaylists.mts",
+    "batch:interpolate": "node --loader ts-node/esm src/batch/interpolate_legacy.mts",
+    "batch:interpolate_legacy": "node --loader ts-node/esm src/batch/interpolate_legacy.mts",
     "docker": "npm run docker:build && npm run docker:run",
     "docker:build": "docker build -t  media-server .",
     "docker:run": "docker run -it -p 7860:7860 media-server"

scripts/channel_comedy.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Starting FFMPEG live stream for channel comedy"
+while true; do
+    if [ -f channel_comedy.txt ] && [ -f channel_1_audio.txt ]; then
+        echo "Files exist, starting stream"
+        # Note: for now we also use channel 1 for audio!
+        ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_comedy.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/comedy
+    else
+        echo "Files do not exist, waiting for files"
+        sleep 1  # check every second
+    fi
+done

scripts/channel_community.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Starting FFMPEG live stream for channel community"
+while true; do
+    if [ -f channel_community.txt ] && [ -f channel_1_audio.txt ]; then
+        echo "Files exist, starting stream"
+        # Note: for now we also use channel 1 for audio!
+        ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_community.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/community
+    else
+        echo "Files do not exist, waiting for files"
+        sleep 1  # check every second
+    fi
+done

scripts/channel_random.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "Starting FFMPEG live stream for channel random"
+while true; do
+    if [ -f channel_random.txt ] && [ -f channel_1_audio.txt ]; then
+        echo "Files exist, starting stream"
+        # Note: for now we also use channel 1 for audio!
+        ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_random.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/random
+    else
+        echo "Files do not exist, waiting for files"
+        sleep 1  # check every second
+    fi
+done

scripts/channel_random_twitch.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+echo "Starting FFMPEG live stream for channel random"
+while true; do
+    if [ -f channel_random.txt ] && [ -f channel_1_audio.txt ]; then
+        echo "Files exist, starting stream"
+        # Note: for now we also use channel 1 for audio!
+        ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_random.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://cdg02.contribute.live-video.net/app/$WEBTV_TWITCH_API_KEY
+        sleep 1
+    else
+        echo "Files do not exist, waiting for files"
+        sleep 1  # check every second
+    fi
+done
+    

scripts/init.sh

@@ -1,46 +1,3 @@
 #!/bin/bash
-
-# ------------- CHANNEL 1 --------------
-echo "creating the storage folders for channel 1.."
-mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_1
 mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_1
-
-echo "creating the playlists for channel 1.." 
-echo "ffconcat version 1.0" > channel_1_video.txt
-echo "ffconcat version 1.0" > channel_1_audio.txt
-
-# ------------- CHANNEL 2 --------------
-echo "creating the storage folders for channel 2.."
-mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2
-mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_2
-
-mkdir -p $WEBTV_VIDEO_ARCHIVE_PATH_CHANNEL_2
-
-
-echo "creating the playlists for channel 2.." 
-echo "ffconcat version 1.0" > channel_2_video.txt
-echo "ffconcat version 1.0" > channel_2_video.txt
-
-
-# ------------- CHANNEL 3 --------------
-echo "creating the storage folders for channel 3.."
 mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3
-mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_3
-
-mkdir -p $WEBTV_VIDEO_ARCHIVE_PATH_CHANNEL_3
-
-echo "creating the playlists for channel 3.." 
-echo "ffconcat version 1.0" > channel_3_video.txt
-echo "ffconcat version 1.0" > channel_3_video.txt
-
-
-# ------------- CHANNEL 4 --------------
-echo "creating the storage folders for channel 4.."
-mkdir -p $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4
-mkdir -p $WEBTV_AUDIO_STORAGE_PATH_CHANNEL_4
-
-mkdir -p $WEBTV_VIDEO_ARCHIVE_PATH_CHANNEL_4
-
-echo "creating the playlists for channel 4.." 
-echo "ffconcat version 1.0" > channel_4_video.txt
-echo "ffconcat version 1.0" > channel_4_video.txt

scripts/stream1.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-COUNT=0
-
-echo "Starting FFMPEG live stream for channel 1"
-while true; do
-    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_1_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/legacy
-done

scripts/stream2.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-COUNT=0
-
-echo "Starting FFMPEG live stream for channel 2"
-while true; do
-    # Note: for now we also use channel 1 for audio!
-    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_2_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/hdtv
-done

scripts/stream3.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-COUNT=0
-
-echo "Starting FFMPEG live stream for channel 3"
-while true; do
-    # Note: for now we also use channel 1 for audio!
-    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_3_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/smooth
-    sleep 1
-done

scripts/stream3_twitch.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-COUNT=0
-
-echo "Starting FFMPEG live stream of channel 3 to Twitch"
-while true; do
-    # Note: for now we also use channel 1 for audio!
-    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_3_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://cdg02.contribute.live-video.net/app/$WEBTV_TWITCH_API_KEY
-    sleep 1
-done

scripts/stream4.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-COUNT=0
-
-echo "Starting FFMPEG live stream for channel 4"
-while true; do
-    # Note: for now we also use channel 1 for audio!
-    ffmpeg -y -nostdin -re -f concat -safe 0 -i channel_4_video.txt -stream_loop -1 -safe 0 -i channel_1_audio.txt -loglevel error -c:v libx264 -preset veryfast -tune zerolatency -c:a aac -ar 44100 -shortest -f flv rtmp://localhost/live/interactive
-done

scripts/video1.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-echo "Starting the video collection stream for channel 1.."
-echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_1*.mp4"
-current_count=0
-
-while true; do
-    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_1*.mp4 2> /dev/null | wc -l)
-
-    if [ $new_count -ne $current_count ]; then
-        echo "there are $new_count videos files for channel 1"
-
-        echo "Updating playlists for channel 1.."
-        current_count=$new_count
-        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_1*.mp4)
-        
-        echo "ffconcat version 1.0" > channel_1_video_tmp.txt
-        for (( i=0; i<${#files[@]}; i++ )); do
-            echo "file '${files[$i]}'" >> channel_1_video_tmp.txt
-        done
-        mv channel_1_video_tmp.txt channel_1_video.txt
-    fi
-
-    # the new playlist will only be updated after the current playlist ended
-    # so there is no emergency here
-    sleep 60
-done

scripts/video2.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-echo "Starting the video collection stream for channel 2.."
-echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2*.mp4"
-current_count=0
-
-while true; do
-    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2*.mp4 2> /dev/null | wc -l)
-
-    if [ $new_count -ne $current_count ]; then
-        echo "there are $new_count videos files for channel 2"
-
-        echo "Updating playlist for channel 2.."
-        current_count=$new_count
-        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_2*.mp4)
-
-        echo "ffconcat version 1.0" > channel_2_video_tmp.txt
-        for (( i=0; i<${#files[@]}; i++ )); do
-            echo "file '${files[$i]}'" >> channel_2_video_tmp.txt
-        done
-        rm channel_2_video.txt
-        mv channel_2_video_tmp.txt channel_2_video.txt
-    fi
-
-    # the new playlist will only be updated after the current playlist ended
-    # so there is no emergency here
-    sleep 60
-done

scripts/video3.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-echo "Starting the video collection stream for channel 3.."
-echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3*.mp4"
-current_count=0
-
-while true; do
-    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3*.mp4 2> /dev/null | wc -l)
-
-    if [ $new_count -ne $current_count ]; then
-        echo "there are $new_count videos files for channel 3"
-
-        echo "Updating playlist for channel 3.."
-        current_count=$new_count
-        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3*.mp4)
-
-        echo "ffconcat version 1.0" > channel_3_video_tmp.txt
-        for (( i=0; i<${#files[@]}; i++ )); do
-            echo "file '${files[$i]}'" >> channel_3_video_tmp.txt
-        done
-        rm channel_3_video.txt
-        mv channel_3_video_tmp.txt channel_3_video.txt
-    fi
-
-    # the new playlist will only be updated after the current playlist ended
-    # so there is no emergency here
-    sleep 60
-done

scripts/video4.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-
-echo "Starting the video collection stream for channel 4.."
-echo "listing files in $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4"
-current_count=0
-
-while true; do
-    new_count=$(ls $WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4 2> /dev/null | wc -l)
-
-    if [ $new_count -ne $current_count ]; then
-        echo "there are $new_count videos files for channel 4"
-
-        echo "Updating playlist for channel 4.."
-        current_count=$new_count
-        files=($WEBTV_VIDEO_STORAGE_PATH_CHANNEL_4*.mp4)
-
-        echo "ffconcat version 1.0" > channel_4_video_tmp.txt
-        for (( i=0; i<${#files[@]}; i++ )); do
-            echo "file '${files[$i]}'" >> channel_4_video_tmp.txt
-        done
-        rm channel_4_video.txt
-        mv channel_4_video_tmp.txt channel_4_video.txt
-    fi
-
-    # channel 4 is an interactive channel, so we have tighter delays
-    sleep 20
-done

src/batch/updatePlaylists.mts

@@ -0,0 +1,95 @@
+import fs from "fs"
+import path from "path"
+
+import { Database } from "../types.mts"
+
+export const updatePlaylists = async (db: Database) => {
+  // TODO we should DROP THE CHANNEL 3 name,
+  // and only keep WEBTV_VIDEO_STORAGE_PATH
+  const directoryPath = process.env.WEBTV_VIDEO_STORAGE_PATH_CHANNEL_3
+
+  // Record of file paths categorized by tags
+  const categoryToFilePaths: Record<string, string[]> = {}
+
+  // Read JSON sequences
+  for (const sequence of db.sequences) {
+    // Read sequence tags and create empty category list if not already present
+    for (const tag of sequence.tags) {
+      if (!categoryToFilePaths[tag]) {
+        categoryToFilePaths[tag] = []
+      }
+    }
+
+    // Read files from the sequence
+    for (const shot of sequence.shots) {
+      const shotId = shot.shotId;
+      const mp4FileName = `${shotId}.mp4`
+      const shotFilePath = path.join(directoryPath, mp4FileName)
+
+      // Check if file exists
+      if (fs.existsSync(shotFilePath)) {
+        // Add the file path to the categories
+        for (const tag of sequence.tags) {
+          categoryToFilePaths[tag].push(shotFilePath)
+        }
+      }
+    }
+  }
+
+  // Print the stats
+  for (const [category, filePaths] of Object.entries(categoryToFilePaths)) {
+    console.log(
+      `Category: ${category}; Number of Files: ${filePaths.length}; Files: ${filePaths.join(
+        ", "
+      )}`
+    )
+  }
+
+  // create the new playlists
+  for (const [category, filePaths] of Object.entries(categoryToFilePaths)) {
+    if (filePaths.length > 0) {
+      console.log(`creating playlist for ${category}..`)
+
+      let data = 'ffconcat version 1.0\n'
+      filePaths.forEach((filePath) => {
+        data += `file '${filePath}'\n`
+      })
+
+      // write file data synchronously
+      fs.writeFileSync(`channel_${category}.txt`, data)
+      console.log(`Created playlist channel_${category}.txt`)
+    }
+  }
+
+  // Get all the file paths in the directory
+  const allFiles = await fs.promises.readdir(directoryPath)
+  const allFilePaths = allFiles
+    .map(file => path.join(directoryPath, file))
+    .filter(filePath => filePath.endsWith('.mp4'))
+
+  // Create channel_random.txt with all files
+  let randomData = 'ffconcat version 1.0\n'
+  allFilePaths.forEach(filePath => {
+    randomData += `file '${filePath}'\n`
+  })
+  await fs.promises.writeFile("channel_random.txt", randomData)
+  console.log("Created playlist channel_random.txt")
+
+  // Filter the list for the last 400 updated files
+  const lastUpdatedFilePaths = allFilePaths
+    .map(filePath => ({
+      filePath,
+      mtime: fs.statSync(filePath).mtime
+    }))
+    .sort((a, b) => b.mtime.getTime() - a.mtime.getTime())
+    .slice(0, 400)
+    .map(file => file.filePath)
+
+  // Create channel_fresh.txt
+  let freshData = 'ffconcat version 1.0\n'
+  lastUpdatedFilePaths.forEach(filePath => {
+    freshData += `file '${filePath}'\n`
+  })
+  await fs.promises.writeFile("channel_fresh.txt", freshData)
+  console.log("Created playlist channel_fresh.txt")
+}

src/index.mts

@@ -9,6 +9,7 @@ import { downloadVideo } from './downloadVideo.mts'
 import { getDatabase } from './getDatabase.mts'
 import { callMusicgen } from './callMusicgen.mts'
 import { interpolateVideo } from './interpolateVideo.mts'
+import { updatePlaylists } from './batch/updatePlaylists.mts'
 
 let hasReachedStartingPoint = false
 
@@ -21,6 +22,9 @@ console.log(`Web TV server status: ${status}`)
 // to add more diversity to the stream, let's cut down on the length
 const maxShotsPerSequence = 10
 
+console.log('Reading prompt database..')
+const db = await getDatabase('./database.json')
+
 const main = async () => {
   console.log('Reading persistent file structure..')
   const stats = await getStats()
@@ -34,9 +38,7 @@ const main = async () => {
     return
   }
 
-
-  console.log('Reading prompt database..')
-  const db = await getDatabase('./database.json')
+  await updatePlaylists(db)
 
   const nbTotalShots = db.sequences.reduce((a, s) => a + s.shots.length, 0)
   console.log(`Prompt database version: ${db.version}`)

start.sh

@@ -15,13 +15,15 @@ bash ./scripts/init.sh
 
 # ------ background processes ---------
 
-bash scripts/interpolate.sh &
-
 # starts the streaming server first, otherwise ffmpeg won't find it
 node ./media-server.js &
 
 sleep 1
 
+bash scripts/interpolate.sh &
+
+sleep 1
+
 # background process that creates an audio stream from audio files
 bash scripts/audio1.sh &
 # bash scripts/audio2.sh &