jbilcke-hf HF staff commited on
Commit
c5b101c
1 Parent(s): f8ca042

add experimental support for 360° videos

Browse files
src/app/interface/equirectangular-video-player/index.tsx ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import AutoSizer from "react-virtualized-auto-sizer"
4
+
5
+ import { cn } from "@/lib/utils"
6
+ import { VideoInfo } from "@/types"
7
+
8
+ import { VideoSphereViewer } from "./viewer"
9
+
10
+ export function EquirectangularVideoPlayer({
11
+ video,
12
+ className = "",
13
+ }: {
14
+ video?: VideoInfo
15
+ className?: string
16
+ }) {
17
+ // we shield the VideeoSphere viewer from bad data
18
+ if (!video?.assetUrl) { return null }
19
+
20
+ return (
21
+ <div
22
+ className={cn(
23
+ `w-full`,
24
+ // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex"
25
+ `aspect-video`,
26
+ className
27
+ )}>
28
+ <AutoSizer>
29
+ {({ height, width }) => (
30
+ <VideoSphereViewer
31
+ video={video}
32
+ className={className}
33
+ width={width}
34
+ height={height}
35
+ />
36
+ )}
37
+ </AutoSizer>
38
+ </div>
39
+ )
40
+ }
src/app/interface/equirectangular-video-player/viewer.tsx ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import { useEffect, useRef, useState } from "react"
4
+ import AutoSizer from "react-virtualized-auto-sizer"
5
+ import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
6
+ import { EquirectangularVideoAdapter, LensflarePlugin, ReactPhotoSphereViewer, ResolutionPlugin, SettingsPlugin, VideoPlugin } from "react-photo-sphere-viewer"
7
+
8
+ import { cn } from "@/lib/utils"
9
+ import { VideoInfo } from "@/types"
10
+
11
+ type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
12
+
13
+ export function VideoSphereViewer({
14
+ video,
15
+ className = "",
16
+ width,
17
+ height,
18
+ muted = false,
19
+ }: {
20
+ video: VideoInfo
21
+ className?: string
22
+ width: number
23
+ height: number
24
+ muted?: boolean
25
+ }) {
26
+ const rootContainerRef = useRef<HTMLDivElement>(null)
27
+ const viewerContainerRef = useRef<HTMLElement>()
28
+ const viewerRef = useRef<Viewer>()
29
+
30
+ useEffect(() => {
31
+ if (!viewerRef.current) { return }
32
+ viewerRef.current.setOptions({
33
+ size: {
34
+ width: `${width}px`,
35
+ height: `${height}px`
36
+ }
37
+ })
38
+ }, [width, height])
39
+
40
+ if (!video.assetUrl) { return null }
41
+
42
+ return (
43
+ <div
44
+ // will be used later, if we need overlays and stuff
45
+ ref={rootContainerRef}
46
+ >
47
+ <ReactPhotoSphereViewer
48
+
49
+ container=""
50
+ containerClass={cn(
51
+ "rounded-xl overflow-hidden",
52
+ className
53
+ )}
54
+
55
+ width={`${width}px`}
56
+ height={`${height}px`}
57
+
58
+ onReady={(instance) => {
59
+ viewerRef.current = instance
60
+ viewerContainerRef.current = instance.container
61
+ }}
62
+
63
+ // to access a plugin we must use viewer.getPlugin()
64
+ // plugins={[[LensflarePlugin, { lensflares: [] }]]}
65
+
66
+ adapter={[EquirectangularVideoAdapter, { muted }]}
67
+ navbar="video"
68
+ src=""
69
+ plugins={[
70
+ [VideoPlugin, {
71
+ muted,
72
+ // progressbar: true,
73
+ bigbutton: false
74
+ }],
75
+ // SettingsPlugin,
76
+ [ResolutionPlugin, {
77
+ defaultResolution: 'HD',
78
+ resolutions: [
79
+ {
80
+ id: 'HD',
81
+ label: 'Standard',
82
+ panorama: { source: video.assetUrl },
83
+ },
84
+ ],
85
+ }],
86
+ ]}
87
+ />
88
+ </div>
89
+ )
90
+ }
91
+
src/app/interface/video-player/cartesian.tsx ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import { Player } from "react-tuby"
4
+ import "react-tuby/css/main.css"
5
+
6
+ import { cn } from "@/lib/utils"
7
+ import { VideoInfo } from "@/types"
8
+
9
+ export function CartesianVideoPlayer({
10
+ video,
11
+ enableShortcuts = true,
12
+ className = "",
13
+ // currentTime,
14
+ }: {
15
+ video: VideoInfo
16
+ enableShortcuts?: boolean
17
+ className?: string
18
+ // currentTime?: number
19
+ }) {
20
+ return (
21
+ <div className={cn(
22
+ `w-full`,
23
+ `flex flex-col items-center justify-center`,
24
+ `rounded-xl overflow-hidden`,
25
+ className
26
+ )}>
27
+ <div className={cn(
28
+ `w-[calc(100%+16px)]`,
29
+ `-ml-2 -mr-2`,
30
+ `flex flex-col items-center justify-center`,
31
+ )}>
32
+ <Player
33
+
34
+ // playerRef={ref}
35
+
36
+ src={[
37
+ {
38
+ quality: "Full HD",
39
+ url: video.assetUrl,
40
+ }
41
+ ]}
42
+
43
+ keyboardShortcut={enableShortcuts}
44
+
45
+ subtitles={[]}
46
+ poster={
47
+ `https://huggingface.co/datasets/jbilcke-hf/ai-tube-index/resolve/main/videos/${video.id}.webp`
48
+ }
49
+ />
50
+ </div>
51
+ </div>
52
+ )
53
+ }
src/app/interface/video-player/equirectangular.tsx ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ "use client"
2
+
3
+ import { useEffect, useRef, useState } from "react"
4
+
5
+ import { PanoramaPosition, PluginConstructor, Point, Position, SphericalPosition, Viewer } from "@photo-sphere-viewer/core"
6
+ import { EquirectangularVideoAdapter, LensflarePlugin, ReactPhotoSphereViewer, ResolutionPlugin, SettingsPlugin, VideoPlugin } from "react-photo-sphere-viewer"
7
+
8
+ import { cn } from "@/lib/utils"
9
+ import { VideoInfo } from "@/types"
10
+
11
+ type PhotoSpherePlugin = (PluginConstructor | [PluginConstructor, any])
12
+
13
+ export function EquirectangularVideoPlayer({
14
+ video,
15
+ className = "",
16
+ width,
17
+ height,
18
+ muted = false,
19
+ }: {
20
+ video: VideoInfo
21
+ className?: string
22
+ width: number
23
+ height: number
24
+ muted?: boolean
25
+ }) {
26
+ const rootContainerRef = useRef<HTMLDivElement>(null)
27
+ const viewerContainerRef = useRef<HTMLElement>()
28
+ const viewerRef = useRef<Viewer>()
29
+
30
+ useEffect(() => {
31
+ if (!viewerRef.current) { return }
32
+ viewerRef.current.setOptions({
33
+ size: {
34
+ width: `${width}px`,
35
+ height: `${height}px`
36
+ }
37
+ })
38
+ }, [width, height])
39
+
40
+ if (!video.assetUrl) { return null }
41
+
42
+ return (
43
+ <div
44
+ // will be used later, if we need overlays and stuff
45
+ ref={rootContainerRef}
46
+ >
47
+ <ReactPhotoSphereViewer
48
+
49
+ container=""
50
+ containerClass={cn(
51
+ "rounded-xl overflow-hidden",
52
+ className
53
+ )}
54
+
55
+ width={`${width}px`}
56
+ height={`${height}px`}
57
+
58
+ onReady={(instance) => {
59
+ viewerRef.current = instance
60
+ viewerContainerRef.current = instance.container
61
+ }}
62
+
63
+ // to access a plugin we must use viewer.getPlugin()
64
+ // plugins={[[LensflarePlugin, { lensflares: [] }]]}
65
+
66
+ adapter={[EquirectangularVideoAdapter, { muted }]}
67
+ navbar="video"
68
+ src=""
69
+ plugins={[
70
+ [VideoPlugin, {
71
+ muted,
72
+ // progressbar: true,
73
+ bigbutton: false
74
+ }],
75
+ // SettingsPlugin,
76
+ [ResolutionPlugin, {
77
+ defaultResolution: 'HD',
78
+ resolutions: [
79
+ {
80
+ id: 'HD',
81
+ label: 'Standard',
82
+ panorama: { source: video.assetUrl },
83
+ },
84
+ ],
85
+ }],
86
+ ]}
87
+ />
88
+ </div>
89
+ )
90
+ }
91
+
src/app/interface/video-player/index.tsx CHANGED
@@ -1,12 +1,13 @@
1
  "use client"
2
 
3
- import { Player } from "react-tuby"
4
- import "react-tuby/css/main.css"
5
 
6
  import { cn } from "@/lib/utils"
7
  import { VideoInfo } from "@/types"
8
- import { MutableRefObject, useEffect, useRef } from "react"
9
- import { isValidNumber } from "@/app/server/actions/utils/isValidNumber"
 
 
10
 
11
  export function VideoPlayer({
12
  video,
@@ -19,51 +20,26 @@ export function VideoPlayer({
19
  className?: string
20
  // currentTime?: number
21
  }) {
 
 
22
 
23
- /*
24
- const ref = useRef(null)
25
-
26
- useEffect(() => {
27
- if (!ref.current) { return }
28
- if (!isValidNumber(currentTime)) { return }
29
-
30
- (ref.current as any).currentTime = currentTime
31
- // $(".tuby-container video").currentTime = 2
32
- }, [currentTime])
33
- */
34
 
35
- // TODO: keep the same form factor?
36
- if (!video) { return null }
 
 
 
 
 
 
 
 
37
 
38
  return (
39
- <div className={cn(
40
- `w-full`,
41
- `flex flex-col items-center justify-center`,
42
- `rounded-xl overflow-hidden`,
43
- className
44
- )}>
45
- <div className={cn(
46
- `w-[calc(100%+16px)]`,
47
- `-ml-2 -mr-2`,
48
- `flex flex-col items-center justify-center`,
49
- )}>
50
- <Player
51
-
52
- // playerRef={ref}
53
-
54
- src={[
55
- {
56
- quality: "Full HD",
57
- url: video.assetUrl,
58
- }
59
- ]}
60
-
61
- keyboardShortcut={enableShortcuts}
62
-
63
- subtitles={[]}
64
- // poster="https://cdn.jsdelivr.net/gh/naptestdev/video-examples@master/poster.png"
65
- />
66
- </div>
67
- </div>
68
  )
69
  }
 
1
  "use client"
2
 
3
+ import AutoSizer from "react-virtualized-auto-sizer"
 
4
 
5
  import { cn } from "@/lib/utils"
6
  import { VideoInfo } from "@/types"
7
+ import { parseProjectionFromLoRA } from "@/app/server/actions/utils/parseProjectionFromLoRA"
8
+
9
+ import { EquirectangularVideoPlayer } from "./equirectangular"
10
+ import { CartesianVideoPlayer } from "./cartesian"
11
 
12
  export function VideoPlayer({
13
  video,
 
20
  className?: string
21
  // currentTime?: number
22
  }) {
23
+ // we should our video players from misssing data
24
+ if (!video?.assetUrl) { return null }
25
 
26
+ const isEquirectangular = (
27
+ video.projection === "equirectangular" ||
28
+ parseProjectionFromLoRA(video.lora) === "equirectangular"
29
+ )
 
 
 
 
 
 
 
30
 
31
+ if (isEquirectangular) {
32
+ // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex"
33
+ return (
34
+ <div className={cn(`w-full aspect-video`, className)}>
35
+ <AutoSizer>{({ height, width }) => (
36
+ <EquirectangularVideoPlayer video={video} className={className} width={width} height={height} />
37
+ )}</AutoSizer>
38
+ </div>
39
+ )
40
+ }
41
 
42
  return (
43
+ <CartesianVideoPlayer video={video} enableShortcuts={enableShortcuts} className={className} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  )
45
  }
src/app/server/actions/ai-tube-hf/getChannelVideos.ts CHANGED
@@ -7,6 +7,7 @@ import { adminApiKey } from "../config"
7
  import { getVideoIndex } from "./getVideoIndex"
8
  import { extendVideosWithStats } from "./extendVideosWithStats"
9
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
 
10
 
11
  // return
12
  export async function getChannelVideos({
@@ -46,6 +47,7 @@ export async function getChannelVideos({
46
  thumbnailUrl: v.thumbnailUrl,
47
  model: v.model,
48
  lora: v.lora,
 
49
  style: v.style,
50
  voice: v.voice,
51
  music: v.music,
 
7
  import { getVideoIndex } from "./getVideoIndex"
8
  import { extendVideosWithStats } from "./extendVideosWithStats"
9
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
10
+ import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
11
 
12
  // return
13
  export async function getChannelVideos({
 
47
  thumbnailUrl: v.thumbnailUrl,
48
  model: v.model,
49
  lora: v.lora,
50
+ projection: parseProjectionFromLoRA(v.lora),
51
  style: v.style,
52
  voice: v.voice,
53
  music: v.music,
src/app/server/actions/ai-tube-hf/getVideoRequestsFromChannel.ts CHANGED
@@ -8,6 +8,7 @@ import { downloadFileAsText } from "./downloadFileAsText"
8
  import { parseDatasetPrompt } from "../utils/parseDatasetPrompt"
9
  import { parseVideoModelName } from "../utils/parseVideoModelName"
10
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
 
11
 
12
  /**
13
  * Return all the videos requests created by a user on their channel
 
8
  import { parseDatasetPrompt } from "../utils/parseDatasetPrompt"
9
  import { parseVideoModelName } from "../utils/parseVideoModelName"
10
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
11
+ import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
12
 
13
  /**
14
  * Return all the videos requests created by a user on their channel
src/app/server/actions/ai-tube-hf/uploadVideoRequestToDataset.ts CHANGED
@@ -6,6 +6,7 @@ import { Credentials, uploadFile, whoAmI } from "@/huggingface/hub/src"
6
  import { ChannelInfo, VideoGenerationModel, VideoInfo, VideoOrientation, VideoRequest } from "@/types"
7
  import { formatPromptFileName } from "../utils/formatPromptFileName"
8
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
 
9
 
10
  /**
11
  * Save the video request to the user's own dataset
@@ -142,6 +143,7 @@ ${prompt}
142
  model,
143
  style,
144
  lora,
 
145
  voice,
146
  music,
147
  thumbnailUrl: channel.thumbnail, // will be generated in async
 
6
  import { ChannelInfo, VideoGenerationModel, VideoInfo, VideoOrientation, VideoRequest } from "@/types"
7
  import { formatPromptFileName } from "../utils/formatPromptFileName"
8
  import { orientationToWidthHeight } from "../utils/orientationToWidthHeight"
9
+ import { parseProjectionFromLoRA } from "../utils/parseProjectionFromLoRA"
10
 
11
  /**
12
  * Save the video request to the user's own dataset
 
143
  model,
144
  style,
145
  lora,
146
+ projection: parseProjectionFromLoRA(lora),
147
  voice,
148
  music,
149
  thumbnailUrl: channel.thumbnail, // will be generated in async
src/app/server/actions/utils/parseProjectionFromLoRA.ts ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { VideoProjection } from "@/types"
2
+
3
+ export function parseProjectionFromLoRA(input?: any): VideoProjection {
4
+ const name = `${input || ""}`.trim().toLowerCase()
5
+
6
+ const isEquirectangular = (
7
+ name.includes("equirectangular") ||
8
+ name.includes("panorama") ||
9
+ name.includes("360")
10
+ )
11
+
12
+ return (
13
+ isEquirectangular
14
+ ? "equirectangular"
15
+ : "cartesian"
16
+ )
17
+ }
src/app/views/user-account-view/index.tsx CHANGED
@@ -42,7 +42,7 @@ export function UserAccountView() {
42
  }
43
  })
44
  }
45
- }, [isLoaded, huggingfaceApiKey])
46
 
47
  return (
48
  <div className={cn(`flex flex-col space-y-4`)}>
 
42
  }
43
  })
44
  }
45
+ }, [isLoaded, huggingfaceApiKey, setUserChannels, setLoaded])
46
 
47
  return (
48
  <div className={cn(`flex flex-col space-y-4`)}>
src/lib/usePlaylist.ts CHANGED
@@ -109,7 +109,7 @@ export function usePlaylist() {
109
  dequeue();
110
  }
111
  }
112
- }, [audio?.currentTime, dequeue, setProgress, isPlaying]);
113
 
114
  const playback = useCallback(async (options?: PlaybackOptions<VideoInfo>): Promise<void> => {
115
  if (!audio) { return }
 
109
  dequeue();
110
  }
111
  }
112
+ }, [audio, audio?.currentTime, dequeue, setProgress, isPlaying]);
113
 
114
  const playback = useCallback(async (options?: PlaybackOptions<VideoInfo>): Promise<void> => {
115
  if (!audio) { return }
src/types.ts CHANGED
@@ -330,6 +330,11 @@ export type VideoOrientation =
330
  | "landscape"
331
  | "square"
332
 
 
 
 
 
 
333
  export type VideoInfo = {
334
  /**
335
  * UUID (v4)
@@ -446,6 +451,11 @@ export type VideoInfo = {
446
  * General video aspect ratio
447
  */
448
  orientation: VideoOrientation
 
 
 
 
 
449
  }
450
 
451
  export type CollectionInfo = {
 
330
  | "landscape"
331
  | "square"
332
 
333
+ export type VideoProjection =
334
+ | "cartesian" // this is the default
335
+ | "equirectangular"
336
+
337
+
338
  export type VideoInfo = {
339
  /**
340
  * UUID (v4)
 
451
  * General video aspect ratio
452
  */
453
  orientation: VideoOrientation
454
+
455
+ /**
456
+ * Video projection (cartesian by default)
457
+ */
458
+ projection: VideoProjection
459
  }
460
 
461
  export type CollectionInfo = {