import os import ast from IPython.display import Audio import nltk # we'll use this to split into sentences import numpy as np from bark import generate_audio, SAMPLE_RATE from moviepy.editor import concatenate_videoclips, concatenate_audioclips from moviepy.editor import VideoFileClip, AudioFileClip, AudioClip, CompositeAudioClip import librosa import soundfile as sf import math def make_audio(en_prompt_file, output_dir): print("Begin to make the aside!") prompt_list = [] with open(en_prompt_file, 'r', encoding='utf-8') as f: video_prompts = f.read() video_fragments = ast.literal_eval(video_prompts) for video_fragment in video_fragments: prompt_list.append(video_fragment["video fragment description"]) if not os.path.exists(output_dir): os.makedirs(output_dir) for i, prompt in enumerate(prompt_list): sentences = nltk.sent_tokenize(prompt) SPEAKER = "v2/en_speaker_1" silence = np.zeros(int(0.25 * SAMPLE_RATE)) # quarter second of silence pieces = [] for sentence in sentences: audio_array = generate_audio(sentence, history_prompt=SPEAKER) # audio_array = generate_audio(sentence) pieces += [audio_array, silence.copy()] audio = Audio(np.concatenate(pieces), rate=SAMPLE_RATE) with open(os.path.join(output_dir, str(i) + ".wav"), 'w+b') as f: f.write(audio.data) def merge_video_audio(video_dir, audio_dir, output_dir): video_fnames = [] for fname in os.listdir(video_dir): if not fname.startswith("result"): video_fnames.append(fname) audio_fnames = [] for fname in os.listdir(audio_dir): if not fname.startswith("result") and not fname.startswith("fast"): audio_fnames.append(fname) video_fnames.sort(key=lambda x: int(x.split('.')[0])) audio_fnames.sort(key=lambda x: int(x.split('.')[0])) assert len(video_fnames) == len(audio_fnames), 'The number of videos is not equal to audios.' if not os.path.exists(output_dir): os.makedirs(output_dir) audios = [] for i, (video_fname, audio_fname) in enumerate(zip(video_fnames, audio_fnames)): video = VideoFileClip(os.path.join(video_dir, video_fname)) audio = AudioFileClip(os.path.join(audio_dir, audio_fname)) video_duration = video.duration audio_duration = audio.duration if audio_duration > video_duration: y, sr = librosa.load(os.path.join(audio_dir, audio_fname)) speed_change = audio_duration / video_duration y_stretched = librosa.effects.time_stretch(y, rate=speed_change) sf.write(os.path.join(audio_dir, "fast_video.wav"), y_stretched, sr) audio = AudioFileClip(os.path.join(audio_dir, "fast_video.wav")) else: silence_len = math.ceil(video_duration * audio.fps) / audio.fps # make sure the silence duration not less than required silence = AudioClip(lambda t: [0] * audio.nchannels, duration=silence_len, fps=audio.fps) audio = CompositeAudioClip([audio, silence]) audios.append(audio) video = video.set_audio(audio) video.write_videofile(os.path.join(output_dir, str(i) + ".mp4")) final_audio = concatenate_audioclips(audios) final_audio.write_audiofile(os.path.join(audio_dir, "result" + ".wav")) def concatenate_videos(video_dir, output_dir=None): if output_dir is None: output_dir = video_dir video_fnames = [] for fname in os.listdir(video_dir): if not fname.startswith("result") and not fname.startswith("audio"): video_fnames.append(fname) video_fnames.sort(key=lambda x: int(x.split('.')[0])) if not os.path.exists(output_dir): os.makedirs(output_dir) video_clips = [VideoFileClip(os.path.join(video_dir, video_fname)) for video_fname in video_fnames] audio_clips = [video.audio for video in video_clips] final_video = concatenate_videoclips(video_clips, method="compose") final_audio = concatenate_audioclips(audio_clips) final_clip = final_video.set_audio(final_audio) final_clip.write_videofile(os.path.join(output_dir, "result.mp4"))