update

README.md

@@ -1,13 +1,21 @@
 ---
-title: Funasr App Clipvideo
-emoji: 🐢
-colorFrom: blue
-colorTo: yellow
-sdk: gradio
-sdk_version: 3.34.0
-app_file: app.py
-pinned: false
-license: mit
+# 详细文档见https://modelscope.cn/docs/%E5%88%9B%E7%A9%BA%E9%97%B4%E5%8D%A1%E7%89%87
+domain: #领域：cv/nlp/audio/multi-modal/AutoML
+- audio
+tags: #自定义标签
+- ClipVideo
+models: #关联模型
+- damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch
+- damo/speech_timestamp_prediction-v1-16k-offline
+deployspec:
+  cpu: 6
+  memory: 87000
+  gpu: 1
+  instance_type: ecs.gn6e-c12g1.3xlarge
+  image_id: mshub-registry-vpc.cn-zhangjiakou.cr.aliyuncs.com/modelscope-repo/studio-service-gpu:funasr-clipvideo-gradio-3.29.0-2
+license: Apache License 2.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+#### Clone with HTTP
+```bash
+ git clone https://www.modelscope.cn/studios/damo/funasr_app_clipvideo.git
+```

app.py

@@ -0,0 +1,137 @@
+import gradio as gr
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from videoclipper import VideoClipper
+
+
+if __name__ == "__main__":
+    inference_pipeline = pipeline(
+        task=Tasks.auto_speech_recognition,
+        model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
+        vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',
+        punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
+    )
+    audio_clipper = VideoClipper(inference_pipeline)
+
+    def audio_recog(audio_input):
+        return audio_clipper.recog(audio_input)
+
+    def audio_clip(dest_text, start_ost, end_ost, state):
+        return audio_clipper.clip(dest_text, start_ost, end_ost, state)
+
+    def video_recog(video_input):
+        return audio_clipper.video_recog(video_input)
+
+    def video_clip(dest_text, start_ost, end_ost, state):
+        return audio_clipper.video_clip(dest_text, start_ost, end_ost, state)
+
+    def video_clip_addsub(dest_text, start_ost, end_ost, state, font_size, font_color):
+        return audio_clipper.video_clip(dest_text, start_ost, end_ost, state, font_size, font_color, add_sub=True)
+
+    '''
+    top_md_1 = ("""
+    基于达摩院自研Paraformer-长音频版的语音识别、端点检测、标点预测、时间戳功能
+
+    准确识别，自由复制所需段落并一键裁剪、添加字幕
+
+    * Step1: 上传视频文件（或使用下方的用例体验），点击 **<font color="#f7802b">识别</font>** 按钮
+    * Step2: 复制识别结果中所需的文字至右上方，设置偏移与字幕配置（可选）
+    * Step3: 点击 **<font color="#f7802b">裁剪</font>** 按钮或 **<font color="#f7802b">裁剪并添加字幕</font>** 按钮获得结果
+    """)
+    '''
+
+    top_md_2 = ("""
+    受到网络传输与服务资源的限制，用于体验的视频最好大小在40mb以下
+    过大的视频可以尝试分离音轨使用音频剪辑，或 **<font color="#1785c4">通过源代码将您的ClipVideo服务部署在本地（推荐）</font>** ：
+    <div align="center">
+    <div style="display:flex; gap: 0.25rem;" align="center">
+    FunASR_APP: <a href='https://github.com/alibaba/funasr-app'><img src='https://img.shields.io/badge/Github-Code-blue'></a> 
+    🌟支持我们: <a href='https://github.com/alibaba/funasr-app/stargazers'><img src='https://img.shields.io/github/stars/alibaba/funasr-app.svg?style=social'></a>
+    </div>
+    </div>
+    """)
+
+    top_md_3 = ("""访问FunASR项目与论文能够帮助您深入了解ClipVideo中所使用的语音处理相关模型：
+    <div align="center">
+    <div style="display:flex; gap: 0.25rem;" align="center">
+        FunASR: <a href='https://github.com/alibaba-damo-academy/FunASR'><img src='https://img.shields.io/badge/Github-Code-blue'></a> 
+        FunASR Paper: <a href="https://arxiv.org/abs/2305.11013"><img src="https://img.shields.io/badge/Arxiv-2305.11013-orange"></a> 
+        🌟Star FunASR: <a href='https://github.com/alibaba-damo-academy/FunASR/stargazers'><img src='https://img.shields.io/github/stars/alibaba-damo-academy/FunASR.svg?style=social'></a>
+    </div>
+    </div>
+    """)
+
+    # gradio interface
+    with gr.Blocks() as demo:
+        #gr.Image("./examples/guide.png", show_label=False)
+        # gr.Markdown(top_md_1)
+        #gr.Markdown(top_md_2)
+        #gr.Markdown(top_md_3)
+        video_state = gr.State()
+        audio_state = gr.State()
+        with gr.Tab("🎥✂️视频裁剪 Video Clipping"):
+            with gr.Row():
+                with gr.Column():
+                    video_input = gr.Video(label="🎥视频输入 Video Input")
+                    gr.Examples(['examples/2022云栖大会_片段2.mp4', 
+                                 'examples/2022云栖大会_片段.mp4', 
+                                 'examples/为什么要多读书？这是我听过最好的答案-片段.mp4', 
+                                 'examples/使用chatgpt_片段.mp4'],
+                                [video_input])
+                    recog_button2 = gr.Button("👂识别 Recognize")
+                    video_text_output = gr.Textbox(label="✏️识别结果 Recognition Result")
+                    video_srt_output = gr.Textbox(label="📖SRT字幕内容 RST Subtitles")
+                with gr.Column():
+                    video_text_input = gr.Textbox(label="✏️待裁剪文本 Text to Clip (多段文本使用'#'连接)")
+                    with gr.Row():
+                        video_start_ost = gr.Slider(minimum=-500, maximum=1000, value=0, step=50, label="⏪开始位置偏移 Start Offset (ms)")
+                        video_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩结束位置偏移 End Offset (ms)")
+                    with gr.Row():
+                        font_size = gr.Slider(minimum=10, maximum=100, value=32, step=2, label="🔠字幕字体大小 Subtitle Font Size")
+                        font_color = gr.Radio(["black", "white", "green", "red"], label="🌈字幕颜色 Subtitle Color", value='white')
+                        # font = gr.Radio(["黑体", "Alibaba Sans"], label="字体 Font")
+                    with gr.Row():
+                        clip_button2 = gr.Button("✂️裁剪\nClip")
+                        clip_button3 = gr.Button("✂️裁剪并添加字幕\nClip and Generate Subtitles")
+                    video_output = gr.Video(label="🎥裁剪结果 Audio Clipped")
+                    video_mess_output = gr.Textbox(label="ℹ️裁剪信息 Clipping Log")
+                    video_srt_clip_output = gr.Textbox(label="📖裁剪部分SRT字幕内容 Clipped RST Subtitles")
+
+        with gr.Tab("🔊✂️音频裁剪 Audio Clipping"):
+            with gr.Row():
+                with gr.Column():
+                    audio_input = gr.Audio(label="🔊音频输入 Audio Input")
+                    gr.Examples(['examples/鲁肃采访片段1.wav'], [audio_input])
+                    recog_button1 = gr.Button("👂识别 Recognize")
+                    audio_text_output = gr.Textbox(label="✏️识别结果 Recognition Result")
+                    audio_srt_output = gr.Textbox(label="📖SRT字幕内容 RST Subtitles")
+                with gr.Column():
+                    audio_text_input = gr.Textbox(label="✏️待裁剪文本 Text to Clip (多段文本使用'#'连接)")
+                    with gr.Row():
+                        audio_start_ost = gr.Slider(minimum=-500, maximum=1000, value=0, step=50, label="⏪开始位置偏移 Start Offset (ms)")
+                        audio_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩结束位置偏移 End Offset (ms)")
+                    with gr.Row():
+                        clip_button1 = gr.Button("✂️裁剪 Clip")
+                    audio_output = gr.Audio(label="🔊裁剪结果 Audio Clipped")
+                    audio_mess_output = gr.Textbox(label="ℹ️裁剪信息 Clipping Log")
+                    audio_srt_clip_output = gr.Textbox(label="📖裁剪部分SRT字幕内容 Clipped RST Subtitles")
+        
+        recog_button1.click(audio_recog, 
+                            inputs=audio_input, 
+                            outputs=[audio_text_output, audio_srt_output, audio_state])
+        clip_button1.click(audio_clip, 
+                           inputs=[audio_text_input, audio_start_ost, audio_end_ost, audio_state], 
+                           outputs=[audio_output, audio_mess_output, audio_srt_clip_output])
+
+        recog_button2.click(video_recog, 
+                            inputs=video_input, 
+                            outputs=[video_text_output, video_srt_output, video_state])
+        clip_button2.click(video_clip, 
+                           inputs=[video_text_input, video_start_ost, video_end_ost, video_state], 
+                           outputs=[video_output, video_mess_output, video_srt_clip_output])
+        clip_button3.click(video_clip_addsub, 
+                           inputs=[video_text_input, video_start_ost, video_end_ost, video_state, font_size, font_color], 
+                           outputs=[video_output, video_mess_output, video_srt_clip_output])
+    
+    # start gradio service in local
+    demo.queue(concurrency_count=3).launch()

examples/2022云栖大会_片段.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d171002592d4b87917349d75c9c4fdfd99e4186f8aa71420d6d98acbeb31ca
+size 36077615

examples/2022云栖大会_片段2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e09b542f45373db284228cdf1507e28d926b4a2ab4697a20e1dca04d5ad0c88
+size 6715481

examples/为什么要多读书？这是我听过最好的答案-片段.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3901cea588badd3ebc502428d983c3d1413edf5159db134ec82e92101535989f
+size 46912031

examples/使用chatgpt_片段.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b7c448b49a96c239d19a0e7e5c14a7a8f558231f700e93722cb0b3da2991cfb
+size 12033342

examples/鲁肃采访片段1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b57d8e54fd1a6d56dd16c969ac89b6570e8f73eedc99b12a18a8112a13377d0
+size 6086432

font/STHeitiMedium.ttc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8fa4a63e2cf500e98e64d4c73260daaba049306cf85dec9e3729bc285b7d645
+size 55754164

subtitle_utils.py

@@ -0,0 +1,105 @@
+def time_convert(ms):
+    ms = int(ms)
+    tail = ms % 1000
+    s = ms // 1000
+    mi = s // 60
+    s = s % 60
+    h = mi // 60
+    mi = mi % 60
+    h = "00" if h == 0 else str(h)
+    mi = "00" if mi == 0 else str(mi)
+    s = "00" if s == 0 else str(s)
+    tail = str(tail)
+    if len(h) == 1: h = '0' + h
+    if len(mi) == 1: mi = '0' + mi
+    if len(s) == 1: s = '0' + s
+    return "{}:{}:{},{}".format(h, mi, s, tail)
+
+
+class Text2SRT():
+    def __init__(self, text_seg, ts_list, offset=0):
+        self.token_list = [i for i in text_seg.split() if len(i)]
+        self.ts_list = ts_list
+        start, end = ts_list[0][0] - offset, ts_list[-1][1] - offset
+        self.start_sec, self.end_sec = start, end
+        self.start_time = time_convert(start)
+        self.end_time = time_convert(end)
+    def text(self):
+        res = ""
+        for word in self.token_list:
+            if '\u4e00' <= word <= '\u9fff':
+                res += word
+            else:
+                res += " " + word
+        return res
+    def len(self):
+        return len(self.token_list)
+    def srt(self):
+        return "{} --> {}\n{}\n".format(self.start_time, self.end_time, self.text())
+    def time(self):
+        return (self.start_sec/1000, self.end_sec/1000)
+
+
+def generate_srt(sentence_list):
+    srt_total = ''
+    for i, d in enumerate(sentence_list):
+        t2s = Text2SRT(d['text_seg'], d['ts_list'])
+        srt_total += "{}\n{}".format(i, t2s.srt())
+    return srt_total
+
+def generate_srt_clip(sentence_list, start, end, begin_index=0):
+    start, end = int(start * 1000), int(end * 1000)
+    srt_total = ''
+    cc = 1 + begin_index
+    subs = []
+    for i, d in enumerate(sentence_list):
+        if d['ts_list'][-1][1] <= start:
+            continue
+        if d['ts_list'][0][0] >= end:
+            break
+        # parts in between
+        if (d['ts_list'][-1][1] < end and d['ts_list'][0][0] > start) or (d['ts_list'][-1][1] == end and d['ts_list'][0][0] == start):
+            t2s = Text2SRT(d['text_seg'], d['ts_list'], offset=start)
+            srt_total += "{}\n{}".format(cc, t2s.srt())
+            subs.append((t2s.time(), t2s.text()))
+            cc += 1
+            continue
+        if d['ts_list'][0][0] <= start:
+            if not d['ts_list'][-1][1] > end:
+                for j, ts in enumerate(d['ts_list']):
+                    if ts[1] > start:
+                        break
+                _text = " ".join(d['text_seg'].split()[j:])
+                _ts = d['ts_list'][j:]
+            else:
+                for j, ts in enumerate(d['ts_list']):
+                    if ts[1] > start:
+                        _start = j
+                        break
+                for j, ts in enumerate(d['ts_list']):
+                    if ts[1] > end:
+                        _end = j
+                        break
+                _text = " ".join(d['text_seg'].split()[_start:_end])
+                _ts = d['ts_list'][_start:_end]
+            if len(ts):
+                t2s = Text2SRT(_text, _ts, offset=start)
+                srt_total += "{}\n{}".format(cc, t2s.srt())
+                subs.append((t2s.time(), t2s.text()))
+                cc += 1
+            continue
+        if d['ts_list'][-1][1] > end:
+            for j, ts in enumerate(d['ts_list']):
+                if ts[1] > end:
+                    break
+            _text = " ".join(d['text_seg'].split()[:j])
+            _ts = d['ts_list'][:j]
+            if len(_ts):
+                t2s = Text2SRT(_text, _ts, offset=start)
+                srt_total += "{}\n{}".format(cc, t2s.srt())
+                subs.append(
+                    (t2s.time(), t2s.text())
+                    )
+                cc += 1
+            continue
+    return srt_total, subs, cc

trans_utils.py

@@ -0,0 +1,57 @@
+PUNC_LIST = ['，', '。', '！', '？', '、']
+
+
+def pre_proc(text):
+    res = ''
+    for i in range(len(text)):
+        if text[i] in PUNC_LIST:
+            continue
+        if '\u4e00' <= text[i] <= '\u9fff':
+            if len(res) and res[-1] != " ":
+                res += ' ' + text[i]+' '
+            else:
+                res += text[i]+' '
+        else:
+            res += text[i]
+    if res[-1] == ' ':
+        res = res[:-1]
+    return res
+
+def proc(raw_text, timestamp, dest_text):
+    # simple matching
+    ld = len(dest_text.split())
+    mi, ts = [], []
+    offset = 0
+    while True:
+        fi = raw_text.find(dest_text, offset, len(raw_text))
+        # import pdb; pdb.set_trace()
+        ti = raw_text[:fi].count(' ')
+        if fi == -1:
+            break
+        offset = fi + ld
+        mi.append(fi)
+        ts.append([timestamp[ti][0]*16, timestamp[ti+ld-1][1]*16])
+        # import pdb; pdb.set_trace()
+    return ts
+
+
+def write_state(output_dir, state):
+    for key in ['/recog_res_raw', '/timestamp', '/sentences']:
+        with open(output_dir+key, 'w') as fout:
+            fout.write(str(state[key[1:]]))
+
+
+def load_state(output_dir):
+    state = {}
+    with open(output_dir+'/recog_res_raw') as fin:
+        line = fin.read()
+        state['recog_res_raw'] = line
+    with open(output_dir+'/timestamp') as fin:
+        line = fin.read()
+        state['timestamp'] = eval(line)
+    with open(output_dir+'/sentences') as fin:
+        line = fin.read()
+        state['sentences'] = eval(line)
+    return state
+        
+    

videoclipper.py

@@ -0,0 +1,146 @@
+import sys
+import copy
+import librosa
+import logging
+import argparse
+import numpy as np
+import soundfile as sf
+import moviepy.editor as mpy
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
+from subtitle_utils import generate_srt, generate_srt_clip
+from trans_utils import pre_proc, proc, write_state, load_state
+
+from moviepy.editor import *
+from moviepy.video.tools.subtitles import SubtitlesClip
+
+
+class VideoClipper():
+    def __init__(self, asr_pipeline):
+        logging.warning("Initializing VideoClipper.")
+        self.asr_pipeline = asr_pipeline
+
+    def recog(self, audio_input, state=None):
+        if state is None:
+            state = {}
+        state['audio_input'] = audio_input
+        _, data = audio_input
+        data = data.astype(np.float64)
+        rec_result = self.asr_pipeline(audio_in=data)
+        state['recog_res_raw'] = rec_result['text_postprocessed']
+        state['timestamp'] = rec_result['time_stamp']
+        state['sentences'] = rec_result['sentences']
+        res_text = rec_result['text']
+        res_srt = generate_srt(rec_result['sentences'])
+        return res_text, res_srt, state
+
+    def clip(self, dest_text, start_ost, end_ost, state):
+        # get from state
+        audio_input = state['audio_input']
+        recog_res_raw = state['recog_res_raw']
+        timestamp = state['timestamp']
+        sentences = state['sentences']
+        sr, data = audio_input
+        data = data.astype(np.float64)
+
+        all_ts = []
+        for _dest_text in dest_text.split('#'):
+            _dest_text = pre_proc(_dest_text)
+            ts = proc(recog_res_raw, timestamp, _dest_text)
+            for _ts in ts: all_ts.append(_ts)
+        ts = all_ts
+        srt_index = 0
+        clip_srt = ""
+        if len(ts):
+            start, end = ts[0]
+            start = min(max(0, start+start_ost*16), len(data))
+            end = min(max(0, end+end_ost*16), len(data))
+            res_audio = data[start:end]
+            start_end_info = "from {} to {}".format(start/16000, end/16000)
+            srt_clip, _, srt_index = generate_srt_clip(sentences, start/16000.0, end/16000.0, begin_index=srt_index)
+            clip_srt += srt_clip
+            for _ts in ts[1:]:  # multiple sentence input or multiple output matched
+                start, end = _ts
+                start = min(max(0, start+start_ost*16), len(data))
+                end = min(max(0, end+end_ost*16), len(data))
+                start_end_info += ", from {} to {}".format(start, end)
+                res_audio = np.concatenate([res_audio, data[start+start_ost*16:end+end_ost*16]], -1)
+                srt_clip, _, srt_index = generate_srt_clip(sentences, start/16000.0, end/16000.0, begin_index=srt_index-1)
+                clip_srt += srt_clip
+        if len(ts):
+            message = "{} periods found in the speech: ".format(len(ts)) + start_end_info
+        else:
+            message = "No period found in the speech, return raw speech. You may check the recognition result and try other destination text."
+        return (sr, res_audio), message, clip_srt
+
+    def video_recog(self, vedio_filename):
+        vedio_filename = vedio_filename
+        clip_video_file = vedio_filename[:-4] + '_clip.mp4'
+        video = mpy.VideoFileClip(vedio_filename)
+        audio_file = vedio_filename[:-3] + 'wav'
+        video.audio.write_audiofile(audio_file)
+        wav = librosa.load(audio_file, 16000)[0]
+        state = {
+            'vedio_filename': vedio_filename,
+            'clip_video_file': clip_video_file,
+            'video': video,
+        }
+        # res_text, res_srt = self.recog((16000, wav), state)
+        return self.recog((16000, wav), state)
+
+    def video_clip(self, dest_text, start_ost, end_ost, state, font_size=32, font_color='white', add_sub=False):
+        # get from state
+        recog_res_raw = state['recog_res_raw']
+        timestamp = state['timestamp']
+        sentences = state['sentences']
+        video = state['video']
+        clip_video_file = state['clip_video_file']
+        vedio_filename = state['vedio_filename']
+        
+        all_ts = []
+        srt_index = 0
+        for _dest_text in dest_text.split('#'):
+            _dest_text = pre_proc(_dest_text)
+            ts = proc(recog_res_raw, timestamp, _dest_text)
+            for _ts in ts: all_ts.append(_ts)
+        ts = all_ts
+        clip_srt = ""
+        if len(ts):
+            start, end = ts[0][0] / 16000, ts[0][1] / 16000
+            start, end = start+start_ost/1000.0, end+end_ost/1000.0
+            video_clip = video.subclip(start, end)
+            clip_video_file = clip_video_file
+            start_end_info = "from {} to {}".format(start, end)
+            # message = "{} periods found in the audio: from {} to {}.".format(len(ts), start, end)
+            srt_clip, subs, srt_index = generate_srt_clip(sentences, start, end, begin_index=srt_index)
+            clip_srt += srt_clip
+            if add_sub:
+                generator = lambda txt: TextClip(txt, font='./font/STHeitiMedium.ttc', fontsize=font_size, color=font_color)
+                subtitles = SubtitlesClip(subs, generator)
+                video_clip = CompositeVideoClip([video_clip, subtitles.set_pos(('center','bottom'))])
+            concate_clip = [video_clip]
+            for _ts in ts[1:]:
+                start, end = _ts[0] / 16000, _ts[1] / 16000
+                start, end = start+start_ost/1000.0, end+end_ost/1000.0
+                _video_clip = video.subclip(start, end)
+                clip_video_file = clip_video_file
+                start_end_info += ", from {} to {}".format(start, end)
+                srt_clip, subs, srt_index = generate_srt_clip(sentences, start, end, begin_index=srt_index-1)
+                clip_srt += srt_clip
+                if add_sub:
+                    generator = lambda txt: TextClip(txt, font='./font/STHeitiMedium.ttc', fontsize=font_size, color=font_color)
+                    subtitles = SubtitlesClip(subs, generator)
+                    _video_clip = CompositeVideoClip([_video_clip, subtitles.set_pos(('center','bottom'))])
+                concate_clip.append(copy.copy(_video_clip))
+            message = "{} periods found in the audio: ".format(len(ts)) + start_end_info
+            logging.warning("Concating...")
+            if len(concate_clip) > 1:
+                video_clip = concatenate_videoclips(concate_clip)
+            video_clip.write_videofile(clip_video_file)
+        else:
+            clip_video_file = vedio_filename
+            message = "No period found in the audio, return raw speech. You may check the recognition result and try other destination text."
+            srt_clip = ''
+        return clip_video_file, message, clip_srt
+
+