Create 3mergeDialogToVideo.py

3mergeDialogToVideo.py

@@ -0,0 +1,272 @@
+# rogerxavier-ocr-with-fastapi.hf.space
+import os
+##这个模型目前只适合确定文本框顺序后再识别,因为如果后面的
+##完整图片处理的反例  现在处理的图片是10\0.jpg
+# [[[953, 743], [987, 743], [987, 867], [953, 867]], [[917, 745], [951, 745], [951, 867], [917, 867]], [[881, 741], [918, 742], [915, 898], [877, 897]], [[843, 743], [879, 743], [879, 809], [843, 809]], [[629, 1058], [669, 1058], [669, 1210], [629, 1210]], [[549, 1227], [583, 1227], [583, 1381], [549, 1381]], [[535, 115], [563, 115], [563, 145], [535, 145]], [[535, 147], [563, 147], [563, 213], [535, 213]], [[507, 443], [539, 443], [539, 579], [507, 579]], [[505, 115], [533, 115], [533, 197], [505, 197]], [[511, 1225], [547, 1225], [547, 1321], [511, 1321]], [[475, 117], [503, 117], [503, 265], [475, 265]], [[467, 421], [503, 421], [503, 575], [467, 575]], [[419, 235], [447, 235], [447, 337], [419, 337]], [[387, 236], [417, 237], [414, 339], [385, 338]], [[209, 796], [242, 797], [239, 921], [206, 920]], [[175, 173], [205, 173], [205, 225], [175, 225]], [[177, 231], [205, 231], [205, 285], [177, 285]], [[103, 1153], [129, 1153], [129, 1223], [103, 1223]], [[41, 100], [108, 101], [104, 549], [36, 548]]]
+# ['就算是你', '没有圣剑', '也不可能有', '胜算', '就算如此', '我也不觉得', '做', ':做个', '·就不觉得', '老好人', '你可怕', '也要有个限度', '我很恐怖吗', '该说真是', '无药可救', '说的是呢', '这个', '但是', '为何?', '第二话让人怜爱']
+
+import requests
+
+import tempfile
+import time
+
+from moviepy.audio.AudioClip import AudioArrayClip
+from moviepy.editor import *
+import cv2
+import azure.cognitiveservices.speech as speechsdk
+import numpy as np
+import io
+import base64
+import json
+from io import BytesIO
+import pandas as pd
+from PIL import Image
+
+
+#通过去水印完整漫画图片->获取相应的对话框图片->获取对话框文字->返回对话框文字
+def get_image_copywrite(image_path:"图片路径(包含后缀)",dialog_cut_path:"对话框切割路径")->"返回漫画关联对话框识别后得到的文案str(原文即可),也可能是none":
+    dialog_texts = ''
+    associate_dialog_img = get_associate_dialog(image_path=image_path,dialog_cut_path=dialog_cut_path)
+    if len(associate_dialog_img)!=0:
+        #如果有对应的对话框
+        for dialog_img_path in associate_dialog_img:
+            cur_dialog_texts = get_sorted_dialog_text(dialog_img_path)#一个对话框的文字list
+            if cur_dialog_texts is not None:
+                for dialog_text in cur_dialog_texts:
+                    dialog_texts += dialog_text
+                    dialog_texts += '\n'
+            else:
+                print(dialog_img_path+"识别是空-可能是有问题")
+        return dialog_texts
+    return None#不规范图片不请求,直接返回none
+
+#通过传入无水印漫画图片对话框路径,得到关联的对话框图片list
+def get_associate_dialog(image_path:"图片路径(包含后缀)",dialog_cut_path:"对话框切割路径")->"返回漫画关联对话框list,也可能是空的list":
+    image_name = os.path.splitext(os.path.basename(image_path))[0]
+    image_name_format = '{:03d}'.format(int(image_name))
+
+    associated_dialogs = []
+    for root, _, files in os.walk(dialog_cut_path):
+        for file in files:
+            if file.startswith(image_name_format) and file.endswith('.jpg'):
+                associated_dialogs.append(os.path.join(root, file))
+
+    return associated_dialogs
+
+
+#通过对话框图片路径,获取对话框文字list
+def get_sorted_dialog_text(image_path:"包含后缀的文件路径")->"返回排序后的text list(一列或者几列话,反正是一个框的内容,几句不清楚,一个框的list当一次文案就行)  或者失败请求返回none":
+    image_bytes = open(image_path, 'rb')
+    headers = {
+        'authority': 'rogerxavier-fastapi-t5-magi.hf.space',
+        'scheme': 'https',
+        'Accept': '*/*',
+        'Accept-Encoding': 'gzip, deflate, br, zstd',
+        'Accept-Language': 'zh-CN,zh;q=0.9',
+        'Cookie': 'spaces-jwt=eyJhbGciOiJFZERTQSJ9.eyJyZWFkIjp0cnVlLCJwZXJtaXNzaW9ucyI6eyJyZXBvLmNvbnRlbnQucmVhZCI6dHJ1ZX0sIm9uQmVoYWxmT2YiOnsia2luZCI6InVzZXIiLCJfaWQiOiI2NDJhNTNiNTE2ZDRkODI5M2M5YjdiNzgiLCJ1c2VyIjoicm9nZXJ4YXZpZXIifSwiaWF0IjoxNzE2Njg3MzU3LCJzdWIiOiIvc3BhY2VzL3JvZ2VyeGF2aWVyL29jcl93aXRoX2Zhc3RhcGkiLCJleHAiOjE3MTY3NzM3NTcsImlzcyI6Imh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28ifQ._sGdEgC-ijbIhLmB6iNSBQ_xHNzb4Ydb9mD0L3ByRmJSbB9ccfGbRgtNmkV1JLLldHp_VEKUSQt9Mwq_q4aGAQ',
+        'Dnt': '1',
+        'Priority': 'u=1, i',
+        'Sec-Ch-Ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Windows"',
+        'Sec-Fetch-Dest': 'empty',
+        'Sec-Fetch-Mode': 'cors',
+        'Sec-Fetch-Site': 'same-origin',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
+    }
+    files = {
+        "image": image_bytes,
+    }
+    try:
+        resp = requests.post("https://rogerxavier-ocr-with-fastapi.hf.space/getCoordinates", files=files,headers=headers)#还是有header才能跑
+        #先json转换，0为坐标list合集,1为 boxid和text合集
+        boxCoordinates , boxInfo = resp.json()[0],resp.json()[1] #分别是list和dict类型
+
+        # 计算文本框的中心点，以便按照从右往左，从上往下的顺序进行排序
+        centers = [((box[0][0] + box[2][0]) / 2, (box[0][1] + box[2][1]) / 2) for box in boxCoordinates]
+
+        # 按照中心点的坐标从右往左，从上往下的顺序对文本框坐标进行排序
+        sorted_indices = sorted(range(len(centers)), key=lambda i: (-centers[i][0], centers[i][1]))
+
+        # 获取排序后的文本框坐标和对应的文字
+        sorted_coordinates = [boxCoordinates[i] for i in sorted_indices]
+        sorted_text = [boxInfo['Text'][str(i)] for i in sorted_indices]
+
+        # 根据x方向偏差要求重新排序同一列的文本框
+        for i in range(len(sorted_indices) - 1):
+            if centers[sorted_indices[i]][0] - centers[sorted_indices[i+1]][0] < (sorted_coordinates[i][2][0] - sorted_coordinates[i][0][0]) / 3:
+                if sorted_coordinates[i][0][1] > sorted_coordinates[i+1][2][1]:
+                    sorted_indices[i], sorted_indices[i+1] = sorted_indices[i+1], sorted_indices[i]
+
+        sorted_coordinates = [boxCoordinates[i] for i in sorted_indices]
+        sorted_text = [boxInfo['Text'][str(i)] for i in sorted_indices]
+
+        print(sorted_coordinates)
+        print(sorted_text)
+        return sorted_text
+    except Exception as e:
+        print("图片请求出现问题")
+        print(e)
+        return None
+
+
+#通过文字获取音频
+def get_audio_data(text:str)-> "返回audio data io句柄, duration":
+    # Creates an instance of a speech config with specified subscription key and service region.
+    speech_key = "5f438c41786144d6b5318317321a2da2"
+    service_region = "eastus"
+
+    speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)
+    # Note: the voice setting will not overwrite the voice element in input SSML.
+    speech_config.speech_synthesis_voice_name = "zh-CN-YunxiNeural" ##云希
+
+    # use the default speaker as audio output.
+    speech_synthesizer = speechsdk.SpeechSynthesizer(speech_config=speech_config)
+
+    result = speech_synthesizer.speak_text_async(text).get()
+    # Check result
+    if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted:
+        print("Speech synthesized for text [{}]".format(text))
+    elif result.reason == speechsdk.ResultReason.Canceled:
+        cancellation_details = result.cancellation_details
+        print("Speech synthesis canceled: {}".format(cancellation_details.reason))
+        if cancellation_details.reason == speechsdk.CancellationReason.Error:
+            print("Error details: {}".format(cancellation_details.error_details))
+
+    # print("音频持续时间是",result.audio_duration)
+    # print("音频数据是",result.audio_data)
+    # 创建临时文件 -当前路径下面
+    with tempfile.NamedTemporaryFile(dir='/',delete=False) as temp_file:
+        temp_file.write(result.audio_data)
+        temp_file.close()
+        # 在这里完成您对文件的操作，比如返回文件名
+        file_name = temp_file.name
+    return file_name, str(result.audio_duration)
+
+
+# 补零函数，将数字部分补齐为指定长度
+def zero_pad(s, length):
+    return s.zfill(length)
+
+
+def gpt_polish(text:str)->"通过gpt润色str文案并返回str新文案,或者gpt请求失败none":
+    # Set your OpenAI API key
+    api_key = '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'
+
+    # Define the headers
+    headers = {
+        'Authorization': f'Bearer {api_key}',
+        'Content-Type': 'application/json',
+    }
+
+    # Chat Completions request data
+    data = {
+        'model': 'gpt-3.5-turbo',  # Replace with your chosen model
+        'messages': [
+            {'role': 'system', 'content': "你是一个assistant,能够根据user发送的漫画中提取的文字,生成一个短视频中一帧的三人称文案(1-2句话)"},
+            {'role': 'user', 'content': text}
+        ]
+    }
+    try:
+
+        response = requests.post('https://api.yingwu.lol/v1/chat/completions', headers=headers, data=json.dumps(data))
+        print("润色后文案是:"+response.json()['choices'][0]['message']['content'])
+        return response.json()['choices'][0]['message']['content']
+    except Exception as e:
+        print("gpt润色文案失败:")
+        print(e)
+        return None
+if __name__ == '__main__':
+    # 获取存放去水印漫画图片的路径 ---放这里是因为获取对话文字时需要和原图关联
+    img_path = 'manga1'
+    # 获取切割后的文本框路径
+    dialog_img_path = 'manga12'
+
+    #获取漫画原图无水印的加入image_files，并排序
+    subdir_path = os.path.join(os.getcwd(), img_path)
+    # 对话图片经过加入list并补0确定顺序
+    image_files = []
+    for root, dirs, files in os.walk(subdir_path):
+        for file in files:
+            if file.endswith(".jpg") or file.endswith(".png"):
+                image_files.append(os.path.relpath(os.path.join(root, file)))
+    # 对对话框文件名中的数字部分进行补零操作-这样顺序会正常
+    image_files.sort(
+        key=lambda x: zero_pad(''.join(filter(str.isdigit, os.path.splitext(os.path.basename(x))[0])), 3))
+
+    dialog_subdir_path = os.path.join(os.getcwd(), dialog_img_path)
+    # 对话图片经过加入list并补0确定顺序
+    dialog_image_files = []
+    for root, dirs, files in os.walk(dialog_subdir_path):
+        for file in files:
+            if file.endswith(".jpg") or file.endswith(".png"):
+                dialog_image_files.append(os.path.relpath(os.path.join(root, file)))
+    # 对对话框文件名中的数字部分进行补零操作-这样顺序会正常
+    dialog_image_files.sort(
+        key=lambda x: zero_pad(''.join(filter(str.isdigit, os.path.splitext(os.path.basename(x))[0])), 3))
+    # 对话图片经过加入list并补0确定顺序
+
+
+    ###音视频相关参数-------------------------------------------------------------------------------------
+    ##这个是临时生成音频文件的全局变量--方便后续删除
+    filename = ''
+    # 视频分辨率和帧率
+    # 获取第一张图片的尺寸
+    image = Image.open(image_files[0])
+    width, height = 1125, 1600  # 无法显示可能是win播放器不支持
+    fps = 30
+    font_path = '1.ttf'  # 设置字体以防默认字体无法同时处理中英文
+    # 创建视频编辑器
+    video_clips = []
+    ###音视频相关参数-------------------------------------------------------------------------------------
+
+
+
+    #因为是根据原图无水印的进行遍历,所以处理前要进行筛选，只处理能找到相应对话框图片的原图
+    filtered_image_files = []
+    for image_path in image_files:
+        dialog_list = get_associate_dialog(image_path, dialog_img_path)
+        if dialog_list:
+            filtered_image_files.append(image_path)
+
+    image_files = filtered_image_files
+
+    for idx, image_file in enumerate(image_files):
+        print("现在处理的图片是"+image_file)
+        #后面是视音频生成部分-这里图片需要用到完整的去水印的而不是对话框用于识别的
+        img = cv2.imread(image_file)
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  ##只支持英文路径
+
+        ##获取当前图片对应的对话框识别文字(还需gpt处理后作为字幕文案)
+        cur_copywrite = get_image_copywrite(image_file,dialog_img_path)  # image_file就是6.jpg了
+        cur_copywrite = gpt_polish(cur_copywrite)
+
+
+
+        if cur_copywrite is not None:
+
+            ##获取当前图片对应的临时音频文件名称和文案时长
+            filename, duration = get_audio_data(cur_copywrite)
+
+            clip = ImageClip(img).set_duration(duration).resize((width, height))  # 初始clip
+
+            txt_clip = TextClip(cur_copywrite, fontsize=40, color='white', bg_color='black',
+                                font=font_path)  ##文本clip后加入视频
+
+            txt_clip = txt_clip.set_pos(('center', 'bottom')).set_duration(duration)
+            # 创建音频剪辑
+            audio_clip = AudioFileClip(filename)
+            clip = clip.set_audio(audio_clip)  # 将音频与视频片段关联
+            clip = CompositeVideoClip([clip, txt_clip])
+            video_clips.append(clip)
+        else:
+            pass  ##图片不规范直接跳过
+    video = concatenate_videoclips(video_clips)
+    # 保存视频
+    video.write_videofile('output_video.mp4', fps=fps)
+    # # 在文件关闭后删除临时文件
+    print("删除临时mp3文件", filename)
+    os.remove(filename)
+
+
+