demo

.gitignore

@@ -0,0 +1,8 @@
+.venv
+.DS_Store
+__pycache__
+flagged
+
+files
+
+.env

app.py

@@ -0,0 +1,149 @@
+import gradio as gr
+import os
+import json
+import uuid
+from pprint import pprint
+import dotenv
+dotenv.load_dotenv()
+
+from PIL import Image
+
+from llm import answer
+from tts import get_audio
+from speechtovid import get_video
+from gender import get_gender
+
+SECRET = os.getenv("SECRET_WORD")
+
+supported_languages = ["English", "Chinese", "Spanish", "Hindi", "Portuguese", "French", "German", "Japanese", "Arabic", "Korean", "Indonesian", "Italian", "Dutch", "Turkish", "Polish", "Swedish", "Filipino", "Malay", "Russian", "Romanian", "Ukrainian", "Greek", "Czech", "Danish", "Finnish", "Bulgarian", "Croatian", "Slovak", "Tamil"]
+
+# create dirs for images, audio and video files if they don't exist
+if not os.path.exists('./files/audio'):
+    os.makedirs('./files/audio')
+if not os.path.exists('./files/images'):
+    os.makedirs('./files/images')
+if not os.path.exists('./files/video'):
+    os.makedirs('./files/video')
+
+# image resizer
+def resize_image(input_image_path, file_name):
+    with Image.open(input_image_path) as im:
+        original_width, original_height = im.size
+        desired_width = 300
+        aspect_ratio = original_width / original_height
+        new_width = desired_width
+        new_height = int(desired_width / aspect_ratio)
+        resized_image = im.resize((new_width, new_height))
+        resized_image.save("./files/images/"+file_name+".png")
+    return "./files/images/"+file_name+".png"
+
+# main func
+def holiday_card(secret, brief, lang, photo):
+
+    if not secret or secret.strip().lower() != SECRET.strip().lower():
+        raise gr.Error("Please use the correct secret word!")
+
+    if not brief:
+        raise gr.Error("Please enter the kind of greeting you want to create!")
+
+    if not photo:
+        raise gr.Error("Please upload a photo!")
+
+
+    # generate a unique id for this greeting
+    uid = str(uuid.uuid4())
+
+    # resize the image, otherwise it will be too big
+    resized_photo = resize_image(photo, uid)
+
+    # get the gender of the person in the photo - so that we can choose the voice
+    gender = get_gender(resized_photo)
+    if gender == 'female':
+        voice = 'Bella'
+    else:
+        voice = 'Antoni'
+
+    # generate the greeting
+    system_prompt = f'''
+You are a native {lang} copywriter with an excellent sense of humour. You help people write text for their holiday voice messages that they will send to their friends and colleagues. You take the user brief and then write a short, joyful, funny and beautiful short speech in {lang} that people say when wishing their colleagues and friends a happy 2024. It shouldn't be more than 2-3 sentences long. Please respond with valid JSON.
+
+If the client brief was good - please return valid JSON like this:
+{{
+    "status": "OK",
+    "text": "your greeting text here"
+}}
+
+If the client brief is inappropriate - please return valid JSON like this:
+{{
+    "status": "ERROR",
+    "reason": "your reason for what is wrong with the brief"
+}}
+
+Please alsways return valid JSON and nothing else
+'''
+
+    # get the answer from the model
+    answer_text = answer(
+        system_message=system_prompt,
+        user_message=brief
+    )
+
+    pprint(answer_text)
+
+    try:
+        answer_data = json.loads(answer_text)
+
+    except Exception as e:
+        pprint(e)
+        raise gr.Error(f"Sorry, something went wrong with the AI. Please try again later")
+
+    if answer_data.get('status') == 'ERROR':
+        raise gr.Error(answer_data.get('reason'))
+
+    text = answer_data.get('text')
+
+    # now get audio
+    try:
+        audio_file = get_audio(uid, text, voice)
+        pprint(audio_file)
+
+    except Exception as e:
+        pprint(e)
+        raise gr.Error(f"Sorry, something went wrong with the audio generation AI. Please try again later")
+
+    # now get video
+    try:
+        video_url = get_video(uid, resized_photo, audio_file)
+    except Exception as e:
+        pprint(e)
+        raise gr.Error(f"Sorry, something went wrong with the video generation AI. Please try again later")
+
+    return text, audio_file, video_url
+
+
+# set up and launch gradio interface
+
+inputs=[
+    gr.Textbox(lines=1, label="What is the secret word?"),
+    gr.Textbox(lines=5, label="What do you want your holiday greeting to be all about?"),
+    gr.Dropdown(supported_languages, label="What language would you like your holiday greeting to be in?", value="French"),
+    gr.Image(type="filepath", label="Upload Image")
+]
+
+outputs=[
+    gr.Textbox(lines=5, label="Your Holiday Greeting Text"),
+    gr.Audio(type="filepath", label="Your Holiday Greeting Audio"),
+    gr.Video(label="Your Holiday Greeting Video")
+]
+
+demo = gr.Interface(
+    holiday_card,
+    inputs,
+    outputs,
+    allow_flagging="never"
+)
+
+demo.queue()
+
+if __name__ == "__main__":
+    demo.launch()

gender.py

@@ -0,0 +1,18 @@
+import requests
+import json
+import os
+from pprint import pprint
+
+HF_TOKEN = os.getenv("HF_TOKEN")
+
+API_URL = "https://api-inference.huggingface.co/models/rizvandwiki/gender-classification-2"
+headers = {"Authorization": "Bearer "+HF_TOKEN}
+
+def get_gender(filename):
+    with open(filename, "rb") as f:
+        data = f.read()
+        response = requests.post(API_URL, headers=headers, data=data)
+
+        gender_prediction = response.json()
+
+        return gender_prediction[0]['label']

llm.py

@@ -0,0 +1,48 @@
+import os
+import openai
+import json
+from pprint import pprint
+from tenacity import retry, wait_random_exponential, stop_after_attempt
+
+openai.api_key = os.environ.get("OPENAI_API_KEY")
+MODEL = 'gpt-4'
+
+'''
+basic underlying method to get completions. retries built in
+'''
+@retry(wait=wait_random_exponential(multiplier=2, max=10), stop=stop_after_attempt(3))
+def get_completion(messages, temperature=1):
+
+    try:
+        chat_completion = openai.ChatCompletion.create(
+            model=MODEL,
+            temperature=temperature,
+            messages=messages
+        )
+
+        return chat_completion.choices[0].message.content
+
+    except Exception as e:
+        print("Unable to generate ChatCompletion response")
+        print(f"Exception: {e}")
+        return e
+
+'''
+basic text completion
+'''
+def answer(system_message, user_message, temperature=1):
+
+    messages = [{
+      "role": "system",
+      "content": system_message
+    },{
+      "role": "user",
+      "content": user_message
+    }]
+
+    completion = get_completion(
+      messages=messages,
+      temperature = temperature
+    )
+
+    return completion

requirements.txt

@@ -0,0 +1,90 @@
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+appnope==0.1.3
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.1.0
+backcall==0.2.0
+certifi==2023.7.22
+charset-normalizer==3.3.1
+click==8.1.7
+colorama==0.4.6
+contourpy==1.1.1
+cycler==0.12.1
+decorator==5.1.1
+elevenlabs==0.2.26
+executing==2.0.0
+fastapi==0.104.0
+ffmpy==0.3.1
+filelock==3.12.4
+fonttools==4.43.1
+frozenlist==1.4.0
+fsspec==2023.10.0
+gradio==3.50.2
+gradio_client==0.6.1
+h11==0.14.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.18.0
+idna==3.4
+importlib-resources==6.1.0
+ipython==8.16.1
+jedi==0.19.1
+Jinja2==3.1.2
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+livereload==2.6.3
+MarkupSafe==2.1.3
+matplotlib==3.8.0
+matplotlib-inline==0.1.6
+multidict==6.0.4
+numpy==1.26.1
+openai==0.28.1
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.2
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==10.1.0
+prompt-toolkit==3.0.39
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-mon==1.1.1
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydub==0.25.1
+Pygments==2.16.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.30.2
+replicate==0.15.4
+requests==2.31.0
+rpds-py==0.10.6
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+stack-data==0.6.3
+starlette==0.27.0
+tenacity==8.2.3
+toolz==0.12.0
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.12.0
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.7
+uvicorn==0.23.2
+watchdog==3.0.0
+wcwidth==0.2.8
+websockets==11.0.3
+yarl==1.9.2

speechtovid.py

@@ -0,0 +1,15 @@
+import os
+import replicate
+from pprint import pprint
+
+def get_video(uid, photo_path, audio_path):
+
+    output = replicate.run(
+        "cjwbw/sadtalker:3aa3dac9353cc4d6bd62a8f95957bd844003b401ca4e4a9b33baa574c549d376",
+        input={
+            "source_image": open(photo_path, "rb"),
+            "driven_audio": open(audio_path, "rb"),
+        }
+    )
+
+    return output

tts.py

@@ -0,0 +1,17 @@
+from elevenlabs import generate, set_api_key, save
+import os
+
+API_KEY = os.getenv('ELEVENLABS_KEY')
+
+set_api_key(API_KEY)
+
+def get_audio(a_uid, text, voice_id):
+
+  audio = generate(
+      text=text,
+      voice=voice_id,
+      model='eleven_multilingual_v2'
+  )
+  file_name = f'./files/audio/{a_uid}.mp3'
+  save(audio, file_name)
+  return file_name