added attribution example, created local pipelines, implemented poc_app.py as a proof of concept

attribution_example.py

@@ -0,0 +1,8 @@
+# Example Code Attribution for AI-Generated Code
+
+# ----ATTRIBUTION-START----
+# LLM: Github Copilot
+# PROMPT: write a hello world example
+# EDITS: changed the wording to make it more personal
+print("Hello, World! This is your Copilot speaking!")
+# -----ATTRIBUTION-END-----

blip_image_caption_large.py

@@ -0,0 +1,13 @@
+# external imports
+from transformers import pipeline
+
+# local imports
+import config
+
+class Blip_Image_Caption_Large:
+    def __init__(self):
+        self.local_pipeline = pipeline("image-to-text", model=config.IMAGE_CAPTION_MODEL)
+
+    def caption_image_local_pipeline(self, image_path):
+        result = self.local_pipeline(image_path)
+        return result

config.py

@@ -0,0 +1,3 @@
+IMAGE_CAPTION_MODEL = "Salesforce/blip-image-captioning-large"
+LLM_MODEL = "microsoft/Phi-3-mini-4k-instruct"
+MUSICGEN_MODEL = "facebook/musicgen-small"

musicgen_small.py

@@ -0,0 +1,14 @@
+# external imports
+from transformers import pipeline
+import scipy
+
+# local imports
+import config
+
+class Musicgen_Small:
+    def __init__(self):
+        self.local_pipeline = pipeline("text-to-audio", model=config.MUSICGEN_MODEL)
+
+    def generate_music_local_pipeline(self, prompt):
+        music = self.local_pipeline(prompt, forward_params={"do_sample": True})
+        scipy.io.wavfile.write("data/musicgen_out.wav", rate=music["sampling_rate"], data=music["audio"])

phi3_mini_4k_instruct.py

@@ -0,0 +1,14 @@
+# external imports
+from transformers import pipeline
+
+# local imports
+import config
+
+
+class Phi3_Mini_4k_Instruct:
+    def __init__(self):
+        self.local_pipeline = pipeline("text-generation", model=config.LLM_MODEL, trust_remote_code=True)
+
+    def generate_text_local_pipeline(self, messages):
+        result = self.local_pipeline(messages)
+        return result

poc_app.py

@@ -0,0 +1,53 @@
+# external imports
+import time
+
+# local imports
+from blip_image_caption_large import Blip_Image_Caption_Large
+from phi3_mini_4k_instruct import Phi3_Mini_4k_Instruct
+from musicgen_small import Musicgen_Small
+
+def main():
+    # test image captioning
+    image_caption_start_time = time.time()
+    image_caption_model = Blip_Image_Caption_Large()
+    test_caption = image_caption_model.caption_image_local_pipeline("data/test3.jpg")
+    print(test_caption)
+    image_caption_end_time = time.time()
+
+    # test text generation
+    text_generation_start_time = time.time()
+    text_generation_model = Phi3_Mini_4k_Instruct()
+
+    #TODO: move this to a config file
+    text_generation_model.local_pipeline.model.config.max_length = 200
+
+    #TODO: move system prompt somewhere else, allow for genre override
+    messages = [
+    {"role": "system", "content": "You are an image caption to song description converter with a deep understanding of Music and Art. You are given the caption of an image. Your task is to generate a textual description of a musical piece that fits the caption. The description should be detailed and vivid, and should include the genre, mood, instruments, tempo, and other relevant information about the music. You should also use your knowledge of art and visual aesthetics to create a musical piece that complements the image. Only output the description of the music, without any explanation or introduction. Be concise."},
+    {"role": "user", "content": test_caption[0]["generated_text"]},
+    ]
+    test_text = text_generation_model.generate_text_local_pipeline(messages)
+    print(test_text)
+    text_generation_end_time = time.time()
+    
+
+    # test audio generation
+    music_generation_start_time = time.time()
+    music_generation_model = Musicgen_Small()
+    music_generation_model.generate_music_local_pipeline(str(test_text[-1]['generated_text'][-1]['content']))
+    music_generation_end_time = time.time()
+
+
+    # calculate durations
+    image_caption_duration = image_caption_end_time - image_caption_start_time
+    text_generation_duration = text_generation_end_time - text_generation_start_time
+    music_generation_duration = music_generation_end_time - music_generation_start_time
+    total_duration = music_generation_end_time - image_caption_start_time
+
+    # output durations
+    print(f"Image Captioning Duration: {image_caption_duration}")
+    print(f"Text Generation Duration: {text_generation_duration}")
+    print(f"Music Generation Duration: {music_generation_duration}")
+    print(f"Total Duration: {total_duration}")
+
+main()

requirements.txt

@@ -0,0 +1,27 @@
+certifi==2024.8.30
+charset-normalizer==3.3.2
+filelock==3.16.0
+fsspec==2024.9.0
+huggingface-hub==0.24.6
+idna==3.8
+Jinja2==3.1.4
+MarkupSafe==2.1.5
+mpmath==1.3.0
+networkx==3.3
+numpy==2.1.1
+packaging==24.1
+pillow==10.4.0
+PyYAML==6.0.2
+regex==2024.9.11
+requests==2.32.3
+safetensors==0.4.5
+scipy==1.14.1
+sympy==1.13.2
+tokenizers==0.19.1
+torch==2.4.1
+torchaudio==2.4.1
+torchvision==0.19.1
+tqdm==4.66.5
+transformers==4.44.2
+typing_extensions==4.12.2
+urllib3==2.2.2