Add application files

.gitignore

@@ -0,0 +1,140 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,14 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="numpy" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (poem2pic)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/poem2pic.iml" filepath="$PROJECT_DIR$/.idea/poem2pic.iml" />
+    </modules>
+  </component>
+</project>

.idea/poem2pic.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.10 (poem2pic)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

app.py

@@ -0,0 +1,66 @@
+import streamlit as st
+
+from config import Config
+from diffusion_image import generate_image_from_text
+from t5_summary import generate_summary
+
+
+st.write('''
+# Poem2Pic: What picture does your poem paint?
+
+Poetry is food for the soul. An image is worth a thousand words. 
+
+With Poem2Pic, generate an image based on a very short summary of your poem!
+''')
+
+footer = """<style>
+.footer {
+    position: fixed;
+    left: 0;
+    bottom: 2%;
+    width: 100%;
+    text-align: center;
+    color: #696969;
+    padding: 2px;
+}
+</style>
+<div class="footer">
+<p><small>Poem2Pic is an experimental project developed for fun. It is not really intended for professional use. 
+Generating an image is an expensive process, so the image quality is kept low in the interest of time and cost. 
+By using Poem2Pic you agree to fair, responsible, and sensible usage of the solution, platform, 
+and the underlying AI models.</small></p>
+</div>
+"""
+st.markdown(footer, unsafe_allow_html=True)
+
+poem = st.text_area(
+    f'''**Type or copy paste a poem (max. {Config.LLM_MAX_INPUT_LENGTH} characters will be considered):**''',
+    ''
+)
+
+if st.button('Generate image'):
+    progress_text = 'Generating image...give it a moment'
+    progress_bar = st.progress(0, text=progress_text)
+    progress_bar.progress(1, text=progress_text)
+
+    poem = poem.strip()
+    input_length = len(poem)
+
+    if input_length > 0:
+        print(poem)
+
+        if input_length <= 5:
+            summary = poem
+        else:
+            target_length = min(input_length, Config.LLM_MAX_INPUT_LENGTH)
+            summary = generate_summary(poem[:target_length])
+
+        print(f'Summary: {summary}')
+        st.write(f'''Summary: {summary}''')
+        progress_bar.progress(25, text='Summary generated...need a few more minutes')
+
+        image = generate_image_from_text(summary)
+        progress_bar.progress(100, text='Done!')
+
+        st.image(image, caption=summary)
+        st.info('(Right-click on the image to save it)', icon='ℹ️ ')

config.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+
+@dataclass(frozen=True)
+class Config:
+    HUGGINGFACEHUB_API_TOKEN: str = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+
+    # Flan-T5
+    LLM_MODEL_NAME: str = 'google/flan-t5-xxl'
+    LLM_TEMPERATURE: float = 0.5
+    LLM_MAX_OUTPUT_LENGTH: int = 100
+    LLM_MAX_INPUT_LENGTH: int = 512
+
+    # Stable Diffusion
+    DIFFUSION_MODEL_NAME: str = 'stabilityai/stable-diffusion-2-1'
+    DIFFUSION_NUM_INFERENCE_STEPS: int = 3

diffusion_image.py

@@ -0,0 +1,29 @@
+import PIL.Image
+from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
+import torch
+
+from config import Config
+
+
+torch.set_grad_enabled(False)
+
+dpm = DPMSolverMultistepScheduler.from_pretrained(Config.DIFFUSION_MODEL_NAME, subfolder='scheduler')
+pipeline = StableDiffusionPipeline.from_pretrained(Config.DIFFUSION_MODEL_NAME, scheduler=dpm)
+# pipeline.enable_xformers_memory_efficient_attention()
+
+
+def generate_image_from_text(text: str) -> PIL.Image.Image:
+    """
+    Generate an image based on the input text.
+
+    :param text: The text
+    :return: An image instance
+    """
+
+    with torch.inference_mode():
+        output_img = pipeline(
+            text,
+            num_inference_steps=Config.DIFFUSION_NUM_INFERENCE_STEPS).images[0]
+        print(output_img)
+
+        return output_img

examples/Ozymandias.txt

@@ -0,0 +1,17 @@
+Ozymandias
+By Percy Bysshe Shelley
+
+I met a traveller from an antique land,
+Who said—“Two vast and trunkless legs of stone
+Stand in the desert. . . . Near them, on the sand,
+Half sunk a shattered visage lies, whose frown,
+And wrinkled lip, and sneer of cold command,
+Tell that its sculptor well those passions read
+Which yet survive, stamped on these lifeless things,
+The hand that mocked them, and the heart that fed;
+And on the pedestal, these words appear:
+My name is Ozymandias, King of Kings;
+Look on my Works, ye Mighty, and despair!
+Nothing beside remains. Round the decay
+Of that colossal Wreck, boundless and bare
+The lone and level sands stretch far away.”

examples/The_Raven.txt

@@ -0,0 +1,128 @@
+The Raven
+By Edgar Allan Poe
+
+Once upon a midnight dreary, while I pondered, weak and weary,
+Over many a quaint and curious volume of forgotten lore,
+While I nodded, nearly napping, suddenly there came a tapping,
+As of some one gently rapping, rapping at my chamber door. “
+“'Tis some visitor,” I muttered, “tapping at my chamber door—
+Only this, and nothing more.”
+
+Ah, distinctly I remember it was in the bleak December,
+And each separate dying ember wrought its ghost upon the floor.
+Eagerly I wished the morrow;—vainly I had sought to borrow
+From my books surcease of sorrow—sorrow for the lost Lenore—
+For the rare and radiant maiden whom the angels name Lenore—
+Nameless here for evermore.
+
+And the silken sad uncertain rustling of each purple curtain
+Thrilled me—filled me with fantastic terrors never felt before;
+So that now, to still the beating of my heart, I stood repeating, “
+“'Tis some visitor entreating entrance at my chamber door—
+Some late visitor entreating entrance at my chamber door;—
+This it is, and nothing more.”
+
+Presently my soul grew stronger; hesitating then no longer,
+“Sir,” said I, “or Madam, truly your forgiveness I implore;
+But the fact is I was napping, and so gently you came rapping,
+And so faintly you came tapping, tapping at my chamber door,
+That I scarce was sure I heard you”—here I opened wide the door;—
+Darkness there, and nothing more.
+
+Deep into that darkness peering, long I stood there wondering, fearing,
+Doubting, dreaming dreams no mortals ever dared to dream before;
+But the silence was unbroken, and the stillness gave no token,
+And the only word there spoken was the whispered word, “Lenore!”
+This I whispered, and an echo murmured back the word, “Lenore!”—
+Merely this, and nothing more.
+
+Back into the chamber turning, all my soul within me burning,
+Soon again I heard a tapping somewhat louder than before.
+“Surely,” said I, “surely that is something at my window lattice,
+Let me see, then, what thereat is, and this mystery explore—
+Let my heart be still a moment and this mystery explore;—
+'Tis the wind and nothing more.”
+
+Open here I flung the shutter, when, with many a flirt and flutter,
+In there stepped a stately raven of the saintly days of yore.
+Not the least obeisance made he; not a minute stopped or stayed he;
+But, with mien of lord or lady, perched above my chamber door—
+Perched upon a bust of Pallas just above my chamber door—
+Perched, and sat, and nothing more.
+
+Then this ebony bird beguiling my sad fancy into smiling,
+By the grave and stern decorum of the countenance it wore.
+“Though thy crest be shorn and shaven, thou,” I said, “art sure no craven,
+Ghastly grim and ancient raven wandering from the Nightly shore—
+Tell me what thy lordly name is on the Night's Plutonian shore!”
+Quoth the Raven, “Nevermore.”
+
+Much I marvelled this ungainly fowl to hear discourse so plainly,
+Though its answer little meaning—little relevancy bore;
+For we cannot help agreeing that no living human being
+Ever yet was blest with seeing bird above his chamber door—
+Bird or beast upon the sculptured bust above his chamber door,
+With such name as “Nevermore.”
+
+But the Raven, sitting lonely on the placid bust, spoke only
+That one word, as if his soul in that one word he did outpour.
+Nothing further then he uttered—not a feather then he fluttered—
+Till I scarcely more than muttered, “other friends have flown before—
+On the morrow he will leave me, as my hopes have flown before.”
+Then the bird said, “Nevermore.”
+
+Startled at the stillness broken by reply so aptly spoken,
+“Doubtless,” said I, “what it utters is its only stock and store,
+Caught from some unhappy master whom unmerciful Disaster
+Followed fast and followed faster till his songs one burden bore—
+Till the dirges of his Hope that melancholy burden bore,
+Of ‘Never—nevermore’.”
+
+But the Raven still beguiling my sad fancy into smiling,
+Straight I wheeled a cushioned seat in front of bird and bust and door;
+Then, upon the velvet sinking, I betook myself to linking
+Fancy unto fancy, thinking what this ominous bird of yore—
+What this grim, ungainly, ghastly, gaunt, and ominous bird of yore
+Meant in croaking “Nevermore.”
+
+This I sat engaged in guessing, but no syllable expressing
+To the fowl whose fiery eyes now burned into my bosom's core;
+This and more I sat divining, with my head at ease reclining
+On the cushion's velvet lining that the lamplight gloated o'er,
+But whose velvet violet lining with the lamplight gloating o'er,
+She shall press, ah, nevermore!
+
+Then, methought, the air grew denser, perfumed from an unseen censer
+Swung by Seraphim whose footfalls tinkled on the tufted floor.
+“Wretch,” I cried, “thy God hath lent thee—by these angels he hath sent thee
+Respite—respite and nepenthe, from thy memories of Lenore;
+Quaff, oh quaff this kind nepenthe and forget this lost Lenore!”
+Quoth the Raven, “Nevermore.”
+
+“Prophet!” said I, “thing of evil!—prophet still, if bird or devil!—
+Whether Tempter sent, or whether tempest tossed thee here ashore,
+Desolate yet all undaunted, on this desert land enchanted—
+On this home by horror haunted—tell me truly, I implore—
+Is there—is there balm in Gilead?—tell me—tell me, I implore!”
+Quoth the Raven, “Nevermore.”
+
+“Prophet!” said I, “thing of evil!—prophet still, if bird or devil!
+By that Heaven that bends above us—by that God we both adore—
+Tell this soul with sorrow laden if, within the distant Aidenn,
+It shall clasp a sainted maiden whom the angels name Lenore—
+Clasp a rare and radiant maiden whom the angels name Lenore.”
+Quoth the Raven, “Nevermore.”
+
+“Be that word our sign in parting, bird or fiend!” I shrieked, upstarting—
+“Get thee back into the tempest and the Night's Plutonian shore!
+Leave no black plume as a token of that lie thy soul hath spoken!
+Leave my loneliness unbroken!—quit the bust above my door!
+Take thy beak from out my heart, and take thy form from off my door!”
+Quoth the Raven, “Nevermore.”
+
+And the Raven, never flitting, still is sitting, still is sitting
+On the pallid bust of Pallas just above my chamber door;
+And his eyes have all the seeming of a demon's that is dreaming,
+And the lamplight o'er him streaming throws his shadow on the floor;
+And my soul from out that shadow that lies floating on the floor
+Shall be lifted—nevermore!

examples/the_old_pond.txt

@@ -0,0 +1,5 @@
+By Matsuo Basho
+
+An old silent pond
+A frog jumps into the pond—
+Splash! Silence again.

requirements.txt

@@ -0,0 +1,9 @@
+python-dotenv[cli]
+langchain
+huggingface_hub
+torch
+transformers
+diffusers[torch]
+accelerate
+streamlit
+Pillow

t5_summary.py

@@ -0,0 +1,37 @@
+from langchain import HuggingFaceHub, PromptTemplate
+
+from config import Config
+
+
+llm = HuggingFaceHub(
+    repo_id=Config.LLM_MODEL_NAME,
+    huggingfacehub_api_token=Config.HUGGINGFACEHUB_API_TOKEN,
+    model_kwargs={'temperature': Config.LLM_TEMPERATURE, 'max_length': Config.LLM_MAX_OUTPUT_LENGTH}
+)
+# print(llm)
+
+template = '''
+You are a helpful assistant who can answer questions about literature and poetry. You are creative. 
+Consider the following poem: 
+
+{poem}
+
+Write a creative summary of the poem in about 20 words. 
+The summary should try to capture the subject, objects, key themes, and sentiments.
+'''
+
+prompt = PromptTemplate.from_template(template)
+
+
+def generate_summary(poem: str) -> str:
+    """
+    Generate the summary of a poem using an LLM.
+
+    :param poem: The text of the poem
+    :return: The summary
+    """
+
+    print(prompt.format(poem=poem))
+    summary = llm(prompt.format(poem=poem))
+
+    return summary