# pip install html2image import base64 import random from io import BytesIO from html2image import Html2Image import os import pathlib import re import gradio as gr import requests from PIL import Image from gradio_client import Client import torch from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline, Pipeline HF_TOKEN = os.getenv("HF_TOKEN") if not HF_TOKEN: raise Exception("HF_TOKEN environment variable is required to call remote API.") API_URL = "https://api-inference.huggingface.co/models/HuggingFaceH4/zephyr-7b-beta" headers = {"Authorization": f"Bearer {HF_TOKEN}"} client = Client("https://latent-consistency-super-fast-lcm-lora-sd1-5.hf.space") def init_speech_to_text_model() -> Pipeline: device = "cuda:0" if torch.cuda.is_available() else "cpu" torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32 model_id = "distil-whisper/distil-medium.en" model = AutoModelForSpeechSeq2Seq.from_pretrained( model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True ) model.to(device) processor = AutoProcessor.from_pretrained(model_id) return pipeline( "automatic-speech-recognition", model=model, tokenizer=processor.tokenizer, feature_extractor=processor.feature_extractor, max_new_tokens=128, torch_dtype=torch_dtype, device=device, ) whisper_pipe = init_speech_to_text_model() def query(payload: dict): response = requests.post(API_URL, headers=headers, json=payload) return response.json() def generate_text(card_text: str, user_request: str) -> (str, str, str): # Prompt must apply the correct chat template for the model see: # https://huggingface.co/docs/transformers/main/en/chat_templating prompt = f"""<|system|> You create Magic the Gathering cards based on the user's request. # RULES - In your response always generate a new card. - Only generate one card, no other dialogue. - Surround card info in triple backticks (```). - Format the card text using headers like in the example below: ``` Name: Band of Brothers ManaCost: {{3}}{{W}}{{W}} Type: Creature — Phyrexian Human Soldier Rarity: rare Text: Vigilance {{W}}, {{T}}: Attach target creature you control to target creature. (Any number of attacking creatures with total power 5 or less can attack in a band. A band deals damage to that creature.) FlavorText: "This time we will be stronger." —Elder brotherhood blessing Power: 2 Toughness: 2 Color: ['W'] ``` <|user|> {user_request} <|assistant|> """ if card_text and card_text != starting_text: prompt = f"""<|system|> You edit Magic the Gathering cards based on the user's request. # RULES - In your response always generate a new card. - Only generate one card, no other dialogue. - Surround card info in triple backticks (```). - Format the card text using headers like in the example below: ``` Name: Band of Brothers ManaCost: {{3}}{{W}}{{W}} Type: Creature — Phyrexian Human Soldier Rarity: rare Text: Vigilance {{W}}, {{T}}: Attach target creature you control to target creature. (Any number of attacking creatures with total power 5 or less can attack in a band. A band deals damage to that creature.) FlavorText: "This time we will be stronger." —Elder brotherhood blessing Power: 2 Toughness: 2 Color: ['W'] ``` <|user|> # CARD TO EDIT ``` {card_text} ``` # EDIT REQUEST {user_request} <|assistant|> """ print(f"Calling API with prompt:\n{prompt}") params = {"max_new_tokens": 512} output = query({"inputs": prompt, "parameters": params}) if 'error' in output: print(f'Language model call failed: {output["error"]}') raise gr.Warning(f'Language model call failed: {output["error"]}') print(f'API RESPONSE SIZE: {len(output[0]["generated_text"])}') assistant_reply = output[0]["generated_text"].split('<|assistant|>')[1] print(f'ASSISTANT REPLY:\n{assistant_reply}') new_card_text = assistant_reply.split('```') if len(new_card_text) > 1: new_card_text = new_card_text[1].strip() + '\n' else: new_card_text = assistant_reply.split('\n\n') if len(new_card_text) < 2: return assistant_reply, card_text, None new_card_text = new_card_text[1].strip() + '\n' return assistant_reply, new_card_text, None def format_html(text, image_data): template = pathlib.Path("./card_template.html").read_text(encoding='utf-8') if "['U']" in text: template = template.replace("{card_color}", 'style="background-color:#5a73ab"') elif "['W']" in text: template = template.replace("{card_color}", 'style="background-color:#f0e3d0"') elif "['G']" in text: template = template.replace("{card_color}", 'style="background-color:#325433"') elif "['B']" in text: template = template.replace("{card_color}", 'style="background-color:#1a1b1e"') elif "['R']" in text: template = template.replace("{card_color}", 'style="background-color:#c2401c"') elif "Type: Land" in text: template = template.replace("{card_color}", 'style="background-color:#aa8c71"') elif "Type: Artifact" in text: template = template.replace("{card_color}", 'style="background-color:#9ba7bc"') else: template = template.replace("{card_color}", 'style="background-color:#edd99d"') pattern = re.compile('Name: (.*)') name = pattern.findall(text)[0] template = template.replace("{name}", name) pattern = re.compile('Mana.?Cost: (.*)') mana_cost = pattern.findall(text)[0] if mana_cost == "None": template = template.replace("{mana_cost}", '') else: symbols = [] for c in mana_cost: if c in {"{", "}"}: continue else: symbols.append(c.lower()) formatted_symbols = [] for s in symbols: formatted_symbols.append(f'') template = template.replace("{mana_cost}", "\n".join(formatted_symbols[::-1])) if not isinstance(image_data, (bytes, bytearray)): template = template.replace('{image_data}', f'{image_data}') else: template = template.replace('{image_data}', f'data:image/png;base64,{image_data.decode("utf-8")}') pattern = re.compile('Type: (.*)') card_type = pattern.findall(text)[0] template = template.replace("{card_type}", card_type) if len(card_type) > 30: template = template.replace("{type_size}", "16") else: template = template.replace("{type_size}", "18") pattern = re.compile('Rarity: (.*)') rarity = pattern.findall(text)[0] template = template.replace("{rarity}", f"ss-{rarity}") pattern = re.compile(r'^Text: (.*)\n\bFlavor.?Text|Power|Color\b', re.MULTILINE | re.DOTALL) card_text = pattern.findall(text)[0] text_lines = [] for line in card_text.splitlines(): line = line.replace('{T}', '') line = line.replace('{UT}', '') line = line.replace('{E}', '') line = re.sub(r"{(.*?)}", r''.lower(), line) line = re.sub(r"ms-(.)/(.)", r''.lower(), line) line = line.replace('(', '(').replace(')', ')') text_lines.append(f"

{line}

") template = template.replace("{card_text}", "\n".join(text_lines)) pattern = re.compile(r'Flavor.?Text: (.*?)\n^.*$', re.MULTILINE | re.DOTALL) flavor_text = pattern.findall(text) if flavor_text: flavor_text = flavor_text[0] flavor_text_lines = [] for line in flavor_text.splitlines(): flavor_text_lines.append(f"

{line}

") template = template.replace("{flavor_text}", "
" + "\n".join(flavor_text_lines) + "
") else: template = template.replace("{flavor_text}", "") if len(card_text) + len(flavor_text or '') > 170 or len(text_lines) > 3: template = template.replace("{text_size}", '16') template = template.replace( 'ms-cost" style="top:0px;float:none;height: 18px;width: 18px;font-size: 13px;">', 'ms-cost" style="top:0px;float:none;height: 16px;width: 16px;font-size: 11px;">') else: template = template.replace("{text_size}", '18') pattern = re.compile('Power: (.*)') power = pattern.findall(text) if power: power = power[0] if not power: template = template.replace("{power_toughness}", "") pattern = re.compile('Toughness: (.*)') toughness = pattern.findall(text)[0] template = template.replace("{power_toughness}", f'

{power}/{toughness}

') else: template = template.replace("{power_toughness}", "") pathlib.Path("scratch.html").write_text(template, encoding='utf-8') return template def get_savename(directory, name, extension): save_name = f"{name}.{extension}" i = 1 while os.path.exists(os.path.join(directory, save_name)): save_name = save_name.replace(f'.{extension}', '').split('-')[0] + f"-{i}.{extension}" i += 1 return save_name def html_to_png(card_name, html): save_name = get_savename('rendered_cards', card_name, 'png') print('CONVERTING HTML CARD TO PNG IMAGE') path = os.path.join('rendered_cards', save_name) try: rendered_card_dir = 'rendered_cards' hti = Html2Image(output_path=rendered_card_dir) paths = hti.screenshot(html_str=html, css_file=['./css/mtg_custom.css', './css/mana.css', './css/keyrune.css'], save_as=save_name, size=(450, 600)) print(paths) path = paths[0] except: pass print('OPENING IMAGE FROM FILE') img = Image.open(path) print('CROPPING BACKGROUND') area = (0, 50, 400, 600) cropped_img = img.crop(area) cropped_img.resize((400, 550)) cropped_img.save(os.path.join(path)) print('CONVERTING HTML CARD TO PNG IMAGE COMPLETE') return cropped_img.convert('RGB') def get_initial_card(): return Image.open('SampleCard.png') def pil_to_base64(image): print('CONVERTING PIL IMAGE TO BASE64 STRING') buffered = BytesIO() image.save(buffered, format="PNG") img_str = base64.b64encode(buffered.getvalue()) print('CONVERTING PIL IMAGE TO BASE64 STRING COMPLETE') return img_str def generate_card(image: str, card_text: str): image_data = pil_to_base64(Image.open(image)) html = format_html(card_text, image_data) pattern = re.compile('Name: (.*)') name = pattern.findall(card_text)[0] card = html_to_png(name, html) return card def transcribe(audio: str) -> (str, str): result = whisper_pipe(audio) return result["text"], None starting_text = """Name: Wizards of the Coast ManaCost: {0} Type: Enchantment Rarity: mythic rare Text: At the beginning of your upkeep, reveal the top card of your library. If it's a card named "Magic: The Gathering", put it into your hand. Otherwise, put it into your graveyard. FlavorText: "We are the guardians of the multiverse, and we will protect it at all costs." Color: ['U']""" def generate_image(card_text: str): pattern = re.compile('Name: (.*)') name = pattern.findall(card_text)[0] pattern = re.compile('Type: (.*)') card_type = pattern.findall(card_text)[0] prompt = f"fantasy illustration of a {card_type} {name}, by Greg Rutkowski" print(f'Calling image generation with prompt: {prompt}') try: result = client.predict( prompt, # str in 'parameter_5' Textbox component 0.3, # float (numeric value between 0.0 and 5) in 'Guidance' Slider component 4, # float (numeric value between 2 and 10) in 'Steps' Slider component random.randint(0, 12013012031030), # float (numeric value between 0 and 12013012031030) in 'Seed' Slider component api_name="/predict" ) print(result) return result except Exception as e: print(f'Failed to generate image from client: {e}') return 'placeholder.png' def add_hotkeys() -> str: return pathlib.Path("hotkeys.js").read_text() with gr.Blocks(title='MagicGen') as demo: gr.Markdown("# 🎴 MagicGenV2") gr.Markdown("## Generate and Edit Magic the Gathering Cards with a Chat Assistant") with gr.Row(): with gr.Column(): with gr.Group(): audio_in = gr.Microphone(label="Record a voice request (click or press ctrl + ` to start/stop)", type='filepath', elem_classes=["record-btn"]) prompt_in = gr.Textbox(label="Or type a text request and press Enter", interactive=True, placeholder="Need an idea? Try one of these:\n- Create a creature card named 'WiFi Elemental'\n- Make it an instant\n- Change the color") with gr.Accordion(label='🤖 Chat Assistant Response', open=False): bot_text = gr.TextArea(label='Response', interactive=False) with gr.Row(): with gr.Column(): in_text = gr.TextArea(label="Card Text (Shift+Enter to submit)", value=starting_text) gen_image_button = gr.Button('🖼️ Generate Card Image') in_image = gr.Image(label="Card Image (400px x 550px)", type='filepath', value='placeholder.png') render_button = gr.Button('🎴 Render Card', variant="primary") gr.ClearButton([audio_in, prompt_in, in_text, in_image]) with gr.Column(): out_image = gr.Image(label="Rendered Card", value=get_initial_card()) transcribe_params = {'fn': transcribe, 'inputs': [audio_in], 'outputs': [prompt_in, audio_in]} generate_text_params = {'fn': generate_text, 'inputs': [in_text, prompt_in], 'outputs': [bot_text, in_text, audio_in]} generate_image_params = {'fn': generate_image, 'inputs': [in_text], 'outputs': [in_image]} generate_card_params = {'fn': generate_card, 'inputs': [in_image, in_text], 'outputs': [out_image]} # Shift + Enter to submit text in TextAreas audio_in.stop_recording(**transcribe_params).then(**generate_text_params).then(**generate_image_params).then( **generate_card_params) prompt_in.submit(**generate_text_params).then(**generate_image_params).then(**generate_card_params) in_text.submit(**generate_card_params) render_button.click(**generate_card_params) gen_image_button.click(**generate_image_params).then(**generate_card_params) demo.load(None, None, None, js=add_hotkeys()) if __name__ == "__main__": demo.queue().launch(favicon_path="favicon-96x96.png")