Upload 10 files

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan21.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan221.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan224.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan24.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan321.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan323.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan422.png filter=lfs diff=lfs merge=lfs -text
+caption_demo/FloorPlan424.png filter=lfs diff=lfs merge=lfs -text

app_test.py

@@ -0,0 +1,120 @@
+import sys
+import time
+import warnings
+from pathlib import Path
+
+
+# 配置hugface环境
+from huggingface_hub import hf_hub_download
+import gradio as gr
+import os
+import glob
+import json
+
+# os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
+# torch.set_float32_matmul_precision("high")
+
+
+
+def instruct_generate(
+    img_path: str = " ",
+    prompt: str = "What food do lamas eat?",
+    input: str = "",
+    max_new_tokens: int = 100,
+    top_k: int = 200,
+    temperature: float = 0.8,
+) -> None:
+    """Generates a response based on a given instruction and an optional input.
+    This script will only work with checkpoints from the instruction-tuned LLaMA-Adapter model.
+    See `finetune_adapter.py`.
+
+    Args:
+        prompt: The prompt/instruction (Alpaca style).
+        adapter_path: Path to the checkpoint with trained adapter weights, which are the output of
+            `finetune_adapter.py`.
+        input: Optional input (Alpaca style).
+        pretrained_path: The path to the checkpoint with pretrained LLaMA weights.
+        tokenizer_path: The tokenizer path to load.
+        quantize: Whether to quantize the model and using which method:
+            ``"llm.int8"``: LLM.int8() mode,
+            ``"gptq.int4"``: GPTQ 4-bit mode.
+        max_new_tokens: The number of generation steps to take.
+        top_k: The number of top most probable tokens to consider in the sampling process.
+        temperature: A value controlling the randomness of the sampling process. Higher values result in more random
+    """
+    output = [prompt, input, max_new_tokens, top_k, temperature]
+    print(output)
+    return output
+
+# 配置具体参数
+
+example_path = "example.json"
+# 1024如果不够, 调整为512
+max_seq_len = 1024
+max_batch_size = 1
+
+with open(example_path, 'r') as f:
+    content = f.read()
+    example_dict = json.loads(content)
+
+
+def create_instruct_demo():
+    with gr.Blocks() as instruct_demo:
+        with gr.Row():
+            with gr.Column():
+                scene_img = gr.Image(label='Scene', type='filepath')
+                object_list = gr.Textbox(
+                    lines=2, label="Input")
+
+                instruction = gr.Textbox(
+                    lines=2, label="Instruction")
+                max_len = gr.Slider(minimum=1, maximum=512,
+                                    value=128, label="Max length")
+                with gr.Accordion(label='Advanced options', open=False):
+                    temp = gr.Slider(minimum=0, maximum=1,
+                                     value=0.8, label="Temperature")
+                    top_k = gr.Slider(minimum=100, maximum=300,
+                                      value=200, label="Top k")
+
+                run_botton = gr.Button("Run")
+
+            with gr.Column():
+                outputs = gr.Textbox(lines=10, label="Output")
+
+        inputs = [instruction, object_list, max_len, top_k, temp]
+
+        # 接下来设定具体的example格式
+        examples_img_list = glob.glob("caption_demo/*.png")
+        examples = []
+        for example_img_one in examples_img_list:
+            scene_name = os.path.basename(example_img_one).split(".")[0]
+            example_object_list = example_dict[scene_name]["input"]
+            example_instruction = example_dict[scene_name]["instruction"]
+            example_one = [example_img_one, example_object_list, example_instruction, 512, 0.8, 200]
+            examples.append(example_one)
+
+        gr.Examples(
+            examples=examples,
+            inputs=inputs,
+            outputs=outputs,
+            fn=instruct_generate,
+            cache_examples=os.getenv('SYSTEM') == 'spaces'
+        )
+        run_botton.click(fn=instruct_generate, inputs=inputs, outputs=outputs)
+    return instruct_demo
+
+
+# Please refer to our [arXiv paper](https://arxiv.org/abs/2303.16199) and [github](https://github.com/ZrrSkywalker/LLaMA-Adapter) for more details.
+description = """
+# TaPA
+The official demo for **Embodied Task Planning with Large Language Models**.
+"""
+
+with gr.Blocks(css='style.css') as demo:
+    gr.Markdown(description)
+    with gr.TabItem("Instruction-Following"):
+        create_instruct_demo()
+
+demo.queue(api_open=True, concurrency_count=1).launch()
+
+

example.json

@@ -0,0 +1,34 @@
+{
+  "FloorPlan21": {
+    "input": "[apple,chair,blender,sink,pottery,oven,egg,table,shredder_(for_paper),seashell,bread,doorknob,fork,plastic_bag,knife,radio_receiver,drawer,person,coffee_maker,inhaler,toaster,plate,cornice,knob,pear,dining_table,tomato,bottle,scale_(measuring_instrument),toilet_tissue,cushion,latch,scissors,soap,handle,balloon,clock,lightbulb,matchbox,refrigerator,trash_can,backpack,alarm_clock,vase,tape_(sticky_cloth_or_paper),printer,cover,faucet,gourd,pan_(for_cooking),ball,spatula,microwave_oven,dispenser,nailfile,cabinet,sweet_potato,lamp,microscope,pot,cup,suitcase,bowl,thermostat,fume_hood,hinge,mirror,spoon,box,]",
+    "instruction": "Can you clean the dishes?"
+  },
+  "FloorPlan24": {
+    "input": "[apple,chair,sink,oven,figurine,shredder_(for_paper),potholder,doorknob,truffle_(chocolate),fork,towel,stove,napkin,knife,drawer,hotplate,coffee_maker,avocado,chopping_board,stool,bolt,toaster,bowling_ball,hand_towel,plate,speaker_(stero_equipment),tag,piggy_bank,knob,dining_table,tomato,scale_(measuring_instrument),toaster_oven,pitcher_(vessel_for_liquid),painting,handle,wineglass,clock,automatic_washer,ice_maker,lightbulb,refrigerator,trash_can,tray,dishwasher,armoire,faucet,gourd,pan_(for_cooking),spatula,microwave_oven,mug,dispenser,cabinet,fire_extinguisher,kitchen_sink,television_set,lamp,cup,bowl,thermostat,water_jug,hinge,spoon,]",
+    "instruction": "Please make me an omelette."
+  },
+  "FloorPlan221": {
+    "input": "[chair,sofa,pen,figurine,table,dog,lampshade,doorknob,bed,toy,drawer,person,statue_(sculpture),flowerpot,stool,monitor_(computer_equipment)computer_monitor,desk,pillow,plate,speaker_(stero_equipment),mouse_(computer_equipment),knob,igniter,dining_table,cushion,painting,dragonfly,laptop_computer,remote_control,vase,trash_can,wall_socket,ashtray,coffee_table,card,computer_keyboard,bird,coaster,television_set,lamp,bowl,thermostat,hinge,curtain,box,]",
+    "instruction": "Could you please close the curtains?"
+  },
+  "FloorPlan224": {
+    "input": "[chair,sofa,figurine,table,crate,necklace,dog,dresser,lampshade,doorknob,horse,frisbee,deer,screwdriver,oil_lamp,drawer,sweater,person,statue_(sculpture),flowerpot,stool,dress,pole,monitor_(computer_equipment)computer_monitor,hat,easel,umbrella,desk,pillow,speaker_(stero_equipment),book,knob,fireplace,ottoman,dining_table,toilet_tissue,cushion,painting,latch,handle,bathtub,laptop_computer,remote_control,clock,lightbulb,candle,vase,trash_can,wall_socket,hose,coffee_table,computer_keyboard,spotlight,bird,cabinet,television_set,lamp,harmonium,cup,thermostat,newspaper,curtain,runner_(carpet),box,]",
+    "instruction": "Can you turn off the light?"
+  },
+  "FloorPlan321": {
+    "input": "[chair,sofa,figurine,table,quilt,bed,lampshade,doorknob,tissue_paper,headboard,button,pencil,drawer,place_mat,cigar_box,knitting_needle,monitor_(computer_equipment)computer_monitor,desk,pillow,chandelier,book,knob,armchair,ottoman,dining_table,notebook,cushion,painting,vent,laptop_computer,blanket,lightbulb,cellular_telephone,trash_can,alarm_clock,tape_(sticky_cloth_or_paper),faucet,card,computer_keyboard,coaster,nailfile,bicycle,mattress,lamp,car_(automobile),magazine,thermostat,heart,mirror,box,]",
+    "instruction": "Can you please hand me the pencil on the desk?"
+  },
+  "FloorPlan323": {
+    "input": "[chair,sofa,sink,dresser,lampshade,bed,doorknob,toy,teddy_bear,towel,headboard,drawer,place_mat,monitor_(computer_equipment)computer_monitor,desk,pillow,speaker_(stero_equipment),mouse_(computer_equipment),piggy_bank,book,cornice,dining_table,cushion,painting,cigarette_case,handle,laptop_computer,remote_control,candle,trash_can,wall_socket,armoire,corkboard,computer_keyboard,lamp,television_set,telephone,cup,hatbox,bowl,thermostat,hinge,mirror,runner_(carpet),box,]",
+    "instruction": "Can you pass me the remote control, please?"
+  },
+  "FloorPlan422": {
+    "input": "[knocker_(on_a_door),sink,hook,clothespin,doorknob,tissue_paper,oil_lamp,drawer,cistern,bottle_cap,desk,hand_towel,knob,bottle,dining_table,toilet_tissue,handle,bathtub,towel_rack,bath_mat,candle_holder,bat_(animal),toilet,wooden_spoon,candle,shower_head,refrigerator,trash_can,cover,hair_dryer,armoire,faucet,scrubbing_brush,dispenser,shower_curtain,cabinet,lamp,bath_towel,cup,thermostat,fume_hood,hinge,mirror,paper_towel,broom,box,]",
+    "instruction": "Open the Cabinet and give me the Soap Bottle"
+  },
+  "FloorPlan424": {
+    "input": "[sink,bucket,doorknob,towel,wine_bucket,cistern,washbasin,pipe,hand_towel,knob,bottle,toilet_tissue,soap,handle,towel_rack,candle_holder,lightbulb,candle,shower_head,crucifix,vase,cover,wall_socket,faucet,scrubbing_brush,dispenser,cabinet,lamp,bath_towel,cup,thermostat,hinge,mirror,toilet,eraser,]",
+    "instruction": "Please clean the sink"
+  }
+}