[Init]

.gitattributes

@@ -31,4 +31,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +0,0 @@
----
-title: AskAnything
-emoji: 📚
-colorFrom: blue
-colorTo: gray
-sdk: gradio
-sdk_version: 3.27.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,172 @@
+import os
+import numpy as np
+import random
+import torch
+import torchvision.transforms as transforms
+from PIL import Image
+from models.tag2text import tag2text_caption
+from util import *
+import gradio as gr
+from chatbot_lv import *
+from load_internvideo import *
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+from simplet5 import SimpleT5
+from models.grit_model import DenseCaptioning
+bot = ConversationBot()
+image_size = 384
+normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                std=[0.229, 0.224, 0.225])
+transform = transforms.Compose([transforms.ToPILImage(),transforms.Resize((image_size, image_size)),transforms.ToTensor(),normalize])
+
+
+# define model
+model = tag2text_caption(pretrained="pretrained_models/tag2text_swin_14m.pth", image_size=image_size, vit='swin_b' )
+model.eval()
+model = model.to(device)
+print("[INFO] initialize caption model success!")
+
+model_T5 = SimpleT5()
+if torch.cuda.is_available():
+    model_T5.load_model(
+        "t5", "./pretrained_models/flan-t5-large-finetuned-openai-summarize_from_feedback", use_gpu=True)
+else:
+    model_T5.load_model(
+        "t5", "./pretrained_models/flan-t5-large-finetuned-openai-summarize_from_feedback", use_gpu=False)
+print("[INFO] initialize summarize model success!")
+# action recognition
+intern_action = load_intern_action(device)
+trans_action = transform_action()
+topil =  T.ToPILImage()
+print("[INFO] initialize InternVideo model success!")
+
+dense_caption_model = DenseCaptioning(device)
+dense_caption_model.initialize_model()
+print("[INFO] initialize dense caption model success!")
+
+def inference(video_path, input_tag, progress=gr.Progress()):
+    video_data = loadvideo_decord_origin(video_path)
+    prediction_list, frame_caption_list, dense_caption_list, tag_1, tag_2 = [],[],[],set(),set()
+    # split video every 60s
+    for start in progress.tqdm(range(0,len(video_data),60)):
+        data = video_data[start:start+60,...]
+        # InternVideo
+        action_index = np.linspace(0, len(data)-1, 8).astype(int)
+        tmp,tmpa = [],[]
+        for i,img in enumerate(data):
+            tmp.append(transform(img).to(device).unsqueeze(0))
+            if i in action_index:
+                tmpa.append(topil(img))
+        action_tensor = trans_action(tmpa)
+        TC, H, W = action_tensor.shape
+        action_tensor = action_tensor.reshape(1, TC//3, 3, H, W).permute(0, 2, 1, 3, 4).to(device)
+        with torch.no_grad():
+            prediction = intern_action(action_tensor)
+            prediction = F.softmax(prediction, dim=1).flatten()
+            prediction = kinetics_classnames[str(int(prediction.argmax()))]
+            prediction_list.append(prediction)
+
+        # dense caption
+        dense_caption = []
+        dense_index = np.arange(0, len(data)-1, 5)
+        original_images = data[dense_index,:,:,::-1]
+        with torch.no_grad():
+            for idx,original_image in zip(dense_index,original_images):
+                dense_caption.append((idx+start,dense_caption_model.run_caption_tensor(original_image)))
+            
+        
+        # Video Caption
+        image = torch.cat(tmp).to(device)   
+        
+        model.threshold = 0.68
+        if input_tag == '' or input_tag == 'none' or input_tag == 'None':
+            input_tag_list = None
+        else:
+            input_tag_list = []
+            input_tag_list.append(input_tag.replace(',',' | '))
+        with torch.no_grad():
+            caption, tag_predict = model.generate(image,tag_input = input_tag_list,max_length = 50, return_tag_predict = True)
+            tag_1.update(tag_predict)
+            tag_2 = ['none']
+    #print(frame_caption, dense_caption, synth_caption)
+    frame_caption_list.extend(caption)
+    dense_caption_list.extend(dense_caption)
+    synth_caption = model_T5.predict('. '.join(caption))
+    frame_caption = ' '.join([f"Second {i+1}:{j}.\n" for i,j in enumerate(frame_caption_list)])
+    dense_caption = ' '.join([f"Second {i+1} : {j}.\n" for (i,j) in dense_caption_list])
+    del data, action_tensor, original_image, image,tmp,tmpa
+    torch.cuda.empty_cache()
+    torch.cuda.ipc_collect()
+    
+    return ' | '.join(tag_1),' | '.join(tag_2), frame_caption, dense_caption, synth_caption[0], gr.update(interactive = True), ','.join(set(prediction_list))
+
+def set_example_video(example: list) -> dict:
+    return gr.Video.update(value=example[0])
+
+
+with gr.Blocks(css="#chatbot {overflow:auto; height:500px;}") as demo:
+    gr.Markdown("<h1><center>Ask Anything with GPT</center></h1>")
+    gr.Markdown(
+        """
+        Ask-Anything is a multifunctional video question answering tool that combines the functions of Action Recognition, Visual Captioning and ChatGPT. Our solution generates dense, descriptive captions for any object and action in a video, offering a range of language styles to suit different user preferences. It supports users to have conversations in different lengths, emotions, authenticity of language.<br>  
+        <p><a href='https://github.com/OpenGVLab/Ask-Anything'><img src='https://img.shields.io/badge/Github-Code-blue'></a></p><p>
+        """
+    )
+    
+    with gr.Row():
+        with gr.Column():
+            input_video_path = gr.inputs.Video(label="Input Video")
+            input_tag = gr.Textbox(lines=1, label="User Prompt (Optional, Enter with commas)",visible=False)
+          
+            with gr.Row():
+                with gr.Column(sclae=0.3, min_width=0):
+                    caption = gr.Button("✍ Upload")
+                    chat_video = gr.Button(" 🎥 Let's Chat! ", interactive=False)
+                with gr.Column(scale=0.7, min_width=0):
+                    loadinglabel = gr.Label(label="State")
+        with gr.Column():
+            openai_api_key_textbox = gr.Textbox(
+                value=os.environ["OPENAI_API_KEY"],
+                placeholder="Paste your OpenAI API key here to start (sk-...)",
+                show_label=False,
+                lines=1,
+                type="password",
+            )
+            chatbot = gr.Chatbot(elem_id="chatbot", label="Chat_with_GPT")
+            state = gr.State([])
+            user_tag_output = gr.State("")
+            image_caption_output = gr.State("")
+            video_caption_output  = gr.State("")
+            model_tag_output = gr.State("")
+            dense_caption_output = gr.State("")
+            with gr.Row(visible=False) as input_raws:
+                with gr.Column(scale=0.8):
+                    txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False)
+                with gr.Column(scale=0.10, min_width=0):
+                    run = gr.Button("🏃‍♂️Run")
+                with gr.Column(scale=0.10, min_width=0):
+                    clear = gr.Button("🔄Clear️")    
+
+    with gr.Row():
+            example_videos = gr.Dataset(components=[input_video_path], samples=[['images/yoga.mp4'], ['images/making_cake.mp4'], ['images/playing_guitar.mp4']])
+
+    example_videos.click(fn=set_example_video, inputs=example_videos, outputs=example_videos.components)
+    caption.click(bot.memory.clear)
+    caption.click(lambda: gr.update(interactive = False), None, chat_video)
+    caption.click(lambda: [], None, chatbot)
+    caption.click(lambda: [], None, state)    
+    caption.click(inference,[input_video_path,input_tag],[model_tag_output, user_tag_output, image_caption_output, dense_caption_output,video_caption_output, chat_video, loadinglabel])
+
+    chat_video.click(bot.init_agent, [openai_api_key_textbox, image_caption_output, dense_caption_output, video_caption_output, model_tag_output, state], [input_raws,chatbot, state, openai_api_key_textbox])
+
+    txt.submit(bot.run_text, [txt, state], [chatbot, state])
+    txt.submit(lambda: "", None, txt)
+    run.click(bot.run_text, [txt, state], [chatbot, state])
+    run.click(lambda: "", None, txt)
+
+    clear.click(bot.memory.clear)
+    clear.click(lambda: [], None, chatbot)
+    clear.click(lambda: [], None, state)
+    
+
+
+demo.launch(server_name="0.0.0.0",enable_queue=True,)#share=True)

chatbot.py

@@ -0,0 +1,82 @@
+from langchain.agents.initialize import initialize_agent
+from langchain.agents.tools import Tool
+from langchain.chains.conversation.memory import ConversationBufferMemory
+from langchain.llms.openai import OpenAI
+import re
+import gradio as gr
+import openai
+
+def cut_dialogue_history(history_memory, keep_last_n_words=400):
+    if history_memory is None or len(history_memory) == 0:
+        return history_memory
+    tokens = history_memory.split()
+    n_tokens = len(tokens)
+    print(f"history_memory:{history_memory}, n_tokens: {n_tokens}")
+    if n_tokens < keep_last_n_words:
+        return history_memory
+    paragraphs = history_memory.split('\n')
+    last_n_tokens = n_tokens
+    while last_n_tokens >= keep_last_n_words:
+        last_n_tokens -= len(paragraphs[0].split(' '))
+        paragraphs = paragraphs[1:]
+    return '\n' + '\n'.join(paragraphs)
+
+
+class ConversationBot:
+    def __init__(self):
+        self.memory = ConversationBufferMemory(memory_key="chat_history", output_key='output')
+        self.tools = []
+
+    def run_text(self, text, state):
+        self.agent.memory.buffer = cut_dialogue_history(self.agent.memory.buffer, keep_last_n_words=500)
+        res = self.agent({"input": text.strip()})
+        res['output'] = res['output'].replace("\\", "/")
+        response = res['output'] #re.sub('(image/\S*png)', lambda m: f'![](/file={m.group(0)})*{m.group(0)}*', res['output'])
+        state = state + [(text, response)]
+        print(f"\nProcessed run_text, Input text: {text}\nCurrent state: {state}\n"
+              f"Current Memory: {self.agent.memory.buffer}")
+        return state, state
+
+
+    def init_agent(self, openai_api_key, image_caption, dense_caption, video_caption, tags, state):
+        chat_history =''
+        PREFIX = "ChatVideo is a chatbot that chats with you based on video descriptions."
+        FORMAT_INSTRUCTIONS = """
+        When you have a response to say to the Human,  you MUST use the format:
+        ```
+        {ai_prefix}: [your response here]
+        ```
+        """
+        SUFFIX = f"""You are a chatbot that conducts conversations based on video descriptions. You mainly answer based on the given video description, and you can also modify the content according to the tag information, and you can also answer the relevant knowledge of the person or object contained in the video. But you don't create a video plot out of nothing.
+
+                Begin!
+
+                Video tags are: {tags}
+
+                The frame description of the video is: {image_caption}
+
+                The dense caption of the video is: {dense_caption}
+
+                The general description of the video is: {video_caption}"""+"""Previous conversation history {chat_history}
+
+                New input: {input}
+
+                {agent_scratchpad}
+                """
+        self.memory.clear()
+        
+        self.llm = OpenAI(temperature=0, openai_api_key=openai_api_key)
+        self.agent = initialize_agent(
+            self.tools,
+            self.llm,
+            agent="conversational-react-description",
+            verbose=True,
+            memory=self.memory,
+            return_intermediate_steps=True,
+            agent_kwargs={'prefix': PREFIX, 'format_instructions': FORMAT_INSTRUCTIONS, 'suffix': SUFFIX}, )
+        state = state + [("I upload a video, Please watch it first! ","I have watch this video, Let's chat!")]
+        return gr.update(visible = True),state, state
+
+if __name__=="__main__":
+    import pdb
+    pdb.set_trace()

configs/med_config.json

@@ -0,0 +1,21 @@
+{
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "type_vocab_size": 2,
+  "vocab_size": 30524,
+  "encoder_width": 768,
+  "add_cross_attention": true   
+}

configs/q2l_config.json

@@ -0,0 +1,23 @@
+{
+    "architectures": [
+      "BertModel"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-12,
+    "max_position_embeddings": 512,
+    "model_type": "bert",
+    "num_attention_heads": 4,
+    "num_hidden_layers": 2,
+    "pad_token_id": 0,
+    "type_vocab_size": 2,
+    "vocab_size": 30522,
+    "encoder_width": 768,
+    "add_cross_attention": true,
+    "add_tag_cross_attention": false
+  }
+  

configs/swin/config_swinB_224.json

@@ -0,0 +1,10 @@
+{
+    "ckpt": "pretrain_model/swin_base_patch4_window7_224_22k.pth",
+    "vision_width": 1024,
+    "image_res": 224,
+    "window_size": 7,
+    "embed_dim": 128,
+    "depths": [ 2, 2, 18, 2 ],
+    "num_heads": [ 4, 8, 16, 32 ]
+  }
+  

configs/swin/config_swinB_384.json

@@ -0,0 +1,10 @@
+{
+    "ckpt": "pretrain_model/swin_base_patch4_window7_224_22k.pth",
+    "vision_width": 1024,
+    "image_res": 384,
+    "window_size": 12,
+    "embed_dim": 128,
+    "depths": [ 2, 2, 18, 2 ],
+    "num_heads": [ 4, 8, 16, 32 ]
+  }
+  

configs/swin/config_swinB_480.json

@@ -0,0 +1,9 @@
+{
+  "ckpt": "pretrain_model/swin_base_patch4_window7_224_22k.pth",
+  "vision_width": 1024,
+  "image_res": 480,
+  "window_size": 15,
+  "embed_dim": 128,
+  "depths": [ 2, 2, 18, 2 ],
+  "num_heads": [ 4, 8, 16, 32 ]
+}

configs/swin/config_swinB_576.json

@@ -0,0 +1,9 @@
+{
+  "ckpt": "pretrain_model/swin_base_patch4_window7_224_22k.pth",
+  "vision_width": 1024,
+  "image_res": 576,
+  "window_size": 18,
+  "embed_dim": 128,
+  "depths": [ 2, 2, 18, 2 ],
+  "num_heads": [ 4, 8, 16, 32 ]
+}

configs/swin/config_swinB_608.json

@@ -0,0 +1,9 @@
+{
+  "ckpt": "pretrain_model/swin_base_patch4_window7_224_22k.pth",
+  "vision_width": 1024,
+  "image_res": 608,
+  "window_size": 19,
+  "embed_dim": 128,
+  "depths": [ 2, 2, 18, 2 ],
+  "num_heads": [ 4, 8, 16, 32 ]
+}

configs/tag2text_caption.yaml

@@ -0,0 +1,33 @@
+image_root: '/home/notebook/data/group/projects/tagging/caption/datasets/public/coco/'
+
+ann_root: 'dataset/caption_dataset'
+coco_gt_root: 'dataset/caption_dataset'
+
+pretrained: '/home/notebook/code/personal/S9049611/BLIP/output/pretrain_caption_tagtotext_v2_bert_asl'
+
+# size of vit model; base or large
+vit: 'swin_b'
+vit_grad_ckpt: False
+vit_ckpt_layer: 0
+
+batch_size: 35
+init_lr: 5e-6 
+
+image_size: 384
+
+# generation configs
+max_length: 20  
+min_length: 5
+num_beams: 3
+prompt: 'a picture of '
+
+# optimizer
+weight_decay: 0.05
+min_lr: 0
+max_epoch: 10
+
+text_pretrain: 'bert'
+
+class_num: 3429
+threshold: 0.7
+

data/tag_class.py

@@ -0,0 +1,3437 @@
+import numpy as np
+
+
+tra_array = ['tennis',
+'bear cub',
+'observatory',
+'bicycle',
+'hillside',
+'judge',
+'watercolor illustration',
+'granite',
+'lobster',
+'livery',
+'stone',
+'ceramic',
+'ranch',
+'cloth',
+'smile',
+'building',
+'tattoo',
+'cricketer',
+'cheek',
+'pear',
+'source',
+'winter',
+'surface',
+'spray',
+'ceremony',
+'magic',
+'curve',
+'container',
+'fair',
+'medicine',
+'baby',
+'tennis racquet',
+'ornament',
+'bamboo',
+'duckling',
+'song',
+'safari',
+'team presentation',
+'daffodil',
+'cross',
+'toothpaste',
+'shield',
+'fashion model',
+'capsule',
+'map',
+'creek',
+'glass house',
+'glass plate',
+'siding',
+'corner',
+'water buffalo',
+'bison',
+'figure skater',
+'diploma',
+'tire',
+'race',
+'cable car',
+'brain',
+'gas stove',
+'soap bubble',
+'palette',
+'snowboard',
+'school child',
+'trench coat',
+'monk',
+'fiber',
+'kitchen window',
+'sunglass',
+'coffee',
+'security',
+'strawberry',
+'penguin',
+'tree root',
+'loaf',
+'engagement ring',
+'lamb',
+'vector cartoon illustration',
+'sandwich',
+'mountain village',
+'shape',
+'charm',
+'fiction',
+'knot',
+'greenhouse',
+'sushi',
+'text',
+'disaster',
+'trophy',
+'gang',
+'strap',
+'soccer game',
+'cardinal',
+'tee',
+'turtle',
+'water surface',
+'grassland',
+'dolphin',
+'store',
+'dirt',
+'iceberg',
+'pergola',
+'farmer market',
+'publicity portrait',
+'tote bag',
+'teenage girl',
+'view mirror',
+'session',
+'commuter',
+'dressing room',
+'tricycle',
+'christmas ball',
+'headlight',
+'police',
+'armchair',
+'chart',
+'yacht',
+'saw',
+'printer',
+'rock band',
+'gingerbread house',
+'tag',
+'table lamp',
+'hockey game',
+'slope',
+'font',
+'wicker basket',
+'jewelry',
+'quarter',
+'software',
+'weapon',
+'pin',
+'worship',
+'painter',
+'goal',
+'morning light',
+'bike',
+'baseball bat',
+'elevator',
+'cuisine',
+'sausage',
+'stunt',
+'wrestler',
+'statue',
+'landing',
+'pillar',
+'willow tree',
+'sea wave',
+'chicken',
+'peanut',
+'muscle',
+'bob',
+'tv genre',
+'bathroom window',
+'radish',
+'textile',
+'pelican',
+'marketplace',
+'crest',
+'elevation map',
+'gift',
+'parish',
+'traffic light',
+'campfire',
+'fog',
+'award winner',
+'beach ball',
+'mat',
+'white house',
+'plaster',
+'moped',
+'football team',
+'solution',
+'bicyclist',
+'bit',
+'playground',
+'darkness',
+'cake',
+'maple leave',
+'mold',
+'cracker',
+'blueberry',
+'rubble',
+'container ship',
+'pedestrian bridge',
+'snail',
+'parrot',
+'form',
+'circuit',
+'highlight',
+'pickup truck',
+'koala',
+'rain',
+'system',
+'weather',
+'raincoat',
+'soccer team',
+'windshield',
+'thunderstorm',
+'mike',
+'bird house',
+'bridge',
+'grandfather',
+'restroom',
+'animation',
+'wilderness',
+'clown',
+'banana',
+'brown',
+'braid',
+'dining room',
+'kindergarten',
+'launch event',
+'purple',
+'school',
+'stairwell',
+'brooch',
+'movie poster image',
+'mountain river',
+'shelf',
+'wicket',
+'headboard',
+'buddha',
+'flower field',
+'dugout',
+'cd',
+'bald eagle',
+'lagoon',
+'seaweed',
+'agriculture',
+'emergency service',
+'maple tree',
+'parachute',
+'continent',
+'amusement park',
+'remote',
+'bun',
+'tackle',
+'hospital',
+'garage door',
+'birthday party',
+'friendship',
+'go',
+'mausoleum',
+'jeep',
+'raccoon',
+'step',
+'ice hockey team',
+'cigarette',
+'lace dress',
+'forest floor',
+'mall',
+'captain',
+'milk',
+'golf course',
+'meal',
+'picnic table',
+'sail',
+'volleyball',
+'canal',
+'terrace',
+'computer desk',
+'caravan',
+'hotel',
+'cheerleader',
+'nurse',
+'museum',
+'marsh',
+'fox',
+'plateau',
+'night',
+'twin',
+'letter logo',
+'autumn tree',
+'powder',
+'convention',
+'creature',
+'lighthouse',
+'shop window',
+'jacket',
+'stork',
+'taxi',
+'trade',
+'blackboard',
+'olive',
+'road sign',
+'resort',
+'snowflake',
+'cemetery',
+'travel',
+'evening dress',
+'picnic',
+'drink',
+'winter morning',
+'football player',
+'snack',
+'boxing glove',
+'dinner party',
+'airline',
+'swing',
+'port',
+'wheelbarrow',
+'bathroom sink',
+'sweater',
+'ambulance',
+'gear',
+'oil',
+'wii controller',
+'array',
+'home office',
+'car show',
+'mixture',
+'profession',
+'tree frog',
+'square',
+'facility',
+'coral reef',
+'sea wall',
+'pizza',
+'exhibit',
+'demolition',
+'trout',
+'ring',
+'coffee shop',
+'bracelet',
+'bean',
+'lip',
+'fencing',
+'landscape',
+'sitting',
+'package',
+'metal',
+'bust',
+'king',
+'hair',
+'window seat',
+'wildlife',
+'trunk',
+'greenery',
+'stencil',
+'fire hydrant',
+'bridesmaid',
+'plaza',
+'alps',
+'tower bridge',
+'crop top',
+'crossing',
+'cinema',
+'pedestrian crossing',
+'family',
+'shopping cart',
+'stomach',
+'church building',
+'screen door',
+'skater',
+'soccer field',
+'kettle',
+'mussel',
+'raindrop',
+'candy cane',
+'water lily',
+'flower girl',
+'desert',
+'enclosure',
+'christmas light',
+'kitchen',
+'caterpillar',
+'plaid',
+'bath',
+'bush',
+'mud',
+'ballet',
+'knee',
+'adult',
+'raft',
+'sea view',
+'cactus',
+'office chair',
+'overall',
+'rim',
+'scaffolding',
+'pig',
+'cover',
+'poster page',
+'sprinkle',
+'chandelier',
+'algae',
+'traffic',
+'surfboard',
+'book',
+'filming',
+'flash',
+'mansion',
+'camouflage',
+'trouser',
+'ticket',
+'weed',
+'cab',
+'trench',
+'elephant',
+'huddle',
+'sphere',
+'christmas decoration',
+'city',
+'launch',
+'doll',
+'christmas ornament',
+'fabric',
+'bikini',
+'biplane',
+'breakfast',
+'neighbourhood',
+'race track',
+'foliage',
+'avocado',
+'school bus',
+'footwear',
+'highway',
+'ocean view',
+'art vector illustration',
+'wall clock',
+'curtain',
+'teenager',
+'kitchen area',
+'robot',
+'tusk',
+'lounge chair',
+'beam',
+'paddle',
+'camel',
+'lid',
+'world map',
+'city view',
+'newlywed',
+'cargo ship',
+'yellow',
+'exhibition',
+'bend',
+'novel',
+'wool',
+'ontario',
+'bread',
+'campus',
+'coastline',
+'cutting board',
+'booth',
+'table top',
+'carpet',
+'beach chair',
+'workout',
+'street food',
+'fun',
+'costumer film designer',
+'gadget',
+'artist',
+'fishing village',
+'builder',
+'violinist',
+'iphone',
+'spider web',
+'traffic sign',
+'ruin',
+'rescue',
+'clipboard',
+'seal',
+'film director',
+'paw',
+'nursery',
+'intersection',
+'tomato sauce',
+'taste',
+'paddy field',
+'christmas tree',
+'wave',
+'stool',
+'watering can',
+'rug',
+'daytime',
+'subway station',
+'craft',
+'pine forest',
+'black',
+'planet',
+'motif',
+'christmas market',
+'glass window',
+'college',
+'wheat',
+'damage',
+'rectangle',
+'picture frame',
+'chess',
+'guest room',
+'street corner',
+'religion',
+'seed',
+'puzzle',
+'freeway',
+'beauty',
+'ocean',
+'watch',
+'mother',
+'garage',
+'quote',
+'dj',
+'supporter',
+'hip hop artist',
+'muffin',
+'eiffel tower',
+'cash',
+'firefighter',
+'cauliflower',
+'bunker',
+'sled',
+'manicure',
+'shark',
+'stall',
+'jungle',
+'family home',
+'tour bus',
+'chimney',
+'touchdown',
+'roundabout',
+'coyote',
+'street scene',
+'tank',
+'wedding dress',
+'mantle',
+'bedroom window',
+'coconut',
+'chapel',
+'goat',
+'living space',
+'rock wall',
+'polka dot',
+'railway',
+'mandala',
+'mango',
+'lesson',
+'mountain landscape',
+'team photo',
+'bookshelf',
+'meter',
+'bulldog',
+'evening sun',
+'stick',
+'card',
+'pink',
+'fish pond',
+'paint',
+'pill',
+'cart',
+'pea',
+'van',
+'album',
+'football college game',
+'mountain pass',
+'doughnut',
+'ski slope',
+'match',
+'official',
+'shadow',
+'organ',
+'celebration',
+'coin',
+'log cabin',
+'firework display',
+'present',
+'twig',
+'chef',
+'confetti',
+'footpath',
+'tour',
+'ponytail',
+'artwork',
+'race car',
+'club',
+'season',
+'hose',
+'pencil',
+'aircraft',
+'rock formation',
+'wardrobe',
+'participant',
+'politician',
+'engineer',
+'peace',
+'filter',
+'sailing boat',
+'water bottle',
+'service dog',
+'poodle',
+'loki',
+'statesman',
+'sleeping bag',
+'outskirt',
+'clock',
+'factory',
+'oak tree',
+'physician',
+'color',
+'room',
+'stairway',
+'company',
+'lady',
+'graph',
+'faucet',
+'tablecloth',
+'subway train',
+'chocolate chip cookie',
+'headquarters',
+'screw',
+'goggle',
+'halloween',
+'city street',
+'swirl',
+'cord',
+'forward',
+'bone',
+'bedding',
+'archway',
+'wig',
+'lobby',
+'mask',
+'attic',
+'kitchen table',
+'skylight',
+'fire',
+'exit',
+'oil painting',
+'passenger',
+'meditation',
+'salmon',
+'fedora',
+'rubber stamp',
+'orange juice',
+'arch',
+'scientist',
+'stroll',
+'manhattan',
+'float',
+'baseball uniform',
+'circle',
+'church',
+'decker bus',
+'competitor',
+'zoo',
+'basketball team',
+'tourist',
+'daughter',
+'silverware',
+'ceiling fan',
+'birth',
+'vase',
+'jack',
+'mushroom',
+'spiral',
+'cage',
+'limb',
+'salad',
+'ad',
+'control',
+'earth',
+'party',
+'bolt',
+'tractor',
+'barley',
+'wedding photo',
+'hawk',
+'warehouse',
+'vegetable garden',
+'chocolate cake',
+'cabbage',
+'floor window',
+'baby shower',
+'magnifying glass',
+'table',
+'stethoscope',
+'reading',
+'mission',
+'croissant',
+'gift box',
+'rocket',
+'forest road',
+'cooking',
+'suite',
+'hill country',
+'motorcycle',
+'baseball player',
+'angle',
+'drug',
+'sport association',
+'championship',
+'family portrait',
+'florist',
+'softball',
+'egret',
+'office',
+'plywood',
+'jockey',
+'mosque',
+'brunch',
+'beanie',
+'office building',
+'pattern',
+'calendar',
+'indoor',
+'pepper',
+'ledge',
+'trail',
+'fuel',
+'laptop computer',
+'tennis shoe',
+'deck chair',
+'guitarist',
+'barn',
+'surgery',
+'cartoon illustration',
+'nebula',
+'railroad',
+'mountain goat',
+'goose',
+'car door',
+'cheer',
+'liquid',
+'hardwood floor',
+'pathway',
+'acorn',
+'gull',
+'airliner',
+'couch',
+'lake house',
+'spaghetti',
+'promenade',
+'collection',
+'garden',
+'bank',
+'robin',
+'tennis ball',
+'peony',
+'gymnast',
+'lavender',
+'deck',
+'test',
+'riverside',
+'rapper',
+'domino',
+'bride',
+'mouse',
+'basil',
+'wedding couple',
+'ocean wave',
+'arm',
+'kitchen floor',
+'grove',
+'family member',
+'backyard',
+'raspberry',
+'forest fire',
+'officer',
+'hibiscus',
+'canyon',
+'composer',
+'signature',
+'olive oil',
+'hibiscus flower',
+'rose',
+'vector icon',
+'sunrise',
+'horseback',
+'motor scooter',
+'office worker',
+'tradition',
+'ingredient',
+'washing machine',
+'lighting',
+'bagel',
+'sailboat',
+'policeman',
+'mare',
+'graphic',
+'halloween pumpkin',
+'stock',
+'pilot',
+'education',
+'team',
+'body',
+'horse',
+'kimono',
+'bazaar',
+'bag',
+'recording studio',
+'parsley',
+'entrance',
+'denim',
+'vet',
+'horse farm',
+'charcoal',
+'architecture',
+'glass vase',
+'puppy',
+'estuary',
+'television show host',
+'city bus',
+'shoulder',
+'beast',
+'balance',
+'golfer',
+'roadside',
+'denim jacket',
+'stone wall',
+'counter top',
+'app icon',
+'toast',
+'head coach',
+'ham',
+'warrior',
+'gem',
+'refrigerator',
+'snowman',
+'construction worker',
+'coal',
+'website',
+'morning fog',
+'mustard',
+'human',
+'owl',
+'puppy dog',
+'piggy bank',
+'vegetation',
+'pirate',
+'action film',
+'marshmallow',
+'thanksgiving',
+'business',
+'disease',
+'signage',
+'greeting',
+'skate park',
+'tile',
+'mouth',
+'spinach',
+'vacation',
+'leader',
+'shrine',
+'walker',
+'science fiction film',
+'bill',
+'rabbit',
+'motor boat',
+'bar',
+'radio',
+'barge',
+'tail',
+'chainsaw',
+'gallery',
+'rainbow',
+'pasta',
+'padlock',
+'web',
+'pastry',
+'ink',
+'reef',
+'school uniform',
+'shawl',
+'treasure',
+'peach',
+'dinner table',
+'injury',
+'harbor',
+'witch',
+'car dealership',
+'litter',
+'gesture',
+'documentary',
+'marriage',
+'sea shell',
+'priest',
+'dome',
+'kit',
+'icon',
+'seaside',
+'bucket',
+'entertainment',
+'stable',
+'hat',
+'puddle',
+'sock',
+'shopper',
+'technology',
+'harbour',
+'orbit',
+'antler',
+'tube',
+'flag waving',
+'cook',
+'tight',
+'commander',
+'farmland',
+'switch',
+'hiker',
+'wedding ceremony',
+'award ceremony',
+'champion',
+'chopstick',
+'farmhouse',
+'performer',
+'spike',
+'accident',
+'cruise ship',
+'passenger train',
+'attraction',
+'entertainer',
+'rear view',
+'sidewalk',
+'parade',
+'racing',
+'plane',
+'ritual',
+'peacock',
+'pocket',
+'plum',
+'drop',
+'carrot',
+'floor',
+'sunset',
+'troop',
+'architect',
+'coffee table',
+'dust',
+'outline',
+'leather',
+'charity event',
+'heat',
+'whale',
+'laundry',
+'coconut tree',
+'crosswalk',
+'pony',
+'ant',
+'pipe',
+'string',
+'coat',
+'angel',
+'beef',
+'church tower',
+'dish',
+'pitch',
+'cupboard',
+'thermometer',
+'dirt field',
+'fireworks',
+'minute',
+'cane',
+'pajama',
+'flower garden',
+'autumn',
+'trash can',
+'dachshund',
+'banana tree',
+'tray',
+'moose',
+'roadway',
+'carnival',
+'antenna',
+'pole',
+'castle wall',
+'ram',
+'cattle',
+'hay',
+'cookie',
+'swimmer',
+'baseball team',
+'strait',
+'hedge',
+'jet',
+'fire pit',
+'octopus',
+'calf',
+'cube',
+'opera',
+'cardboard box',
+'tiara',
+'kitchen sink',
+'prairie',
+'bowl',
+'galaxy',
+'straw hat',
+'linen',
+'ski resort',
+'stitch',
+'street lamp',
+'motorist',
+'icicle',
+'stain',
+'flora',
+'drain',
+'kitchen cabinet',
+'decor',
+'bouquet',
+'pound',
+'interior design',
+'nail polish',
+'figurine',
+'tomb',
+'disc',
+'twist',
+'blouse',
+'ribbon',
+'figure',
+'burger',
+'cork',
+'soccer goalkeeper',
+'train bridge',
+'drinking water',
+'dew',
+'baker',
+'storm cloud',
+'tarmac',
+'tv drama',
+'sponge',
+'magnet',
+'sailor',
+'entry',
+'swan',
+'exercise',
+'sloth',
+'jewel',
+'scuba diver',
+'bite',
+'cat tree',
+'tent',
+'can',
+'tennis match',
+'ecosystem',
+'picket fence',
+'palm',
+'train car',
+'frying pan',
+'rally',
+'tablet pc',
+'reindeer',
+'image',
+'wolf',
+'chin',
+'conservatory',
+'flood water',
+'cityscape',
+'beach sand',
+'car park',
+'pavement',
+'farm field',
+'swimming',
+'winter storm',
+'stem',
+'pillow',
+'inning',
+'gorilla',
+'desk',
+'avenue',
+'fern',
+'money',
+'pearl',
+'train station',
+'skillet',
+'nap',
+'barber',
+'library',
+'freezer',
+'label',
+'rainforest',
+'parking sign',
+'mirror',
+'wing',
+'noodle',
+'press room',
+'sculpture',
+'tablet',
+'viewer',
+'prayer',
+'mini',
+'mechanic',
+'laugh',
+'rice field',
+'hand',
+'mustache',
+'mountain road',
+'catwalk',
+'conference',
+'cape',
+'installation',
+'musician',
+'stream',
+'machine',
+'speech',
+'crocodile',
+'soccer match',
+'town square',
+'passport',
+'post box',
+'point',
+'stone building',
+'motorway',
+'mix',
+'dentist',
+'businessperson',
+'happiness',
+'boat',
+'vineyard',
+'treadmill',
+'glass wall',
+'water droplet',
+'coffee mug',
+'graduate',
+'sunflower',
+'parliament',
+'shepherd',
+'movie',
+'wine',
+'orchard',
+'tulip',
+'motherboard',
+'cup',
+'broom',
+'spot',
+'drawing',
+'polo shirt',
+'graduation',
+'film producer',
+'moonlight',
+'glow',
+'film format',
+'t shirt',
+'rock face',
+'sword',
+'clinic',
+'festival day',
+'meadow',
+'staple',
+'pupil',
+'training ground',
+'rider',
+'flower',
+'foal',
+'wharf',
+'foot bridge',
+'shooting',
+'top',
+'mast',
+'police car',
+'robe',
+'wedding bouquet',
+'stop sign',
+'birthday cake',
+'glitter',
+'butter',
+'scooter',
+'tundra',
+'superhero',
+'pocket watch',
+'inscription',
+'youngster',
+'fruit tree',
+'movie poster',
+'engine',
+'foundation',
+'motorcyclist',
+'take',
+'woman',
+'antelope',
+'country artist',
+'road trip',
+'typewriter',
+'tuxedo',
+'brand',
+'pine',
+'bathroom',
+'paradise',
+'texture',
+'balloon',
+'dining table',
+'home',
+'computer screen',
+'actor',
+'clip',
+'tv tower',
+'panorama',
+'summit',
+'cat',
+'plot',
+'eagle',
+'dancer',
+'pup',
+'studio shot',
+'tear',
+'bird bath',
+'classroom',
+'bookstore',
+'city wall',
+'tv programme',
+'blade',
+'easel',
+'buttercream',
+'sweet',
+'designer',
+'diamond',
+'handshake',
+'herb',
+'corn field',
+'seafront',
+'concrete',
+'street artist',
+'gas',
+'stamp',
+'window display',
+'paper',
+'note',
+'pint',
+'quarry',
+'research',
+'fixture',
+'manager',
+'soil',
+'leopard',
+'board game',
+'ladder',
+'stop light',
+'island',
+'ramp',
+'football match',
+'icing',
+'drill',
+'currency',
+'summer evening',
+'topping',
+'pyramid',
+'pomegranate',
+'cell',
+'ivy',
+'squad',
+'scenery',
+'computer',
+'locomotive',
+'surf',
+'mascot',
+'dune',
+'path',
+'duck',
+'twilight',
+'wire',
+'bow tie',
+'strike',
+'cormorant',
+'car wash',
+'crane',
+'market',
+'philosopher',
+'alarm clock',
+'camera',
+'birch',
+'greeting card',
+'plain',
+'clay',
+'donut',
+'lock',
+'moth',
+'laboratory',
+'fan',
+'violin',
+'jazz fusion artist',
+'mountain biker',
+'terrain',
+'magazine',
+'pickup',
+'comedy film',
+'smartphone',
+'film',
+'bed',
+'microwave oven',
+'tournament',
+'lawn',
+'car window',
+'alligator',
+'screen',
+'jetty',
+'shopping bag',
+'landscape view',
+'cabinetry',
+'friendly match',
+'thing',
+'petal',
+'shopping center',
+'transport',
+'ballet dancer',
+'shoreline',
+'princess',
+'car seat',
+'parking meter',
+'green',
+'vodka',
+'band',
+'rock',
+'costume',
+'warning sign',
+'strip',
+'plaque',
+'wheelchair',
+'headband',
+'ginger',
+'dice',
+'media',
+'hairdresser',
+'press',
+'living room',
+'stove',
+'player',
+'cherry',
+'workshop',
+'carving',
+'embroidery',
+'doodle',
+'adventure',
+'rugby player',
+'monument',
+'brush',
+'marker',
+'loft',
+'postcard',
+'collage',
+'ball',
+'professor',
+'dresser',
+'gig',
+'festival',
+'blackbird',
+'makeup artist',
+'video camera',
+'sticker',
+'peak',
+'wildflower',
+'santa hat',
+'rodeo',
+'wedding photographer',
+'guy',
+'staff',
+'waterfall',
+'operation',
+'defender',
+'falcon',
+'haze',
+'individual',
+'gentleman',
+'greyhound',
+'rocking chair',
+'rice',
+'garbage',
+'platter',
+'chocolate',
+'splash',
+'business suit',
+'cheetah',
+'valley',
+'maze',
+'trampoline',
+'garland',
+'slalom',
+'unicorn',
+'tree stump',
+'painting',
+'romance',
+'fight',
+'alcohol',
+'ghost',
+'fondant',
+'spa',
+'shutter',
+'death',
+'demonstration',
+'cotton',
+'pier',
+'flea market',
+'history',
+'savannah',
+'fist',
+'aisle',
+'crew',
+'jug',
+'pose',
+'anchor',
+'teapot',
+'boat house',
+'business team',
+'tripod',
+'bee',
+'pebble',
+'mattress',
+'canvas',
+'hallway',
+'campaign',
+'pod',
+'lake district',
+'article',
+'white',
+'sofa',
+'honey',
+'marathon',
+'pancake',
+'tourist attraction',
+'wedding gown',
+'battle',
+'shelving',
+'sea',
+'sheet music',
+'pie',
+'yarn',
+'construction site',
+'flyer',
+'tie',
+'star',
+'lettuce',
+'martial artist',
+'dart',
+'straw',
+'reflection',
+'conference room',
+'temperature',
+'rugby',
+'mosquito',
+'physicist',
+'rock climber',
+'crash',
+'backdrop',
+'toilet seat',
+'sand castle',
+'water park',
+'toy car',
+'waste',
+'luxury',
+'hangar',
+'rv',
+'tree trunk',
+'board',
+'gold',
+'project picture',
+'cap',
+'cottage',
+'relief',
+'attire',
+'microscope',
+'battery',
+'roll',
+'line',
+'parking garage',
+'crystal',
+'broadcasting',
+'brick wall',
+'lab',
+'flooring',
+'meeting',
+'3d cg rendering',
+'desktop computer',
+'cowboy',
+'sailing ship',
+'junction',
+'hairstyle',
+'homework',
+'profile',
+'model',
+'flower pot',
+'street light',
+'salt lake',
+'maple',
+'space',
+'blizzard',
+'throw',
+'zebras',
+'brochure',
+'constellation',
+'beak',
+'kilt',
+'pond',
+'blue sky',
+'sneaker',
+'sand dune',
+'morning sun',
+'almond',
+'grill',
+'curl',
+'basketball girl game',
+'chameleon',
+'toilet bowl',
+'prince',
+'keyboard',
+'queen',
+'computer monitor',
+'writing',
+'crown',
+'basilica',
+'kiss',
+'house',
+'parking',
+'football competition',
+'shell',
+'sport equipment',
+'comedy',
+'baboon',
+'vendor',
+'rise building',
+'wrap',
+'food truck',
+'cat bed',
+'rickshaw',
+'flare',
+'teal',
+'nectar',
+'eclipse',
+'vehicle',
+'steam locomotive',
+'gorge',
+'cow',
+'christmas card',
+'demonstrator',
+'memorial',
+'towel',
+'jewellery',
+'train',
+'frisbee',
+'baseball game',
+'fur',
+'afternoon sun',
+'community',
+'sparkler',
+'bandage',
+'firework',
+'dollar',
+'pasture',
+'video',
+'bus',
+'tree house',
+'seashore',
+'field',
+'hamburger',
+'souvenir',
+'hedgehog',
+'worm',
+'pine cone',
+'osprey',
+'dinosaur',
+'vegetable',
+'junk',
+'poster',
+'army',
+'winger',
+'bundle',
+'stage',
+'growth',
+'wedding party',
+'service',
+'blanket',
+'ruler',
+'eye',
+'credit card',
+'castle',
+'diner',
+'hut',
+'elk',
+'hard rock artist',
+'nun',
+'dog breed',
+'nest',
+'drama film',
+'number icon',
+'water tank',
+'giraffe',
+'altar',
+'pavilion',
+'tv personality',
+'suv',
+'street vendor',
+'street sign',
+'ditch',
+'debris',
+'foam',
+'takeoff',
+'spice',
+'mountain lake',
+'tea',
+'orchestra',
+'spacecraft',
+'counter',
+'abbey',
+'mountain',
+'hydrangea',
+'racer',
+'orange tree',
+'tide',
+'cowboy hat',
+'rapid',
+'town',
+'wild',
+'herd',
+'vein',
+'driveway',
+'jar',
+'bark',
+'illustration',
+'horror film',
+'corn',
+'stroller',
+'industry',
+'mountain stream',
+'gym',
+'neckline',
+'pan',
+'client',
+'spectator',
+'eggplant',
+'camper',
+'fawn',
+'hoodie',
+'meat',
+'lemonade',
+'food market',
+'slum',
+'comic book character',
+'flower market',
+'love',
+'palace',
+'gun',
+'heel',
+'shopping street',
+'shooting basketball guard',
+'family photo',
+'rooftop',
+'laundry basket',
+'airport runway',
+'horn',
+'face mask',
+'flight',
+'appetizer',
+'violet',
+'country lane',
+'cement',
+'instrument',
+'tv actor',
+'spark',
+'celebrity',
+'award',
+'country house',
+'standing',
+'auction',
+'date',
+'engagement',
+'puck',
+'advertisement',
+'chair',
+'zebra',
+'driftwood',
+'bumblebee',
+'maple leaf',
+'bonnet',
+'orange',
+'water tower',
+'door',
+'singer',
+'floor plan',
+'discussion',
+'theatre',
+'pilgrim',
+'mug',
+'branch',
+'window sill',
+'baseball pitcher',
+'bakery',
+'lollipop',
+'basketball player',
+'toilet paper',
+'chalkboard',
+'cabin',
+'sign',
+'night sky',
+'cannon',
+'fishing net',
+'submarine',
+'suit',
+'fur coat',
+'wine bottle',
+'folder',
+'street art',
+'suspension bridge',
+'evening sky',
+'billboard',
+'postage stamp',
+'newspaper',
+'transportation',
+'surgeon',
+'light',
+'park',
+'horizon',
+'road',
+'sand bar',
+'trumpet',
+'lounge',
+'cloud forest',
+'birthday celebration',
+'balcony',
+'anime',
+'beehive',
+'umbrella',
+'goldfish',
+'baseball cap',
+'waterhole',
+'ceiling',
+'carousel',
+'backpack',
+'plant pot',
+'atmosphere',
+'sunflower field',
+'spire',
+'vision',
+'woodpecker',
+'chip',
+'pool table',
+'lotus flower',
+'cone',
+'humpback whale',
+'reservoir',
+'hunt',
+'piano',
+'plate',
+'dining area',
+'luggage',
+'skier',
+'dance floor',
+'crow',
+'stair',
+'overpass',
+'opera house',
+'bear',
+'jazz artist',
+'water',
+'vessel',
+'cast',
+'yard',
+'cathedral',
+'basketball hoop',
+'graveyard',
+'sound',
+'berry',
+'onlooker',
+'fauna',
+'birch tree',
+'retail',
+'hill',
+'skeleton',
+'journalist',
+'frost',
+'basket',
+'nail',
+'dusk',
+'trash',
+'dawn',
+'clover',
+'hen',
+'volcano',
+'basketball coach',
+'home decor',
+'charge',
+'haircut',
+'sense',
+'university',
+'lizard',
+'daisy',
+'tablet computer',
+'grass field',
+'prison',
+'metal artist',
+'bathroom mirror',
+'window frame',
+'chest',
+'flavor',
+'pop country artist',
+'market square',
+'monkey',
+'blog',
+'deer',
+'speech bubble',
+'dog',
+'independence day',
+'girl',
+'boy',
+'tartan',
+'furniture',
+'appliance',
+'office window',
+'fish boat',
+'sand box',
+'tv sitcom',
+'drama',
+'sleigh',
+'depression',
+'paper towel',
+'baseball',
+'protestor',
+'grape',
+'wedding cake',
+'invitation',
+'accessory',
+'pick',
+'grandparent',
+'racket',
+'tea plantation',
+'outdoors',
+'egg',
+'glass bowl',
+'sun',
+'organization',
+'lion',
+'panel',
+'station',
+'wallpaper',
+'helicopter',
+'salt',
+'vanity',
+'patio',
+'lunch',
+'street performer',
+'mountain range',
+'soup',
+'bacon',
+'power station',
+'cantilever bridge',
+'hummingbird',
+'shirt',
+'rope',
+'hip',
+'chalk',
+'pendant',
+'choir',
+'tv',
+'lichen',
+'railway bridge',
+'art gallery',
+'bartender',
+'wagon',
+'baby elephant',
+'accordion',
+'horseshoe',
+'building site',
+'clutch',
+'harvest',
+'savanna',
+'geranium',
+'business woman',
+'paddock',
+'patch',
+'beech tree',
+'war',
+'suburbs',
+'hospital bed',
+'motorcycle racer',
+'moss',
+'gravel',
+'government agency',
+'dollar bill',
+'father',
+'fjord',
+'concert',
+'nut',
+'wedding photography',
+'finish line',
+'home plate',
+'food',
+'nose',
+'thumb',
+'village',
+'dining room table',
+'bumper',
+'monster',
+'blackberry',
+'lime',
+'conflict',
+'gala',
+'wallet',
+'wrist',
+'hug',
+'mermaid',
+'lava',
+'lawyer',
+'folk rock artist',
+'arena',
+'onion',
+'toothbrush',
+'fashion',
+'perfume',
+'flip',
+'triangle',
+'woodland',
+'mail',
+'grasshopper',
+'studio',
+'wood floor',
+'den',
+'racquet',
+'cello',
+'lemur',
+'astronaut',
+'glass table',
+'blood',
+'dvd',
+'planter',
+'silver',
+'leash',
+'master bedroom',
+'forest',
+'batter',
+'shoe',
+'engraving',
+'opening',
+'product',
+'toe',
+'cocktail',
+'mallard duck',
+'bike ride',
+'oasis',
+'wedding ring',
+'cinematographer',
+'holly',
+'autograph',
+'fence',
+'ice cube',
+'cove',
+'pineapple',
+'aurora',
+'glass bead',
+'produce',
+'apartment building',
+'cob',
+'miniature',
+'cockpit',
+'flashlight',
+'frog',
+'sheep',
+'groom',
+'steel',
+'watermelon',
+'clip art',
+'paper plate',
+'ostrich',
+'contour',
+'mural',
+'cub',
+'paisley bandanna',
+'winery',
+'turn',
+'handle',
+'satellite',
+'post',
+'pork',
+'child',
+'asphalt',
+'grocery store',
+'vulture',
+'trolley',
+'nightclub',
+'brick',
+'trailer',
+'compass',
+'cereal',
+'cafe',
+'cartoon character',
+'sugar',
+'fiction book',
+'glass floor',
+'umpire',
+'guitar',
+'hamster',
+'protester',
+'airplane',
+'garment',
+'blazer',
+'railway line',
+'wedding',
+'shoe box',
+'parking lot',
+'construction',
+'graduation ceremony',
+'tram',
+'telescope',
+'copper',
+'pain',
+'autumn forest',
+'guest house',
+'partner',
+'crayon',
+'dip',
+'boot',
+'corridor',
+'computer keyboard',
+'hockey player',
+'chicken coop',
+'bus station',
+'gathering',
+'ankle',
+'bunk bed',
+'wood table',
+'football coach',
+'monarch',
+'pharmacy',
+'legging',
+'mannequin',
+'female',
+'train track',
+'stack',
+'canopy',
+'design element',
+'grandmother',
+'symbol',
+'beach hut',
+'zucchini',
+'bomb',
+'businessman',
+'skyscraper',
+'tongue',
+'case',
+'sparkle',
+'highland',
+'ballroom',
+'prom',
+'estate',
+'customer',
+'archipelago',
+'cheese',
+'debate',
+'carriage',
+'bulldozer',
+'pumpkin',
+'sitting room',
+'gas station',
+'wedding reception',
+'camp',
+'dog bed',
+'tower',
+'property',
+'river bed',
+'pop latin artist',
+'fridge',
+'wine glass',
+'coast',
+'beer',
+'tow truck',
+'fire truck',
+'mountain bike',
+'thigh',
+'heron',
+'boat ride',
+'gondola',
+'turquoise',
+'lake',
+'llama',
+'kitty',
+'tin',
+'waiting room',
+'coffee cup',
+'socialite',
+'guard',
+'tap',
+'waterway',
+'forehead',
+'list',
+'erosion',
+'box',
+'sea lion',
+'pollen',
+'dam',
+'wasp',
+'salon',
+'tennis tournament',
+'flower box',
+'aquarium',
+'rain cloud',
+'clothing store',
+'lead singer',
+'cupcake',
+'tortoise',
+'lettering',
+'sport facility',
+'dance',
+'dog house',
+'nature',
+'football',
+'rooster',
+'footballer',
+'railway track',
+'crowd',
+'fishing rod',
+'silhouette',
+'wind turbine',
+'sari',
+'bus window',
+'cloud',
+'charity',
+'medal',
+'yoga',
+'event',
+'veil',
+'fashion menswear milan week',
+'news',
+'knife',
+'print',
+'screen tv',
+'walnut',
+'fungus',
+'ice cream',
+'computer mouse',
+'play',
+'tribe',
+'picture',
+'video game',
+'business card',
+'music festival',
+'rack',
+'envelope',
+'shower',
+'dirt road',
+'mine',
+'oyster',
+'monarch butterfly',
+'dude',
+'fruit salad',
+'podium',
+'fork',
+'lace',
+'test match',
+'boulder',
+'cricket player',
+'staircase',
+'peninsula',
+'shopping',
+'popcorn',
+'oak',
+'market stall',
+'pine tree',
+'mountaineer',
+'student',
+'closet',
+'hood',
+'handstand',
+'centerpiece',
+'insect',
+'patient',
+'makeover',
+'tennis player',
+'sheet',
+'park bench',
+'apple',
+'organism',
+'hook',
+'turkey',
+'tangerine',
+'sibling',
+'shopping mall',
+'bird',
+'scarf',
+'smoothie',
+'net',
+'grass',
+'napkin',
+'ray',
+'eyebrow',
+'laptop keyboard',
+'motorbike',
+'woman hand',
+'oven',
+'book cover',
+'easter egg',
+'microwave',
+'sand',
+'snapshot',
+'soccer ball',
+'makeup',
+'knight',
+'bowling ball',
+'shower curtain',
+'flame',
+'lightning',
+'running',
+'power plant',
+'crib',
+'cartoon',
+'moat',
+'fashion girl',
+'wedding invitation',
+'bottle',
+'cliff',
+'monastery',
+'file photo',
+'apartment',
+'casino',
+'cream',
+'sweatshirt',
+'storm',
+'cruise',
+'teddy bear',
+'shovel',
+'wind farm',
+'writer',
+'dock',
+'professional',
+'hotel room',
+'job',
+'monitor',
+'donkey',
+'pass',
+'interview',
+'duchess',
+'mark',
+'plank',
+'beard',
+'zombie',
+'trio',
+'channel',
+'cricket team',
+'windmill',
+'vest',
+'diagram',
+'cable',
+'winter scene',
+'golden gate bridge',
+'buffalo',
+'studio portrait',
+'pagoda',
+'whiskey',
+'freight train',
+'kite',
+'future',
+'steam train',
+'phone box',
+'headset',
+'wood',
+'snowboarder',
+'paper bag',
+'slide',
+'grapefruit',
+'seating',
+'morning',
+'bronze sculpture',
+'theatre actor',
+'stump',
+'jean',
+'landmark',
+'jam',
+'waist',
+'watercolor',
+'hammock',
+'light fixture',
+'ice',
+'basin',
+'beverage',
+'shelter',
+'premiere',
+'mound',
+'ear',
+'bronze',
+'sunlight',
+'street',
+'energy',
+'barn door',
+'hike',
+'fleet',
+'claw',
+'beach',
+'pepperoni',
+'bin',
+'trainer',
+'buffet',
+'archive',
+'toddler',
+'referee',
+'bay window',
+'dove',
+'production company',
+'evening light',
+'gate',
+'farm',
+'reed',
+'fruit stand',
+'explorer',
+'snow storm',
+'throw pillow',
+'button',
+'display case',
+'bookcase',
+'lead',
+'lipstick',
+'basketball court',
+'cargo',
+'ensemble',
+'pope',
+'clock tower',
+'teen',
+'speaker',
+'rat',
+'laptop',
+'ski',
+'mess',
+'stadium',
+'ferry boat',
+'bunny',
+'waterfront',
+'downtown',
+'sink',
+'press conference',
+'dinner',
+'condiment',
+'thread',
+'audience',
+'grid',
+'car',
+'plastic',
+'people',
+'barbecue',
+'pigeon',
+'urinal',
+'seagull',
+'volunteer',
+'hockey',
+'fir tree',
+'pollution',
+'trial',
+'collar',
+'area',
+'meeting room',
+'circus',
+'yogurt',
+'orangutan',
+'viaduct',
+'comedian',
+'drone',
+'scissor',
+'pop rock artist',
+'biscuit',
+'panda',
+'water feature',
+'air balloon',
+'remote control',
+'watercolor painting',
+'show',
+'walk',
+'post office',
+'bike path',
+'rap gangsta artist',
+'microphone',
+'crack',
+'sunset sky',
+'glass',
+'tv show',
+'cartoon style',
+'stripe',
+'foyer',
+'signal',
+'calligraphy',
+'bulb',
+'gardener',
+'coffee bean',
+'spider',
+'tapestry',
+'city skyline',
+'necklace',
+'kitten',
+'traveler',
+'veteran',
+'frosting',
+'fry',
+'tennis court',
+'tank top',
+'butterfly house',
+'mist',
+'drummer',
+'water level',
+'scale',
+'baseball glove',
+'music video performer',
+'champagne',
+'camping',
+'clothing',
+'water drop',
+'telephone box',
+'pen',
+'morning mist',
+'fire engine',
+'porch',
+'opening ceremony',
+'style',
+'palm tree',
+'fashion show',
+'universe',
+'scratch',
+'axe',
+'ottoman',
+'explosion',
+'rib',
+'boutique',
+'game',
+'cucumber',
+'fruit',
+'stone bridge',
+'nature reserve',
+'track',
+'train window',
+'punch',
+'telephone pole',
+'velvet',
+'sauce',
+'moon',
+'contrast',
+'flamingo',
+'bat',
+'vending machine',
+'ship',
+'equestrian',
+'shade',
+'comforter',
+'pallet',
+'sparrow',
+'wii',
+'glaze',
+'grocery',
+'steeple',
+'soccer player',
+'contract',
+'advertising',
+'runner',
+'chimpanzee',
+'world',
+'seat',
+'project',
+'chihuahua',
+'bubble',
+'willow',
+'pedestal',
+'soul hip hop artist',
+'curb',
+'drawer',
+'leaf',
+'banner',
+'launch party',
+'coach',
+'government',
+'snowball',
+'toy',
+'portrait',
+'doctor',
+'whiteboard',
+'electronic',
+'tiger',
+'graffiti',
+'column',
+'nightstand',
+'whistle',
+'maxi dress',
+'bench',
+'wetsuit',
+'bird feeder',
+'football game',
+'basketball',
+'class',
+'bathroom door',
+'store window',
+'text message',
+'wreath',
+'street view',
+'binocular',
+'pet',
+'facade',
+'drought',
+'lemon',
+'new year',
+'night view',
+'airplane window',
+'specie',
+'rule',
+'jaw',
+'wheat field',
+'diet',
+'pop artist',
+'habitat',
+'screenshot',
+'scoreboard',
+'shore',
+'mane',
+'quilt',
+'ski lift',
+'orchid',
+'turban',
+'christmas',
+'airport',
+'marina',
+'glass door',
+'glass bottle',
+'restaurant',
+'conductor',
+'logo',
+'sleep',
+'tape',
+'tomato',
+'river bank',
+'lilac',
+'tooth',
+'training',
+'pottery',
+'shop',
+'steam engine',
+'mason jar',
+'base',
+'procession',
+'border',
+'shoot',
+'footprint',
+'hotdog',
+'bull',
+'stocking',
+'recreation',
+'automobile model',
+'design',
+'country pop artist',
+'river',
+'retriever',
+'department store',
+'auditorium',
+'sport car',
+'supermarket',
+'belt',
+'cricket',
+'window box',
+'dress shirt',
+'letter',
+'residence',
+'megaphone',
+'pant',
+'wildfire',
+'bird nest',
+'crab',
+'swimsuit',
+'candle',
+'funeral',
+'mill',
+'national park',
+'plant',
+'cop',
+'power line',
+'perch',
+'blue',
+'finger',
+'ferris wheel',
+'globe',
+'skateboard',
+'helmet',
+'movie theater',
+'uniform',
+'hammer',
+'material',
+'kid',
+'well',
+'butterfly',
+'sideline',
+'fashion fall show',
+'planet earth',
+'lift',
+'male',
+'sauna',
+'gray',
+'flour',
+'sand sculpture',
+'program',
+'cabinet',
+'infant',
+'wheel',
+'aircraft model',
+'dough',
+'garlic',
+'skate',
+'arrow',
+'wrapping paper',
+'ripple',
+'lamp',
+'iron',
+'banknote',
+'beaver',
+'ferry',
+'courtyard',
+'bassist',
+'countryside',
+'steak',
+'comfort',
+'boxer',
+'laundry room',
+'campsite',
+'brick building',
+'golf',
+'subway',
+'headphone',
+'fort',
+'handbag',
+'drum',
+'flood',
+'saddle',
+'bass',
+'labyrinth',
+'needle',
+'sun ray',
+'app',
+'menu',
+'president',
+'cardigan',
+'dandelion',
+'wetland',
+'ice hockey player',
+'number',
+'city hall',
+'fishing',
+'portrait session',
+'pug',
+'key',
+'art print',
+'minister',
+'hurdle',
+'emergency',
+'painting artist',
+'flag pole',
+'evening',
+'purse',
+'recipe',
+'golf ball',
+'coloring book',
+'mountain peak',
+'senior',
+'holiday',
+'bud',
+'cousin',
+'pantry',
+'lap',
+'skin',
+'flag',
+'tissue paper',
+'ridge',
+'wire fence',
+'surfer',
+'climber',
+'photograph',
+'sewing machine',
+'cooler',
+'actress',
+'apple tree',
+'cancer',
+'starfish',
+'automobile make',
+'dumbbell',
+'brace',
+'tunnel',
+'window',
+'paint artist',
+'composition',
+'school student',
+'condo',
+'convertible',
+'cushion',
+'selfie',
+'territory',
+'guide',
+'tree',
+'court',
+'shrimp',
+'stone house',
+'dress',
+'eyelash',
+'juice',
+'broccoli',
+'chain',
+'tourism',
+'mountain top',
+'concept car',
+'film premiere',
+'light bulb',
+'cafeteria',
+'badge',
+'flower bed',
+'theater',
+'root',
+'racecar driver',
+'basketball boy game',
+'glove',
+'skyline',
+'wall',
+'glacier',
+'airport terminal',
+'bug',
+'trim',
+'railway station',
+'briefcase',
+'flat',
+'fountain',
+'person',
+'lane',
+'asparagus',
+'art',
+'lantern',
+'dishwasher',
+'director',
+'snake',
+'lecture',
+'game controller',
+'tree branch',
+'pub',
+'bathing suit',
+'queue',
+'belly',
+'poppy',
+'bow',
+'pitcher',
+'ice cream cone',
+'cave',
+'candy',
+'road bridge',
+'host',
+'traffic jam',
+'earring',
+'file',
+'foot',
+'watermark overlay stamp',
+'mailbox',
+'supercar',
+'railing',
+'bedroom',
+'seafood',
+'waffle',
+'bronze statue',
+'plan',
+'flow',
+'marble',
+'basketball game',
+'automobile',
+'scene',
+'cypress tree',
+'soldier',
+'skateboarder',
+'glass building',
+'cherry tree',
+'pump',
+'grain',
+'wildebeest',
+'loop',
+'frame',
+'bathtub',
+'saxophone',
+'diver',
+'stalk',
+'lily',
+'bead',
+'alley',
+'flock',
+'family room',
+'manufacturing',
+'pointer',
+'worker',
+'navy',
+'potato',
+'teacher',
+'photography',
+'dolly',
+'boardwalk',
+'water fountain',
+'athlete',
+'side dish',
+'bay',
+'ice hockey',
+'phone',
+'hero',
+'face',
+'gold medal',
+'blind',
+'swamp',
+'researcher',
+'swim',
+'meatball',
+'iguana',
+'leather jacket',
+'jellyfish',
+'site',
+'smoke',
+'traffic signal',
+'melon',
+'beetle',
+'calculator',
+'skirt',
+'plantation',
+'sculptor',
+'barrier',
+'catcher',
+'security guard',
+'sketch',
+'awning',
+'steering wheel',
+'mountain view',
+'bus stop',
+'pool',
+'leg',
+'spotlight',
+'apron',
+'mineral',
+'inlet',
+'sleeve',
+'torch',
+'emotion',
+'march',
+'police officer',
+'performance',
+'lamp post',
+'fishing boat',
+'summer',
+'presentation',
+'saucer',
+'suitcase',
+'supermodel',
+'goalkeeper',
+'shrub',
+'rock artist',
+'document',
+'beach house',
+'man',
+'blue artist',
+'cigar',
+'railroad track',
+'gown',
+'mosaic',
+'bungalow',
+'alphabet',
+'baseball field',
+'shed',
+'pedestrian',
+'rail',
+'soap',
+'kitchen counter',
+'dessert',
+'dunk',
+'blossom',
+'conversation',
+'fruit market',
+'glass jar',
+'military',
+'beer bottle',
+'photographer',
+'tennis racket',
+'competition',
+'escalator',
+'bell tower',
+'stilt',
+'ballerina',
+'television',
+'feather',
+'fence post',
+'rear',
+'dahlia',
+'red carpet',
+'tub',
+'hole',
+'fortress',
+'pack',
+'telephone',
+'cardboard',
+'city park',
+'platform',
+'college student',
+'arch bridge',
+'wind',
+'blender',
+'bloom',
+'ice rink',
+'birthday',
+'raven',
+'fairy',
+'embankment',
+'hall',
+'flower shop',
+'suburb',
+'barrel',
+'biker',
+'steam',
+'dragonfly',
+'formation',
+'electricity',
+'business people',
+'symmetry',
+'walkway',
+'fisherman',
+'gas mask',
+'loch',
+'youth',
+'hanger',
+'dot',
+'fish',
+'street market',
+'animation film',
+'crime fiction film',
+'boar',
+'emblem',
+'halloween costume',
+'kangaroo',
+'couple',
+'spoon',
+'squirrel',
+'neon sign',
+'sky',
+'office desk',
+'beauty salon',
+'breakwater',
+'fashion look',
+'toaster',
+'author',
+'news conference',
+'outdoor',
+'canoe',
+'dragon',
+'tool',
+'shopping centre',
+'ladybug',
+'swimming pool',
+'landscaping',
+'ski pole',
+'red',
+'truck',
+'fly',
+'temple',
+'level',
+'sunday',
+'railroad bridge',
+'car mirror',
+'lawn mower',
+'flute',
+'aircraft carrier',
+'fashion menswear london week',
+'sunshine',
+'tile floor',
+'skull',
+'fossil',
+'flower arrangement',
+'diaper',
+'sea turtle',
+'cherry blossom',
+'fireman',
+'shack',
+'lens',
+'waiter',
+'animal',
+'basement',
+'snow',
+'autumn park',
+'glass box',
+'kick',
+'head',
+'anniversary',
+'vine',
+'back',
+'paper lantern',
+'fish tank',
+'cellphone',
+'silk',
+'coral',
+'notebook',
+'photo',
+'gazebo',
+'ketchup',
+'driver',
+'farmer',
+'bonfire',
+'chestnut',
+'photoshoot',
+'football field',
+'olive tree',
+'pheasant',
+'sandal',
+'toilet',
+'fireplace',
+'music',
+'deity',
+'fish market',
+'fig',
+'bell',
+'neck',
+'grave',
+'villa',
+'cyclist',
+'crate',
+'grey',
+'asphalt road',
+'soccer',
+'hostel',
+'municipality',
+'courthouse',
+'roof',
+'end table',
+'pot',
+'sedan',
+'structure',
+'folk artist',
+'sport',
+'sport team',
+'protest',
+'syringe',
+'fashion designer',
+'jersey',
+'heart shape',
+'kayak',
+'stare',
+'sit with',
+'direct',
+'read',
+'photograph',
+'spin',
+'teach',
+'laugh',
+'carve',
+'grow on',
+'warm',
+'watch',
+'stretch',
+'smell',
+'decorate',
+'shine',
+'light',
+'dance',
+'send',
+'park',
+'chase',
+'collect',
+'lead',
+'kiss',
+'lead to',
+'lick',
+'smile',
+'cheer',
+'sit',
+'point',
+'block',
+'rock',
+'drop',
+'cut',
+'ski',
+'wrap',
+'lose',
+'serve',
+'provide',
+'sleep',
+'dress',
+'embrace',
+'burn',
+'pack',
+'stir',
+'create',
+'touch',
+'wash',
+'stick',
+'reveal',
+'shop',
+'train',
+'paint',
+'groom',
+'hunt',
+'bloom',
+'play',
+'pay',
+'brush',
+'shoot',
+'hold',
+'picture',
+'carry',
+'sip',
+'contain',
+'turn',
+'pour',
+'pitch',
+'give',
+'add',
+'blow',
+'look in',
+'show',
+'walk',
+'illuminate',
+'kneel',
+'cover',
+'drag',
+'post',
+'present',
+'fit',
+'operate',
+'fish',
+'race',
+'write',
+'deliver',
+'peel',
+'push',
+'run',
+'sit around',
+'buy',
+'jump',
+'walk on',
+'attend',
+'clean',
+'sell',
+'ride on',
+'mount',
+'host',
+'dry',
+'plant',
+'sing',
+'row',
+'shake',
+'perch',
+'ride',
+'fight',
+'skateboard',
+'live',
+'call',
+'surround',
+'practice',
+'play on',
+'work on',
+'step',
+'relax',
+'hit',
+'fall in',
+'flow',
+'greet',
+'launch',
+'wear',
+'hang on',
+'drive',
+'sit in',
+'break',
+'learn',
+'fly',
+'connect',
+'display',
+'locate',
+'compete',
+'go for',
+'sail',
+'lift',
+'toast',
+'help',
+'run on',
+'reflect',
+'pose',
+'scratch',
+'frame',
+'dribble',
+'herd',
+'enter',
+'exit',
+'place',
+'inspect',
+'build',
+'pick',
+'fill',
+'grind',
+'skate',
+'offer',
+'float',
+'sit by',
+'stand',
+'release',
+'rest',
+'singe',
+'climb',
+'tie',
+'mark',
+'lay',
+'stand around',
+'capture',
+'set',
+'land',
+'swinge',
+'run in',
+'kick',
+'lean',
+'head',
+'sign',
+'approach',
+'swim',
+'close',
+'crash',
+'control',
+'fall',
+'remove',
+'repair',
+'open',
+'appear',
+'travel',
+'load',
+'miss',
+'check',
+'surf',
+'moor',
+'smoke',
+'drink',
+'board',
+'seat',
+'feed',
+'rise',
+'sit on',
+'swing',
+'grow',
+'strike',
+'date',
+'slide',
+'share',
+'graze',
+'jump in',
+'lie',
+'extrude',
+'roll',
+'move',
+'gather',
+'eat',
+'pull',
+'run through',
+'squeeze',
+'lay on',
+'draw',
+'play with',
+'wave',
+'assemble',
+'perform',
+'march',
+'score',
+'attach',
+'adjust',
+'hang',
+'hug',
+'sleep on',
+'throw',
+'live in',
+'talk',
+'pet',
+'work',
+'run with',
+'see',
+'flip',
+'catch',
+'cook',
+'receive',
+'celebrate',
+'look',
+'classic',
+'bridal',
+'indoor',
+'industrial',
+'teenage',
+'mini',
+'grassy',
+'aged',
+'long',
+'warm',
+'light',
+'handsome',
+'happy',
+'three',
+'pregnant',
+'circular',
+'urban',
+'silver',
+'ceramic',
+'3d',
+'green',
+'blonde',
+'golden',
+'dark',
+'tropical',
+'ripe',
+'deep',
+'fat',
+'musical',
+'giant',
+'medical',
+'medieval',
+'bare',
+'stunning',
+'bold',
+'geographical',
+'huge',
+'plastic',
+'foggy',
+'stormy',
+'gothic',
+'biological',
+'empty',
+'clear',
+'antique',
+'pink',
+'steep',
+'brown',
+'striped',
+'aerial',
+'rainy',
+'cool',
+'flying',
+'commercial',
+'purple',
+'trendy',
+'blank',
+'haired',
+'dead',
+'wooden',
+'flat',
+'high',
+'beige',
+'panoramic',
+'angry',
+'dozen',
+'rural',
+'solar',
+'big',
+'small',
+'stained',
+'thick',
+'many',
+'fresh',
+'clean',
+'strong',
+'abstract',
+'crowded',
+'retro',
+'dry',
+'gorgeous',
+'martial',
+'modern',
+'blue',
+'cloudy',
+'low',
+'four',
+'outdoor',
+'single',
+'much',
+'beautiful',
+'snowy',
+'pretty',
+'new',
+'short',
+'sunny',
+'closed',
+'rocky',
+'red',
+'two',
+'double',
+'male',
+'gray',
+'five',
+'colorful',
+'automotive',
+'various',
+'one',
+'old',
+'rusty',
+'tall',
+'wild',
+'narrow',
+'natural',
+'several',
+'frozen',
+'textured',
+'lush',
+'young',
+'hot',
+'mixed',
+'white',
+'float',
+'quiet',
+'round',
+'bright',
+'religious',
+'female',
+'historical',
+'shiny',
+'traditional',
+'tourist',
+'yellow',
+'bald',
+'coastal',
+'lovely',
+'little',
+'broken',
+'romantic',
+'wide',
+'royal',
+'rich',
+'open',
+'cute',
+'ancient',
+'cold',
+'political',
+'elderly',
+'gold',
+'full',
+'rustic',
+'metallic',
+'floral',
+'sad',
+'wet',
+'fancy',
+'senior',
+'tiny',
+'stylish',
+'large',
+'frosty',
+'orange',
+'transparent',
+'electronic',
+'shallow',
+'scared',
+'armed',
+'dirty',
+'historic',
+'black',
+'few',
+'windy',
+'some',
+'square',
+'ornamental',
+'sandy',
+'thin']
+
+
+tra_array = np.array(tra_array)
+
+

environment.yaml

@@ -0,0 +1,288 @@
+name: py38
+channels:
+  - pytorch
+  - nvidia
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - appdirs=1.4.4=pyhd3eb1b0_0
+  - asttokens=2.0.5=pyhd3eb1b0_0
+  - backcall=0.2.0=pyhd3eb1b0_0
+  - blas=1.0=mkl
+  - brotlipy=0.7.0=py38h27cfd23_1003
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2023.01.10=h06a4308_0
+  - certifi=2022.12.7=py38h06a4308_0
+  - cffi=1.15.1=py38h5eee18b_3
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - comm=0.1.2=py38h06a4308_0
+  - cryptography=39.0.1=py38h9ce1e76_0
+  - cuda-cudart=11.7.99=0
+  - cuda-cupti=11.7.101=0
+  - cuda-libraries=11.7.1=0
+  - cuda-nvrtc=11.7.99=0
+  - cuda-nvtx=11.7.91=0
+  - cuda-runtime=11.7.1=0
+  - debugpy=1.5.1=py38h295c915_0
+  - executing=0.8.3=pyhd3eb1b0_0
+  - ffmpeg=4.3=hf484d3e_0
+  - flit-core=3.8.0=py38h06a4308_0
+  - freetype=2.12.1=h4a9f257_0
+  - giflib=5.2.1=h5eee18b_3
+  - gmp=6.2.1=h295c915_3
+  - gnutls=3.6.15=he1e5248_0
+  - idna=3.4=py38h06a4308_0
+  - importlib_metadata=6.0.0=hd3eb1b0_0
+  - intel-openmp=2021.4.0=h06a4308_3561
+  - ipykernel=6.19.2=py38hb070fc8_0
+  - ipython=8.12.0=py38h06a4308_0
+  - jedi=0.18.1=py38h06a4308_1
+  - jpeg=9e=h5eee18b_1
+  - jupyter_client=8.1.0=py38h06a4308_0
+  - jupyter_core=5.3.0=py38h06a4308_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libcublas=11.10.3.66=0
+  - libcufft=10.7.2.124=h4fbf590_0
+  - libcufile=1.6.0.25=0
+  - libcurand=10.3.2.56=0
+  - libcusolver=11.4.0.1=0
+  - libcusparse=11.7.4.91=0
+  - libdeflate=1.17=h5eee18b_0
+  - libffi=3.4.2=h6a678d5_6
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgfortran-ng=11.2.0=h00389a5_1
+  - libgfortran5=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libiconv=1.16=h7f8727e_2
+  - libidn2=2.3.2=h7f8727e_0
+  - libnpp=11.7.4.75=0
+  - libnvjpeg=11.8.0.2=0
+  - libpng=1.6.39=h5eee18b_0
+  - libsodium=1.0.18=h7b6447c_0
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.5.0=h6a678d5_2
+  - libunistring=0.9.10=h27cfd23_0
+  - libwebp=1.2.4=h11a3e52_1
+  - libwebp-base=1.2.4=h5eee18b_1
+  - lz4-c=1.9.4=h6a678d5_0
+  - matplotlib-inline=0.1.6=py38h06a4308_0
+  - mkl=2021.4.0=h06a4308_640
+  - mkl-service=2.4.0=py38h7f8727e_0
+  - mkl_fft=1.3.1=py38hd3c417c_0
+  - mkl_random=1.2.2=py38h51133e4_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.6=py38h06a4308_0
+  - nettle=3.7.3=hbbd107a_1
+  - numpy=1.23.5=py38h14f4228_0
+  - numpy-base=1.23.5=py38h31eccc5_0
+  - openh264=2.1.1=h4ff587b_0
+  - openssl=1.1.1t=h7f8727e_0
+  - packaging=23.0=py38h06a4308_0
+  - parso=0.8.3=pyhd3eb1b0_0
+  - pexpect=4.8.0=pyhd3eb1b0_3
+  - pickleshare=0.7.5=pyhd3eb1b0_1003
+  - pillow=9.4.0=py38h6a678d5_0
+  - pip=23.0.1=py38h06a4308_0
+  - platformdirs=2.5.2=py38h06a4308_0
+  - pooch=1.4.0=pyhd3eb1b0_0
+  - prompt-toolkit=3.0.36=py38h06a4308_0
+  - psutil=5.9.0=py38h5eee18b_0
+  - ptyprocess=0.7.0=pyhd3eb1b0_2
+  - pure_eval=0.2.2=pyhd3eb1b0_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyopenssl=23.0.0=py38h06a4308_0
+  - pysocks=1.7.1=py38h06a4308_0
+  - python=3.8.16=h7a1cb2a_3
+  - python-dateutil=2.8.2=pyhd3eb1b0_0
+  - pytorch=1.13.1=py3.8_cuda11.7_cudnn8.5.0_0
+  - pytorch-cuda=11.7=h778d358_3
+  - pytorch-mutex=1.0=cuda
+  - pyzmq=23.2.0=py38h6a678d5_0
+  - readline=8.2=h5eee18b_0
+  - requests=2.28.1=py38h06a4308_1
+  - scipy=1.10.1=py38h14f4228_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.2.0=pyhd3eb1b0_0
+  - tk=8.6.12=h1ccaba5_0
+  - torchaudio=0.13.1=py38_cu117
+  - torchvision=0.14.1=py38_cu117
+  - tornado=6.2=py38h5eee18b_0
+  - traitlets=5.7.1=py38h06a4308_0
+  - typing_extensions=4.4.0=py38h06a4308_0
+  - urllib3=1.26.15=py38h06a4308_0
+  - wcwidth=0.2.5=pyhd3eb1b0_0
+  - wheel=0.38.4=py38h06a4308_0
+  - xz=5.2.10=h5eee18b_1
+  - zeromq=4.3.4=h2531618_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+    - absl-py==1.4.0
+    - accelerate==0.18.0
+    - addict==2.4.0
+    - aiofiles==23.1.0
+    - aiohttp==3.8.4
+    - aiosignal==1.3.1
+    - altair==4.2.2
+    - antlr4-python3-runtime==4.9.3
+    - anyio==3.6.2
+    - async-timeout==4.0.2
+    - attrs==22.2.0
+    - av==10.0.0
+    - bitsandbytes==0.37.0
+    - black==23.3.0
+    - blis==0.7.9
+    - boto3==1.26.111
+    - botocore==1.29.111
+    - braceexpand==0.1.7
+    - cachetools==5.3.0
+    - catalogue==2.0.8
+    - click==8.1.3
+    - cloudpickle==2.2.1
+    - colorama==0.4.6
+    - confection==0.0.4
+    - contourpy==1.0.7
+    - cycler==0.11.0
+    - cymem==2.0.7
+    - cython==0.29.34
+    - dataclasses-json==0.5.7
+    - decorator==4.4.2
+    - decord==0.6.0
+    - detectron2==0.6
+    - einops==0.3.0
+    - entrypoints==0.4
+    - fairscale==0.4.13
+    - fastapi==0.95.0
+    - ffmpy==0.3.0
+    - filelock==3.11.0
+    - fire==0.5.0
+    - fonttools==4.39.3
+    - frozenlist==1.3.3
+    - fsspec==2023.3.0
+    - future==0.18.3
+    - fvcore==0.1.5.post20221221
+    - google-auth==2.17.3
+    - google-auth-oauthlib==0.4.6
+    - gradio==3.25.0
+    - gradio-client==0.1.0
+    - greenlet==2.0.2
+    - grpcio==1.53.0
+    - h11==0.14.0
+    - httpcore==0.16.3
+    - httpx==0.23.3
+    - huggingface-hub==0.13.3
+    - hydra-core==1.3.2
+    - imageio==2.9.0
+    - imageio-ffmpeg==0.4.2
+    - importlib-metadata==6.1.0
+    - importlib-resources==5.12.0
+    - iopath==0.1.9
+    - ipdb==0.13.13
+    - jinja2==3.1.2
+    - jmespath==1.0.1
+    - joblib==1.2.0
+    - jsonschema==4.17.3
+    - kiwisolver==1.4.4
+    - langchain==0.0.101
+    - langcodes==3.3.0
+    - lightning-utilities==0.3.0
+    - linkify-it-py==2.0.0
+    - lvis==0.5.3
+    - markdown==3.4.3
+    - markdown-it-py==2.2.0
+    - markupsafe==2.1.2
+    - marshmallow==3.19.0
+    - marshmallow-enum==1.5.1
+    - matplotlib==3.7.1
+    - mdit-py-plugins==0.3.3
+    - mdurl==0.1.2
+    - mmcv==1.7.1
+    - model-index==0.1.11
+    - moviepy==1.0.3
+    - multidict==6.0.4
+    - murmurhash==1.0.9
+    - mypy-extensions==1.0.0
+    - nltk==3.8.1
+    - oauthlib==3.2.2
+    - omegaconf==2.3.0
+    - openai==0.27.2
+    - openapi-schema-pydantic==1.2.4
+    - opencv-python==4.1.2.30
+    - openmim==0.3.7
+    - ordered-set==4.1.0
+    - orjson==3.8.9
+    - pandas==1.5.3
+    - pathspec==0.11.1
+    - pathy==0.10.1
+    - pkgutil-resolve-name==1.3.10
+    - portalocker==2.7.0
+    - preshed==3.0.8
+    - proglog==0.1.10
+    - protobuf==3.20.3
+    - pyasn1==0.4.8
+    - pyasn1-modules==0.2.8
+    - pycocotools==2.0.2
+    - pydantic==1.10.7
+    - pydeprecate==0.3.1
+    - pydub==0.25.1
+    - pygments==2.14.0
+    - pyparsing==3.0.9
+    - pyre-extensions==0.0.23
+    - pyrsistent==0.19.3
+    - python-multipart==0.0.6
+    - pytorch-lightning==1.8.3
+    - pytz==2023.3
+    - pyyaml==6.0
+    - regex==2023.3.23
+    - requests-oauthlib==1.3.1
+    - rfc3986==1.5.0
+    - rich==13.3.3
+    - rsa==4.9
+    - s3transfer==0.6.0
+    - sacremoses==0.0.53
+    - semantic-version==2.10.0
+    - sentencepiece==0.1.97
+    - setuptools==59.5.0
+    - simplet5==0.1.4
+    - smart-open==6.3.0
+    - sniffio==1.3.0
+    - spacy==3.5.1
+    - spacy-legacy==3.0.12
+    - spacy-loggers==1.0.4
+    - sqlalchemy==1.4.47
+    - srsly==2.4.6
+    - starlette==0.26.1
+    - tabulate==0.9.0
+    - tenacity==8.2.2
+    - tensorboard==2.12.0
+    - tensorboard-data-server==0.7.0
+    - tensorboard-plugin-wit==1.8.1
+    - tensorboardx==2.6
+    - termcolor==2.2.0
+    - thinc==8.1.9
+    - timm==0.6.13
+    - tokenizers==0.10.3
+    - tomli==2.0.1
+    - toolz==0.12.0
+    - torchmetrics==0.11.4
+    - tqdm==4.65.0
+    - transformers==4.15.0
+    - typer==0.7.0
+    - typing-inspect==0.8.0
+    - uc-micro-py==1.0.1
+    - uvicorn==0.21.1
+    - wasabi==1.1.1
+    - webdataset==0.2.48
+    - websockets==10.4
+    - werkzeug==2.2.3
+    - xformers==0.0.13
+    - yacs==0.1.8
+    - yapf==0.32.0
+    - yarl==1.8.2
+    - zipp==3.15.0

intern_action.py

@@ -0,0 +1,510 @@
+#!/usr/bin/env python
+import os
+from collections import OrderedDict
+
+from timm.models.layers import DropPath
+import torch
+from torch import nn
+from torch.nn import MultiheadAttention
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+
+
+MODEL_PATH = './'
+_MODELS = {
+    "ViT-B/16": os.path.join(MODEL_PATH, "vit_b16.pth"),
+    "ViT-L/14": os.path.join(MODEL_PATH, "vit_l14.pth"),
+    "ViT-L/14_336": os.path.join(MODEL_PATH, "vit_l14_336.pth"),
+}
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class Local_MHRA(nn.Module):
+    def __init__(self, d_model, dw_reduction=1.5, pos_kernel_size=3):
+        super().__init__() 
+
+        padding = pos_kernel_size // 2
+        re_d_model = int(d_model // dw_reduction)
+        self.pos_embed = nn.Sequential(
+            nn.BatchNorm3d(d_model),
+            nn.Conv3d(d_model, re_d_model, kernel_size=1, stride=1, padding=0),
+            nn.Conv3d(re_d_model, re_d_model, kernel_size=(pos_kernel_size, 1, 1), stride=(1, 1, 1), padding=(padding, 0, 0), groups=re_d_model),
+            nn.Conv3d(re_d_model, d_model, kernel_size=1, stride=1, padding=0),
+        )
+
+        # init zero
+        print('Init zero for Conv in pos_emb')
+        nn.init.constant_(self.pos_embed[3].weight, 0)
+        nn.init.constant_(self.pos_embed[3].bias, 0)
+
+    def forward(self, x):
+        return self.pos_embed(x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(
+            self, d_model, n_head, attn_mask=None, drop_path=0.0, 
+            dw_reduction=1.5, no_lmhra=False, double_lmhra=True
+        ):
+        super().__init__() 
+        
+        self.n_head = n_head
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        print(f'Drop path rate: {drop_path}')
+
+        self.no_lmhra = no_lmhra
+        self.double_lmhra = double_lmhra
+        print(f'No L_MHRA: {no_lmhra}')
+        print(f'Double L_MHRA: {double_lmhra}')
+        if not no_lmhra:
+            self.lmhra1 = Local_MHRA(d_model, dw_reduction=dw_reduction)
+            if double_lmhra:
+                self.lmhra2 = Local_MHRA(d_model, dw_reduction=dw_reduction)
+
+        # spatial
+        self.attn = MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x):
+        self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
+
+    def forward(self, x, T=8, use_checkpoint=False):
+        # x: 1+HW, NT, C
+        if not self.no_lmhra:
+            # Local MHRA
+            tmp_x = x[1:, :, :]
+            L, NT, C = tmp_x.shape
+            N = NT // T
+            H = W = int(L ** 0.5)
+            tmp_x = tmp_x.view(H, W, N, T, C).permute(2, 4, 3, 0, 1).contiguous()
+            tmp_x = tmp_x + self.drop_path(self.lmhra1(tmp_x))
+            tmp_x = tmp_x.view(N, C, T, L).permute(3, 0, 2, 1).contiguous().view(L, NT, C)
+            x = torch.cat([x[:1, :, :], tmp_x], dim=0)
+        # MHSA
+        if use_checkpoint:
+            attn_out = checkpoint.checkpoint(self.attention, self.ln_1(x))
+            x = x + self.drop_path(attn_out)
+        else:
+            x = x + self.drop_path(self.attention(self.ln_1(x)))
+        # Local MHRA
+        if not self.no_lmhra and self.double_lmhra:
+            tmp_x = x[1:, :, :]
+            tmp_x = tmp_x.view(H, W, N, T, C).permute(2, 4, 3, 0, 1).contiguous()
+            tmp_x = tmp_x + self.drop_path(self.lmhra2(tmp_x))
+            tmp_x = tmp_x.view(N, C, T, L).permute(3, 0, 2, 1).contiguous().view(L, NT, C)
+            x = torch.cat([x[:1, :, :], tmp_x], dim=0)
+        # FFN
+        if use_checkpoint:
+            mlp_out = checkpoint.checkpoint(self.mlp, self.ln_2(x))
+            x = x + self.drop_path(mlp_out)
+        else:
+            x = x + self.drop_path(self.mlp(self.ln_2(x)))
+        return x
+
+
+class Extractor(nn.Module):
+    def __init__(
+            self, d_model, n_head, attn_mask=None,
+            mlp_factor=4.0, dropout=0.0, drop_path=0.0,
+        ):
+        super().__init__()
+
+        self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+        print(f'Drop path rate: {drop_path}')
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = nn.LayerNorm(d_model)
+        d_mlp = round(mlp_factor * d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_mlp)),
+            ("gelu", QuickGELU()),
+            ("dropout", nn.Dropout(dropout)),
+            ("c_proj", nn.Linear(d_mlp, d_model))
+        ]))
+        self.ln_2 = nn.LayerNorm(d_model)
+        self.ln_3 = nn.LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+        # zero init
+        nn.init.xavier_uniform_(self.attn.in_proj_weight)
+        nn.init.constant_(self.attn.out_proj.weight, 0.)
+        nn.init.constant_(self.attn.out_proj.bias, 0.)
+        nn.init.xavier_uniform_(self.mlp[0].weight)
+        nn.init.constant_(self.mlp[-1].weight, 0.)
+        nn.init.constant_(self.mlp[-1].bias, 0.)
+
+    def attention(self, x, y):
+        d_model = self.ln_1.weight.size(0)
+        q = (x @ self.attn.in_proj_weight[:d_model].T) + self.attn.in_proj_bias[:d_model]
+
+        k = (y @ self.attn.in_proj_weight[d_model:-d_model].T) + self.attn.in_proj_bias[d_model:-d_model]
+        v = (y @ self.attn.in_proj_weight[-d_model:].T) + self.attn.in_proj_bias[-d_model:]
+        Tx, Ty, N = q.size(0), k.size(0), q.size(1)
+        q = q.view(Tx, N, self.attn.num_heads, self.attn.head_dim).permute(1, 2, 0, 3)
+        k = k.view(Ty, N, self.attn.num_heads, self.attn.head_dim).permute(1, 2, 0, 3)
+        v = v.view(Ty, N, self.attn.num_heads, self.attn.head_dim).permute(1, 2, 0, 3)
+        aff = (q @ k.transpose(-2, -1) / (self.attn.head_dim ** 0.5))
+
+        aff = aff.softmax(dim=-1)
+        out = aff @ v
+        out = out.permute(2, 0, 1, 3).flatten(2)
+        out = self.attn.out_proj(out)
+        return out
+
+    def forward(self, x, y):
+        x = x + self.drop_path(self.attention(self.ln_1(x), self.ln_3(y)))
+        x = x + self.drop_path(self.mlp(self.ln_2(x)))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(
+            self, width, layers, heads, attn_mask=None, backbone_drop_path_rate=0., 
+            use_checkpoint=False, checkpoint_num=[0], t_size=8, dw_reduction=2,
+            no_lmhra=False, double_lmhra=True,
+            return_list=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
+            n_layers=12, n_dim=768, n_head=12, mlp_factor=4.0, drop_path_rate=0.,
+            mlp_dropout=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], 
+            cls_dropout=0.5, num_classes=400,
+        ):
+        super().__init__()
+        self.T = t_size
+        self.return_list = return_list
+        # backbone
+        b_dpr = [x.item() for x in torch.linspace(0, backbone_drop_path_rate, layers)]
+        self.resblocks = nn.ModuleList([
+            ResidualAttentionBlock(
+                width, heads, attn_mask, 
+                drop_path=b_dpr[i],
+                dw_reduction=dw_reduction,
+                no_lmhra=no_lmhra,
+                double_lmhra=double_lmhra,
+            ) for i in range(layers)
+        ])
+        # checkpoint
+        self.use_checkpoint = use_checkpoint
+        self.checkpoint_num = checkpoint_num
+        self.n_layers = n_layers
+        print(f'Use checkpoint: {self.use_checkpoint}')
+        print(f'Checkpoint number: {self.checkpoint_num}')
+
+        # global block
+        assert n_layers == len(return_list)
+        if n_layers > 0:
+            self.temporal_cls_token = nn.Parameter(torch.zeros(1, 1, n_dim))
+            self.dpe = nn.ModuleList([
+                nn.Conv3d(n_dim, n_dim, kernel_size=3, stride=1, padding=1, bias=True, groups=n_dim)
+                for i in range(n_layers)
+            ])
+            for m in self.dpe:
+                nn.init.constant_(m.bias, 0.)
+            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, n_layers)]
+            self.dec = nn.ModuleList([
+                Extractor(
+                    n_dim, n_head, mlp_factor=mlp_factor, 
+                    dropout=mlp_dropout[i], drop_path=dpr[i],
+                ) for i in range(n_layers)
+            ])
+            self.balance = nn.Parameter(torch.zeros((n_dim)))
+            self.sigmoid = nn.Sigmoid()
+        # projection
+        self.proj = nn.Sequential(
+            nn.LayerNorm(n_dim),
+            nn.Dropout(cls_dropout),
+            nn.Linear(n_dim, num_classes),
+        )
+
+    def forward(self, x):
+        T_down = self.T
+        L, NT, C = x.shape
+        N = NT // T_down
+        H = W = int((L - 1) ** 0.5)
+
+        if self.n_layers > 0:
+            cls_token = self.temporal_cls_token.repeat(1, N, 1)
+
+        j = -1
+        for i, resblock in enumerate(self.resblocks):
+            if self.use_checkpoint and i < self.checkpoint_num[0]:
+                x = resblock(x, self.T, use_checkpoint=True)
+            else:
+                x = resblock(x, T_down)
+            if i in self.return_list:
+                j += 1
+                tmp_x = x.clone()
+                tmp_x = tmp_x.view(L, N, T_down, C)
+                # dpe
+                _, tmp_feats = tmp_x[:1], tmp_x[1:]
+                tmp_feats = tmp_feats.permute(1, 3, 2, 0).reshape(N, C, T_down, H, W)
+                tmp_feats = self.dpe[j](tmp_feats).view(N, C, T_down, L - 1).permute(3, 0, 2, 1).contiguous()
+                tmp_x[1:] = tmp_x[1:] + tmp_feats
+                # global block
+                tmp_x = tmp_x.permute(2, 0, 1, 3).flatten(0, 1)  # T * L, N, C
+                cls_token = self.dec[j](cls_token, tmp_x)
+
+        if self.n_layers > 0:
+            weight = self.sigmoid(self.balance)
+            residual = x.view(L, N, T_down, C)[0].mean(1)  # L, N, T, C
+            return self.proj((1 - weight) * cls_token[0, :, :] + weight * residual)
+        else:
+            residual = x.view(L, N, T_down, C)[0].mean(1)  # L, N, T, C
+            return self.proj(residual)
+
+
+class VisionTransformer(nn.Module):
+    def __init__(
+        self, 
+        # backbone
+        input_resolution, patch_size, width, layers, heads, output_dim, backbone_drop_path_rate=0.,
+        use_checkpoint=False, checkpoint_num=[0], t_size=8, kernel_size=3, dw_reduction=1.5,
+        temporal_downsample=True,
+        no_lmhra=-False, double_lmhra=True,
+        # global block
+        return_list=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
+        n_layers=12, n_dim=768, n_head=12, mlp_factor=4.0, drop_path_rate=0.,
+        mlp_dropout=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], 
+        cls_dropout=0.5, num_classes=400,
+    ):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+        padding = (kernel_size - 1) // 2
+        if temporal_downsample:
+            self.conv1 = nn.Conv3d(3, width, (kernel_size, patch_size, patch_size), (2, patch_size, patch_size), (padding, 0, 0), bias=False)
+            t_size = t_size // 2
+        else:
+            self.conv1 = nn.Conv3d(3, width, (1, patch_size, patch_size), (1, patch_size, patch_size), (0, 0, 0), bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+        
+        self.transformer = Transformer(
+            width, layers, heads, dw_reduction=dw_reduction, 
+            backbone_drop_path_rate=backbone_drop_path_rate, 
+            use_checkpoint=use_checkpoint, checkpoint_num=checkpoint_num, t_size=t_size,
+            no_lmhra=no_lmhra, double_lmhra=double_lmhra,
+            return_list=return_list, n_layers=n_layers, n_dim=n_dim, n_head=n_head, 
+            mlp_factor=mlp_factor, drop_path_rate=drop_path_rate, mlp_dropout=mlp_dropout, 
+            cls_dropout=cls_dropout, num_classes=num_classes,
+        )
+
+    def forward(self, x):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        N, C, T, H, W = x.shape
+        x = x.permute(0, 2, 3, 4, 1).reshape(N * T, H * W, C)
+        
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        out = self.transformer(x)
+        return out
+
+
+def inflate_weight(weight_2d, time_dim, center=True):
+    print(f'Init center: {center}')
+    if center:
+        weight_3d = torch.zeros(*weight_2d.shape)
+        weight_3d = weight_3d.unsqueeze(2).repeat(1, 1, time_dim, 1, 1)
+        middle_idx = time_dim // 2
+        weight_3d[:, :, middle_idx, :, :] = weight_2d
+    else:
+        weight_3d = weight_2d.unsqueeze(2).repeat(1, 1, time_dim, 1, 1)
+        weight_3d = weight_3d / time_dim
+    return weight_3d
+
+
+def load_state_dict(model, state_dict):
+    state_dict_3d = model.state_dict()
+    for k in state_dict.keys():
+        if state_dict[k].shape != state_dict_3d[k].shape:
+            if len(state_dict_3d[k].shape) <= 2:
+                print(f'Ignore: {k}')
+                continue
+            print(f'Inflate: {k}, {state_dict[k].shape} => {state_dict_3d[k].shape}')
+            time_dim = state_dict_3d[k].shape[2]
+            state_dict[k] = inflate_weight(state_dict[k], time_dim)
+    model.load_state_dict(state_dict, strict=False)
+
+
+def intern_action_b16(
+    pretrained=True, use_checkpoint=False, checkpoint_num=[0],
+    t_size=16, dw_reduction=1.5, backbone_drop_path_rate=0., 
+    temporal_downsample=True,
+    no_lmhra=False, double_lmhra=True,
+    return_list=[8, 9, 10, 11], 
+    n_layers=4, n_dim=768, n_head=12, mlp_factor=4.0, drop_path_rate=0.,
+    mlp_dropout=[0.5, 0.5, 0.5, 0.5], 
+    cls_dropout=0.5, num_classes=400,
+):
+    model = VisionTransformer(
+        input_resolution=224,
+        patch_size=16,
+        width=768,
+        layers=12,
+        heads=12,
+        output_dim=512,
+        use_checkpoint=use_checkpoint,
+        checkpoint_num=checkpoint_num,
+        t_size=t_size,
+        dw_reduction=dw_reduction, 
+        backbone_drop_path_rate=backbone_drop_path_rate, 
+        temporal_downsample=temporal_downsample,
+        no_lmhra=no_lmhra,
+        double_lmhra=double_lmhra,
+        return_list=return_list, 
+        n_layers=n_layers, 
+        n_dim=n_dim, 
+        n_head=n_head, 
+        mlp_factor=mlp_factor, 
+        drop_path_rate=drop_path_rate, 
+        mlp_dropout=mlp_dropout, 
+        cls_dropout=cls_dropout, 
+        num_classes=num_classes,
+    )
+
+    if pretrained:
+        print('load pretrained weights')
+        state_dict = torch.load(_MODELS["ViT-B/16"], map_location='cpu')
+        load_state_dict(model, state_dict)
+    return model.eval()
+
+
+def intern_action_l14(
+    pretrained=True, use_checkpoint=False, checkpoint_num=[0],
+    t_size=16, dw_reduction=1.5, backbone_drop_path_rate=0., 
+    temporal_downsample=True,
+    no_lmhra=False, double_lmhra=True,
+    return_list=[20, 21, 22, 23],
+    n_layers=4, n_dim=1024, n_head=16, mlp_factor=4.0, drop_path_rate=0.,
+    mlp_dropout=[0.5, 0.5, 0.5, 0.5], 
+    cls_dropout=0.5, num_classes=400,
+):
+    model = VisionTransformer(
+        input_resolution=224,
+        patch_size=14,
+        width=1024,
+        layers=24,
+        heads=16,
+        output_dim=768,
+        use_checkpoint=use_checkpoint,
+        checkpoint_num=checkpoint_num,
+        t_size=t_size,
+        dw_reduction=dw_reduction, 
+        backbone_drop_path_rate=backbone_drop_path_rate, 
+        temporal_downsample=temporal_downsample,
+        no_lmhra=no_lmhra,
+        double_lmhra=double_lmhra,
+        return_list=return_list, 
+        n_layers=n_layers, 
+        n_dim=n_dim, 
+        n_head=n_head, 
+        mlp_factor=mlp_factor, 
+        drop_path_rate=drop_path_rate, 
+        mlp_dropout=mlp_dropout, 
+        cls_dropout=cls_dropout, 
+        num_classes=num_classes,
+    )
+
+    if pretrained:
+        print('load pretrained weights')
+        state_dict = torch.load(_MODELS["ViT-L/14"], map_location='cpu')
+        load_state_dict(model, state_dict)
+    return model.eval()
+
+
+def intern_action_l14_336(
+    pretrained=True, use_checkpoint=False, checkpoint_num=[0],
+    t_size=16, dw_reduction=1.5, backbone_drop_path_rate=0., 
+    no_temporal_downsample=True,
+    no_lmhra=False, double_lmhra=True,
+    return_list=[20, 21, 22, 23],
+    n_layers=4, n_dim=1024, n_head=16, mlp_factor=4.0, drop_path_rate=0.,
+    mlp_dropout=[0.5, 0.5, 0.5, 0.5], 
+    cls_dropout=0.5, num_classes=400,
+):
+    model = VisionTransformer(
+        input_resolution=336,
+        patch_size=14,
+        width=1024,
+        layers=24,
+        heads=16,
+        output_dim=768,
+        use_checkpoint=use_checkpoint,
+        checkpoint_num=checkpoint_num,
+        t_size=t_size,
+        dw_reduction=dw_reduction, 
+        backbone_drop_path_rate=backbone_drop_path_rate, 
+        no_temporal_downsample=no_temporal_downsample,
+        no_lmhra=no_lmhra,
+        double_lmhra=double_lmhra,
+        return_list=return_list, 
+        n_layers=n_layers, 
+        n_dim=n_dim, 
+        n_head=n_head, 
+        mlp_factor=mlp_factor, 
+        drop_path_rate=drop_path_rate, 
+        mlp_dropout=mlp_dropout, 
+        cls_dropout=cls_dropout, 
+        num_classes=num_classes,
+    )
+
+    if pretrained:
+        print('load pretrained weights')
+        state_dict = torch.load(_MODELS["ViT-L/14_336"], map_location='cpu')
+        load_state_dict(model, state_dict)
+    return model.eval()
+
+
+if __name__ == '__main__':
+    import time
+    from fvcore.nn import FlopCountAnalysis
+    from fvcore.nn import flop_count_table
+    import numpy as np
+
+    seed = 4217
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    num_frames = 16
+
+    model = intern_action_l14(
+        pretrained=False, 
+        t_size=num_frames, backbone_drop_path_rate=0., drop_path_rate=0.,
+        dw_reduction=1.5,
+        no_lmhra=False,
+        temporal_downsample=True,
+        return_list=[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
+        mlp_dropout=[0.5]*16,
+        n_layers=16
+    )
+    print(model)
+
+    flops = FlopCountAnalysis(model, torch.rand(1, 3, num_frames, 224, 224))
+    s = time.time()
+    print(flop_count_table(flops, max_depth=1))
+    print(time.time()-s)

load_internvideo.py

@@ -0,0 +1,469 @@
+from intern_action import intern_action_b16
+from huggingface_hub import hf_hub_download
+# from kinetics_class_index import kinetics_classnames
+import torch
+import torch.nn as nn
+import torchvision.transforms as T
+import torch.nn.functional as F
+import numpy as np
+
+from transforms import (
+    GroupNormalize, GroupScale, GroupCenterCrop, 
+    Stack, ToTorchFormatTensor
+)
+
+class Intern_Action(nn.Module):
+    def __init__(self, model):
+        super().__init__()
+        self.backbone = model
+    
+    def forward(self, x):
+        return self.backbone(x)
+
+def get_index(num_frames, num_segments=8):
+    seg_size = float(num_frames - 1) / num_segments
+    start = int(seg_size / 2)
+    offsets = np.array([
+        start + int(np.round(seg_size * idx)) for idx in range(num_segments)
+    ])
+    return offsets
+
+def transform_action():
+    # transform
+    crop_size = 224
+    scale_size = 256
+    input_mean = [0.485, 0.456, 0.406]
+    input_std = [0.229, 0.224, 0.225]
+
+    return T.Compose([
+        # T.ToPILImage(),
+        GroupScale(int(scale_size)),
+        GroupCenterCrop(crop_size),
+        Stack(),
+        ToTorchFormatTensor(),
+        GroupNormalize(input_mean, input_std) 
+    ])
+
+def load_intern_action(device):
+    # Create an id to label name mapping
+    kinetics_id_to_classname = {}
+    for k, v in kinetics_classnames.items():
+        kinetics_id_to_classname[k] = v
+
+    model_path = hf_hub_download(repo_id="Andy1621/uniformerv2", filename="k400+k710_uniformerv2_b16_8x224.pyth")
+    # Pick a pretrained model 
+    model = Intern_Action(intern_action_b16(pretrained=False, t_size=8, no_lmhra=True, temporal_downsample=False))
+    state_dict = torch.load(model_path, map_location=device)
+    model.load_state_dict(state_dict)
+    # Set to eval mode and move to desired device
+    model = model.to(device)
+    model = model.eval()
+    return model
+
+def cut_frame_to_8(data):
+    index = np.linspace(0, len(data)-1, 8).astype(int)
+    return data[index]
+
+kinetics_classnames = {
+    "0": "riding a bike", 
+    "1": "marching", 
+    "2": "dodgeball", 
+    "3": "playing cymbals", 
+    "4": "checking tires", 
+    "5": "roller skating", 
+    "6": "tasting beer", 
+    "7": "clapping", 
+    "8": "drawing", 
+    "9": "juggling fire", 
+    "10": "bobsledding", 
+    "11": "petting animal (not cat)", 
+    "12": "spray painting", 
+    "13": "training dog", 
+    "14": "eating watermelon", 
+    "15": "building cabinet", 
+    "16": "applauding", 
+    "17": "playing harp", 
+    "18": "balloon blowing", 
+    "19": "sled dog racing", 
+    "20": "wrestling", 
+    "21": "pole vault", 
+    "22": "hurling (sport)", 
+    "23": "riding scooter", 
+    "24": "shearing sheep", 
+    "25": "sweeping floor", 
+    "26": "eating carrots", 
+    "27": "skateboarding", 
+    "28": "dunking basketball", 
+    "29": "disc golfing", 
+    "30": "eating spaghetti", 
+    "31": "playing flute", 
+    "32": "riding mechanical bull", 
+    "33": "making sushi", 
+    "34": "trapezing", 
+    "35": "picking fruit", 
+    "36": "stretching leg", 
+    "37": "playing ukulele", 
+    "38": "tying tie", 
+    "39": "skydiving", 
+    "40": "playing cello", 
+    "41": "jumping into pool", 
+    "42": "shooting goal (soccer)", 
+    "43": "trimming trees", 
+    "44": "bookbinding", 
+    "45": "ski jumping", 
+    "46": "walking the dog", 
+    "47": "riding unicycle", 
+    "48": "shaving head", 
+    "49": "hopscotch", 
+    "50": "playing piano", 
+    "51": "parasailing", 
+    "52": "bartending", 
+    "53": "kicking field goal", 
+    "54": "finger snapping", 
+    "55": "dining", 
+    "56": "yawning", 
+    "57": "peeling potatoes", 
+    "58": "canoeing or kayaking", 
+    "59": "front raises", 
+    "60": "laughing", 
+    "61": "dancing macarena", 
+    "62": "digging", 
+    "63": "reading newspaper", 
+    "64": "hitting baseball", 
+    "65": "clay pottery making", 
+    "66": "exercising with an exercise ball", 
+    "67": "playing saxophone", 
+    "68": "shooting basketball", 
+    "69": "washing hair", 
+    "70": "lunge", 
+    "71": "brushing hair", 
+    "72": "curling hair", 
+    "73": "kitesurfing", 
+    "74": "tapping guitar", 
+    "75": "bending back", 
+    "76": "skipping rope", 
+    "77": "situp", 
+    "78": "folding paper", 
+    "79": "cracking neck", 
+    "80": "assembling computer", 
+    "81": "cleaning gutters", 
+    "82": "blowing out candles", 
+    "83": "shaking hands", 
+    "84": "dancing gangnam style", 
+    "85": "windsurfing", 
+    "86": "tap dancing", 
+    "87": "skiing (not slalom or crosscountry)", 
+    "88": "bandaging", 
+    "89": "push up", 
+    "90": "doing nails", 
+    "91": "punching person (boxing)", 
+    "92": "bouncing on trampoline", 
+    "93": "scrambling eggs", 
+    "94": "singing", 
+    "95": "cleaning floor", 
+    "96": "krumping", 
+    "97": "drumming fingers", 
+    "98": "snowmobiling", 
+    "99": "gymnastics tumbling", 
+    "100": "headbanging", 
+    "101": "catching or throwing frisbee", 
+    "102": "riding elephant", 
+    "103": "bee keeping", 
+    "104": "feeding birds", 
+    "105": "snatch weight lifting", 
+    "106": "mowing lawn", 
+    "107": "fixing hair", 
+    "108": "playing trumpet", 
+    "109": "flying kite", 
+    "110": "crossing river", 
+    "111": "swinging legs", 
+    "112": "sanding floor", 
+    "113": "belly dancing", 
+    "114": "sneezing", 
+    "115": "clean and jerk", 
+    "116": "side kick", 
+    "117": "filling eyebrows", 
+    "118": "shuffling cards", 
+    "119": "recording music", 
+    "120": "cartwheeling", 
+    "121": "feeding fish", 
+    "122": "folding clothes", 
+    "123": "water skiing", 
+    "124": "tobogganing", 
+    "125": "blowing leaves", 
+    "126": "smoking", 
+    "127": "unboxing", 
+    "128": "tai chi", 
+    "129": "waxing legs", 
+    "130": "riding camel", 
+    "131": "slapping", 
+    "132": "tossing salad", 
+    "133": "capoeira", 
+    "134": "playing cards", 
+    "135": "playing organ", 
+    "136": "playing violin", 
+    "137": "playing drums", 
+    "138": "tapping pen", 
+    "139": "vault", 
+    "140": "shoveling snow", 
+    "141": "playing tennis", 
+    "142": "getting a tattoo", 
+    "143": "making a sandwich", 
+    "144": "making tea", 
+    "145": "grinding meat", 
+    "146": "squat", 
+    "147": "eating doughnuts", 
+    "148": "ice fishing", 
+    "149": "snowkiting", 
+    "150": "kicking soccer ball", 
+    "151": "playing controller", 
+    "152": "giving or receiving award", 
+    "153": "welding", 
+    "154": "throwing discus", 
+    "155": "throwing axe", 
+    "156": "ripping paper", 
+    "157": "swimming butterfly stroke", 
+    "158": "air drumming", 
+    "159": "blowing nose", 
+    "160": "hockey stop", 
+    "161": "taking a shower", 
+    "162": "bench pressing", 
+    "163": "planting trees", 
+    "164": "pumping fist", 
+    "165": "climbing tree", 
+    "166": "tickling", 
+    "167": "high kick", 
+    "168": "waiting in line", 
+    "169": "slacklining", 
+    "170": "tango dancing", 
+    "171": "hurdling", 
+    "172": "carrying baby", 
+    "173": "celebrating", 
+    "174": "sharpening knives", 
+    "175": "passing American football (in game)", 
+    "176": "headbutting", 
+    "177": "playing recorder", 
+    "178": "brush painting", 
+    "179": "garbage collecting", 
+    "180": "robot dancing", 
+    "181": "shredding paper", 
+    "182": "pumping gas", 
+    "183": "rock climbing", 
+    "184": "hula hooping", 
+    "185": "braiding hair", 
+    "186": "opening present", 
+    "187": "texting", 
+    "188": "decorating the christmas tree", 
+    "189": "answering questions", 
+    "190": "playing keyboard", 
+    "191": "writing", 
+    "192": "bungee jumping", 
+    "193": "sniffing", 
+    "194": "eating burger", 
+    "195": "playing accordion", 
+    "196": "making pizza", 
+    "197": "playing volleyball", 
+    "198": "tasting food", 
+    "199": "pushing cart", 
+    "200": "spinning poi", 
+    "201": "cleaning windows", 
+    "202": "arm wrestling", 
+    "203": "changing oil", 
+    "204": "swimming breast stroke", 
+    "205": "tossing coin", 
+    "206": "deadlifting", 
+    "207": "hoverboarding", 
+    "208": "cutting watermelon", 
+    "209": "cheerleading", 
+    "210": "snorkeling", 
+    "211": "washing hands", 
+    "212": "eating cake", 
+    "213": "pull ups", 
+    "214": "surfing water", 
+    "215": "eating hotdog", 
+    "216": "holding snake", 
+    "217": "playing harmonica", 
+    "218": "ironing", 
+    "219": "cutting nails", 
+    "220": "golf chipping", 
+    "221": "shot put", 
+    "222": "hugging", 
+    "223": "playing clarinet", 
+    "224": "faceplanting", 
+    "225": "trimming or shaving beard", 
+    "226": "drinking shots", 
+    "227": "riding mountain bike", 
+    "228": "tying bow tie", 
+    "229": "swinging on something", 
+    "230": "skiing crosscountry", 
+    "231": "unloading truck", 
+    "232": "cleaning pool", 
+    "233": "jogging", 
+    "234": "ice climbing", 
+    "235": "mopping floor", 
+    "236": "making bed", 
+    "237": "diving cliff", 
+    "238": "washing dishes", 
+    "239": "grooming dog", 
+    "240": "weaving basket", 
+    "241": "frying vegetables", 
+    "242": "stomping grapes", 
+    "243": "moving furniture", 
+    "244": "cooking sausages", 
+    "245": "doing laundry", 
+    "246": "dying hair", 
+    "247": "knitting", 
+    "248": "reading book", 
+    "249": "baby waking up", 
+    "250": "punching bag", 
+    "251": "surfing crowd", 
+    "252": "cooking chicken", 
+    "253": "pushing car", 
+    "254": "springboard diving", 
+    "255": "swing dancing", 
+    "256": "massaging legs", 
+    "257": "beatboxing", 
+    "258": "breading or breadcrumbing", 
+    "259": "somersaulting", 
+    "260": "brushing teeth", 
+    "261": "stretching arm", 
+    "262": "juggling balls", 
+    "263": "massaging person's head", 
+    "264": "eating ice cream", 
+    "265": "extinguishing fire", 
+    "266": "hammer throw", 
+    "267": "whistling", 
+    "268": "crawling baby", 
+    "269": "using remote controller (not gaming)", 
+    "270": "playing cricket", 
+    "271": "opening bottle", 
+    "272": "playing xylophone", 
+    "273": "motorcycling", 
+    "274": "driving car", 
+    "275": "exercising arm", 
+    "276": "passing American football (not in game)", 
+    "277": "playing kickball", 
+    "278": "sticking tongue out", 
+    "279": "flipping pancake", 
+    "280": "catching fish", 
+    "281": "eating chips", 
+    "282": "shaking head", 
+    "283": "sword fighting", 
+    "284": "playing poker", 
+    "285": "cooking on campfire", 
+    "286": "doing aerobics", 
+    "287": "paragliding", 
+    "288": "using segway", 
+    "289": "folding napkins", 
+    "290": "playing bagpipes", 
+    "291": "gargling", 
+    "292": "skiing slalom", 
+    "293": "strumming guitar", 
+    "294": "javelin throw", 
+    "295": "waxing back", 
+    "296": "riding or walking with horse", 
+    "297": "plastering", 
+    "298": "long jump", 
+    "299": "parkour", 
+    "300": "wrapping present", 
+    "301": "egg hunting", 
+    "302": "archery", 
+    "303": "cleaning toilet", 
+    "304": "swimming backstroke", 
+    "305": "snowboarding", 
+    "306": "catching or throwing baseball", 
+    "307": "massaging back", 
+    "308": "blowing glass", 
+    "309": "playing guitar", 
+    "310": "playing chess", 
+    "311": "golf driving", 
+    "312": "presenting weather forecast", 
+    "313": "rock scissors paper", 
+    "314": "high jump", 
+    "315": "baking cookies", 
+    "316": "using computer", 
+    "317": "washing feet", 
+    "318": "arranging flowers", 
+    "319": "playing bass guitar", 
+    "320": "spraying", 
+    "321": "cutting pineapple", 
+    "322": "waxing chest", 
+    "323": "auctioning", 
+    "324": "jetskiing", 
+    "325": "drinking", 
+    "326": "busking", 
+    "327": "playing monopoly", 
+    "328": "salsa dancing", 
+    "329": "waxing eyebrows", 
+    "330": "watering plants", 
+    "331": "zumba", 
+    "332": "chopping wood", 
+    "333": "pushing wheelchair", 
+    "334": "carving pumpkin", 
+    "335": "building shed", 
+    "336": "making jewelry", 
+    "337": "catching or throwing softball", 
+    "338": "bending metal", 
+    "339": "ice skating", 
+    "340": "dancing charleston", 
+    "341": "abseiling", 
+    "342": "climbing a rope", 
+    "343": "crying", 
+    "344": "cleaning shoes", 
+    "345": "dancing ballet", 
+    "346": "driving tractor", 
+    "347": "triple jump", 
+    "348": "throwing ball", 
+    "349": "getting a haircut", 
+    "350": "running on treadmill", 
+    "351": "climbing ladder", 
+    "352": "blasting sand", 
+    "353": "playing trombone", 
+    "354": "drop kicking", 
+    "355": "country line dancing", 
+    "356": "changing wheel", 
+    "357": "feeding goats", 
+    "358": "tying knot (not on a tie)", 
+    "359": "setting table", 
+    "360": "shaving legs", 
+    "361": "kissing", 
+    "362": "riding mule", 
+    "363": "counting money", 
+    "364": "laying bricks", 
+    "365": "barbequing", 
+    "366": "news anchoring", 
+    "367": "smoking hookah", 
+    "368": "cooking egg", 
+    "369": "peeling apples", 
+    "370": "yoga", 
+    "371": "sharpening pencil", 
+    "372": "dribbling basketball", 
+    "373": "petting cat", 
+    "374": "playing ice hockey", 
+    "375": "milking cow", 
+    "376": "shining shoes", 
+    "377": "juggling soccer ball", 
+    "378": "scuba diving", 
+    "379": "playing squash or racquetball", 
+    "380": "drinking beer", 
+    "381": "sign language interpreting", 
+    "382": "playing basketball", 
+    "383": "breakdancing", 
+    "384": "testifying", 
+    "385": "making snowman", 
+    "386": "golf putting", 
+    "387": "playing didgeridoo", 
+    "388": "biking through snow", 
+    "389": "sailing", 
+    "390": "jumpstyle dancing", 
+    "391": "water sliding", 
+    "392": "grooming horse", 
+    "393": "massaging feet", 
+    "394": "playing paintball", 
+    "395": "making a cake", 
+    "396": "bowling", 
+    "397": "contact juggling", 
+    "398": "applying cream", 
+    "399": "playing badminton"
+}
+

models/grit_model.py

@@ -0,0 +1,46 @@
+import os
+import sys
+
+from models.grit_src.image_dense_captions import image_caption_api, init_demo, dense_pred_to_caption, dense_pred_to_caption_only_name
+from detectron2.data.detection_utils import read_image
+
+class DenseCaptioning():
+    def __init__(self, device):
+        self.device = device
+        self.demo =  None
+
+
+    def initialize_model(self):
+        self.demo = init_demo(self.device)
+
+    def image_dense_caption_debug(self, image_src):
+        dense_caption = """
+        1. the broccoli is green, [0, 0, 333, 325]; 
+        2. a piece of broccoli, [0, 147, 143, 324]; 
+        3. silver fork on plate, [4, 547, 252, 612];
+        """
+        return dense_caption
+    
+    def image_dense_caption(self, image_src):
+        dense_caption = image_caption_api(image_src, self.device)
+        print('\033[1;35m' + '*' * 100 + '\033[0m')
+        print("Step2, Dense Caption:\n")
+        print(dense_caption)
+        print('\033[1;35m' + '*' * 100 + '\033[0m')
+        return dense_caption
+    
+    def run_caption_api(self,image_src):
+        img = read_image(image_src, format="BGR")
+        print(img.shape)
+        predictions, visualized_output = self.demo.run_on_image(img)
+        new_caption = dense_pred_to_caption_only_name(predictions)
+        return new_caption
+
+    def run_caption_tensor(self,img):
+        # img = read_image(image_src, format="BGR")
+        # print(img.shape)
+        predictions, visualized_output = self.demo.run_on_image(img)
+        new_caption = dense_pred_to_caption_only_name(predictions)
+        return new_caption
+    
+

models/grit_src/configs/Base.yaml

@@ -0,0 +1,77 @@
+MODEL:
+  META_ARCHITECTURE: "GRiT"
+  MASK_ON: True
+  PROPOSAL_GENERATOR:
+    NAME: "CenterNet"
+  FPN:
+    IN_FEATURES: ["layer3", "layer4", "layer5"]
+  PIXEL_MEAN: [123.675, 116.280, 103.530]
+  PIXEL_STD: [58.395, 57.12, 57.375]
+  ROI_HEADS:
+    NAME: GRiTROIHeadsAndTextDecoder
+    IN_FEATURES: ["p3", "p4", "p5"]
+    IOU_THRESHOLDS: [0.6]
+    NUM_CLASSES: 1
+    SCORE_THRESH_TEST: 0.02
+    NMS_THRESH_TEST: 0.5
+    OBJECT_FEAT_POOLER_RES: 14
+  ROI_BOX_CASCADE_HEAD:
+    IOUS: [0.6, 0.7, 0.8]
+  ROI_BOX_HEAD:
+    NAME: "FastRCNNConvFCHead"
+    NUM_FC: 2
+    POOLER_RESOLUTION: 7
+    CLS_AGNOSTIC_BBOX_REG: True
+    MULT_PROPOSAL_SCORE: True
+  ROI_MASK_HEAD:
+    NAME: "MaskRCNNConvUpsampleHead"
+    NUM_CONV: 4
+    POOLER_RESOLUTION: 14
+    CLS_AGNOSTIC_MASK: True
+  CENTERNET:
+    NUM_CLASSES: 1
+    REG_WEIGHT: 1.
+    NOT_NORM_REG: True
+    ONLY_PROPOSAL: True
+    WITH_AGN_HM: True
+    INFERENCE_TH: 0.0001
+    PRE_NMS_TOPK_TRAIN: 4000
+    POST_NMS_TOPK_TRAIN: 2000
+    PRE_NMS_TOPK_TEST: 1000
+    POST_NMS_TOPK_TEST: 256
+    NMS_TH_TRAIN: 0.9
+    NMS_TH_TEST: 0.9
+    POS_WEIGHT: 0.5
+    NEG_WEIGHT: 0.5
+    IGNORE_HIGH_FP: 0.85
+DATASETS:
+  TRAIN: ("coco_2017_train",)
+  TEST: ("coco_2017_val",)
+DATALOADER:
+  SAMPLER_TRAIN: "MultiDatasetSampler"
+  DATASET_RATIO: [1]
+  DATASET_INPUT_SIZE: [1024]
+  DATASET_INPUT_SCALE: [[0.1, 2.0]]
+  FILTER_EMPTY_ANNOTATIONS: False
+  NUM_WORKERS: 8
+TEST:
+  DETECTIONS_PER_IMAGE: 256
+SOLVER:
+  LR_SCHEDULER_NAME: "WarmupCosineLR"
+  CHECKPOINT_PERIOD: 10000
+  WARMUP_ITERS: 1000
+  WARMUP_FACTOR: 0.001
+  USE_CUSTOM_SOLVER: True
+  OPTIMIZER: "ADAMW"
+  MAX_ITER: 180000
+  IMS_PER_BATCH: 64
+  BASE_LR: 0.00008
+  VIT_LAYER_DECAY: True
+  CLIP_GRADIENTS:
+    ENABLED: True
+INPUT:
+  FORMAT: RGB
+  CUSTOM_AUG: EfficientDetResizeCrop
+  TRAIN_SIZE: 640
+USE_ACT_CHECKPOINT: True
+VERSION: 2

models/grit_src/configs/GRiT_B_DenseCap.yaml

@@ -0,0 +1,20 @@
+_BASE_: "Base.yaml"
+MODEL:
+  TRAIN_TASK: ["DenseCap"]
+  TEST_TASK: "DenseCap"
+  MASK_ON: False
+  ROI_HEADS:
+    SOFT_NMS_ENABLED: False
+  BEAM_SIZE: 1
+  WEIGHTS: "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_base.pth"
+  BACKBONE:
+    NAME: build_vit_fpn_backbone
+  VIT_LAYERS: 12
+SOLVER:
+  VIT_LAYER_DECAY_RATE: 0.7
+DATASETS:
+  TRAIN: ("vg_train",)
+  TEST: ("vg_test",)
+DATALOADER:
+  DATASET_BS: 2
+OUTPUT_DIR: "./output/GRiT_B_DenseCap"

models/grit_src/configs/GRiT_B_DenseCap_ObjectDet.yaml

@@ -0,0 +1,23 @@
+_BASE_: "Base.yaml"
+MODEL:
+  TRAIN_TASK: ["ObjectDet", "DenseCap"]
+  TEST_TASK: "DenseCap" # DenseCap or ObjectDet: Choose one for testing
+  MASK_ON: True
+  ROI_HEADS:
+    SOFT_NMS_ENABLED: False
+  BEAM_SIZE: 1
+  WEIGHTS: "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_base.pth"
+  BACKBONE:
+    NAME: build_vit_fpn_backbone
+  VIT_LAYERS: 12
+SOLVER:
+  VIT_LAYER_DECAY_RATE: 0.7
+DATASETS:
+  TRAIN: ("GRiT_coco2017_train", "vg_train")
+  TEST: ("coco_2017_test-dev",)
+DATALOADER:
+  DATASET_RATIO: [1, 1]
+  DATASET_BS: 2
+  DATASET_INPUT_SIZE: [1024, 1024]
+  DATASET_INPUT_SCALE: [[0.1, 2.0], [0.1, 2.0]]
+OUTPUT_DIR: "./output/GRiT_B_DenseCap_ObjectDet"

models/grit_src/configs/GRiT_B_ObjectDet.yaml

@@ -0,0 +1,20 @@
+_BASE_: "Base.yaml"
+MODEL:
+  TRAIN_TASK: ["ObjectDet"]
+  TEST_TASK: "ObjectDet"
+  MASK_ON: True
+  ROI_HEADS:
+    SOFT_NMS_ENABLED: True
+  BEAM_SIZE: 3
+  WEIGHTS: "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_base.pth"
+  BACKBONE:
+    NAME: build_vit_fpn_backbone
+  VIT_LAYERS: 12
+SOLVER:
+  VIT_LAYER_DECAY_RATE: 0.7
+DATASETS:
+  TRAIN: ("GRiT_coco2017_train",)
+  TEST: ("coco_2017_val",)
+DATALOADER:
+  DATASET_BS: 2
+OUTPUT_DIR: "./output/GRiT_B_ObjectDet"

models/grit_src/configs/GRiT_H_ObjectDet.yaml

@@ -0,0 +1,21 @@
+_BASE_: "Base.yaml"
+MODEL:
+  TRAIN_TASK: ["ObjectDet"]
+  TEST_TASK: "ObjectDet"
+  MASK_ON: True
+  ROI_HEADS:
+    SOFT_NMS_ENABLED: True
+  BEAM_SIZE: 3
+  WEIGHTS: "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_huge_p14to16.pth"
+  BACKBONE:
+    NAME: build_vit_fpn_backbone_huge
+  VIT_LAYERS: 32
+SOLVER:
+  MAX_ITER: 135000
+  VIT_LAYER_DECAY_RATE: 0.9
+DATASETS:
+  TRAIN: ("GRiT_coco2017_train",)
+  TEST: ("coco_2017_val",)
+DATALOADER:
+  DATASET_BS: 1
+OUTPUT_DIR: "./output/GRiT_H_ObjectDet"

models/grit_src/configs/GRiT_L_ObjectDet.yaml

@@ -0,0 +1,20 @@
+_BASE_: "Base.yaml"
+MODEL:
+  TRAIN_TASK: ["ObjectDet"]
+  TEST_TASK: "ObjectDet"
+  MASK_ON: True
+  ROI_HEADS:
+    SOFT_NMS_ENABLED: True
+  BEAM_SIZE: 3
+  WEIGHTS: "detectron2://ImageNetPretrained/MAE/mae_pretrain_vit_large.pth"
+  BACKBONE:
+    NAME: build_vit_fpn_backbone_large
+  VIT_LAYERS: 24
+SOLVER:
+  VIT_LAYER_DECAY_RATE: 0.8
+DATASETS:
+  TRAIN: ("GRiT_coco2017_train",)
+  TEST: ("coco_2017_val",)
+DATALOADER:
+  DATASET_BS: 1
+OUTPUT_DIR: "./output/GRiT_L_ObjectDet"

models/grit_src/grit/__init__.py

@@ -0,0 +1,7 @@
+from .modeling.meta_arch import grit
+from .modeling.roi_heads import grit_roi_heads
+from .modeling.backbone import vit
+
+from .data.datasets import object365
+from .data.datasets import vg
+from .data.datasets import grit_coco

models/grit_src/grit/config.py

@@ -0,0 +1,50 @@
+from detectron2.config import CfgNode as CN
+
+
+def add_grit_config(cfg):
+    _C = cfg
+
+    _C.MODEL.BEAM_SIZE = 1
+    _C.MODEL.TRAIN_TASK = ["ObjectDet", "DenseCap"]
+    _C.MODEL.TEST_TASK = "DenseCap"  # This can be varied if the model is jointly trained on multiple tasks
+
+    _C.MODEL.ROI_BOX_HEAD.USE_BIAS = 0.0 # >= 0: not use
+    _C.MODEL.ROI_BOX_HEAD.MULT_PROPOSAL_SCORE = False
+
+    _C.MODEL.ROI_HEADS.MASK_WEIGHT = 1.0
+    _C.MODEL.ROI_HEADS.OBJECT_FEAT_POOLER_RES = 14
+    _C.MODEL.ROI_HEADS.SOFT_NMS_ENABLED = False
+
+    # Backbones
+    _C.MODEL.VIT_LAYERS = 12
+
+    # Text Decoder
+    _C.TEXT_DECODER = CN()
+    _C.TEXT_DECODER.VOCAB_SIZE = 30522
+    _C.TEXT_DECODER.HIDDEN_SIZE = 768
+    _C.TEXT_DECODER.NUM_LAYERS = 6
+    _C.TEXT_DECODER.ATTENTION_HEADS = 12
+    _C.TEXT_DECODER.FEEDFORWARD_SIZE = 768 * 4
+    
+    # Multi-dataset dataloader
+    _C.DATALOADER.DATASET_RATIO = [1, 1]  # sample ratio
+    _C.DATALOADER.DATASET_BS = 1
+    _C.DATALOADER.DATASET_INPUT_SIZE = [1024, 1024]
+    _C.DATALOADER.DATASET_INPUT_SCALE = [(0.1, 2.0), (0.1, 2.0)]
+    _C.DATALOADER.DATASET_MIN_SIZES = [(640, 800), (640, 800)]
+    _C.DATALOADER.DATASET_MAX_SIZES = [1333, 1333]
+    
+    _C.SOLVER.USE_CUSTOM_SOLVER = True
+    _C.SOLVER.OPTIMIZER = 'ADAMW'
+    _C.SOLVER.VIT_LAYER_DECAY = True
+    _C.SOLVER.VIT_LAYER_DECAY_RATE = 0.7
+
+    _C.INPUT.CUSTOM_AUG = 'EfficientDetResizeCrop'
+    _C.INPUT.TRAIN_SIZE = 1024
+    _C.INPUT.TEST_SIZE = 1024
+    _C.INPUT.SCALE_RANGE = (0.1, 2.)
+    # 'default' for fixed short / long edge
+    _C.INPUT.TEST_INPUT_TYPE = 'default' 
+
+    _C.FIND_UNUSED_PARAM = True
+    _C.USE_ACT_CHECKPOINT = True

models/grit_src/grit/custom_solver.py

@@ -0,0 +1,88 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Modified by Jialian Wu from https://github.com/facebookresearch/Detic/blob/main/detic/custom_solver.py
+import itertools
+from typing import Any, Callable, Dict, Iterable, List, Set, Type, Union
+import torch
+
+from detectron2.config import CfgNode
+
+from detectron2.solver.build import maybe_add_gradient_clipping
+
+
+def build_custom_optimizer(cfg: CfgNode, model: torch.nn.Module) -> torch.optim.Optimizer:
+    params: List[Dict[str, Any]] = []
+    memo: Set[torch.nn.parameter.Parameter] = set()
+    optimizer_type = cfg.SOLVER.OPTIMIZER
+
+    for key, value in model.named_parameters(recurse=True):
+        if not value.requires_grad:
+            continue
+        # Avoid duplicating parameters
+        if value in memo:
+            continue
+        memo.add(value)
+        lr = cfg.SOLVER.BASE_LR
+        weight_decay = cfg.SOLVER.WEIGHT_DECAY
+
+        if cfg.SOLVER.VIT_LAYER_DECAY:
+            lr = lr * get_vit_lr_decay_rate(key, cfg.SOLVER.VIT_LAYER_DECAY_RATE, cfg.MODEL.VIT_LAYERS)
+
+        param = {"params": [value], "lr": lr}
+        if optimizer_type != 'ADAMW':
+            param['weight_decay'] = weight_decay
+        params += [param]
+
+    def maybe_add_full_model_gradient_clipping(optim):  # optim: the optimizer class
+        # detectron2 doesn't have full model gradient clipping now
+        clip_norm_val = cfg.SOLVER.CLIP_GRADIENTS.CLIP_VALUE
+        enable = (
+            cfg.SOLVER.CLIP_GRADIENTS.ENABLED
+            and cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model"
+            and clip_norm_val > 0.0
+        )
+
+        class FullModelGradientClippingOptimizer(optim):
+            def step(self, closure=None):
+                all_params = itertools.chain(*[x["params"] for x in self.param_groups])
+                torch.nn.utils.clip_grad_norm_(all_params, clip_norm_val)
+                super().step(closure=closure)
+
+        return FullModelGradientClippingOptimizer if enable else optim
+
+    
+    if optimizer_type == 'SGD':
+        optimizer = maybe_add_full_model_gradient_clipping(torch.optim.SGD)(
+            params, cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM, 
+            nesterov=cfg.SOLVER.NESTEROV
+        )
+    elif optimizer_type == 'ADAMW':
+        optimizer = maybe_add_full_model_gradient_clipping(torch.optim.AdamW)(
+            params, cfg.SOLVER.BASE_LR, 
+            weight_decay=cfg.SOLVER.WEIGHT_DECAY
+        )
+    else:
+        raise NotImplementedError(f"no optimizer type {optimizer_type}")
+    if not cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model":
+        optimizer = maybe_add_gradient_clipping(cfg, optimizer)
+    return optimizer
+
+
+def get_vit_lr_decay_rate(name, lr_decay_rate=1.0, num_layers=12):
+    """
+    Calculate lr decay rate for different ViT blocks.
+    Args:
+        name (string): parameter name.
+        lr_decay_rate (float): base lr decay rate.
+        num_layers (int): number of ViT blocks.
+
+    Returns:
+        lr decay rate for the given parameter.
+    """
+    layer_id = num_layers + 1
+    if name.startswith("backbone"):
+        if ".pos_embed" in name or ".patch_embed" in name:
+            layer_id = 0
+        elif ".blocks." in name and ".residual." not in name:
+            layer_id = int(name[name.find(".blocks.") :].split(".")[2]) + 1
+
+    return lr_decay_rate ** (num_layers + 1 - layer_id)