AskAnything / load_internvideo.py
ynhe's picture
[Init]
5423e78
from intern_action import intern_action_b16
from huggingface_hub import hf_hub_download
# from kinetics_class_index import kinetics_classnames
import torch
import torch.nn as nn
import torchvision.transforms as T
import torch.nn.functional as F
import numpy as np
from transforms import (
GroupNormalize, GroupScale, GroupCenterCrop,
Stack, ToTorchFormatTensor
)
class Intern_Action(nn.Module):
def __init__(self, model):
super().__init__()
self.backbone = model
def forward(self, x):
return self.backbone(x)
def get_index(num_frames, num_segments=8):
seg_size = float(num_frames - 1) / num_segments
start = int(seg_size / 2)
offsets = np.array([
start + int(np.round(seg_size * idx)) for idx in range(num_segments)
])
return offsets
def transform_action():
# transform
crop_size = 224
scale_size = 256
input_mean = [0.485, 0.456, 0.406]
input_std = [0.229, 0.224, 0.225]
return T.Compose([
# T.ToPILImage(),
GroupScale(int(scale_size)),
GroupCenterCrop(crop_size),
Stack(),
ToTorchFormatTensor(),
GroupNormalize(input_mean, input_std)
])
def load_intern_action(device):
# Create an id to label name mapping
kinetics_id_to_classname = {}
for k, v in kinetics_classnames.items():
kinetics_id_to_classname[k] = v
model_path = hf_hub_download(repo_id="Andy1621/uniformerv2", filename="k400+k710_uniformerv2_b16_8x224.pyth")
# Pick a pretrained model
model = Intern_Action(intern_action_b16(pretrained=False, t_size=8, no_lmhra=True, temporal_downsample=False))
state_dict = torch.load(model_path, map_location=device)
model.load_state_dict(state_dict)
# Set to eval mode and move to desired device
model = model.to(device)
model = model.eval()
return model
def cut_frame_to_8(data):
index = np.linspace(0, len(data)-1, 8).astype(int)
return data[index]
kinetics_classnames = {
"0": "riding a bike",
"1": "marching",
"2": "dodgeball",
"3": "playing cymbals",
"4": "checking tires",
"5": "roller skating",
"6": "tasting beer",
"7": "clapping",
"8": "drawing",
"9": "juggling fire",
"10": "bobsledding",
"11": "petting animal (not cat)",
"12": "spray painting",
"13": "training dog",
"14": "eating watermelon",
"15": "building cabinet",
"16": "applauding",
"17": "playing harp",
"18": "balloon blowing",
"19": "sled dog racing",
"20": "wrestling",
"21": "pole vault",
"22": "hurling (sport)",
"23": "riding scooter",
"24": "shearing sheep",
"25": "sweeping floor",
"26": "eating carrots",
"27": "skateboarding",
"28": "dunking basketball",
"29": "disc golfing",
"30": "eating spaghetti",
"31": "playing flute",
"32": "riding mechanical bull",
"33": "making sushi",
"34": "trapezing",
"35": "picking fruit",
"36": "stretching leg",
"37": "playing ukulele",
"38": "tying tie",
"39": "skydiving",
"40": "playing cello",
"41": "jumping into pool",
"42": "shooting goal (soccer)",
"43": "trimming trees",
"44": "bookbinding",
"45": "ski jumping",
"46": "walking the dog",
"47": "riding unicycle",
"48": "shaving head",
"49": "hopscotch",
"50": "playing piano",
"51": "parasailing",
"52": "bartending",
"53": "kicking field goal",
"54": "finger snapping",
"55": "dining",
"56": "yawning",
"57": "peeling potatoes",
"58": "canoeing or kayaking",
"59": "front raises",
"60": "laughing",
"61": "dancing macarena",
"62": "digging",
"63": "reading newspaper",
"64": "hitting baseball",
"65": "clay pottery making",
"66": "exercising with an exercise ball",
"67": "playing saxophone",
"68": "shooting basketball",
"69": "washing hair",
"70": "lunge",
"71": "brushing hair",
"72": "curling hair",
"73": "kitesurfing",
"74": "tapping guitar",
"75": "bending back",
"76": "skipping rope",
"77": "situp",
"78": "folding paper",
"79": "cracking neck",
"80": "assembling computer",
"81": "cleaning gutters",
"82": "blowing out candles",
"83": "shaking hands",
"84": "dancing gangnam style",
"85": "windsurfing",
"86": "tap dancing",
"87": "skiing (not slalom or crosscountry)",
"88": "bandaging",
"89": "push up",
"90": "doing nails",
"91": "punching person (boxing)",
"92": "bouncing on trampoline",
"93": "scrambling eggs",
"94": "singing",
"95": "cleaning floor",
"96": "krumping",
"97": "drumming fingers",
"98": "snowmobiling",
"99": "gymnastics tumbling",
"100": "headbanging",
"101": "catching or throwing frisbee",
"102": "riding elephant",
"103": "bee keeping",
"104": "feeding birds",
"105": "snatch weight lifting",
"106": "mowing lawn",
"107": "fixing hair",
"108": "playing trumpet",
"109": "flying kite",
"110": "crossing river",
"111": "swinging legs",
"112": "sanding floor",
"113": "belly dancing",
"114": "sneezing",
"115": "clean and jerk",
"116": "side kick",
"117": "filling eyebrows",
"118": "shuffling cards",
"119": "recording music",
"120": "cartwheeling",
"121": "feeding fish",
"122": "folding clothes",
"123": "water skiing",
"124": "tobogganing",
"125": "blowing leaves",
"126": "smoking",
"127": "unboxing",
"128": "tai chi",
"129": "waxing legs",
"130": "riding camel",
"131": "slapping",
"132": "tossing salad",
"133": "capoeira",
"134": "playing cards",
"135": "playing organ",
"136": "playing violin",
"137": "playing drums",
"138": "tapping pen",
"139": "vault",
"140": "shoveling snow",
"141": "playing tennis",
"142": "getting a tattoo",
"143": "making a sandwich",
"144": "making tea",
"145": "grinding meat",
"146": "squat",
"147": "eating doughnuts",
"148": "ice fishing",
"149": "snowkiting",
"150": "kicking soccer ball",
"151": "playing controller",
"152": "giving or receiving award",
"153": "welding",
"154": "throwing discus",
"155": "throwing axe",
"156": "ripping paper",
"157": "swimming butterfly stroke",
"158": "air drumming",
"159": "blowing nose",
"160": "hockey stop",
"161": "taking a shower",
"162": "bench pressing",
"163": "planting trees",
"164": "pumping fist",
"165": "climbing tree",
"166": "tickling",
"167": "high kick",
"168": "waiting in line",
"169": "slacklining",
"170": "tango dancing",
"171": "hurdling",
"172": "carrying baby",
"173": "celebrating",
"174": "sharpening knives",
"175": "passing American football (in game)",
"176": "headbutting",
"177": "playing recorder",
"178": "brush painting",
"179": "garbage collecting",
"180": "robot dancing",
"181": "shredding paper",
"182": "pumping gas",
"183": "rock climbing",
"184": "hula hooping",
"185": "braiding hair",
"186": "opening present",
"187": "texting",
"188": "decorating the christmas tree",
"189": "answering questions",
"190": "playing keyboard",
"191": "writing",
"192": "bungee jumping",
"193": "sniffing",
"194": "eating burger",
"195": "playing accordion",
"196": "making pizza",
"197": "playing volleyball",
"198": "tasting food",
"199": "pushing cart",
"200": "spinning poi",
"201": "cleaning windows",
"202": "arm wrestling",
"203": "changing oil",
"204": "swimming breast stroke",
"205": "tossing coin",
"206": "deadlifting",
"207": "hoverboarding",
"208": "cutting watermelon",
"209": "cheerleading",
"210": "snorkeling",
"211": "washing hands",
"212": "eating cake",
"213": "pull ups",
"214": "surfing water",
"215": "eating hotdog",
"216": "holding snake",
"217": "playing harmonica",
"218": "ironing",
"219": "cutting nails",
"220": "golf chipping",
"221": "shot put",
"222": "hugging",
"223": "playing clarinet",
"224": "faceplanting",
"225": "trimming or shaving beard",
"226": "drinking shots",
"227": "riding mountain bike",
"228": "tying bow tie",
"229": "swinging on something",
"230": "skiing crosscountry",
"231": "unloading truck",
"232": "cleaning pool",
"233": "jogging",
"234": "ice climbing",
"235": "mopping floor",
"236": "making bed",
"237": "diving cliff",
"238": "washing dishes",
"239": "grooming dog",
"240": "weaving basket",
"241": "frying vegetables",
"242": "stomping grapes",
"243": "moving furniture",
"244": "cooking sausages",
"245": "doing laundry",
"246": "dying hair",
"247": "knitting",
"248": "reading book",
"249": "baby waking up",
"250": "punching bag",
"251": "surfing crowd",
"252": "cooking chicken",
"253": "pushing car",
"254": "springboard diving",
"255": "swing dancing",
"256": "massaging legs",
"257": "beatboxing",
"258": "breading or breadcrumbing",
"259": "somersaulting",
"260": "brushing teeth",
"261": "stretching arm",
"262": "juggling balls",
"263": "massaging person's head",
"264": "eating ice cream",
"265": "extinguishing fire",
"266": "hammer throw",
"267": "whistling",
"268": "crawling baby",
"269": "using remote controller (not gaming)",
"270": "playing cricket",
"271": "opening bottle",
"272": "playing xylophone",
"273": "motorcycling",
"274": "driving car",
"275": "exercising arm",
"276": "passing American football (not in game)",
"277": "playing kickball",
"278": "sticking tongue out",
"279": "flipping pancake",
"280": "catching fish",
"281": "eating chips",
"282": "shaking head",
"283": "sword fighting",
"284": "playing poker",
"285": "cooking on campfire",
"286": "doing aerobics",
"287": "paragliding",
"288": "using segway",
"289": "folding napkins",
"290": "playing bagpipes",
"291": "gargling",
"292": "skiing slalom",
"293": "strumming guitar",
"294": "javelin throw",
"295": "waxing back",
"296": "riding or walking with horse",
"297": "plastering",
"298": "long jump",
"299": "parkour",
"300": "wrapping present",
"301": "egg hunting",
"302": "archery",
"303": "cleaning toilet",
"304": "swimming backstroke",
"305": "snowboarding",
"306": "catching or throwing baseball",
"307": "massaging back",
"308": "blowing glass",
"309": "playing guitar",
"310": "playing chess",
"311": "golf driving",
"312": "presenting weather forecast",
"313": "rock scissors paper",
"314": "high jump",
"315": "baking cookies",
"316": "using computer",
"317": "washing feet",
"318": "arranging flowers",
"319": "playing bass guitar",
"320": "spraying",
"321": "cutting pineapple",
"322": "waxing chest",
"323": "auctioning",
"324": "jetskiing",
"325": "drinking",
"326": "busking",
"327": "playing monopoly",
"328": "salsa dancing",
"329": "waxing eyebrows",
"330": "watering plants",
"331": "zumba",
"332": "chopping wood",
"333": "pushing wheelchair",
"334": "carving pumpkin",
"335": "building shed",
"336": "making jewelry",
"337": "catching or throwing softball",
"338": "bending metal",
"339": "ice skating",
"340": "dancing charleston",
"341": "abseiling",
"342": "climbing a rope",
"343": "crying",
"344": "cleaning shoes",
"345": "dancing ballet",
"346": "driving tractor",
"347": "triple jump",
"348": "throwing ball",
"349": "getting a haircut",
"350": "running on treadmill",
"351": "climbing ladder",
"352": "blasting sand",
"353": "playing trombone",
"354": "drop kicking",
"355": "country line dancing",
"356": "changing wheel",
"357": "feeding goats",
"358": "tying knot (not on a tie)",
"359": "setting table",
"360": "shaving legs",
"361": "kissing",
"362": "riding mule",
"363": "counting money",
"364": "laying bricks",
"365": "barbequing",
"366": "news anchoring",
"367": "smoking hookah",
"368": "cooking egg",
"369": "peeling apples",
"370": "yoga",
"371": "sharpening pencil",
"372": "dribbling basketball",
"373": "petting cat",
"374": "playing ice hockey",
"375": "milking cow",
"376": "shining shoes",
"377": "juggling soccer ball",
"378": "scuba diving",
"379": "playing squash or racquetball",
"380": "drinking beer",
"381": "sign language interpreting",
"382": "playing basketball",
"383": "breakdancing",
"384": "testifying",
"385": "making snowman",
"386": "golf putting",
"387": "playing didgeridoo",
"388": "biking through snow",
"389": "sailing",
"390": "jumpstyle dancing",
"391": "water sliding",
"392": "grooming horse",
"393": "massaging feet",
"394": "playing paintball",
"395": "making a cake",
"396": "bowling",
"397": "contact juggling",
"398": "applying cream",
"399": "playing badminton"
}