|
from .intern_action import intern_action_b16 |
|
from huggingface_hub import hf_hub_download |
|
|
|
import torch |
|
import torch.nn as nn |
|
import torchvision.transforms as T |
|
import torch.nn.functional as F |
|
import numpy as np |
|
|
|
from .processing import ( |
|
GroupNormalize, GroupScale, GroupCenterCrop, |
|
Stack, ToTorchFormatTensor |
|
) |
|
|
|
class Intern_Action(nn.Module): |
|
def __init__(self, model): |
|
super().__init__() |
|
self.backbone = model |
|
|
|
def forward(self, x): |
|
return self.backbone(x) |
|
|
|
def get_index(num_frames, num_segments=8): |
|
seg_size = float(num_frames - 1) / num_segments |
|
start = int(seg_size / 2) |
|
offsets = np.array([ |
|
start + int(np.round(seg_size * idx)) for idx in range(num_segments) |
|
]) |
|
return offsets |
|
|
|
def transform_action(): |
|
|
|
crop_size = 224 |
|
scale_size = 256 |
|
input_mean = [0.485, 0.456, 0.406] |
|
input_std = [0.229, 0.224, 0.225] |
|
|
|
return T.Compose([ |
|
|
|
GroupScale(int(scale_size)), |
|
GroupCenterCrop(crop_size), |
|
Stack(), |
|
ToTorchFormatTensor(), |
|
GroupNormalize(input_mean, input_std) |
|
]) |
|
|
|
def load_intern_action(device): |
|
|
|
kinetics_id_to_classname = {} |
|
for k, v in kinetics_classnames.items(): |
|
kinetics_id_to_classname[k] = v |
|
|
|
model_path = hf_hub_download(repo_id="Andy1621/uniformerv2", filename="k400+k710_uniformerv2_b16_8x224.pyth") |
|
|
|
model = Intern_Action(intern_action_b16(pretrained=False, t_size=8, no_lmhra=True, temporal_downsample=False)) |
|
state_dict = torch.load(model_path, map_location=device) |
|
model.load_state_dict(state_dict) |
|
|
|
model = model.to(device) |
|
model = model.eval() |
|
return model |
|
|
|
def cut_frame_to_8(data): |
|
index = np.linspace(0, len(data)-1, 8).astype(int) |
|
return data[index] |
|
|
|
kinetics_classnames = { |
|
"0": "riding a bike", |
|
"1": "marching", |
|
"2": "dodgeball", |
|
"3": "playing cymbals", |
|
"4": "checking tires", |
|
"5": "roller skating", |
|
"6": "tasting beer", |
|
"7": "clapping", |
|
"8": "drawing", |
|
"9": "juggling fire", |
|
"10": "bobsledding", |
|
"11": "petting animal (not cat)", |
|
"12": "spray painting", |
|
"13": "training dog", |
|
"14": "eating watermelon", |
|
"15": "building cabinet", |
|
"16": "applauding", |
|
"17": "playing harp", |
|
"18": "balloon blowing", |
|
"19": "sled dog racing", |
|
"20": "wrestling", |
|
"21": "pole vault", |
|
"22": "hurling (sport)", |
|
"23": "riding scooter", |
|
"24": "shearing sheep", |
|
"25": "sweeping floor", |
|
"26": "eating carrots", |
|
"27": "skateboarding", |
|
"28": "dunking basketball", |
|
"29": "disc golfing", |
|
"30": "eating spaghetti", |
|
"31": "playing flute", |
|
"32": "riding mechanical bull", |
|
"33": "making sushi", |
|
"34": "trapezing", |
|
"35": "picking fruit", |
|
"36": "stretching leg", |
|
"37": "playing ukulele", |
|
"38": "tying tie", |
|
"39": "skydiving", |
|
"40": "playing cello", |
|
"41": "jumping into pool", |
|
"42": "shooting goal (soccer)", |
|
"43": "trimming trees", |
|
"44": "bookbinding", |
|
"45": "ski jumping", |
|
"46": "walking the dog", |
|
"47": "riding unicycle", |
|
"48": "shaving head", |
|
"49": "hopscotch", |
|
"50": "playing piano", |
|
"51": "parasailing", |
|
"52": "bartending", |
|
"53": "kicking field goal", |
|
"54": "finger snapping", |
|
"55": "dining", |
|
"56": "yawning", |
|
"57": "peeling potatoes", |
|
"58": "canoeing or kayaking", |
|
"59": "front raises", |
|
"60": "laughing", |
|
"61": "dancing macarena", |
|
"62": "digging", |
|
"63": "reading newspaper", |
|
"64": "hitting baseball", |
|
"65": "clay pottery making", |
|
"66": "exercising with an exercise ball", |
|
"67": "playing saxophone", |
|
"68": "shooting basketball", |
|
"69": "washing hair", |
|
"70": "lunge", |
|
"71": "brushing hair", |
|
"72": "curling hair", |
|
"73": "kitesurfing", |
|
"74": "tapping guitar", |
|
"75": "bending back", |
|
"76": "skipping rope", |
|
"77": "situp", |
|
"78": "folding paper", |
|
"79": "cracking neck", |
|
"80": "assembling computer", |
|
"81": "cleaning gutters", |
|
"82": "blowing out candles", |
|
"83": "shaking hands", |
|
"84": "dancing gangnam style", |
|
"85": "windsurfing", |
|
"86": "tap dancing", |
|
"87": "skiing (not slalom or crosscountry)", |
|
"88": "bandaging", |
|
"89": "push up", |
|
"90": "doing nails", |
|
"91": "punching person (boxing)", |
|
"92": "bouncing on trampoline", |
|
"93": "scrambling eggs", |
|
"94": "singing", |
|
"95": "cleaning floor", |
|
"96": "krumping", |
|
"97": "drumming fingers", |
|
"98": "snowmobiling", |
|
"99": "gymnastics tumbling", |
|
"100": "headbanging", |
|
"101": "catching or throwing frisbee", |
|
"102": "riding elephant", |
|
"103": "bee keeping", |
|
"104": "feeding birds", |
|
"105": "snatch weight lifting", |
|
"106": "mowing lawn", |
|
"107": "fixing hair", |
|
"108": "playing trumpet", |
|
"109": "flying kite", |
|
"110": "crossing river", |
|
"111": "swinging legs", |
|
"112": "sanding floor", |
|
"113": "belly dancing", |
|
"114": "sneezing", |
|
"115": "clean and jerk", |
|
"116": "side kick", |
|
"117": "filling eyebrows", |
|
"118": "shuffling cards", |
|
"119": "recording music", |
|
"120": "cartwheeling", |
|
"121": "feeding fish", |
|
"122": "folding clothes", |
|
"123": "water skiing", |
|
"124": "tobogganing", |
|
"125": "blowing leaves", |
|
"126": "smoking", |
|
"127": "unboxing", |
|
"128": "tai chi", |
|
"129": "waxing legs", |
|
"130": "riding camel", |
|
"131": "slapping", |
|
"132": "tossing salad", |
|
"133": "capoeira", |
|
"134": "playing cards", |
|
"135": "playing organ", |
|
"136": "playing violin", |
|
"137": "playing drums", |
|
"138": "tapping pen", |
|
"139": "vault", |
|
"140": "shoveling snow", |
|
"141": "playing tennis", |
|
"142": "getting a tattoo", |
|
"143": "making a sandwich", |
|
"144": "making tea", |
|
"145": "grinding meat", |
|
"146": "squat", |
|
"147": "eating doughnuts", |
|
"148": "ice fishing", |
|
"149": "snowkiting", |
|
"150": "kicking soccer ball", |
|
"151": "playing controller", |
|
"152": "giving or receiving award", |
|
"153": "welding", |
|
"154": "throwing discus", |
|
"155": "throwing axe", |
|
"156": "ripping paper", |
|
"157": "swimming butterfly stroke", |
|
"158": "air drumming", |
|
"159": "blowing nose", |
|
"160": "hockey stop", |
|
"161": "taking a shower", |
|
"162": "bench pressing", |
|
"163": "planting trees", |
|
"164": "pumping fist", |
|
"165": "climbing tree", |
|
"166": "tickling", |
|
"167": "high kick", |
|
"168": "waiting in line", |
|
"169": "slacklining", |
|
"170": "tango dancing", |
|
"171": "hurdling", |
|
"172": "carrying baby", |
|
"173": "celebrating", |
|
"174": "sharpening knives", |
|
"175": "passing American football (in game)", |
|
"176": "headbutting", |
|
"177": "playing recorder", |
|
"178": "brush painting", |
|
"179": "garbage collecting", |
|
"180": "robot dancing", |
|
"181": "shredding paper", |
|
"182": "pumping gas", |
|
"183": "rock climbing", |
|
"184": "hula hooping", |
|
"185": "braiding hair", |
|
"186": "opening present", |
|
"187": "texting", |
|
"188": "decorating the christmas tree", |
|
"189": "answering questions", |
|
"190": "playing keyboard", |
|
"191": "writing", |
|
"192": "bungee jumping", |
|
"193": "sniffing", |
|
"194": "eating burger", |
|
"195": "playing accordion", |
|
"196": "making pizza", |
|
"197": "playing volleyball", |
|
"198": "tasting food", |
|
"199": "pushing cart", |
|
"200": "spinning poi", |
|
"201": "cleaning windows", |
|
"202": "arm wrestling", |
|
"203": "changing oil", |
|
"204": "swimming breast stroke", |
|
"205": "tossing coin", |
|
"206": "deadlifting", |
|
"207": "hoverboarding", |
|
"208": "cutting watermelon", |
|
"209": "cheerleading", |
|
"210": "snorkeling", |
|
"211": "washing hands", |
|
"212": "eating cake", |
|
"213": "pull ups", |
|
"214": "surfing water", |
|
"215": "eating hotdog", |
|
"216": "holding snake", |
|
"217": "playing harmonica", |
|
"218": "ironing", |
|
"219": "cutting nails", |
|
"220": "golf chipping", |
|
"221": "shot put", |
|
"222": "hugging", |
|
"223": "playing clarinet", |
|
"224": "faceplanting", |
|
"225": "trimming or shaving beard", |
|
"226": "drinking shots", |
|
"227": "riding mountain bike", |
|
"228": "tying bow tie", |
|
"229": "swinging on something", |
|
"230": "skiing crosscountry", |
|
"231": "unloading truck", |
|
"232": "cleaning pool", |
|
"233": "jogging", |
|
"234": "ice climbing", |
|
"235": "mopping floor", |
|
"236": "making bed", |
|
"237": "diving cliff", |
|
"238": "washing dishes", |
|
"239": "grooming dog", |
|
"240": "weaving basket", |
|
"241": "frying vegetables", |
|
"242": "stomping grapes", |
|
"243": "moving furniture", |
|
"244": "cooking sausages", |
|
"245": "doing laundry", |
|
"246": "dying hair", |
|
"247": "knitting", |
|
"248": "reading book", |
|
"249": "baby waking up", |
|
"250": "punching bag", |
|
"251": "surfing crowd", |
|
"252": "cooking chicken", |
|
"253": "pushing car", |
|
"254": "springboard diving", |
|
"255": "swing dancing", |
|
"256": "massaging legs", |
|
"257": "beatboxing", |
|
"258": "breading or breadcrumbing", |
|
"259": "somersaulting", |
|
"260": "brushing teeth", |
|
"261": "stretching arm", |
|
"262": "juggling balls", |
|
"263": "massaging person's head", |
|
"264": "eating ice cream", |
|
"265": "extinguishing fire", |
|
"266": "hammer throw", |
|
"267": "whistling", |
|
"268": "crawling baby", |
|
"269": "using remote controller (not gaming)", |
|
"270": "playing cricket", |
|
"271": "opening bottle", |
|
"272": "playing xylophone", |
|
"273": "motorcycling", |
|
"274": "driving car", |
|
"275": "exercising arm", |
|
"276": "passing American football (not in game)", |
|
"277": "playing kickball", |
|
"278": "sticking tongue out", |
|
"279": "flipping pancake", |
|
"280": "catching fish", |
|
"281": "eating chips", |
|
"282": "shaking head", |
|
"283": "sword fighting", |
|
"284": "playing poker", |
|
"285": "cooking on campfire", |
|
"286": "doing aerobics", |
|
"287": "paragliding", |
|
"288": "using segway", |
|
"289": "folding napkins", |
|
"290": "playing bagpipes", |
|
"291": "gargling", |
|
"292": "skiing slalom", |
|
"293": "strumming guitar", |
|
"294": "javelin throw", |
|
"295": "waxing back", |
|
"296": "riding or walking with horse", |
|
"297": "plastering", |
|
"298": "long jump", |
|
"299": "parkour", |
|
"300": "wrapping present", |
|
"301": "egg hunting", |
|
"302": "archery", |
|
"303": "cleaning toilet", |
|
"304": "swimming backstroke", |
|
"305": "snowboarding", |
|
"306": "catching or throwing baseball", |
|
"307": "massaging back", |
|
"308": "blowing glass", |
|
"309": "playing guitar", |
|
"310": "playing chess", |
|
"311": "golf driving", |
|
"312": "presenting weather forecast", |
|
"313": "rock scissors paper", |
|
"314": "high jump", |
|
"315": "baking cookies", |
|
"316": "using computer", |
|
"317": "washing feet", |
|
"318": "arranging flowers", |
|
"319": "playing bass guitar", |
|
"320": "spraying", |
|
"321": "cutting pineapple", |
|
"322": "waxing chest", |
|
"323": "auctioning", |
|
"324": "jetskiing", |
|
"325": "drinking", |
|
"326": "busking", |
|
"327": "playing monopoly", |
|
"328": "salsa dancing", |
|
"329": "waxing eyebrows", |
|
"330": "watering plants", |
|
"331": "zumba", |
|
"332": "chopping wood", |
|
"333": "pushing wheelchair", |
|
"334": "carving pumpkin", |
|
"335": "building shed", |
|
"336": "making jewelry", |
|
"337": "catching or throwing softball", |
|
"338": "bending metal", |
|
"339": "ice skating", |
|
"340": "dancing charleston", |
|
"341": "abseiling", |
|
"342": "climbing a rope", |
|
"343": "crying", |
|
"344": "cleaning shoes", |
|
"345": "dancing ballet", |
|
"346": "driving tractor", |
|
"347": "triple jump", |
|
"348": "throwing ball", |
|
"349": "getting a haircut", |
|
"350": "running on treadmill", |
|
"351": "climbing ladder", |
|
"352": "blasting sand", |
|
"353": "playing trombone", |
|
"354": "drop kicking", |
|
"355": "country line dancing", |
|
"356": "changing wheel", |
|
"357": "feeding goats", |
|
"358": "tying knot (not on a tie)", |
|
"359": "setting table", |
|
"360": "shaving legs", |
|
"361": "kissing", |
|
"362": "riding mule", |
|
"363": "counting money", |
|
"364": "laying bricks", |
|
"365": "barbequing", |
|
"366": "news anchoring", |
|
"367": "smoking hookah", |
|
"368": "cooking egg", |
|
"369": "peeling apples", |
|
"370": "yoga", |
|
"371": "sharpening pencil", |
|
"372": "dribbling basketball", |
|
"373": "petting cat", |
|
"374": "playing ice hockey", |
|
"375": "milking cow", |
|
"376": "shining shoes", |
|
"377": "juggling soccer ball", |
|
"378": "scuba diving", |
|
"379": "playing squash or racquetball", |
|
"380": "drinking beer", |
|
"381": "sign language interpreting", |
|
"382": "playing basketball", |
|
"383": "breakdancing", |
|
"384": "testifying", |
|
"385": "making snowman", |
|
"386": "golf putting", |
|
"387": "playing didgeridoo", |
|
"388": "biking through snow", |
|
"389": "sailing", |
|
"390": "jumpstyle dancing", |
|
"391": "water sliding", |
|
"392": "grooming horse", |
|
"393": "massaging feet", |
|
"394": "playing paintball", |
|
"395": "making a cake", |
|
"396": "bowling", |
|
"397": "contact juggling", |
|
"398": "applying cream", |
|
"399": "playing badminton" |
|
} |
|
|
|
|
