load_internvideo.py

from intern_action import intern_action_b16
from huggingface_hub import hf_hub_download
# from kinetics_class_index import kinetics_classnames
import torch
import torch.nn as nn
import torchvision.transforms as T
import torch.nn.functional as F
import numpy as np

from transforms import (
    GroupNormalize, GroupScale, GroupCenterCrop, 
    Stack, ToTorchFormatTensor
)

class Intern_Action(nn.Module):
    def __init__(self, model):
        super().__init__()
        self.backbone = model
    
    def forward(self, x):
        return self.backbone(x)

def get_index(num_frames, num_segments=8):
    seg_size = float(num_frames - 1) / num_segments
    start = int(seg_size / 2)
    offsets = np.array([
        start + int(np.round(seg_size * idx)) for idx in range(num_segments)
    ])
    return offsets

def transform_action():
    # transform
    crop_size = 224
    scale_size = 256
    input_mean = [0.485, 0.456, 0.406]
    input_std = [0.229, 0.224, 0.225]

    return T.Compose([
        # T.ToPILImage(),
        GroupScale(int(scale_size)),
        GroupCenterCrop(crop_size),
        Stack(),
        ToTorchFormatTensor(),
        GroupNormalize(input_mean, input_std) 
    ])

def load_intern_action(device):
    # Create an id to label name mapping
    kinetics_id_to_classname = {}
    for k, v in kinetics_classnames.items():
        kinetics_id_to_classname[k] = v

    model_path = hf_hub_download(repo_id="Andy1621/uniformerv2", filename="k400+k710_uniformerv2_b16_8x224.pyth")
    # Pick a pretrained model 
    model = Intern_Action(intern_action_b16(pretrained=False, t_size=8, no_lmhra=True, temporal_downsample=False))
    state_dict = torch.load(model_path, map_location=device)
    model.load_state_dict(state_dict)
    # Set to eval mode and move to desired device
    model = model.to(device)
    model = model.eval()
    return model

def cut_frame_to_8(data):
    index = np.linspace(0, len(data)-1, 8).astype(int)
    return data[index]

kinetics_classnames = {
    "0": "riding a bike", 
    "1": "marching", 
    "2": "dodgeball", 
    "3": "playing cymbals", 
    "4": "checking tires", 
    "5": "roller skating", 
    "6": "tasting beer", 
    "7": "clapping", 
    "8": "drawing", 
    "9": "juggling fire", 
    "10": "bobsledding", 
    "11": "petting animal (not cat)", 
    "12": "spray painting", 
    "13": "training dog", 
    "14": "eating watermelon", 
    "15": "building cabinet", 
    "16": "applauding", 
    "17": "playing harp", 
    "18": "balloon blowing", 
    "19": "sled dog racing", 
    "20": "wrestling", 
    "21": "pole vault", 
    "22": "hurling (sport)", 
    "23": "riding scooter", 
    "24": "shearing sheep", 
    "25": "sweeping floor", 
    "26": "eating carrots", 
    "27": "skateboarding", 
    "28": "dunking basketball", 
    "29": "disc golfing", 
    "30": "eating spaghetti", 
    "31": "playing flute", 
    "32": "riding mechanical bull", 
    "33": "making sushi", 
    "34": "trapezing", 
    "35": "picking fruit", 
    "36": "stretching leg", 
    "37": "playing ukulele", 
    "38": "tying tie", 
    "39": "skydiving", 
    "40": "playing cello", 
    "41": "jumping into pool", 
    "42": "shooting goal (soccer)", 
    "43": "trimming trees", 
    "44": "bookbinding", 
    "45": "ski jumping", 
    "46": "walking the dog", 
    "47": "riding unicycle", 
    "48": "shaving head", 
    "49": "hopscotch", 
    "50": "playing piano", 
    "51": "parasailing", 
    "52": "bartending", 
    "53": "kicking field goal", 
    "54": "finger snapping", 
    "55": "dining", 
    "56": "yawning", 
    "57": "peeling potatoes", 
    "58": "canoeing or kayaking", 
    "59": "front raises", 
    "60": "laughing", 
    "61": "dancing macarena", 
    "62": "digging", 
    "63": "reading newspaper", 
    "64": "hitting baseball", 
    "65": "clay pottery making", 
    "66": "exercising with an exercise ball", 
    "67": "playing saxophone", 
    "68": "shooting basketball", 
    "69": "washing hair", 
    "70": "lunge", 
    "71": "brushing hair", 
    "72": "curling hair", 
    "73": "kitesurfing", 
    "74": "tapping guitar", 
    "75": "bending back", 
    "76": "skipping rope", 
    "77": "situp", 
    "78": "folding paper", 
    "79": "cracking neck", 
    "80": "assembling computer", 
    "81": "cleaning gutters", 
    "82": "blowing out candles", 
    "83": "shaking hands", 
    "84": "dancing gangnam style", 
    "85": "windsurfing", 
    "86": "tap dancing", 
    "87": "skiing (not slalom or crosscountry)", 
    "88": "bandaging", 
    "89": "push up", 
    "90": "doing nails", 
    "91": "punching person (boxing)", 
    "92": "bouncing on trampoline", 
    "93": "scrambling eggs", 
    "94": "singing", 
    "95": "cleaning floor", 
    "96": "krumping", 
    "97": "drumming fingers", 
    "98": "snowmobiling", 
    "99": "gymnastics tumbling", 
    "100": "headbanging", 
    "101": "catching or throwing frisbee", 
    "102": "riding elephant", 
    "103": "bee keeping", 
    "104": "feeding birds", 
    "105": "snatch weight lifting", 
    "106": "mowing lawn", 
    "107": "fixing hair", 
    "108": "playing trumpet", 
    "109": "flying kite", 
    "110": "crossing river", 
    "111": "swinging legs", 
    "112": "sanding floor", 
    "113": "belly dancing", 
    "114": "sneezing", 
    "115": "clean and jerk", 
    "116": "side kick", 
    "117": "filling eyebrows", 
    "118": "shuffling cards", 
    "119": "recording music", 
    "120": "cartwheeling", 
    "121": "feeding fish", 
    "122": "folding clothes", 
    "123": "water skiing", 
    "124": "tobogganing", 
    "125": "blowing leaves", 
    "126": "smoking", 
    "127": "unboxing", 
    "128": "tai chi", 
    "129": "waxing legs", 
    "130": "riding camel", 
    "131": "slapping", 
    "132": "tossing salad", 
    "133": "capoeira", 
    "134": "playing cards", 
    "135": "playing organ", 
    "136": "playing violin", 
    "137": "playing drums", 
    "138": "tapping pen", 
    "139": "vault", 
    "140": "shoveling snow", 
    "141": "playing tennis", 
    "142": "getting a tattoo", 
    "143": "making a sandwich", 
    "144": "making tea", 
    "145": "grinding meat", 
    "146": "squat", 
    "147": "eating doughnuts", 
    "148": "ice fishing", 
    "149": "snowkiting", 
    "150": "kicking soccer ball", 
    "151": "playing controller", 
    "152": "giving or receiving award", 
    "153": "welding", 
    "154": "throwing discus", 
    "155": "throwing axe", 
    "156": "ripping paper", 
    "157": "swimming butterfly stroke", 
    "158": "air drumming", 
    "159": "blowing nose", 
    "160": "hockey stop", 
    "161": "taking a shower", 
    "162": "bench pressing", 
    "163": "planting trees", 
    "164": "pumping fist", 
    "165": "climbing tree", 
    "166": "tickling", 
    "167": "high kick", 
    "168": "waiting in line", 
    "169": "slacklining", 
    "170": "tango dancing", 
    "171": "hurdling", 
    "172": "carrying baby", 
    "173": "celebrating", 
    "174": "sharpening knives", 
    "175": "passing American football (in game)", 
    "176": "headbutting", 
    "177": "playing recorder", 
    "178": "brush painting", 
    "179": "garbage collecting", 
    "180": "robot dancing", 
    "181": "shredding paper", 
    "182": "pumping gas", 
    "183": "rock climbing", 
    "184": "hula hooping", 
    "185": "braiding hair", 
    "186": "opening present", 
    "187": "texting", 
    "188": "decorating the christmas tree", 
    "189": "answering questions", 
    "190": "playing keyboard", 
    "191": "writing", 
    "192": "bungee jumping", 
    "193": "sniffing", 
    "194": "eating burger", 
    "195": "playing accordion", 
    "196": "making pizza", 
    "197": "playing volleyball", 
    "198": "tasting food", 
    "199": "pushing cart", 
    "200": "spinning poi", 
    "201": "cleaning windows", 
    "202": "arm wrestling", 
    "203": "changing oil", 
    "204": "swimming breast stroke", 
    "205": "tossing coin", 
    "206": "deadlifting", 
    "207": "hoverboarding", 
    "208": "cutting watermelon", 
    "209": "cheerleading", 
    "210": "snorkeling", 
    "211": "washing hands", 
    "212": "eating cake", 
    "213": "pull ups", 
    "214": "surfing water", 
    "215": "eating hotdog", 
    "216": "holding snake", 
    "217": "playing harmonica", 
    "218": "ironing", 
    "219": "cutting nails", 
    "220": "golf chipping", 
    "221": "shot put", 
    "222": "hugging", 
    "223": "playing clarinet", 
    "224": "faceplanting", 
    "225": "trimming or shaving beard", 
    "226": "drinking shots", 
    "227": "riding mountain bike", 
    "228": "tying bow tie", 
    "229": "swinging on something", 
    "230": "skiing crosscountry", 
    "231": "unloading truck", 
    "232": "cleaning pool", 
    "233": "jogging", 
    "234": "ice climbing", 
    "235": "mopping floor", 
    "236": "making bed", 
    "237": "diving cliff", 
    "238": "washing dishes", 
    "239": "grooming dog", 
    "240": "weaving basket", 
    "241": "frying vegetables", 
    "242": "stomping grapes", 
    "243": "moving furniture", 
    "244": "cooking sausages", 
    "245": "doing laundry", 
    "246": "dying hair", 
    "247": "knitting", 
    "248": "reading book", 
    "249": "baby waking up", 
    "250": "punching bag", 
    "251": "surfing crowd", 
    "252": "cooking chicken", 
    "253": "pushing car", 
    "254": "springboard diving", 
    "255": "swing dancing", 
    "256": "massaging legs", 
    "257": "beatboxing", 
    "258": "breading or breadcrumbing", 
    "259": "somersaulting", 
    "260": "brushing teeth", 
    "261": "stretching arm", 
    "262": "juggling balls", 
    "263": "massaging person's head", 
    "264": "eating ice cream", 
    "265": "extinguishing fire", 
    "266": "hammer throw", 
    "267": "whistling", 
    "268": "crawling baby", 
    "269": "using remote controller (not gaming)", 
    "270": "playing cricket", 
    "271": "opening bottle", 
    "272": "playing xylophone", 
    "273": "motorcycling", 
    "274": "driving car", 
    "275": "exercising arm", 
    "276": "passing American football (not in game)", 
    "277": "playing kickball", 
    "278": "sticking tongue out", 
    "279": "flipping pancake", 
    "280": "catching fish", 
    "281": "eating chips", 
    "282": "shaking head", 
    "283": "sword fighting", 
    "284": "playing poker", 
    "285": "cooking on campfire", 
    "286": "doing aerobics", 
    "287": "paragliding", 
    "288": "using segway", 
    "289": "folding napkins", 
    "290": "playing bagpipes", 
    "291": "gargling", 
    "292": "skiing slalom", 
    "293": "strumming guitar", 
    "294": "javelin throw", 
    "295": "waxing back", 
    "296": "riding or walking with horse", 
    "297": "plastering", 
    "298": "long jump", 
    "299": "parkour", 
    "300": "wrapping present", 
    "301": "egg hunting", 
    "302": "archery", 
    "303": "cleaning toilet", 
    "304": "swimming backstroke", 
    "305": "snowboarding", 
    "306": "catching or throwing baseball", 
    "307": "massaging back", 
    "308": "blowing glass", 
    "309": "playing guitar", 
    "310": "playing chess", 
    "311": "golf driving", 
    "312": "presenting weather forecast", 
    "313": "rock scissors paper", 
    "314": "high jump", 
    "315": "baking cookies", 
    "316": "using computer", 
    "317": "washing feet", 
    "318": "arranging flowers", 
    "319": "playing bass guitar", 
    "320": "spraying", 
    "321": "cutting pineapple", 
    "322": "waxing chest", 
    "323": "auctioning", 
    "324": "jetskiing", 
    "325": "drinking", 
    "326": "busking", 
    "327": "playing monopoly", 
    "328": "salsa dancing", 
    "329": "waxing eyebrows", 
    "330": "watering plants", 
    "331": "zumba", 
    "332": "chopping wood", 
    "333": "pushing wheelchair", 
    "334": "carving pumpkin", 
    "335": "building shed", 
    "336": "making jewelry", 
    "337": "catching or throwing softball", 
    "338": "bending metal", 
    "339": "ice skating", 
    "340": "dancing charleston", 
    "341": "abseiling", 
    "342": "climbing a rope", 
    "343": "crying", 
    "344": "cleaning shoes", 
    "345": "dancing ballet", 
    "346": "driving tractor", 
    "347": "triple jump", 
    "348": "throwing ball", 
    "349": "getting a haircut", 
    "350": "running on treadmill", 
    "351": "climbing ladder", 
    "352": "blasting sand", 
    "353": "playing trombone", 
    "354": "drop kicking", 
    "355": "country line dancing", 
    "356": "changing wheel", 
    "357": "feeding goats", 
    "358": "tying knot (not on a tie)", 
    "359": "setting table", 
    "360": "shaving legs", 
    "361": "kissing", 
    "362": "riding mule", 
    "363": "counting money", 
    "364": "laying bricks", 
    "365": "barbequing", 
    "366": "news anchoring", 
    "367": "smoking hookah", 
    "368": "cooking egg", 
    "369": "peeling apples", 
    "370": "yoga", 
    "371": "sharpening pencil", 
    "372": "dribbling basketball", 
    "373": "petting cat", 
    "374": "playing ice hockey", 
    "375": "milking cow", 
    "376": "shining shoes", 
    "377": "juggling soccer ball", 
    "378": "scuba diving", 
    "379": "playing squash or racquetball", 
    "380": "drinking beer", 
    "381": "sign language interpreting", 
    "382": "playing basketball", 
    "383": "breakdancing", 
    "384": "testifying", 
    "385": "making snowman", 
    "386": "golf putting", 
    "387": "playing didgeridoo", 
    "388": "biking through snow", 
    "389": "sailing", 
    "390": "jumpstyle dancing", 
    "391": "water sliding", 
    "392": "grooming horse", 
    "393": "massaging feet", 
    "394": "playing paintball", 
    "395": "making a cake", 
    "396": "bowling", 
    "397": "contact juggling", 
    "398": "applying cream", 
    "399": "playing badminton"
}