MN/Models/PhysicalAI
Collection
PhysicalAI models such as CLIP, VLMs, VLAs, World Action Models, etc. for EmbodiedAI, autonomous vehicles using deep learning. • 40 items • Updated
This repository contains weights or code derived from the XVLA foundational architecture developed by Hugging Face and the LeRobot/XVLA Authors.
This is XVLA-Base model cloned from Hugginface "lerobot/xvla-base" repository. This was createed for ready-to-use custom model for easy inference during Hackathon challenge.
import numpy as np
from PIL import Image
import torch
import lerobot
from lerobot.policies.xvla.modeling_xvla import XVLAPolicy
from lerobot.policies.factory import make_pre_post_processors
from lerobot.datasets.lerobot_dataset import LeRobotDataset
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model_id = "Man1103/XVLA-Base-0.88B"
policy = XVLAPolicy.from_pretrained(model_id).to(device)
print("XVLA model successfully loaded!")
from lerobot.policies.factory import make_pre_post_processors
preprocess, postprocess = make_pre_post_processors(
policy.config,
model_id,
preprocessor_overrides={"device_processor": {"device": str(device)}},
)
print("XVLA pre-processor and post-processor successfully loaded!")
from lerobot.datasets.lerobot_dataset import LeRobotDataset
# load a lerobotdataset (we will replace with a simpler dataset)
dataset = LeRobotDataset("lerobot/libero")
# pick an episode
episode_index = 0
# each episode corresponds to a contiguous range of frame indices
from_idx = dataset.meta.episodes["dataset_from_index"][episode_index]
to_idx = dataset.meta.episodes["dataset_to_index"][episode_index]
# get a single frame from that episode (e.g. the first frame)
frame_index = from_idx
frame = dict(dataset[frame_index])
batch = preprocess(frame)
with torch.inference_mode():
pred_action = policy.select_action(batch)
# use your policy postprocess, this post process the action
# for instance unnormalize the actions, detokenize it etc..
pred_action = postprocess(pred_action)
Base model
lerobot/xvla-base