Question: I'm trying to run inference on demand for yolo-nas using sagemaker batch transformer. Using pre trained model with pre trained weights. But I am getting this error:  python3: can't open file '//serve': [Errno 2] No such file or directory  I have no idea what this '//serve' is. I have no ref or use of it at all, and can't find any docs about it Some more data about my case:  Data are just images with single object in them (known as 'crop')  seq-gil-transformer.py sits in s3://benny-test/indexer/yolo-nas/1/code  /opt/ml/processing/input is not the right path. I know about it, I'm in the process of understanding which path I should use but the code crashes way before it...   I have my own custom image as follows: FROM python:3.10-bullseye RUN apt update && \ apt install ffmpeg libsm6 libxext6 libgl1 -y && \ rm -rf /var/lib/apt/lists/* ADD req.txt req.txt RUN pip3 install -r req.txt ENTRYPOINT ["python3"]  I have my own inference code known as 'seq-gil-transformer.py': (I know that I can improve it, for now trying to make the logic work & understand sagemaker and then I will improve the code) import super_gradients import torch from torch import nn import json from collections import OrderedDict from typing import List from pathlib import Path import PIL import numpy as np import time import cProfile import trace import io import psutil import numpy as np import matplotlib.pyplot as plt import sys import pandas as pd import argparse # getting images from local path def get_images(path, device): crops_path_list = Path(path).glob("*.png") images = [] for crop in crops_path_list: image = PIL.Image.open(str(crop)).convert("RGB") image = np.asarray(image) image = np.ascontiguousarray(image.transpose(2, 0, 1)) image = torch.from_numpy(np.array(image)).unsqueeze(0).to(dtype=torch.float32, device=device) images.append(image) return images # saving model def create_yolo_nas_indexer(device="cpu"): yolo_nas = super_gradients.training.models.get("yolo_nas_l", pretrained_weights="coco").to(device) torch.save(yolo_nas.backbone, f"yolo_nas_l_{device}.tar.gz") # loading model. # using our own module named 'modelwrapper', # you can see it in the end of this post def load_yolo_nas(device="cpu",output_layers=["stage2","stage3","context_module"]): yolo_nas = torch.load(f"yolo_nas_l_{device}.pth.tar").to(device) wraped_yolo_nas = ModelWrapper(yolo_nas,device=device).to(device) wraped_yolo_nas.add_output_layers(output_layers) return wraped_yolo_nas # making the inference using 'modelWrapper' # and the loaded yolo-nas super-gredients of # deci-ai model per image(crop) def primitive_c2v(crop:torch.Tensor, model)->list: output = model(crop)[0] return output if __name__ == "__main__": engine = "cpu" crops = get_images("/opt/ml/processing/input", engine) H = W = 64 C = 3 outputs = [] create_yolo_nas_indexer(engine) model = load_yolo_nas(engine) model.eval() list_o = [] for crop in crops: start_time = time.time() output = primitive_c2v(crop, model) output = [l.detach().numpy() for l in output] list_o.append(output) # printing a bit of the output just to be sure # everything went well. # in my data i got list of lists of numpy tensors. # each list in the grand list is of size of 4 for l in list_o: print(len(l))  And the sagemaker code itself is: import boto3 import sagemaker from sagemaker import get_execution_role sagemaker_session = sagemaker.Session() role = get_execution_role() bucket = 'benny-test' model = sagemaker.model.Model( source_dir="s3://benny-test/indexer/yolo-nas/1/code", entry_point="seq-gil-transformer.py", image_uri='*.dkr.ecr.eu-west-1.amazonaws.com/sagemaker:super-gredients-0.1', role=role, name="yolo-nas-cpu") transformer = model.transformer( instance_count=1, instance_type="ml.m5.4xlarge" ) transformer.transform( data="s3://benny-test/indexer/Raw/dummy_set/images" ) transformer.wait()  Model wrapper ref: this code is not necessarily needed but I'm showing it here for reference and reproducibility :) def remove_all_hooks_recursive(model: nn.Module) -> None: for name, child in model.named_children(): if child is not None: if hasattr(child, "_forward_hooks"): child._forward_hooks = OrderedDict() elif hasattr(child, "_forward_pre_hooks"): child._forward_pre_hooks = OrderedDict() elif hasattr(child, "_backward_hooks"): child._backward_hooks = OrderedDict() remove_all_hooks_recursive(child) def add_all_modules_to_model_dict_recursive(model, module_dict, prefix=''): """Recursively adds all modules in a PyTorch model to a hierarchical dictionary.""" for name, module in model.named_children(): full_name = prefix + '.' + name if prefix else name full_name = full_name if full_name != "_model" else "" module_dict[full_name] = module if isinstance(module, nn.Module): add_all_modules_to_model_dict_recursive(module, module_dict, full_name) class StopModel(Exception): def __init__(self): super().__init__() def forward_hook(model_wrapper, layer_name, model=None): def hook(module, input, output): model_wrapper.selected_out[layer_name] = output if model is not None: _, code = model(output) model_wrapper.selected_out[f"code_{layer_name}"] = code if model_wrapper.stop_at_last_hook and layer_name == model_wrapper.last_layer: raise StopModel() return hook class ModelWrapper(nn.Module): def __init__(self, model, stop_at_last_hook=False, device=None): super().__init__() self.stop_at_last_hook = stop_at_last_hook self.model = model self.model.eval() self.output_layers = [] self.selected_out = OrderedDict() self.fhooks = [] self.layer_size_dict = {} self.layer_stride_dict = {} self.model_dict = self.add_all_modules_to_model_dict() self.device = device or torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.last_layer = None @classmethod def from_cfg(cls, cfg_path: str): with open(cfg_path, "r") as f: cfg_dict = json.load(f) cls(**cfg_dict) def add_output_layers(self, output_layers: List[str]): self.last_layer = output_layers[-1] for output_layer in output_layers: if output_layer not in self.model_dict: raise ValueError(f"Model does not have layer: {output_layer}") self.output_layers = output_layers for layer_name, module in self.model_dict.items(): if layer_name in self.output_layers: self.fhooks.append(module.register_forward_hook(forward_hook(self, layer_name))) self.compute_output_layer_parameters() def compute_output_layer_parameters(self): random_input = torch.rand(1, 3, 64, 64) #TODO: CHANGE TO ZEROS and avoid seed usage random_input = random_input.to(self.device) self.forward(random_input) for layer_name, output_value in self.selected_out.items(): if isinstance(output_value, (list, tuple)): self.layer_size_dict[layer_name] = None self.layer_stride_dict[layer_name] = None else: self.layer_size_dict[layer_name] = output_value.shape[1] self.layer_stride_dict[layer_name] = int(64 / output_value.shape[2]) def print_all_modules(self, print_module_str=False): for layer_name, module in self.model_dict.items(): layer_txt = layer_name if print_module_str: layer_txt += f": {str(module)}" if layer_name in self.output_layers: layer_txt += " (SET AS AN OUTPUT LAYER)" print(layer_txt) def forward(self, x): # TODO: find a way to run the model only for the selected out if self.stop_at_last_hook: try: self.model(x) except Exception as e: if not isinstance(e, StopModel): raise e out = None else: out = self.model(x) return out, self.selected_out def inference(self, image, name): bbox_list = self.model.inference(image, name) return bbox_list, self.selected_out def add_all_modules_to_model_dict(self): model_dict = {} add_all_modules_to_model_dict_recursive(self.model, model_dict) return model_dict def remove_all_hooks(self): remove_all_hooks_recursive(self.model) self.selected_out = OrderedDict() 

Answer: Sagemaker batch inference and endpoints work in same way. They are expecting web server with get[ping] and post[invocations] to start working. The thing is, when sagemaker runs, he is running a file named "serve" which was missing in my case. To be clear, as far as i understood. "my case" is when we dont use estimator before the inference. you can see that im defining my own pre trained model like that: model = sagemaker.model.Model( image_uri='*.dkr.ecr.eu-west-1.amazonaws.com/sagemaker:yolo-nas-cpu-infra-0.1.28', role=role, name="yolo-nas-cpu-infra-v0-1-28-dt"+str(date_time_in_numbers), ) transformer = model.transformer( instance_count=1, instance_type="ml.m5.4xlarge", ) transformer.transform( data="s3://benny-test/indexer/yolo-nas/sagemaker-bs/", content_type="application/json" )  TL;DR used flask, ngnix, gunicorn. serve file configuring and running ngnix and gunicorn