Can not use randomDataset call in spawned calls...?

GGPYTORCH000: <p>I am learning the FSDP example here but they used example that are not downloadable (has download restiction). I am sick and tired of poorly written tutorial like this whereas they take examples of undownloadable dataset.</p>
<p><a href="https://pytorch.org/tutorials/intermediate/FSDP_tutorial.html" class="onebox" target="_blank" rel="noopener nofollow ugc">https://pytorch.org/tutorials/intermediate/FSDP_tutorial.html</a></p>
<p>I tried to remedy by substiting the mnist dataset by writing completely randomized dataset creation (whole code pasted below) but I must be doing something with train_size which I set to 30,000 for randomized data creation:</p>
<pre><code>WORLD_SIZE = torch.cuda.device_count()
mp.spawn(fsdp_main,
    args=(WORLD_SIZE, args),
    nprocs=WORLD_SIZE,
    join=True)
</code></pre>
<p>whole code:</p>
<blockquote>
<p>import os<br>
import argparse<br>
import functools<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
import torch.optim as optim</p>
<p>from torchvision import datasets, transforms<br>
from torch.utils.data import Dataset, DataLoader</p>
<p>from torch.optim.lr_scheduler import StepLR<br>
import torch.distributed as dist<br>
import torch.multiprocessing as mp</p>
<p>from torch.nn.parallel import DistributedDataParallel as DDP<br>
from torch.utils.data.distributed import DistributedSampler<br>
from torch.distributed.fsdp import FullyShardedDataParallel as FSDP<br>
from torch.distributed.fsdp.fully_sharded_data_parallel import (<br>
CPUOffload,<br>
BackwardPrefetch,<br>
)<br>
from torch.distributed.fsdp.wrap import (<br>
size_based_auto_wrap_policy,<br>
enable_wrap,<br>
wrap,<br>
)</p>
<p>CONFIG_USE_RANDOM_DATA=1</p>
<h1>Parameters and DataLoaders</h1>
<p>input_size = 1000<br>
output_size = 10<br>
batch_size = 1000<br>
data_size = 60000<br>
<strong>train_size = 30000</strong><br>
valid_size = 20000<br>
tset_size = 10000<br>
def setup(rank, world_size):<br>
os.environ[‘MASTER_ADDR’] = ‘localhost’<br>
os.environ[‘MASTER_PORT’] = ‘12355’</p>
<pre><code># initialize the process group
dist.init_process_group("nccl", rank=rank, world_size=world_size)
</code></pre>
<p>def cleanup():<br>
dist.destroy_process_group()</p>
<p>class Net(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(Net, self).<strong>init</strong>()<br>
self.conv1 = nn.Conv2d(1, 32, 3, 1)<br>
self.conv2 = nn.Conv2d(32, 64, 3, 1)<br>
self.dropout1 = nn.Dropout(0.25)<br>
self.dropout2 = nn.Dropout(0.5)<br>
self.fc1 = nn.Linear(9216, 128)<br>
self.fc2 = nn.Linear(128, 10)</p>
<pre><code>def forward(self, x):

    x = self.conv1(x)
    x = F.relu(x)
    x = self.conv2(x)
    x = F.relu(x)
    x = F.max_pool2d(x, 2)
    x = self.dropout1(x)
    x = torch.flatten(x, 1)
    x = self.fc1(x)
    x = F.relu(x)
    x = self.dropout2(x)
    x = self.fc2(x)
    output = F.log_softmax(x, dim=1)
    return output
</code></pre>
<p>def train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=None):<br>
model.train()<br>
ddp_loss = torch.zeros(2).to(rank)<br>
if sampler:<br>
sampler.set_epoch(epoch)<br>
for batch_idx, (data, target) in enumerate(train_loader):<br>
data, target = data.to(rank), target.to(rank)<br>
optimizer.zero_grad()<br>
output = model(data)<br>
loss = F.nll_loss(output, target, reduction=‘sum’)<br>
loss.backward()<br>
optimizer.step()<br>
ddp_loss[0] += loss.item()<br>
ddp_loss[1] += len(data)</p>
<pre><code>dist.all_reduce(ddp_loss, op=dist.ReduceOp.SUM)
if rank == 0:
    print('Train Epoch: {} \tLoss: {:.6f}'.format(epoch, ddp_loss[0] / ddp_loss[1]))
</code></pre>
<p>def test(model, rank, world_size, test_loader):<br>
model.eval()<br>
correct = 0<br>
ddp_loss = torch.zeros(3).to(rank)<br>
with torch.no_grad():<br>
for data, target in test_loader:<br>
data, target = data.to(rank), target.to(rank)<br>
output = model(data)<br>
ddp_loss[0] += F.nll_loss(output, target, reduction=‘sum’).item()  # sum up batch loss<br>
pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability<br>
ddp_loss[1] += pred.eq(target.view_as(pred)).sum().item()<br>
ddp_loss[2] += len(data)</p>
<pre><code>dist.all_reduce(ddp_loss, op=dist.ReduceOp.SUM)

if rank == 0:
    test_loss = ddp_loss[0] / ddp_loss[2]
    print('Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
        test_loss, int(ddp_loss[1]), int(ddp_loss[2]),
        100. * ddp_loss[1] / ddp_loss[2]))
</code></pre>
<p>def fsdp_main(rank, world_size, args):<br>
setup(rank, world_size)</p>
<pre><code>transform=transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.1307,), (0.3081,))
])

if not CONFIG_USE_RANDOM_DATA:
    train_kwargs = {'batch_size': args.batch_size}
    test_kwargs = {'batch_size': args.test_batch_size}
    cuda_kwargs = {'num_workers': 2,
                    'pin_memory': True,
                    'shuffle': False}
    train_kwargs.update(cuda_kwargs)
    test_kwargs.update(cuda_kwargs)
else:
    dataset1 = datasets.MNIST('../data', train=True, download=True,
                        transform=transform)
    dataset2 = datasets.MNIST('../data', train=False,
    sampler1 = DistributedSampler(dataset1, rank=rank, num_replicas=world_size, shuffle=True)
    sampler2 = DistributedSampler(dataset2, rank=rank, num_replicas=world_size)

    train_kwargs = {'batch_size': args.batch_size, 'sampler': sampler1}
    test_kwargs = {'batch_size': args.test_batch_size, 'sampler': sampler2}
    cuda_kwargs = {'num_workers': 2,
                    'pin_memory': True,
                    'shuffle': False}
    train_kwargs.update(cuda_kwargs)
    test_kwargs.update(cuda_kwargs)

if CONFIG_USE_RANDOM_DATA:
    train_loader = torch.utils.data.DataLoader(dataset=RandomDataset(input_size, train_size), **train_kwargs)
    test_loader = torch.utul.data.DataLoader(dataset=RandomDataset(input_size, test_size), **test_kwargs)
else:
    train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs)
    test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)

my_auto_wrap_policy = functools.partial(
    size_based_auto_wrap_policy, min_num_params=100
)
torch.cuda.set_device(rank)


init_start_event = torch.cuda.Event(enable_timing=True)
init_end_event = torch.cuda.Event(enable_timing=True)

model = Net().to(rank)

model = FSDP(model)

optimizer = optim.Adadelta(model.parameters(), lr=args.lr)

scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
init_start_event.record()
for epoch in range(1, args.epochs + 1):
    train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=sampler1)
    test(model, rank, world_size, test_loader)
    scheduler.step()

init_end_event.record()

if rank == 0:
    print(f"CUDA event elapsed time: {init_start_event.elapsed_time(init_end_event) / 1000}sec")
    print(f"{model}")

if args.save_model:
    # use a barrier to make sure training is done on all ranks
    dist.barrier()
    states = model.state_dict()
    if rank == 0:
        torch.save(states, "mnist_cnn.pt")

cleanup()
</code></pre>
<p><span class="hashtag-raw">#2</span>.5 Finally, parse the arguments and set the main function</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
# Training settings<br>
parser = argparse.ArgumentParser(description=‘PyTorch MNIST Example’)<br>
parser.add_argument(‘–batch-size’, type=int, default=64, metavar=‘N’,<br>
help=‘input batch size for training (default: 64)’)<br>
parser.add_argument(‘–test-batch-size’, type=int, default=1000, metavar=‘N’,<br>
help=‘input batch size for testing (default: 1000)’)<br>
parser.add_argument(‘–epochs’, type=int, default=10, metavar=‘N’,<br>
help=‘number of epochs to train (default: 14)’)<br>
parser.add_argument(‘–lr’, type=float, default=1.0, metavar=‘LR’,<br>
help=‘learning rate (default: 1.0)’)<br>
parser.add_argument(‘–gamma’, type=float, default=0.7, metavar=‘M’,<br>
help=‘Learning rate step gamma (default: 0.7)’)<br>
parser.add_argument(‘–no-cuda’, action=‘store_true’, default=False,<br>
help=‘disables CUDA training’)<br>
parser.add_argument(‘–seed’, type=int, default=1, metavar=‘S’,<br>
help=‘random seed (default: 1)’)<br>
parser.add_argument(‘–save-model’, action=‘store_true’, default=False,<br>
help=‘For Saving the current Model’)<br>
args = parser.parse_args()</p>
<pre><code>torch.manual_seed(args.seed)

WORLD_SIZE = torch.cuda.device_count()
mp.spawn(fsdp_main,
    args=(WORLD_SIZE, args),
    nprocs=WORLD_SIZE,
    join=True)
</code></pre>
</blockquote>
<p>Executing gives following error:</p>
<blockquote>
<p>[root@gt-pla-u25-08 3-fsdp]# python3 ex1-fsdp.py<br>
W0528 06:38:51.695650 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54881 via signal SIGTERM<br>
W0528 06:38:51.696214 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54882 via signal SIGTERM<br>
W0528 06:38:51.697674 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54883 via signal SIGTERM<br>
W0528 06:38:51.699138 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54884 via signal SIGTERM<br>
W0528 06:38:51.701952 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54886 via signal SIGTERM<br>
W0528 06:38:51.702436 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54887 via signal SIGTERM<br>
W0528 06:38:51.705028 140375835883328 torch/multiprocessing/spawn.py:145] Terminating process 54888 via signal SIGTERM<br>
Traceback (most recent call last):<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 225, in <br>
mp.spawn(fsdp_main,<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 281, in spawn<br>
return start_processes(fn, args, nprocs, join, daemon, start_method=“spawn”)<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 237, in start_processes<br>
while not context.join():<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 188, in join<br>
raise ProcessRaisedException(msg, error_index, failed_process.pid)<br>
torch.multiprocessing.spawn.ProcessRaisedException:</p>
<p>– Process 4 terminated with the following error:<br>
Traceback (most recent call last):<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 75, in _wrap<br>
fn(i, *args)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 180, in fsdp_main<br>
train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=sampler1)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 87, in train<br>
for batch_idx, (data, target) in enumerate(train_loader):<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/dataloader.py”, line 631, in <strong>next</strong><br>
data = self._next_data()<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/dataloader.py”, line 1346, in _next_data<br>
return self._process_data(data)<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/dataloader.py”, line 1372, in _process_data<br>
data.reraise()<br>
File “/usr/local/lib64/python3.9/site-packages/torch/_utils.py”, line 705, in reraise<br>
raise exception<br>
IndexError: Caught IndexError in DataLoader worker process 0.<br>
Original Traceback (most recent call last):<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/_utils/worker.py”, line 308, in _worker_loop<br>
data = fetcher.fetch(index)  # type: ignore[possibly-undefined]<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/_utils/fetch.py”, line 51, in fetch<br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “/usr/local/lib64/python3.9/site-packages/torch/utils/data/_utils/fetch.py”, line 51, in <br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/…/classes.py”, line 13, in <strong>getitem</strong><br>
return self.data[index]<br>
IndexError: index 41548 is out of bounds for dimension 0 with size 30000</p>
</blockquote>
<p>Increating train_size to 60000 gave following error:</p>
<blockquote>
<p>[root@gt-pla-u25-08 3-fsdp]# python3 ex1-fsdp.py<br>
W0528 06:41:33.675591 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59623 via signal SIGTERM<br>
W0528 06:41:33.675977 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59624 via signal SIGTERM<br>
W0528 06:41:33.676160 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59626 via signal SIGTERM<br>
W0528 06:41:33.677502 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59627 via signal SIGTERM<br>
W0528 06:41:33.679066 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59628 via signal SIGTERM<br>
W0528 06:41:33.681031 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59629 via signal SIGTERM<br>
W0528 06:41:33.683310 139771223697216 torch/multiprocessing/spawn.py:145] Terminating process 59630 via signal SIGTERM<br>
Traceback (most recent call last):<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 225, in <br>
mp.spawn(fsdp_main,<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 281, in spawn<br>
return start_processes(fn, args, nprocs, join, daemon, start_method=“spawn”)<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 237, in start_processes<br>
while not context.join():<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 188, in join<br>
raise ProcessRaisedException(msg, error_index, failed_process.pid)<br>
torch.multiprocessing.spawn.ProcessRaisedException:</p>
<p>– Process 2 terminated with the following error:<br>
Traceback (most recent call last):<br>
File “/usr/local/lib64/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 75, in _wrap<br>
fn(i, *args)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 180, in fsdp_main<br>
train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=sampler1)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 87, in train<br>
for batch_idx, (data, target) in enumerate(train_loader):<br>
ValueError: too many values to unpack (expected 2)</p>
</blockquote>

ptrblck: <aside class="quote no-group" data-username="GGPYTORCH000" data-post="1" data-topic="203607">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ggpytorch000/48/47787_2.png" class="avatar"> GGPYTORCH000:</div>
<blockquote>
<p>for batch_idx, (data, target) in enumerate(train_loader):<br>
ValueError: too many values to unpack (expected 2)</p>
</blockquote>
</aside>
<p>It seems your custom <code>Dataset</code> returns a single tensor:</p>
<aside class="quote no-group" data-username="GGPYTORCH000" data-post="1" data-topic="203607">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ggpytorch000/48/47787_2.png" class="avatar"> GGPYTORCH000:</div>
<blockquote>
<p>ine 13, in <strong>getitem</strong><br>
return self.data[index]</p>
</blockquote>
</aside>
<p>so either assign the data only or additionally return a target.</p>
<aside class="quote no-group" data-username="GGPYTORCH000" data-post="1" data-topic="203607">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ggpytorch000/48/47787_2.png" class="avatar"> GGPYTORCH000:</div>
<blockquote>
<p>I am sick and tired of poorly written tutorial like this whereas they take examples of undownloadable dataset.</p>
</blockquote>
</aside>
<p>Contributions are more than welcome in case you want to improve tutorials. <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=12" title=":slight_smile:" class="emoji" alt=":slight_smile:" loading="lazy" width="20" height="20"></p>

GGPYTORCH000: <p>actually title is misnomer, initially randomdataset was unrecognized and i was scratching my head where it was imported from. turns out another example somewhere in their site had class defined so i picked up that code but still does not work because other code example uses it differently:</p>
<p>for data in rand_loader:</p>
<p>after defining as:</p>
<p>class RandomDataset(Dataset):</p>
<pre><code>def __init__(self, size, length):
    print("GG: RandomDataset.__init__(size=", size, "length: ", length, ")")
    self.len = length
    self.data = torch.randn(length, size)

def __getitem__(self, index):
    print("GG: RandomDataset.__getitem__(index=", index, ")")
    return self.data[index]

def __len__(self):
    print("GG: RandomDataset.__len__() returning self.len: ", self.len)
    return self.len
</code></pre>
<p>In this example,<br>
for batch_idx, (data, target) in enumerate(train_loader):</p>
<p>So i made following mods for this example:</p>
<pre><code>#for batch_idx, (data, target) in enumerate(train_loader):
for data, target in train_loader:
</code></pre>
<p>since batch_idx does not seem to be used anywhere anyways.<br>
and defined randomdataset1 slightly modded version as well:</p>
<p>class RandomDataset1(Dataset):</p>
<pre><code>def __init__(self, size, length):
    self.len = length
    self.data = torch.randn(length, size)
    self.label = torch.randn(length, 1)

def __getitem__(self, index):
    return self.data[index], self.label[index]

def __len__(self):
    return self.len
</code></pre>
<p>to include label and return both data, and label.<br>
After run, similar error, so still looking at what went wrong…</p>
<p>GG: train entered: rank:  1 : world_size:  2 , train_loader:  &lt;torch.utils.data.dataloader.DataLoader object at 0x7f25c54c2cd0&gt; , epoch:  1<br>
W0529 05:57:28.003877 140141330396992 torch/multiprocessing/spawn.py:145] Terminating process 147 via signal SIGTERM<br>
Traceback (most recent call last):<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 205, in <br>
mp.spawn(fsdp_main,<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 281, in spawn<br>
return start_processes(fn, args, nprocs, join, daemon, start_method=“spawn”)<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 237, in start_processes<br>
while not context.join():<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 188, in join<br>
raise ProcessRaisedException(msg, error_index, failed_process.pid)<br>
torch.multiprocessing.spawn.ProcessRaisedException:</p>
<p>– Process 0 terminated with the following error:<br>
Traceback (most recent call last):<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/multiprocessing/spawn.py”, line 75, in _wrap<br>
fn(i, *args)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 159, in fsdp_main<br>
train(args, model, rank, world_size, train_loader, optimizer, epoch, sampler=sampler1)<br>
File “/root/extdir/gg/git/codelab/gpu/ml/pytorch/distributed/tutorials/3-fsdp/ex1-fsdp.py”, line 63, in train<br>
for data, target in train_loader:<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/dataloader.py”, line 631, in <strong>next</strong><br>
data = self._next_data()<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/dataloader.py”, line 1346, in _next_data<br>
return self._process_data(data)<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/dataloader.py”, line 1372, in _process_data<br>
data.reraise()<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/_utils.py”, line 705, in reraise<br>
raise exception<br>
IndexError: Caught IndexError in DataLoader worker process 0.<br>
Original Traceback (most recent call last):<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/_utils/worker.py”, line 308, in _worker_loop<br>
data = fetcher.fetch(index)  # type: ignore[possibly-undefined]<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py”, line 51, in fetch<br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py”, line 51, in <br>
data = [self.dataset[idx] for idx in possibly_batched_index]<br>
File “…/classes.py”, line 55, in <strong>getitem</strong><br>
return self.data[index], self.label[index]<br>
IndexError: index 35845 is out of bounds for dimension 0 with size 30000</p>

ptrblck: <aside class="quote no-group" data-username="GGPYTORCH000" data-post="3" data-topic="203607">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ggpytorch000/48/47787_2.png" class="avatar"> GGPYTORCH000:</div>
<blockquote>
<p>return self.data[index], self.label[index]<br>
IndexError: index 35845 is out of bounds for dimension 0 with size 30000</p>
</blockquote>
</aside>
<p>The index used in <code>__getitem__</code> is invalid so instead of defining the <code>self.len</code> manually, compute it from the actually used data:</p>
<pre data-code-wrap="python"><code class="lang-python">def __len__(self):
    return len(self.data)
</code></pre>

GGPYTORCH000: <p>nice catch! seems like working. I would try investigate why self.len worked on other example and here is not. Might have to do with actual sharding.</p>
<p>I am getting shape error now but that should be taken care of creating randomdataset to be same shape as mnist.</p>
<pre><code>return self._conv_forward(input, self.weight, self.bias)
</code></pre>
<p>File “/home/miniconda3/envs/root-test/lib/python3.9/site-packages/torch/nn/modules/conv.py”, line 456, in _conv_forward<br>
return F.conv2d(input, weight, bias, self.stride,<br>
RuntimeError: Expected 3D (unbatched) or 4D (batched) input to conv2d, but got input of size: [64, 10]</p>