Repository: pytorch/vision

Issue:
MNIST dataset no longer downloads (repeat of old fixed March 2020 problem)
## 🐛 Bug

This seems to be a recurrence of an issue spotted in #1938 which was fixed back in March 2020 and then closed, but has now reappeared.  There are a number of people in #1938 reporting that the issue appeared somewhere in the last 12 hours.

## To Reproduce

Steps to reproduce the behavior:

1. Open a fresh google colab
2. Try something like the following:

```
import torchvision
from torchvision import datasets, transforms
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5,), (0.5,)),
                              ])
trainset = datasets.MNIST('PATH_TO_STORE_TRAINSET', download=True, train=True, transform=transform)

```

Results in:

```
Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to PATH_TO_STORE_TRAINSET/MNIST/raw/train-images-idx3-ubyte.gz
0/? [00:00<?, ?it/s]
---------------------------------------------------------------------------
HTTPError                                 Traceback (most recent call last)
<ipython-input-16-492e382ce34e> in <module>()
      4                               transforms.Normalize((0.5,), (0.5,)),
      5                               ])
----> 6 trainset = datasets.MNIST('PATH_TO_STORE_TRAINSET', download=True, train=True, transform=transform)

11 frames
/usr/local/lib/python3.7/dist-packages/torchvision/datasets/mnist.py in __init__(self, root, train, transform, target_transform, download)
     77 
     78         if download:
---> 79             self.download()
     80 
     81         if not self._check_exists():

/usr/local/lib/python3.7/dist-packages/torchvision/datasets/mnist.py in download(self)
    144         for url, md5 in self.resources:
    145             filename = url.rpartition('/')[2]
--> 146             download_and_extract_archive(url, download_root=self.raw_folder, filename=filename, md5=md5)
    147 
    148         # process and save as torch files

/usr/local/lib/python3.7/dist-packages/torchvision/datasets/utils.py in download_and_extract_archive(url, download_root, extract_root, filename, md5, remove_finished)
    254         filename = os.path.basename(url)
    255 
--> 256     download_url(url, download_root, filename, md5)
    257 
    258     archive = os.path.join(download_root, filename)

/usr/local/lib/python3.7/dist-packages/torchvision/datasets/utils.py in download_url(url, root, filename, md5)
     82                 )
     83             else:
---> 84                 raise e
     85         # check integrity of downloaded file
     86         if not check_integrity(fpath, md5):

/usr/local/lib/python3.7/dist-packages/torchvision/datasets/utils.py in download_url(url, root, filename, md5)
     70             urllib.request.urlretrieve(
     71                 url, fpath,
---> 72                 reporthook=gen_bar_updater()
     73             )
     74         except (urllib.error.URLError, IOError) as e:  # type: ignore[attr-defined]

/usr/lib/python3.7/urllib/request.py in urlretrieve(url, filename, reporthook, data)
    245     url_type, path = splittype(url)
    246 
--> 247     with contextlib.closing(urlopen(url, data)) as fp:
    248         headers = fp.info()
    249 

/usr/lib/python3.7/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

/usr/lib/python3.7/urllib/request.py in open(self, fullurl, data, timeout)
    529         for processor in self.process_response.get(protocol, []):
    530             meth = getattr(processor, meth_name)
--> 531             response = meth(req, response)
    532 
    533         return response

/usr/lib/python3.7/urllib/request.py in http_response(self, request, response)
    639         if not (200 <= code < 300):
    640             response = self.parent.error(
--> 641                 'http', request, response, code, msg, hdrs)
    642 
    643         return response

/usr/lib/python3.7/urllib/request.py in error(self, proto, *args)
    567         if http_err:
    568             args = (dict, 'default', 'http_error_default') + orig_args
--> 569             return self._call_chain(*args)
    570 
    571 # XXX probably also want an abstract factory that knows when it makes

/usr/lib/python3.7/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
    501         for handler in handlers:
    502             func = getattr(handler, meth_name)
--> 503             result = func(*args)
    504             if result is not None:
    505                 return result

/usr/lib/python3.7/urllib/request.py in http_error_default(self, req, fp, code, msg, hdrs)
    647 class HTTPDefaultErrorHandler(BaseHandler):
    648     def http_error_default(self, req, fp, code, msg, hdrs):
--> 649         raise HTTPError(req.full_url, code, msg, hdrs, fp)
    650 
    651 class HTTPRedirectHandler(BaseHandler):

HTTPError: HTTP Error 403: Forbidden
```


## Expected behavior

The dataset should just be loaded (as indeed it was this morning).

## Environment

```
Collecting environment information...
PyTorch version: 1.7.1+cu101
Is debug build: False
CUDA used to build PyTorch: 10.1
ROCM used to build PyTorch: N/A

OS: Ubuntu 18.04.5 LTS (x86_64)
GCC version: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Clang version: 6.0.0-1ubuntu2 (tags/RELEASE_600/final)
CMake version: version 3.12.0

Python version: 3.7 (64-bit runtime)
Is CUDA available: False
CUDA runtime version: 11.0.221
GPU models and configuration: Could not collect
Nvidia driver version: Could not collect
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.7.6.5
/usr/lib/x86_64-linux-gnu/libcudnn.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.0.4
/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.0.4
HIP runtime version: N/A
MIOpen runtime version: N/A

Versions of relevant libraries:
[pip3] numpy==1.19.5
[pip3] torch==1.7.1+cu101
[pip3] torchsummary==1.5.1
[pip3] torchtext==0.3.1
[pip3] torchvision==0.8.2+cu101
[conda] Could not collect```



cc @pmeier


Discussion:
Also posted  issue here:  https://github.com/googlecolab/colabtools/issues/1889
Yes, I am seeing this issue as of today (it was not happening yesterday)
Commenting here, for updates! 👀 
I have faced this problem too.
The colab team at google closed my report https://github.com/googlecolab/colabtools/issues/1889 there with the following statement:

> This isn't a colab issue; the data you're trying to download is refusing based on user-agent:
>
> `!curl -s -O -H 'xUser-agent: Python-urllib/3.7' http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz`
> successfully downloads ~9.9MiB of data (note the leading 'x' in the User-agent header) but
> 
> `!curl -s -O -H 'User-agent: Python-urllib/3.7' http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz `
> only gets the 403 page.
> 
> You probably want to report this to whoever hosts the data.

It's an interesting observation, if nothing else. Still, I think it's hard to pin things on yann.lecun ... since any solution that depends critically on the upness or downness of his website is probably fragile anyway, even if MNIST is "his".   Surely there is a better place than there for torchvision to use as a more future proof place from which to download its data?
Thanks for the heads up, we will be investigating this issue
copy this snippet at the top of your notebook, run it, and then just load your datasets as usual...

`from six.moves import urllib`
`opener = urllib.request.build_opener()`
`opener.addheaders = [('User-agent', 'Mozilla/5.0')]`
`urllib.request.install_opener(opener)`
Yeah, I'm pretty sure they blacklisted the standard `urllib` user agent.
Can confirm that the default user agent is blacklisted. We will get in touch with the author and see if the change is intentional or not. 

- If it is **unintentional** this should resolve itself in the next few days.
- If it is **intentional** we could simply change the default user agent.

Problem with the latter is that we are going to publish a new release tomorrow and any patch we provide will not be included. Thus, lets hope it is the former. 

Anyway, until this is resolved follow https://github.com/pytorch/vision/issues/3497#issuecomment-789983415 for a work around.

Resolving patch:
diff --git a/torchvision/datasets/utils.py b/torchvision/datasets/utils.py
--- a/torchvision/datasets/utils.py
+++ b/torchvision/datasets/utils.py
@@ -7,11 +7,28 @@
 from typing import Any, Callable, List, Iterable, Optional, TypeVar
 from urllib.parse import urlparse
 import zipfile
+import urllib
+import urllib.request
+import urllib.error
 
 import torch
 from torch.utils.model_zoo import tqdm
 
 
+USER_AGENT = "pytorch/vision"
+
+
+def _urlretrieve(url: str, filename: str, chunk_size: int = 1024) -> None:
+    with open(filename, "wb") as fh:
+        with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": USER_AGENT})) as response:
+            with tqdm(total=response.length) as pbar:
+                for chunk in iter(lambda: response.read(chunk_size), ""):
+                    if not chunk:
+                        break
+                    pbar.update(chunk_size)
+                    fh.write(chunk)
+
+
 def gen_bar_updater() -> Callable[[int, int, int], None]:
     pbar = tqdm(total=None)
 
@@ -83,8 +100,6 @@ def download_url(
         md5 (str, optional): MD5 checksum of the download. If None, do not check
         max_redirect_hops (int, optional): Maximum number of redirect hops allowed
     """
-    import urllib
-
     root = os.path.expanduser(root)
     if not filename:
         filename = os.path.basename(url)
@@ -108,19 +123,13 @@ def download_url(
     # download the file
     try:
         print('Downloading ' + url + ' to ' + fpath)
-        urllib.request.urlretrieve(
-            url, fpath,
-            reporthook=gen_bar_updater()
-        )
+        _urlretrieve(url, fpath)
     except (urllib.error.URLError, IOError) as e:  # type: ignore[attr-defined]
         if url[:5] == 'https':
             url = url.replace('https:', 'http:')
             print('Failed download. Trying https -> http instead.'
                   ' Downloading ' + url + ' to ' + fpath)
-            urllib.request.urlretrieve(
-                url, fpath,
-                reporthook=gen_bar_updater()
-            )
+            _urlretrieve(url, fpath)
         else:
             raise e
     # check integrity of downloaded file


Test patch:
diff --git a/test/test_datasets_download.py b/test/test_datasets_download.py
--- a/test/test_datasets_download.py
+++ b/test/test_datasets_download.py
@@ -14,7 +14,7 @@
 import pytest
 
 from torchvision import datasets
-from torchvision.datasets.utils import download_url, check_integrity, download_file_from_google_drive
+from torchvision.datasets.utils import download_url, check_integrity, download_file_from_google_drive, USER_AGENT
 
 from common_utils import get_tmp_dir
 from fakedata_generation import places365_root
@@ -150,7 +150,7 @@ def assert_server_response_ok():
 
 
 def assert_url_is_accessible(url, timeout=5.0):
-    request = Request(url, headers=dict(method="HEAD"))
+    request = Request(url, headers={"method": "HEAD", "User-Agent": USER_AGENT})
     with assert_server_response_ok():
         urlopen(request, timeout=timeout)
 
@@ -160,7 +160,8 @@ def assert_file_downloads_correctly(url, md5, timeout=5.0):
         file = path.join(root, path.basename(url))
         with assert_server_response_ok():
             with open(file, "wb") as fh:
-                response = urlopen(url, timeout=timeout)
+                request = Request(url, headers={"User-Agent": USER_AGENT})
+                response = urlopen(request, timeout=timeout)
                 fh.write(response.read())
 
         assert check_integrity(file, md5=md5), "The MD5 checksums mismatch"