Repository: pytorch/vision

Issue:
Package libjpeg and libpng within torchvision
With the addition of image-reading functions enabled by default in torchvision #1632 #1881 #1909, we will need to properly package libjpeg and libpng (and other libraries will be used soon as well, like ffmpeg) so that users of {Windows, Linux, OSX} x {pip, conda, source} can use those new functionalities in torchvision without issues.

I believe the current recommended way that PyTorch deals with 3rd-party libraries is the following:
- for conda, we use conda's package manager to handle the dependencies. Both [libjpeg](https://anaconda.org/conda-forge/libjpeg-turbo) and [libpng](https://anaconda.org/anaconda/libpng) are available in standard channels, although I'm not sure if using the `conda-forge` channel would bring problems due to conflicts with other libraries we use.
- for pip, I believe the preferred way is to include the libraries together with the torchvision wheel
- for source, we would need to handle the common installation locations, or at least provide clear instructions on how to provide the path to the libraries, either via CMake or performing the handling via python like [Pillow does (although the logic seems quite intricate)](https://github.com/python-pillow/Pillow/blob/master/setup.py).

We will also need to make sure our CI (which is fairly tied to binary builds) is updated.

cc @seemethere @rgommers 


Discussion:
> although I'm not sure if using the `conda-forge` channel would bring problems due to conflicts with other libraries we use.

It may right now I think, we really need to improve that situation. It would also force the users to mix channels. `libpng` is available in `defaults`. `libjpeg-turbo` is in `defaults` as well, but it looks outdated. Needs a more detailed look at what the best option is here.
> for pip, I believe the preferred way is to include the libraries together with the torchvision wheel

@fmassa said that size of the wheels isn't critical, and just bundling in `libpng` and `libjpeg` should be fine for now. So using [auditwheel](https://github.com/pypa/auditwheel) (Linux) and [delocate](https://github.com/matthew-brett/delocate) (macOS) seems like the way to go. There's no such tool for Windows, so there the right libs need to be copied in by hand.
@fmassa, is it possible to have assignment permissions on this repository?
@andfoy I gave you and @rgommers triage permissions to torchvision

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@
 import re
 import sys
 from setuptools import setup, find_packages
-from pkg_resources import get_distribution, DistributionNotFound
+from pkg_resources import parse_version, get_distribution, DistributionNotFound
 import subprocess
 import distutils.command.clean
 import distutils.spawn
@@ -76,7 +76,76 @@ def write_version_file():
 requirements.append(pillow_req + pillow_ver)
 
 
+def find_library(name, vision_include):
+    this_dir = os.path.dirname(os.path.abspath(__file__))
+    build_prefix = os.environ.get('BUILD_PREFIX', None)
+    is_conda_build = build_prefix is not None
+
+    library_found = False
+    conda_installed = False
+    lib_folder = None
+    include_folder = None
+    library_header = '{0}.h'.format(name)
+
+    print('Running build on conda-build: {0}'.format(is_conda_build))
+    if is_conda_build:
+        # Add conda headers/libraries
+        if os.name == 'nt':
+            build_prefix = os.path.join(build_prefix, 'Library')
+        include_folder = os.path.join(build_prefix, 'include')
+        lib_folder = os.path.join(build_prefix, 'lib')
+        library_header_path = os.path.join(
+            include_folder, library_header)
+        library_found = os.path.isfile(library_header_path)
+        conda_installed = library_found
+    else:
+        # Check if using Anaconda to produce wheels
+        conda = distutils.spawn.find_executable('conda')
+        is_conda = conda is not None
+        print('Running build on conda: {0}'.format(is_conda))
+        if is_conda:
+            python_executable = sys.executable
+            py_folder = os.path.dirname(python_executable)
+            if os.name == 'nt':
+                env_path = os.path.join(py_folder, 'Library')
+            else:
+                env_path = os.path.dirname(py_folder)
+            lib_folder = os.path.join(env_path, 'lib')
+            include_folder = os.path.join(env_path, 'include')
+            library_header_path = os.path.join(
+                include_folder, library_header)
+            library_found = os.path.isfile(library_header_path)
+            conda_installed = library_found
+
+    # Try to locate turbojpeg in Linux standard paths
+    if not library_found:
+        if sys.platform == 'linux':
+            library_found = os.path.exists('/usr/include/{0}'.format(
+                library_header))
+            library_found = library_found or os.path.exists(
+                '/usr/local/include/{0}'.format(library_header))
+        else:
+            # Lookup in TORCHVISION_INCLUDE or in the package file
+            package_path = [os.path.join(this_dir, 'torchvision')]
+            for folder in vision_include + package_path:
+                candidate_path = os.path.join(folder, library_header)
+                library_found = os.path.exists(candidate_path)
+                if library_found:
+                    break
+
+    return library_found, conda_installed, include_folder, lib_folder
+
+
 def get_extensions():
+    vision_include = os.environ.get('TORCHVISION_INCLUDE', None)
+    vision_library = os.environ.get('TORCHVISION_LIBRARY', None)
+    vision_include = (vision_include.split(os.pathsep)
+                      if vision_include is not None else [])
+    vision_library = (vision_library.split(os.pathsep)
+                      if vision_library is not None else [])
+    include_dirs = vision_include
+    library_dirs = vision_library
+
     this_dir = os.path.dirname(os.path.abspath(__file__))
     extensions_dir = os.path.join(this_dir, 'torchvision', 'csrc')
 
@@ -149,13 +218,14 @@ def get_extensions():
 
     sources = [os.path.join(extensions_dir, s) for s in sources]
 
-    include_dirs = [extensions_dir]
+    include_dirs += [extensions_dir]
 
     ext_modules = [
         extension(
             'torchvision._C',
             sources,
             include_dirs=include_dirs,
+            library_dirs=library_dirs,
             define_macros=define_macros,
             extra_compile_args=extra_compile_args,
         )
@@ -171,6 +241,65 @@ def get_extensions():
             )
         )
 
+    # Image reading extension
+    image_macros = []
+    image_include = [extensions_dir]
+    image_library = []
+    image_link_flags = []
+
+    # Locating libPNG
+    libpng = distutils.spawn.find_executable('libpng-config')
+    png_found = libpng is not None
+    image_macros += [('PNG_FOUND', str(int(png_found)))]
+    print('PNG found: {0}'.format(png_found))
+    if png_found:
+        png_version = subprocess.run([libpng, '--version'],
+                                     stdout=subprocess.PIPE)
+        png_version = png_version.stdout.strip().decode('utf-8')
+        print('libpng version: {0}'.format(png_version))
+        png_version = parse_version(png_version)
+        if png_version >= parse_version("1.6.0"):
+            print('Building torchvision with PNG image support')
+            png_lib = subprocess.run([libpng, '--libdir'],
+                                     stdout=subprocess.PIPE)
+            png_include = subprocess.run([libpng, '--I_opts'],
+                                         stdout=subprocess.PIPE)
+            image_library += [png_lib.stdout.strip().decode('utf-8')]
+            image_include += [png_include.stdout.strip().decode('utf-8')]
+            image_link_flags.append('png' if os.name != 'nt' else 'libpng')
+        else:
+            print('libpng installed version is less than 1.6.0, '
+                  'disabling PNG support')
+            png_found = False
+
+    # Locating libjpegturbo
+    turbojpeg_info = find_library('turbojpeg', vision_include)
+    (turbojpeg_found, conda_installed,
+     turbo_include_folder, turbo_lib_folder) = turbojpeg_info
+
+    image_macros += [('JPEG_FOUND', str(int(turbojpeg_found)))]
+    print('turboJPEG found: {0}'.format(turbojpeg_found))
+    if turbojpeg_found:
+        print('Building torchvision with JPEG image support')
+        image_link_flags.append('turbojpeg')
+        if conda_installed:
+            image_library += [turbo_lib_folder]
+            image_include += [turbo_include_folder]
+
+    image_path = os.path.join(extensions_dir, 'cpu', 'image')
+    image_src = glob.glob(os.path.join(image_path, '*.cpp'))
+
+    if png_found or turbojpeg_found:
+        ext_modules.append(extension(
+            'torchvision.image',
+            image_src,
+            include_dirs=include_dirs + [image_path] + image_include,
+            library_dirs=library_dirs + image_library,
+            define_macros=image_macros,
+            libraries=image_link_flags,
+            extra_compile_args=extra_compile_args
+        ))
+
     ffmpeg_exe = distutils.spawn.find_executable('ffmpeg')
     has_ffmpeg = ffmpeg_exe is not None
 
@@ -243,7 +372,9 @@ def run(self):
 
     # Package info
     packages=find_packages(exclude=('test',)),
-
+    package_data={
+        package_name: ['*.lib', '*.dylib', '*.so']
+    },
     zip_safe=False,
     install_requires=requirements,
     extras_require={
diff --git a/torchvision/io/__init__.py b/torchvision/io/__init__.py
--- a/torchvision/io/__init__.py
+++ b/torchvision/io/__init__.py
@@ -30,5 +30,5 @@
     "_read_video_clip_from_memory",
     "_read_video_meta_data",
     "VideoMetaData",
-    "Timebase",
+    "Timebase"
 ]
diff --git a/torchvision/io/image.py b/torchvision/io/image.py
new file mode 100644
--- /dev/null
+++ b/torchvision/io/image.py
@@ -0,0 +1,109 @@
+import torch
+from torch import nn, Tensor
+
+import os
+import os.path as osp
+import importlib
+
+_HAS_IMAGE_OPT = False
+
+try:
+    lib_dir = osp.join(osp.dirname(__file__), "..")
+
+    loader_details = (
+        importlib.machinery.ExtensionFileLoader,
+        importlib.machinery.EXTENSION_SUFFIXES
+    )
+
+    extfinder = importlib.machinery.FileFinder(lib_dir, loader_details)
+    ext_specs = extfinder.find_spec("image")
+    if ext_specs is not None:
+        torch.ops.load_library(ext_specs.origin)
+        _HAS_IMAGE_OPT = True
+except (ImportError, OSError):
+    pass
+
+
+def decode_png(input):
+    # type: (Tensor) -> Tensor
+    """
+    Decodes a PNG image into a 3 dimensional RGB Tensor.
+    The values of the output tensor are uint8 between 0 and 255.
+
+    Arguments:
+        input (Tensor[1]): a one dimensional int8 tensor containing
+    the raw bytes of the PNG image.
+
+    Returns:
+        output (Tensor[image_width, image_height, 3])
+    """
+    if not isinstance(input, torch.Tensor) or len(input) == 0:
+        raise ValueError("Expected a non empty 1-dimensional tensor.")
+
+    if not input.dtype == torch.uint8:
+        raise ValueError("Expected a torch.uint8 tensor.")
+    output = torch.ops.image.decode_png(input)
+    return output
+
+
+def read_png(path):
+    # type: (str) -> Tensor
+    """
+    Reads a PNG image into a 3 dimensional RGB Tensor.
+    The values of the output tensor are uint8 between 0 and 255.
+
+    Arguments:
+        path (str): path of the PNG image.
+
+    Returns:
+        output (Tensor[image_width, image_height, 3])
+    """
+    if not os.path.isfile(path):
+        raise ValueError("Expected a valid file path.")
+
+    size = os.path.getsize(path)
+    if size == 0:
+        raise ValueError("Expected a non empty file.")
+    data = torch.from_file(path, dtype=torch.uint8, size=size)
+    return decode_png(data)
+
+
+def decode_jpeg(input):
+    # type: (Tensor) -> Tensor
+    """
+    Decodes a JPEG image into a 3 dimensional RGB Tensor.
+    The values of the output tensor are uint8 between 0 and 255.
+    Arguments:
+        input (Tensor[1]): a one dimensional int8 tensor containing
+    the raw bytes of the JPEG image.
+    Returns:
+        output (Tensor[image_width, image_height, 3])
+    """
+    if not isinstance(input, torch.Tensor) or len(input) == 0 or input.ndim != 1:
+        raise ValueError("Expected a non empty 1-dimensional tensor.")
+
+    if not input.dtype == torch.uint8:
+        raise ValueError("Expected a torch.uint8 tensor.")
+
+    output = torch.ops.image.decode_jpeg(input)
+    return output
+
+
+def read_jpeg(path):
+    # type: (str) -> Tensor
+    """
+    Reads a JPEG image into a 3 dimensional RGB Tensor.
+    The values of the output tensor are uint8 between 0 and 255.
+    Arguments:
+        path (str): path of the JPEG image.
+    Returns:
+        output (Tensor[image_width, image_height, 3])
+    """
+    if not os.path.isfile(path):
+        raise ValueError("Expected a valid file path.")
+
+    size = os.path.getsize(path)
+    if size == 0:
+        raise ValueError("Expected a non empty file.")
+    data = torch.from_file(path, dtype=torch.uint8, size=size)
+    return decode_jpeg(data)


Test patch:
diff --git a/test/test_image.py b/test/test_image.py
new file mode 100644
--- /dev/null
+++ b/test/test_image.py
@@ -0,0 +1,72 @@
+import os
+import unittest
+import sys
+
+import torch
+import torchvision
+from PIL import Image
+from torchvision.io.image import read_png, decode_png, read_jpeg, decode_jpeg
+import numpy as np
+
+IMAGE_ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets")
+IMAGE_DIR = os.path.join(IMAGE_ROOT, "fakedata", "imagefolder")
+
+
+def get_images(directory, img_ext):
+    assert os.path.isdir(directory)
+    for root, _, files in os.walk(directory):
+        for fl in files:
+            _, ext = os.path.splitext(fl)
+            if ext == img_ext:
+                yield os.path.join(root, fl)
+
+
+class ImageTester(unittest.TestCase):
+    def test_read_jpeg(self):
+        for img_path in get_images(IMAGE_ROOT, "jpg"):
+            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
+            img_ljpeg = read_jpeg(img_path)
+
+            norm = img_ljpeg.shape[0] * img_ljpeg.shape[1] * img_ljpeg.shape[2] * 255
+            err = torch.abs(img_ljpeg.flatten().float() - img_pil.flatten().float()).sum().float() / (norm)
+            self.assertLessEqual(err, 1e-2)
+
+    def test_decode_jpeg(self):
+        for img_path in get_images(IMAGE_ROOT, "jpg"):
+            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
+            size = os.path.getsize(img_path)
+            img_ljpeg = decode_jpeg(torch.from_file(img_path, dtype=torch.uint8, size=size))
+
+            norm = img_ljpeg.shape[0] * img_ljpeg.shape[1] * img_ljpeg.shape[2] * 255
+            err = torch.abs(img_ljpeg.flatten().float() - img_pil.flatten().float()).sum().float() / (norm)
+
+            self.assertLessEqual(err, 1e-2)
+
+        with self.assertRaisesRegex(ValueError, "Expected a non empty 1-dimensional tensor."):
+            decode_jpeg(torch.empty((100, 1), dtype=torch.uint8))
+
+        with self.assertRaisesRegex(ValueError, "Expected a torch.uint8 tensor."):
+            decode_jpeg(torch.empty((100, ), dtype=torch.float16))
+
+        with self.assertRaisesRegex(RuntimeError, "Error while reading jpeg headers"):
+            decode_jpeg(torch.empty((100), dtype=torch.uint8))
+
+    def test_read_png(self):
+        for img_path in get_images(IMAGE_DIR, "png"):
+            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
+            img_lpng = read_png(img_path)
+            self.assertEqual(img_lpng, img_pil)
+
+    def test_decode_png(self):
+        for img_path in get_images(IMAGE_DIR, "png"):
+            img_pil = torch.from_numpy(np.array(Image.open(img_path)))
+            size = os.path.getsize(img_path)
+            img_lpng = decode_png(torch.from_file(img_path, dtype=torch.uint8, size=size))
+            self.assertEqual(img_lpng, img_pil)
+
+            self.assertEqual(decode_png(torch.empty()), torch.empty())
+            self.assertEqual(decode_png(torch.randint(3, 5, (300,))), torch.empty())
+
+
+if __name__ == '__main__':
+    unittest.main()