Repository: chezou/tabula-py
Pull request #196 (2019-12-16 00:34:05)

Support non-ascii URL and saving too long file name
This PR enables non-ascii file fetching and long file name downloading

## Description

This PR fixes two types of issues to fetch remote PDF:
1) Non-ASCII file name
2) Too long file name i.e., file name longer than 250 will be trimmed


## Motivation and Context

Currently, tabula-py doesn't work well with the following URL:

```
import tabula
pdf_path = "http://202.107.205.11:8612/doc/浙高企认办〔2018〕3号关于浙江省2018年第一批拟更名高新技术企业公示名单.pdf"

tabula.read_pdf(pdf_path)
```

## How Has This Been Tested?
Unit test and manual test

## Screenshots (if appropriate):

## Types of changes

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)

## Checklist:
- [x] I read [the contributing document](https://tabula-py.readthedocs.io/en/latest/contributing.html).
- [x] My code follows the code style of this project with running linter.
- [ ] My change requires a change to the documentation.
- [ ] I have updated the documentation accordingly.



Patch:
diff --git a/tabula/file_util.py b/tabula/file_util.py
index 9174375..a608d60 100644
--- a/tabula/file_util.py
+++ b/tabula/file_util.py
@@ -1,11 +1,11 @@
 import os
 import shutil
-from urllib.parse import urlparse as parse_url
-from urllib.parse import uses_netloc, uses_params, uses_relative
+from urllib.parse import quote, urlparse, uses_netloc, uses_params, uses_relative
 from urllib.request import Request, urlopen
 
 _VALID_URLS = set(uses_relative + uses_netloc + uses_params)
 _VALID_URLS.discard("")
+MAX_FILE_SIZE = 250
 
 
 def localize_file(path_or_buffer, user_agent=None, suffix=".pdf"):
@@ -31,14 +31,17 @@ def localize_file(path_or_buffer, user_agent=None, suffix=".pdf"):
     path_or_buffer = _stringify_path(path_or_buffer)
 
     if _is_url(path_or_buffer):
+        path_or_buffer = quote(path_or_buffer, safe="/:")
         if user_agent:
             req = urlopen(_create_request(path_or_buffer, user_agent))
         else:
             req = urlopen(path_or_buffer)
 
-        parsed_url = parse_url(req.geturl())
+        parsed_url = urlparse(req.geturl())
         filename = os.path.basename(parsed_url.path)
-        if os.path.splitext(filename)[-1] != suffix:
+        fname, ext = os.path.splitext(filename)
+        filename = "{}{}".format(fname[:MAX_FILE_SIZE], ext)
+        if ext != suffix:
             pid = os.getpid()
             filename = "{}{}".format(pid, suffix)
 
@@ -63,7 +66,7 @@ def localize_file(path_or_buffer, user_agent=None, suffix=".pdf"):
 
 def _is_url(url):
     try:
-        return parse_url(url).scheme in _VALID_URLS
+        return urlparse(url).scheme in _VALID_URLS
 
     except Exception:
         return False


Test patch:
diff --git a/tests/test_util.py b/tests/test_util.py
index df8b50f..9f800c5 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -1,3 +1,4 @@
+import os
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -5,6 +6,9 @@ import tabula
 
 
 class TestUtil(unittest.TestCase):
+    def setUp(self):
+        tabula.file_util.MAX_FILE_SIZE = 30
+
     def test_environment_info(self):
         self.assertEqual(tabula.environment_info(), None)
 
@@ -26,8 +30,51 @@ class TestUtil(unittest.TestCase):
         cm.geturl.return_value = uri
         mock_urlopen.return_value = cm
 
-        tabula.file_util.localize_file(uri, user_agent=user_agent)
+        fname, _ = tabula.file_util.localize_file(uri, user_agent=user_agent)
         mock_fun.assert_called_with(uri, user_agent)
+        self.addCleanup(os.remove, fname)
+
+    @patch("tabula.file_util.shutil.copyfileobj")
+    @patch("tabula.file_util.urlopen")
+    def test_localize_file_with_non_ascii_url(self, mock_urlopen, mock_copyfileobj):
+        uri = (
+            "https://github.com/tabulapdf/tabula-java/raw/"
+            "master/src/test/resources/technology/tabula/日本語.pdf"
+        )
+        expected_uri = (
+            "https://github.com/tabulapdf/tabula-java/raw/master/src/test/"
+            "resources/technology/tabula/%E6%97%A5%E6%9C%AC%E8%AA%9E.pdf"
+        )
+
+        cm = MagicMock()
+        cm.getcode.return_value = 200
+        cm.read.return_value = b"contents"
+        cm.geturl.return_value = uri
+        mock_urlopen.return_value = cm
+
+        fname, _ = tabula.file_util.localize_file(uri)
+        mock_urlopen.assert_called_with(expected_uri)
+        self.addCleanup(os.remove, fname)
+
+    @patch("tabula.file_util.shutil.copyfileobj")
+    @patch("tabula.file_util.urlopen")
+    def test_localize_file_with_long_url(self, mock_urlopen, mock_copyfileobj):
+        uri = (
+            "https://github.com/tabulapdf/tabula-py/raw/"
+            "master/src/tests/resources/"
+            "12345678901234567890123456789012345.pdf"
+        )
+
+        cm = MagicMock()
+        cm.getcode.return_value = 200
+        cm.read.return_value = b"contents"
+        cm.geturl.return_value = uri
+        mock_urlopen.return_value = cm
+
+        fname, _ = tabula.file_util.localize_file(uri)
+        mock_urlopen.assert_called_with(uri)
+        self.assertEqual(fname, "123456789012345678901234567890.pdf")
+        self.addCleanup(os.remove, fname)
 
 
 if __name__ == "__main__":