Repository: Duke-GCB/DukeDSClient

Issue:
ddsclient upload stuck
A user reported an upload that appeared to be stuck.
I was able to look at the process in linux.
Using `lsof -u <user>` I saw that there were several TCP connections back to DukeDS API (heroku) that were stuck in CLOSE_WAIT.


Discussion:

Patch:
diff --git a/ddsc/core/localstore.py b/ddsc/core/localstore.py
index d8d38ec..27e0866 100644
--- a/ddsc/core/localstore.py
+++ b/ddsc/core/localstore.py
@@ -2,6 +2,7 @@ import hashlib
 import math
 import mimetypes
 import os
+from os.path import isfile, isdir
 from ddsc.core.ignorefile import FileFilter, IgnoreFilePatterns
 from ddsc.core.util import KindType, ProjectWalker, plural_fmt, join_with_commas_and_and
 
@@ -30,7 +31,9 @@ class LocalProject(object):
         :param path: str path to add
         """
         abspath = os.path.abspath(path)
-        self.children.append(_build_project_tree(abspath, self.followsymlinks, self.file_filter))
+        child = _build_project_tree(abspath, self.followsymlinks, self.file_filter)
+        if child:
+            self.children.append(child)
 
     def add_paths(self, path_list):
         """
@@ -205,13 +208,19 @@ def _build_project_tree(path, followsymlinks, file_filter):
     :return: the top node of the tree LocalFile or LocalFolder
     """
     result = None
-    if os.path.isfile(path):
+    if isfile(path):
         result = LocalFile(path)
-    else:
+    elif isdir(path):
         result = _build_folder_tree(os.path.abspath(path), followsymlinks, file_filter)
+    else:
+        _on_non_regular_file(path)
     return result
 
 
+def _on_non_regular_file(path):
+    print("Warning: Skipping {}. This is an unsupported type of file.".format(path))
+
+
 def _build_folder_tree(top_abspath, followsymlinks, file_filter):
     """
     Build a tree of LocalFolder with children based on a path.
@@ -244,8 +253,11 @@ def _build_folder_tree(top_abspath, followsymlinks, file_filter):
             child_dirs.remove(remove_child_dir)
         for child_filename in child_files:
             abs_child_filename = os.path.join(dir_name, child_filename)
-            if ignore_file_patterns.include(abs_child_filename, is_file=True):
-                folder.add_child(LocalFile(abs_child_filename))
+            if isfile(abs_child_filename):
+                if ignore_file_patterns.include(abs_child_filename, is_file=True):
+                    folder.add_child(LocalFile(abs_child_filename))
+            else:
+                _on_non_regular_file(abs_child_filename)
     return path_to_content.get(top_abspath)
 
 


Test patch:
diff --git a/ddsc/core/tests/test_localstore.py b/ddsc/core/tests/test_localstore.py
index ba0172f..e6de966 100644
--- a/ddsc/core/tests/test_localstore.py
+++ b/ddsc/core/tests/test_localstore.py
@@ -106,6 +106,16 @@ class TestProjectContent(TestCase):
             '/note.txt'
         ])
 
+    @patch('ddsc.core.localstore.isfile')
+    @patch('ddsc.core.localstore.print')
+    def test_top_level_non_regular_file(self, mock_print, mock_isfile):
+        mock_isfile.return_value = False
+        content = LocalProject(False, file_exclude_regex=INCLUDE_ALL)
+        content.add_path('/tmp/DukeDsClientTestFolder/note.txt')
+        self.assertEqual(get_file_or_folder_paths(content), [])
+        mock_print.assert_called_with('Warning: Skipping /tmp/DukeDsClientTestFolder/note.txt. '
+                                      'This is an unsupported type of file.')
+
     def test_empty_folder_str(self):
         content = LocalProject(False, file_exclude_regex=INCLUDE_ALL)
         content.add_path('/tmp/DukeDsClientTestFolder/emptyfolder')
@@ -130,6 +140,18 @@ class TestProjectContent(TestCase):
             '/scripts/makemoney.sh',
         ])
 
+    @patch('ddsc.core.localstore.isfile')
+    @patch('ddsc.core.localstore.print')
+    def test_one_folder_containing_non_regular_file(self, mock_print, mock_isfile):
+        mock_isfile.return_value = False
+        content = LocalProject(False, file_exclude_regex=INCLUDE_ALL)
+        content.add_path('/tmp/DukeDsClientTestFolder/scripts')
+        self.assertEqual(get_file_or_folder_paths(content), [
+            '/scripts'
+        ])
+        mock_print.assert_called_with('Warning: Skipping /tmp/DukeDsClientTestFolder/scripts/makemoney.sh. '
+                                      'This is an unsupported type of file.')
+
     def test_nested_folder_str(self):
         content = LocalProject(False, file_exclude_regex=INCLUDE_ALL)
         content.add_path('/tmp/DukeDsClientTestFolder/results')