Repository: Flexget/Flexget

Issue:
Thread should be created after deamonizing
There is a long existing bug related to `Task Queue has died unexpectedly` in daemon mode. #1947 #2656 The cause of this bug is really simple. **Do not** create thread before `os.fork()` (demonize). Here is a small code snippet that demonstrates the problem:
```python
from threading import Thread
import os
import time
import sys


def func():
    for _ in range(5):
        print('running in thread')
        time.sleep(0.5)

t = Thread(target=func)

pid = os.fork()
if pid > 0:
    sys.exit(0)

t.start()
print("pid:", os.getpid())
print('is thread alive after fork:', t.is_alive())
t.join()
```
The running result is as follows:
```
# lym @ mig186 in ~ [11:46:22]
$ python main.py
running in thread
pid: 122253
is thread alive after fork: False
# lym @ mig186 in ~ [11:46:23]
$ running in thread
running in thread
running in thread
running in thread
```
The thread resource created before `os.fork()` is only alive in the parent process (which is exited to daemonize). In the child process, the `t.is_alive()` always return False. Here is the right way to do:
```python
from threading import Thread
import os
import time
import sys


def func():
    for _ in range(5):
        print('running in thread')
        time.sleep(0.5)


pid = os.fork()
if pid > 0:
    sys.exit(0)

t = Thread(target=func)
t.start()
print("pid:", os.getpid())
print('is thread alive after fork:', t.is_alive())
t.join()
```
By the way, the process of daemonize usually involves double fork, in which case we should create all the threading after the second fork.

And here is the cause of bug in `flexget/manager.py`, the task queue is initialized in line 231 in the `def initialize(self)` function, and the damonize happens afterwards in line 476 `self.daemonize()`, which involves double forking:
```python
    def daemonize(self) -> None:
        """Daemonizes the current process. Returns the new pid"""
        if sys.platform.startswith('win'):
            logger.error('Cannot daemonize on windows')
            return
        if threading.active_count() != 1:
            logger.critical(
                'There are {!r} active threads. Daemonizing now may cause strange failures.',
                threading.enumerate(),
            )

        logger.info('Daemonizing...')

        try:
            pid = os.fork()
            if pid > 0:
                # Don't run the exit handlers on the parent
                atexit._exithandlers = []
                # exit first parent
                sys.exit(0)
        except OSError as e:
            sys.stderr.write(f'fork #1 failed: {e.errno} ({e.strerror})\n')
            sys.exit(1)

        # decouple from parent environment
        os.chdir('/')
        os.setsid()
        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # Don't run the exit handlers on the parent
                atexit._exithandlers = []
                # exit from second parent
                sys.exit(0)
        except OSError as e:
            sys.stderr.write(f'fork #2 failed: {e.errno} ({e.strerror})\n')
            sys.exit(1)
```

Therefore after daemonizing, when we try to execute the command, this function always shows the not alive error (line 415):
```python
    def execute_command(self, options: argparse.Namespace) -> None:
        """
        Handles the 'execute' CLI command.
        If there is already a task queue running in this process, adds the execution to the queue.
        If FlexGet is being invoked with this command, starts up a task queue and runs the execution.
        Fires events:
        * manager.execute.started
        * manager.execute.completed
        :param options: argparse options
        """
        fire_event('manager.execute.started', self, options)
        if self.task_queue.is_alive() or self.is_daemon:
            if not self.task_queue.is_alive():
                logger.error(
                    'Task queue has died unexpectedly. Restarting it. Please open an issue on Github and include'
                    ' any previous error logs.'
                )
```
And it creates a new task queue and leaving the old one hanging, which causes other strange problems like the flexget daemon can not be stopped.





Discussion:

Resolving patch:
diff --git a/flexget/plugins/input/filesystem.py b/flexget/plugins/input/filesystem.py
--- a/flexget/plugins/input/filesystem.py
+++ b/flexget/plugins/input/filesystem.py
@@ -1,7 +1,7 @@
 from __future__ import unicode_literals, division, absolute_import
 import logging
 import re
-import os
+from datetime import datetime
 
 from path import Path
 
@@ -104,14 +104,14 @@ def create_entry(self, filepath, test_mode):
         """
         entry = Entry()
         entry['location'] = filepath
-        entry['url'] = 'file://{}'.format(filepath)
+        entry['url'] = 'file://%s' % filepath
         entry['filename'] = filepath.name
         if filepath.isfile():
             entry['title'] = filepath.namebase
         else:
             entry['title'] = filepath.name
         try:
-            entry['timestamp'] = os.path.getmtime(filepath)
+            entry['timestamp'] = datetime.fromtimestamp(filepath.getmtime())
         except Exception as e:
             log.warning('Error setting timestamp for %s: %s' % (filepath, e))
             entry['timestamp'] = None
@@ -125,7 +125,7 @@ def create_entry(self, filepath, test_mode):
                 log.info("    Timestamp: %s" % entry["timestamp"])
             return entry
         else:
-            log.error('Non valid entry created: {}'.format(entry))
+            log.error('Non valid entry created: %s '% entry)
             return
 
     def get_max_depth(self, recursion, base_depth):
@@ -146,18 +146,18 @@ def get_entries_from_path(self, path_list, match, recursion, test_mode, get_file
         entries = []
 
         for folder in path_list:
-            log.verbose('Scanning folder {}. Recursion is set to {}.'.format(folder, recursion))
+            log.verbose('Scanning folder %s. Recursion is set to %s.' % (folder, recursion))
             folder = Path(folder).expanduser()
             log.debug('Scanning %s' % folder)
             base_depth = len(folder.splitall())
             max_depth = self.get_max_depth(recursion, base_depth)
             folder_objects = self.get_folder_objects(folder, recursion)
             for path_object in folder_objects:
-                log.debug('Checking if {} qualifies to be added as an entry.'.format(path_object))
+                log.debug('Checking if %s qualifies to be added as an entry.' % path_object)
                 try:
                     path_object.exists()
                 except UnicodeError:
-                    log.error('File %s not decodable with filesystem encoding: {}'.format(path_object))
+                    log.error('File %s not decodable with filesystem encoding: %s' % path_object)
                     continue
                 entry = None
                 object_depth = len(path_object.splitall())
@@ -168,7 +168,7 @@ def get_entries_from_path(self, path_list, match, recursion, test_mode, get_file
                             path_object.isfile() and get_files):
                             entry = self.create_entry(path_object, test_mode)
                         else:
-                            log.debug("Path object's {} type doesn't match requested object types.".format(path_object))
+                            log.debug("Path object's %s type doesn't match requested object types." % path_object)
                         if entry and entry not in entries:
                             entries.append(entry)
 


Test patch:
diff --git a/tests/filesystem_test_dir/Test1/dir1/dir4/dir6/dir7/dir8/file11.txt b/tests/filesystem_test_dir/Test1/dir1/dir4/dir6/dir7/dir8/file11.txt
new file mode 100644
diff --git a/tests/filesystem_test_dir/Test1/dir1/dir4/file4.avi b/tests/filesystem_test_dir/Test1/dir1/dir4/file4.avi
new file mode 100644
diff --git a/tests/filesystem_test_dir/Test1/dir1/file3.xlsx b/tests/filesystem_test_dir/Test1/dir1/file3.xlsx
new file mode 100644
Binary files /dev/null and b/tests/filesystem_test_dir/Test1/dir1/file3.xlsx differ
diff --git a/tests/filesystem_test_dir/Test1/dir2/file5.mkv b/tests/filesystem_test_dir/Test1/dir2/file5.mkv
new file mode 100644
diff --git a/tests/filesystem_test_dir/Test1/file1.mkv b/tests/filesystem_test_dir/Test1/file1.mkv
new file mode 100644
diff --git a/tests/filesystem_test_dir/Test1/file2.txt b/tests/filesystem_test_dir/Test1/file2.txt
new file mode 100644
diff --git a/tests/filesystem_test_dir/Test2/file10.mkv b/tests/filesystem_test_dir/Test2/file10.mkv
new file mode 100644
diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py
new file mode 100644
--- /dev/null
+++ b/tests/test_filesystem.py
@@ -0,0 +1,196 @@
+from __future__ import unicode_literals, division, absolute_import
+from tests import FlexGetBase
+from path import Path
+
+
+class TestFilesystem(FlexGetBase):
+    base = "filesystem_test_dir/"
+    test1 = base + '/Test1'
+    test2 = base + '/Test2'
+
+    __yaml__ = """
+        tasks:
+          string:
+            filesystem: """ + test1 + """
+
+          list:
+           filesystem:
+             - """ + test1 + """
+             - """ + test2 + """
+
+          object_string:
+            filesystem:
+              path: """ + test1 + """
+
+          object_list:
+            filesystem:
+              path:
+                - """ + test1 + """
+                - """ + test2 + """
+
+          file_mask:
+            filesystem:
+              path: """ + test1 + """
+              mask: '*.mkv'
+
+          regexp_test:
+            filesystem:
+              path: """ + test1 + """
+              regexp: '.*\.(mkv)$'
+
+          recursive_true:
+            filesystem:
+              path: """ + test1 + """
+              recursive: yes
+
+          recursive_2_levels:
+            filesystem:
+              path: """ + test1 + """
+              recursive: 2
+
+          retrieve_files:
+            filesystem:
+              path: """ + test1 + """
+              retrieve: files
+
+          retrieve_files_and_dirs:
+            filesystem:
+              path: """ + test1 + """
+              retrieve:
+                - files
+                - dirs
+
+          combine_1:
+            filesystem:
+              path: """ + test1 + """
+              mask: '*.mkv'
+              recursive: 2
+
+          combine_2:
+            filesystem:
+              path: """ + test1 + """
+              recursive: yes
+              retrieve: dirs
+        """
+
+    item_list = ['file1.mkv', 'file2.txt', 'file10.mkv', 'file11.txt', 'file4.avi', 'file3.xlsx', 'file5.mkv', 'dir1',
+                 'dir2', 'dir4', 'dir6', 'dir7', 'dir8']
+
+    def assert_check(self, task_name, test_type, filenames):
+        for file in filenames:
+            file = Path(file)
+            if test_type == 'positive':
+                assertion_error = 'Failed %s %s test, did not find %s' % (test_type, task_name, file)
+                assert self.task.find_entry(title=file.namebase), assertion_error
+            else:
+                assertion_error = 'Failed %s %s test, found %s' % (test_type, task_name, file)
+                assert not self.task.find_entry(title=file.namebase), assertion_error
+
+    def test_string(self):
+        task_name = 'string'
+        should_exist = 'dir1', 'dir2', 'file1.mkv', 'file2.txt'
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_list(self):
+        task_name = 'list'
+        should_exist = ['dir1', 'dir2', 'file1.mkv', 'file2.txt', 'file10.mkv']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_object_string(self):
+        task_name = 'object_string'
+        should_exist = ['dir1', 'dir2', 'file1.mkv', 'file2.txt']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_object_list(self):
+        task_name = 'object_list'
+        should_exist = ['dir1', 'dir2', 'file1.mkv', 'file2.txt', 'file10.mkv']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_file_mask(self):
+        task_name = 'file_mask'
+        should_exist = ['file1.mkv']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_regexp_test(self):
+        task_name = 'regexp_test'
+        should_exist = ['file1.mkv']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_recursive_true(self):
+        task_name = 'recursive_true'
+        should_exist = ['dir1', 'dir4', 'dir6', 'dir7', 'dir8', 'file11.txt', 'file4.avi', 'file3.xlsx', 'dir2',
+                        'file5.mkv', 'file1.mkv', 'file2.txt']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_recursive_2_levels(self):
+        task_name = 'recursive_2_levels'
+        should_exist = ['dir1', 'dir4', 'file3.xlsx', 'dir2', 'file5.mkv', 'file1.mkv', 'file2.txt']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_retrieve_files(self):
+        task_name = 'retrieve_files'
+        should_exist = ['file1.mkv', 'file2.txt']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_retrieve_files_and_dirs(self):
+        task_name = 'retrieve_files_and_dirs'
+        should_exist = ['dir1', 'dir2', 'file1.mkv', 'file2.txt']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_combine_1(self):
+        task_name = 'combine_1'
+        should_exist = ['file5.mkv', 'file1.mkv']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)
+
+    def test_combine_2(self):
+        task_name = 'combine_2'
+        should_exist = ['dir1', 'dir4', 'dir2', 'dir6', 'dir7', 'dir8']
+        should_not_exist = [item for item in self.item_list if item not in should_exist]
+        self.execute_task(task_name)
+
+        self.assert_check(task_name, 'positive', should_exist)
+        self.assert_check(task_name, 'negative', should_not_exist)