Repository: pymedusa/Medusa

Issue:
[py-3.7.0] Log viewer first line is always cut short
**Describe the bug**
Let's say the last log line logged to `application.log` is:
```
2018-12-29 19:22:08 DEBUG    CHECKVERSION :: [5af2f5f] User-Agent: Medusa/0.2.14 (Windows; 7; 3e7bf898-0b8e-11e9-843c-f38c5b096e82)
```
This line would show up first on the logviewer page (when the level is set to DEBUG).
However, with the bug, that first line is always cut short.
```
2018-12-29 19:22:08 DEBUG    CHECKVERSION :: [5af2f5f] User-Agent: Medusa/0.2.14 (Windows; 7; 3e7bf898-0b8e-
```

As you can see in the screenshots below, the log line is logged correctly to the console window and to the `application.log` file.
The bug is also visible on the "text view" of the logviewer (when clicking the notepad icon on the page), so it's not a UI bug.

**Additional context**
I think the problem lies somewhere [in the function that reads the lines from the file](https://github.com/pymedusa/Medusa/blob/5af2f5fe7ff0f31449113617fd0dda75eb8bcf07/medusa/logger/__init__.py#L134), for the logviewer page.

I think this only started with the merge of #4982...
So, with git blame I found the function was last tweaked in https://github.com/pymedusa/Medusa/commit/96937a28fd8382f2e8f298e50d9ed43a096e1f50#diff-ce0671ac2b513f7cce2afda1a576e33c (part of #4982 / 9751a934d5155673523704a415b89c0e5356a12d)

**Medusa:**
- **Python**: 3.7.0 (v3.7.0:1bf9cc5093, Jun 27 2018, 04:59:51) [MSC v.1914 64 bit (AMD64)]
- **OS:** Windows-7-6.1.7601-SP1
- **Branch:** develop
- **Commit:** 5af2f5fe7ff0f31449113617fd0dda75eb8bcf07 


**To Reproduce**
1. Go to http://localhost:8081/errorlogs/viewlog/?min_level=10
2. See bug as explained before

**Screenshots**
### Logviewer page
![image](https://user-images.githubusercontent.com/10238474/50540724-b8d23400-0b9f-11e9-8836-165305ee4bac.png)

### Console window
![image](https://user-images.githubusercontent.com/10238474/50540730-cc7d9a80-0b9f-11e9-9ac0-f7912972ed10.png)

### `application.log`
![image](https://user-images.githubusercontent.com/10238474/50540752-53cb0e00-0ba0-11e9-8181-fb7be9adbbcb.png)




Discussion:

Resolving patch:
diff --git a/medusa/logger/__init__.py b/medusa/logger/__init__.py
--- a/medusa/logger/__init__.py
+++ b/medusa/logger/__init__.py
@@ -68,7 +68,6 @@
 }
 
 FORMATTER_PATTERN = '%(asctime)s %(levelname)-8s %(threadName)s :: [%(curhash)s] %(message)s'
-default_encoding = 'utf-8'
 censored_items = {}
 censored = []
 
@@ -92,13 +91,13 @@ def rebuild_censored_list():
     def quote_unicode(value):
         """Quote a unicode value by encoding it to bytes first."""
         if isinstance(value, text_type):
-            return quote(value.encode(default_encoding, 'replace'))
+            return quote(value.encode('utf-8', 'replace'))
         return quote(value)
 
     # set of censored items and urlencoded counterparts
     results |= {quote_unicode(item) for item in results}
     # convert set items to unicode and typecast to list
-    results = list({item.decode(default_encoding, 'replace')
+    results = list({item.decode('utf-8', 'replace')
                     if not isinstance(item, text_type) else item for item in results})
     # sort the list in order of descending length so that entire item is censored
     # e.g. password and password_1 both get censored instead of getting ********_1
@@ -204,54 +203,47 @@ def read_loglines(log_file=None, modification_time=None, start_index=0, max_line
                 yield formatter(logline)
 
 
-def blocks_r(file_obj, size=64 * 1024, reset_offset=True, encoding=None):
+def blocks_r(filename, size=64 * 1024, reset_offset=True):
     """
     Yields the data within a file in reverse-ordered blocks of given size.
 
     This code is part of the flyingcircus package: https://pypi.org/project/flyingcircus/
+    The code was adapted for our use case.
     All credits go to the original author.
 
     Note that:
      - the content of the block is NOT reversed.
-     - if the file is open in text mode, the actual size of the block may be
-       shorter for multi-byte encodings (such as `utf8`, which is the default).
 
     Args:
-        file_obj (file): The input file.
+        filename (str): The input file name.
         size (int|None): The block size.
             If int, the file is yielded in blocks of the specified size.
             If None, the file is yielded at once.
         reset_offset (bool): Reset the file offset.
             If True, starts reading from the end of the file.
             Otherwise, starts reading from where the file current position is.
-        encoding (str|None): The encoding for correct block size computation.
-            If `str`, must be a valid string encoding.
-            If None, the default encoding is used.
 
     Yields:
-        block (bytes|str): The data within the blocks.
+        block (bytes): The data within the blocks.
 
     """
-    offset = 0
-    if reset_offset:
-        file_size = remaining_size = file_obj.seek(0, os.SEEK_END)
-    else:
-        file_size = remaining_size = file_obj.tell()
-    rounding = 0
-    while remaining_size > 0:
-        offset = min(file_size, offset + size)
-        file_obj.seek(file_size - offset)
-        block = file_obj.read(min(remaining_size, size))
-        if not isinstance(block, bytes):
-            real_size = len(
-                block.encode(encoding) if encoding else block.encode())
-            rounding = len(block) - real_size
-        remaining_size -= size
-        yield block[:len(block) + rounding] if rounding else block
+    with io.open(filename, 'rb') as file_obj:
+        offset = 0
+        if reset_offset:
+            file_size = remaining_size = file_obj.seek(0, os.SEEK_END)
+        else:
+            file_size = remaining_size = file_obj.tell()
+        rounding = 0
+        while remaining_size > 0:
+            offset = min(file_size, offset + size)
+            file_obj.seek(file_size - offset)
+            block = file_obj.read(min(remaining_size, size))
+            remaining_size -= size
+            yield block[:len(block) + rounding] if rounding else block
 
 
 def reverse_readlines(filename, skip_empty=True, append_newline=False, block_size=512 * 1024,
-                      reset_offset=True, encoding=default_encoding):
+                      reset_offset=True, encoding='utf-8'):
     """
     Flexible function for reversing read lines incrementally.
 
@@ -266,29 +258,33 @@ def reverse_readlines(filename, skip_empty=True, append_newline=False, block_siz
         block_size (int|None): The block size.
             If int, the file is processed in blocks of the specified size.
             If None, the file is processed at once.
+        reset_offset (bool): Reset the file offset.
+            If True, starts reading from the end of the file.
+            Otherwise, starts reading from where the file current position is.
+        encoding (str|None): The encoding for correct block size computation.
+            If `str`, must be a valid string encoding.
+            If None, the default encoding is used.
 
     Yields:
         line (str): The next line.
 
     """
-    with io.open(filename, 'r', encoding=encoding) as fo:
-        newline = '\n'
-        empty = ''
-        remainder = empty
-        block_generator_kws = dict(size=block_size, reset_offset=reset_offset)
-        block_generator = blocks_r
-        block_generator_kws.update(dict(encoding=encoding))
-        for block in block_generator(fo, **block_generator_kws):
-            lines = block.split(newline)
-            if remainder:
-                lines[-1] = lines[-1] + remainder
-            remainder = lines[0]
-            mask = slice(-1, 0, -1)
-            for line in lines[mask]:
-                if line or not skip_empty:
-                    yield line + (newline if append_newline else empty)
-        if remainder or not skip_empty:
-            yield remainder + (newline if append_newline else empty)
+    newline = '\n'
+    empty = ''
+    remainder = empty
+    block_generator_kws = dict(size=block_size, reset_offset=reset_offset)
+    block_generator = blocks_r
+    for block in block_generator(filename, **block_generator_kws):
+        lines = block.decode(encoding).split(newline)
+        if remainder:
+            lines[-1] = lines[-1] + remainder
+        remainder = lines[0]
+        mask = slice(-1, 0, -1)
+        for line in lines[mask]:
+            if line or not skip_empty:
+                yield line + (newline if append_newline else empty)
+    if remainder or not skip_empty:
+        yield remainder + (newline if append_newline else empty)
 
 
 def filter_logline(logline, min_level=None, thread_name=None, search_query=None):
@@ -579,7 +575,7 @@ class CensoredFormatter(logging.Formatter, object):
     # Needed because Newznab apikey isn't stored as key=value in a section.
     apikey_re = re.compile(r'(?P<before>[&?]r|[&?]apikey|[&?]api_key)(?:=|%3D)([^&]*)(?P<after>[&\w]?)', re.IGNORECASE)
 
-    def __init__(self, fmt=None, datefmt=None, encoding=default_encoding):
+    def __init__(self, fmt=None, datefmt=None, encoding='utf-8'):
         """Constructor."""
         super(CensoredFormatter, self).__init__(fmt, datefmt)
         self.encoding = encoding
@@ -687,7 +683,7 @@ def reconfigure_file_handler(self):
         target_size = int(app.LOG_SIZE * 1024 * 1024)
         target_number = int(app.LOG_NR)
         if not self.file_handler or self.log_file != target_file or self.file_handler.backupCount != target_number or self.file_handler.maxBytes != target_size:
-            file_handler = RotatingFileHandler(target_file, maxBytes=target_size, backupCount=target_number, encoding=default_encoding)
+            file_handler = RotatingFileHandler(target_file, maxBytes=target_size, backupCount=target_number, encoding='utf-8')
             file_handler.setFormatter(CensoredFormatter(FORMATTER_PATTERN, dateTimeFormat))
             file_handler.setLevel(self.log_level)
 
diff --git a/medusa/server/web/core/error_logs.py b/medusa/server/web/core/error_logs.py
--- a/medusa/server/web/core/error_logs.py
+++ b/medusa/server/web/core/error_logs.py
@@ -140,17 +140,17 @@ def viewlog(self, min_level=logging.INFO, log_filter=None, log_search=None, max_
 
         period = log_periods.get(log_period)
         modification_time = datetime.now() - period if period else None
-        data = [line for line in read_loglines(modification_time=modification_time, formatter=text_type, max_lines=max_lines,
-                                               predicate=lambda l: filter_logline(l, min_level=min_level,
-                                                                                  thread_name=thread_names.get(log_filter, log_filter),
-                                                                                  search_query=log_search))]
+        data = (html_escape(line) for line in read_loglines(modification_time=modification_time, formatter=text_type, max_lines=max_lines,
+                                                            predicate=lambda l: filter_logline(l, min_level=min_level,
+                                                                                               thread_name=thread_names.get(log_filter, log_filter),
+                                                                                               search_query=log_search)))
 
         if not text_view:
-            return t.render(log_lines='\n'.join([html_escape(line) for line in data]),
+            return t.render(log_lines='\n'.join(data),
                             min_level=min_level, log_name_filters=log_name_filters, log_filter=log_filter, log_search=log_search, log_period=log_period,
                             controller='errorlogs', action='viewlogs')
         else:
-            return '<br/>'.join([html_escape(line) for line in data])
+            return '<br/>'.join(data)
 
     def submit_errors(self):
         """Create an issue in medusa issue tracker."""


Test patch: