Repository: ray-project/ray

Issue:
log monitor report UnicodeDecodeError
<!--
General questions should be asked on the mailing list ray-dev@googlegroups.com.
Questions about how to use Ray should be asked on
[StackOverflow](https://stackoverflow.com/questions/tagged/ray).

Before submitting an issue, please fill out the following form.
-->

### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: centos 7
- **Ray installed from (source or binary)**: binay
- **Ray version**: 0.7.2 0.8
- **Python version**: python 3.7
- **Exact command to reproduce**:

<!--
You can obtain the Ray version with

python -c "import ray; print(ray.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->
Now the `log_monitor` monitor all the files under logs. This could be causing the following errors when we read those log file with `VIM` because of the `.*.swap` file will be created. 
```Traceback (most recent call last):
  File "/home/xianyang/anaconda3/lib/python3.7/site-packages/ray/log_monitor.py", line 278, in <module>
    raise e
  File "/home/xianyang/anaconda3/lib/python3.7/site-packages/ray/log_monitor.py", line 268, in <module>
    log_monitor.run()
  File "/home/xianyang/anaconda3/lib/python3.7/site-packages/ray/log_monitor.py", line 219, in run
    anything_published = self.check_log_files_and_publish_updates()
  File "/home/xianyang/anaconda3/lib/python3.7/site-packages/ray/log_monitor.py", line 175, in check_log_files_and_publish_updates
    next_line = file_info.file_handle.readline()
  File "/home/xianyang/anaconda3/lib/python3.7/codecs.py", line 322, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xca in position 21: invalid continuation byte

```
### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/python/ray/log_monitor.py b/python/ray/log_monitor.py
--- a/python/ray/log_monitor.py
+++ b/python/ray/log_monitor.py
@@ -4,6 +4,7 @@
 
 import argparse
 import errno
+import glob
 import json
 import logging
 import os
@@ -89,18 +90,20 @@ def close_all_files(self):
 
     def update_log_filenames(self):
         """Update the list of log files to monitor."""
-        log_filenames = os.listdir(self.logs_dir)
-
-        for log_filename in log_filenames:
-            full_path = os.path.join(self.logs_dir, log_filename)
-            if full_path not in self.log_filenames:
-                self.log_filenames.add(full_path)
+        # we only monior worker log files
+        log_file_paths = glob.glob("{}/worker*[.out|.err]".format(
+            self.logs_dir))
+        for file_path in log_file_paths:
+            if os.path.isfile(
+                    file_path) and file_path not in self.log_filenames:
+                self.log_filenames.add(file_path)
                 self.closed_file_infos.append(
                     LogFileInfo(
-                        filename=full_path,
+                        filename=file_path,
                         size_when_last_opened=0,
                         file_position=0,
                         file_handle=None))
+                log_filename = os.path.basename(file_path)
                 logger.info("Beginning to track file {}".format(log_filename))
 
     def open_closed_files(self):
@@ -172,20 +175,21 @@ def check_log_files_and_publish_updates(self):
             lines_to_publish = []
             max_num_lines_to_read = 100
             for _ in range(max_num_lines_to_read):
-                next_line = file_info.file_handle.readline()
-                if next_line == "":
-                    break
-                if next_line[-1] == "\n":
-                    next_line = next_line[:-1]
-                lines_to_publish.append(next_line)
-
-            # Publish the lines if this is a worker process.
-            filename = file_info.filename.split("/")[-1]
-            is_worker = (filename.startswith("worker")
-                         and (filename.endswith("out")
-                              or filename.endswith("err")))
-
-            if is_worker and file_info.file_position == 0:
+                try:
+                    next_line = file_info.file_handle.readline()
+                    if next_line == "":
+                        break
+                    if next_line[-1] == "\n":
+                        next_line = next_line[:-1]
+                    lines_to_publish.append(next_line)
+                except Exception:
+                    logger.error("Error: Reading file: {}, position: {} "
+                                 "failed.".format(
+                                     file_info.full_path,
+                                     file_info.file_info.file_handle.tell()))
+                    raise
+
+            if file_info.file_position == 0:
                 if (len(lines_to_publish) > 0 and
                         lines_to_publish[0].startswith("Ray worker pid: ")):
                     file_info.worker_pid = int(
@@ -195,7 +199,7 @@ def check_log_files_and_publish_updates(self):
             # Record the current position in the file.
             file_info.file_position = file_info.file_handle.tell()
 
-            if len(lines_to_publish) > 0 and is_worker:
+            if len(lines_to_publish) > 0:
                 self.redis_client.publish(
                     ray.gcs_utils.LOG_FILE_CHANNEL,
                     json.dumps({


Test patch: