Repository: stevenblack/hosts
Issue: Extra newlines in generated hosts file around comments

When I run `python3 updateHostsFile.py`  then scroll to the bottom of the generated `hosts` file I see this:

```
#

# The contents of this file (containing a listing of additional domains in

# 'hosts' file format) are appended to the unified hosts file during the

# update process.



0.0.0.0 someblacklisteddomain.com
```

Which comes from the blacklist file, however in the blacklist file those lines look like this:

```
# blacklist
#
# The contents of this file (containing a listing of additional domains in
# 'hosts' file format) are appended to the unified hosts file during the
# update process.

0.0.0.0 someblacklisteddomain.com
```

Somehow the script is inserting extra newline characters, but only around empty lines (`/^\\s?\\n/` in regex parlance)  and lines that start with comments  (`/\\s?#.*/` in regex parlance). 

Additionally comments after a valid line are being stripped, so for example.

In blacklist...
```
0.0.0.0 someblacklisteddomain.com  # some comment here
```
Becomes...
```
0.0.0.0 someblacklisteddomain.com
```

Interestingly, `myhosts` is not affected by this issue. Inline comments at the end of valid hosts rules are left as-is, and there are no extra newlines between normal comment blocks.

all 3-rd party blocklists and the included custom `blacklist` are both affected by this. Example:

```
# Start adaway.org

# AdAway default blocklist

# Blocking mobile ad providers and some analytics providers

# 
```

Changed files:
testUpdateHostsFile.py, updateHostsFile.py

diff --git a/testUpdateHostsFile.py b/testUpdateHostsFile.py
index 171cb21b..6b13b567 100644
--- a/testUpdateHostsFile.py
+++ b/testUpdateHostsFile.py
@@ -86,7 +86,7 @@ class TestGetDefaults(Base):
                         "replace": False,
                         "backup": False,
                         "skipstatichosts": False,
-                        "keepdomaincomments": False,
+                        "keepdomaincomments": True,
                         "extensionspath": "foo" + self.sep + "extensions",
                         "extensions": [],
                         "compress": False,
@@ -753,10 +753,12 @@ class TestStripRule(Base):
             self.assertEqual(output, line)
 
     def test_strip_more_than_two(self):
+        comment = " # comments here galore"
+
         for line in ["0.0.0.0 twitter.com", "127.0.0.1 facebook.com",
                      "8.8.8.8 google.com", "1.2.3.4 foo.bar.edu"]:
-            output = strip_rule(line + " # comments here galore")
-            self.assertEqual(output, line)
+            output = strip_rule(line + comment)
+            self.assertEqual(output, line + comment)
 
 
 class TestWriteOpeningHeader(BaseMockDir):
diff --git a/updateHostsFile.py b/updateHostsFile.py
index e1e1e715..5647db44 100644
--- a/updateHostsFile.py
+++ b/updateHostsFile.py
@@ -60,7 +60,7 @@ def get_defaults():
         "replace": False,
         "backup": False,
         "skipstatichosts": False,
-        "keepdomaincomments": False,
+        "keepdomaincomments": True,
         "extensionspath": path_join_robust(BASEDIR_PATH, "extensions"),
         "extensions": [],
         "compress": False,
@@ -99,8 +99,8 @@ def main():
     parser.add_argument("--ip", "-i", dest="targetip", default="0.0.0.0",
                         help="Target IP address. Default is 0.0.0.0.")
     parser.add_argument("--keepdomaincomments", "-k",
-                        dest="keepdomaincomments", default=False,
-                        help="Keep domain line comments.")
+                        dest="keepdomaincomments", action="store_false", default=True,
+                        help="Do not keep domain line comments.")
     parser.add_argument("--noupdate", "-n", dest="noupdate", default=False,
                         action="store_true", help="Don't update from "
                                                   "host data sources.")
@@ -628,8 +628,8 @@ def create_initial_file():
     for source in recursive_glob(settings["datapath"],
                                  settings["hostfilename"]):
 
-        start = "# Start {}\\n".format(os.path.basename(os.path.dirname(source)))
-        end = "# End {}\\n".format(os.path.basename(os.path.dirname(source)))
+        start = "# Start {}\\n\\n".format(os.path.basename(os.path.dirname(source)))
+        end = "# End {}\\n\\n".format(os.path.basename(os.path.dirname(source)))
 
         with open(source, "r") as curFile:
             write_data(merge_file, start + curFile.read() + end)
@@ -772,7 +772,7 @@ def remove_dups_and_excl(merge_file, exclusion_regexes, output_file=None):
         line = line.replace("\\t+", " ")
 
         # see gh-271: trim trailing whitespace, periods
-        line = line.rstrip(' .') + "\\n"
+        line = line.rstrip(' .')
 
         # Testing the first character doesn't require startswith
         if line[0] == "#" or re.match(r'^\\s*$', line[0]):
@@ -843,7 +843,10 @@ def normalize_rule(rule, target_ip, keep_domain_comments):
         rule = "%s %s" % (target_ip, hostname)
 
         if suffix and keep_domain_comments:
-            rule += " #%s" % suffix
+            if not suffix.strip().startswith('#'):
+                rule += " #%s" % suffix
+            else:
+                rule += " %s" % suffix
 
         return hostname, rule + "\\n"
 
@@ -860,7 +863,10 @@ def normalize_rule(rule, target_ip, keep_domain_comments):
         rule = "%s %s" % (target_ip, ip_host)
 
         if suffix and keep_domain_comments:
-            rule += " #%s" % suffix
+            if not suffix.strip().startswith('#'):
+                rule += " #%s" % suffix
+            else:
+                rule += " %s" % suffix
 
         return ip_host, rule + "\\n"
 
@@ -875,9 +881,6 @@ def strip_rule(line):
     """
     Sanitize a rule string provided before writing it to the output hosts file.
 
-    Some sources put comments around their rules, for accuracy we need
-    to strip them the comments are preserved in the output hosts file.
-
     Parameters
     ----------
     line : str
@@ -894,7 +897,7 @@ def strip_rule(line):
         # just return blank
         return ""
     else:
-        return split_line[0] + " " + split_line[1]
+        return " ".join(split_line)
 
 
 def write_opening_header(final_file, **header_params):