Repository: randombit/botan
Pull request #3089 (2022-12-02 11:59:43)

Fix Python unspecified-encoding lint
@reneme Taking a bite at a time at #3076


Patch:
diff --git a/configure.py b/configure.py
index 1c906069d..6bf5bda95 100755
--- a/configure.py
+++ b/configure.py
@@ -55,7 +55,7 @@ def normalize_source_path(source):
     return os.path.normpath(source).replace('\\', '/')
 
 def parse_version_file(version_path):
-    version_file = open(version_path)
+    version_file = open(version_path, encoding='utf8')
     key_and_val = re.compile(r"([a-z_]+) = ([a-zA-Z0-9:\-\']+)")
 
     results = {}
@@ -736,7 +736,7 @@ def lex_me_harder(infofile, allowed_groups, allowed_maps, name_val_pairs):
     def py_var(group):
         return group.replace(':', '_')
 
-    lexer = shlex.shlex(open(infofile), infofile, posix=True)
+    lexer = shlex.shlex(open(infofile, encoding='utf8'), infofile, posix=True)
     lexer.wordchars += '=:.<>/,-!?+*' # handle various funky chars in info.txt
 
     groups = allowed_groups + allowed_maps
@@ -1695,7 +1695,7 @@ def read_textfile(filepath):
     if filepath is None:
         return ''
 
-    with open(filepath) as f:
+    with open(filepath, encoding='utf8') as f:
         return ''.join(f.readlines())
 
 
@@ -2717,7 +2717,7 @@ class AmalgamationHeader:
                     yield line
 
     def write_to_file(self, filepath, include_guard):
-        with open(filepath, 'w') as f:
+        with open(filepath, 'w', encoding='utf8') as f:
             AmalgamationHelper.write_banner(f)
             f.write("\n#ifndef %s\n#define %s\n\n" % (include_guard, include_guard))
             f.write(self.header_includes)
@@ -3280,7 +3280,7 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou
                 logging.error('Error while creating "%s": %s', build_dir, e)
 
     def write_template_with_variables(sink, template, variables):
-        with open(sink, 'w') as f:
+        with open(sink, 'w', encoding='utf8') as f:
             f.write(process_template(template, variables))
 
     def write_template(sink, template):
@@ -3337,7 +3337,7 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou
 
     template_vars.update(generate_build_info(build_paths, using_mods, cc, arch, osinfo, options))
 
-    with open(os.path.join(build_paths.build_dir, 'build_config.json'), 'w') as f:
+    with open(os.path.join(build_paths.build_dir, 'build_config.json'), 'w', encoding='utf8') as f:
         json.dump(template_vars, f, sort_keys=True, indent=2)
 
     if options.with_compilation_database:
@@ -3373,7 +3373,7 @@ def do_io_for_build(cc, arch, osinfo, using_mods, info_modules, build_paths, sou
         rst2man_file = os.path.join(build_paths.build_dir, 'botan.rst')
         cli_doc = os.path.join(source_paths.doc_dir, 'cli.rst')
 
-        cli_doc_contents = open(cli_doc).readlines()
+        cli_doc_contents = open(cli_doc, encoding='utf8').readlines()
 
         while cli_doc_contents[0] != "\n":
             cli_doc_contents.pop(0)
@@ -3387,7 +3387,7 @@ botan
 
         """.strip()
 
-        with open(rst2man_file, 'w') as f:
+        with open(rst2man_file, 'w', encoding='utf8') as f:
             f.write(rst2man_header)
             f.write("\n")
             for line in cli_doc_contents:
diff --git a/src/editors/sublime/build.py b/src/editors/sublime/build.py
index 3081fdc50..8b20eecdf 100755
--- a/src/editors/sublime/build.py
+++ b/src/editors/sublime/build.py
@@ -41,7 +41,7 @@ def _find_regex_in_makefile(regex):
         raise BuildError(
             'No Makefile found. Initial ./configure.py invocation must be performed manually.')
 
-    with open('Makefile', 'r') as f:
+    with open('Makefile', 'r', encoding='utf8') as f:
         return re.search(regex, f.read())
 
 
@@ -81,7 +81,7 @@ def _parse_test_file(test_file):
         raise BuildError(
             'Given file path is not a Botan unit test: ' + test_file)
 
-    with open(test_file, 'r') as f:
+    with open(test_file, 'r', encoding='utf8') as f:
         find_test_registration = \
             re.compile(
                 r'BOTAN_REGISTER_TEST(_FN)?\s*\(\s*\"(.+)\",\s*\"(.+)\",[^)]+\)')
diff --git a/src/scripts/build_docs.py b/src/scripts/build_docs.py
index c5623ed71..904bd2534 100755
--- a/src/scripts/build_docs.py
+++ b/src/scripts/build_docs.py
@@ -52,7 +52,7 @@ def touch(fname):
     try:
         os.utime(fname, None)
     except OSError:
-        open(fname, 'a').close()
+        open(fname, 'a', encoding='utf8').close()
 
 def copy_files(src_path, dest_dir):
 
@@ -126,7 +126,7 @@ def parse_options(args):
 
 def read_config(config):
     try:
-        f = open(config)
+        f = open(config, encoding='utf8')
         cfg = json.load(f)
         f.close()
     except OSError:
diff --git a/src/scripts/check.py b/src/scripts/check.py
index b8c5882be..6a39d50c3 100755
--- a/src/scripts/check.py
+++ b/src/scripts/check.py
@@ -60,7 +60,7 @@ def parse_options(args):
 
 def read_config(config):
     try:
-        with open(config) as f:
+        with open(config, encoding='utf8') as f:
             return json.load(f)
     except OSError:
         raise Exception('Failed to load build config %s - is build dir correct?' % (config))
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index 8b443863c..dfc72a0c2 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -365,7 +365,7 @@ def run_cmd(cmd, root_dir):
 
     redirect_stdout = None
     if len(cmd) >= 3 and cmd[-2] == '>':
-        redirect_stdout = open(cmd[-1], 'w')
+        redirect_stdout = open(cmd[-1], 'w', encoding='utf8')
         cmd = cmd[:-2]
     if len(cmd) > 1 and cmd[0].startswith('indir:'):
         cwd = cmd[0][6:]
diff --git a/src/scripts/ci_check_install.py b/src/scripts/ci_check_install.py
index c07356182..01187521b 100755
--- a/src/scripts/ci_check_install.py
+++ b/src/scripts/ci_check_install.py
@@ -83,7 +83,7 @@ def main(args=None):
         print("Usage: %s <build_config.json>" % args[0])
         return 1
 
-    with open(os.path.join(args[1])) as f:
+    with open(os.path.join(args[1]), encoding='utf8') as f:
         build_config = json.load(f)
 
     install_prefix = build_config['prefix']
diff --git a/src/scripts/cleanup.py b/src/scripts/cleanup.py
index aee514367..ae8e54c05 100755
--- a/src/scripts/cleanup.py
+++ b/src/scripts/cleanup.py
@@ -87,7 +87,7 @@ def main(args=None):
     build_config_str = None
 
     try:
-        build_config_file = open(build_config_path)
+        build_config_file = open(build_config_path, encoding='utf8')
         build_config_str = build_config_file.read()
         build_config_file.close()
     except Exception: # pylint: disable=broad-except
diff --git a/src/scripts/dist.py b/src/scripts/dist.py
index 5a7c5dfd1..874bb1620 100755
--- a/src/scripts/dist.py
+++ b/src/scripts/dist.py
@@ -164,7 +164,7 @@ def rewrite_version_file(version_file, target_version, snapshot_branch, rev_id,
     if snapshot_branch:
         assert target_version == snapshot_branch
 
-    contents = open(version_file).readlines()
+    contents = open(version_file, encoding='utf8').readlines()
 
     version_re = re.compile('release_(major|minor|patch) = ([0-9]+)')
     version_suffix_re = re.compile('release_suffix = \'(-(alpha|beta|rc)[0-9]+)\'')
@@ -222,7 +222,7 @@ def rewrite_version_file(version_file, target_version, snapshot_branch, rev_id,
                     marked_version, target_version))
 
     new_contents = ''.join(list(content_rewriter(target_version)))
-    open(version_file, 'w').write(new_contents)
+    open(version_file, 'w', encoding='utf8').write(new_contents)
 
 def write_archive(version, output_basename, archive_type, rel_epoch, all_files, hash_file):
     # pylint: disable=too-many-locals
@@ -424,7 +424,7 @@ def main(args=None):
 
     hash_file = None
     if options.write_hash_file is not None:
-        hash_file = open(options.write_hash_file, 'w')
+        hash_file = open(options.write_hash_file, 'w', encoding='utf8')
 
     for archive_type in archives:
         output_files.append(write_archive(target_version,
diff --git a/src/scripts/install.py b/src/scripts/install.py
index 632270905..09b615ee6 100755
--- a/src/scripts/install.py
+++ b/src/scripts/install.py
@@ -132,7 +132,7 @@ def main(args):
         logging.debug('Make %s executable', dst)
         os.chmod(dst, exe_mode)
 
-    with open(os.path.join(build_dir, 'build_config.json')) as f:
+    with open(os.path.join(build_dir, 'build_config.json'), encoding='utf8') as f:
         cfg = json.load(f)
 
     ver_major = int(cfg['version_major'])
diff --git a/src/scripts/website.py b/src/scripts/website.py
index 8c45f816f..d24e79211 100755
--- a/src/scripts/website.py
+++ b/src/scripts/website.py
@@ -72,7 +72,7 @@ def run_sphinx(botan_dir, tmp_dir, output_dir):
    API Reference <https://botan.randombit.net/doxygen>
 """
 
-    contents_rst = open(os.path.join(sphinx_dir, 'contents.rst'), 'w')
+    contents_rst = open(os.path.join(sphinx_dir, 'contents.rst'), 'w', encoding='utf8')
     contents_rst.write(toc)
     contents_rst.close()
 
@@ -156,7 +156,7 @@ def main(args):
     for f in ['doc/pgpkey.txt', 'license.txt']:
         shutil.copy(os.path.join(botan_dir, f), output_dir)
 
-    favicon = open(os.path.join(output_dir, 'favicon.ico'), 'w')
+    favicon = open(os.path.join(output_dir, 'favicon.ico'), 'w', encoding='utf8')
     # Create an empty favicon.ico file so it gets cached by browsers
     favicon.close()
 


Test patch:
diff --git a/src/scripts/test_cli.py b/src/scripts/test_cli.py
index 2d61a83d3..af0ba8eba 100755
--- a/src/scripts/test_cli.py
+++ b/src/scripts/test_cli.py
@@ -474,7 +474,7 @@ def cli_compress_tests(tmp_dir):
     input_file = os.path.join(tmp_dir, 'input.txt')
     output_file = os.path.join(tmp_dir, 'input.txt.gz')
 
-    with open(input_file, 'w') as f:
+    with open(input_file, 'w', encoding='utf8') as f:
         f.write("hi there")
         f.close()
 
@@ -495,7 +495,7 @@ def cli_compress_tests(tmp_dir):
     if not os.access(input_file, os.R_OK):
         logging.error("Decompression did not created expected output file")
 
-    recovered = open(input_file).read()
+    recovered = open(input_file, encoding='utf8').read()
     if recovered != "hi there":
         logging.error("Decompression did not recover original input")
 
@@ -533,7 +533,7 @@ def cli_roughtime_check_tests(tmp_dir):
         return
     chain = os.path.join(tmp_dir, 'roughtime-chain')
 
-    with open(chain, 'w') as f:
+    with open(chain, 'w', encoding='utf8') as f:
         f.write("""\
 ed25519 bbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA
 ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= uLeTON9D+2HqJMzK6sYWLNDEdtBl9t/9yw1cVAOm0/sONH5Oqdq9dVPkC9syjuWbglCiCPVF+FbOtcxCkrgMmA== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWOw1jl0uSiBEH9HE8/6r7zxoSc01f48vw+UzH8+VJoPelnvVJBj4lnH8uRLh5Aw0i4Du7XM1dp2u0r/I5PzhMQoDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AUBo+tEqPBQC47l77to7ESFTVhlw1SC74P5ssx6gpuJ6eP+1916GuUiySGE/x3Fp0c3otUGAdsRQou5p9PDTeane/YEeVq4/8AgAAAEAAAABTSUcAREVMRe5T1ml8wHyWAcEtHP/U5Rg/jFXTEXOSglngSa4aI/CECVdy4ZNWeP6vv+2//ZW7lQsrWo7ZkXpvm9BdBONRSQIDAAAAIAAAACgAAABQVUJLTUlOVE1BWFQpXlenV0OfVisvp9jDHXLw8vymZVK9Pgw9k6Edf8ZEhUgSGEc5jwUASHLvZE2PBQAAAAAA
@@ -549,11 +549,11 @@ ed25519 cj8GsiNlRkqiDElAeNMSBBMwrAl15hYPgX50+GWX/lA= Tsy82BBU2xxVqNe1ip11OyEGoKW
   4: UTC 2019-08-04T13:38:18 (+-1000000us)
   5: UTC 2019-08-04T13:38:18 (+-1000000us)""")
 
-    with open(chain, 'w') as f:
+    with open(chain, 'w', encoding='utf8') as f:
         f.write("ed25519 bbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA")
     test_cli("roughtime_check", [chain, "--raw-time"], "1: UTC 1564925897781286 (+-1000000us)")
 
-    with open(chain, 'w') as f:
+    with open(chain, 'w', encoding='utf8') as f:
         f.write("ed25519 cbT+RPS7zKX6w71ssPibzmwWqU9ffRV5oj2OresSmhE= eu9yhsJfVfguVSqGZdE8WKIxaBBM0ZG3Vmuc+IyZmG2YVmrIktUByDdwIFw6F4rZqmSFsBO85ljoVPz5bVPCOw== BQAAAEAAAABAAAAApAAAADwBAABTSUcAUEFUSFNSRVBDRVJUSU5EWBnGOEajOwPA6G7oL47seBP4C7eEpr57H43C2/fK/kMA0UGZVUdf4KNX8oxOK6JIcsbVk8qhghTwA70qtwpYmQkDAAAABAAAAAwAAABSQURJTUlEUFJPT1RAQg8AJrA8tEqPBQAqisiuAxgy2Pj7UJAiWbCdzGz1xcCnja3T+AqhC8fwpeIwW4GPy/vEb/awXW2DgSLKJfzWIAz+2lsR7t4UjNPvAgAAAEAAAABTSUcAREVMRes9Ch4X0HIw5KdOTB8xK4VDFSJBD/G9t7Et/CU7UW61OiTBXYYQTG2JekWZmGa0OHX1JPGG+APkpbsNw0BKUgYDAAAAIAAAACgAAABQVUJLTUlOVE1BWFR/9BWjpsWTQ1f6iUJea3EfZ1MkX3ftJiV3ABqNLpncFwAAAAAAAAAA//////////8AAAAA")
     test_cli("roughtime_check", chain, expected_stderr='Error: Roughtime Invalid signature or public key')
 
@@ -617,7 +617,7 @@ ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= 2A+I9q2+ZayxDDYC5n2YW8Bn/zB
     server_response = response[0]
     test_cli("roughtime", [], expected_stderr='Please specify either --servers-file or --host and --pubkey')
 
-    with open(ecosystem, 'w') as f:
+    with open(ecosystem, 'w', encoding='utf8') as f:
         f.write("Cloudflare-Roughtime ed25519 gD63hSj4ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + str(server_port))
 
     test_cli("roughtime", [
@@ -626,7 +626,7 @@ ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= 2A+I9q2+ZayxDDYC5n2YW8Bn/zB
         "--servers-file=" + ecosystem]
              , expected_stderr='ERROR: Public key does not match!')
 
-    with open(ecosystem, 'w') as f:
+    with open(ecosystem, 'w', encoding='utf8') as f:
         f.write("Cloudflare-Roughtime ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= udp 127.0.0.1:" + str(server_port))
 
     test_cli("roughtime", [
@@ -640,7 +640,7 @@ ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= 2A+I9q2+ZayxDDYC5n2YW8Bn/zB
         "--servers-file=" + ecosystem]
              , "Cloudflare-Roughtime     : UTC 2019-09-12T08:00:11 (+-1000000us)")
 
-    with open(chain_file, 'r') as f:
+    with open(chain_file, 'r', encoding='utf8') as f:
         read_data = f.read()
     if read_data != chain[0]:
         logging.error("unexpected chain")
@@ -655,7 +655,7 @@ ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= 2A+I9q2+ZayxDDYC5n2YW8Bn/zB
         "--raw-time"]
              , "UTC 1568275214691000 (+-1000000us)")
 
-    with open(chain_file, 'r') as f:
+    with open(chain_file, 'r', encoding='utf8') as f:
         read_data = f.read()
     if read_data != chain[1]:
         logging.error("unexpected chain")
@@ -670,7 +670,7 @@ ed25519 gD63hSj3ScS+wuOeGrubXlq35N1c5Lby/S+T7MNTjxo= 2A+I9q2+ZayxDDYC5n2YW8Bn/zB
         "--max-chain-size=2"]
              , "UTC 2019-09-12T08:00:42 (+-1000000us)")
 
-    with open(chain_file, 'r') as f:
+    with open(chain_file, 'r', encoding='utf8') as f:
         read_data = f.read()
     if read_data != chain[2]:
         logging.error("unexpected chain")
diff --git a/src/scripts/test_fuzzers.py b/src/scripts/test_fuzzers.py
index 9048cde64..37a7dff9b 100755
--- a/src/scripts/test_fuzzers.py
+++ b/src/scripts/test_fuzzers.py
@@ -32,7 +32,7 @@ def run_fuzzer_gdb(args):
 
 def run_fuzzer(args):
     (fuzzer_bin, corpus_file) = args
-    corpus_fd = open(corpus_file, 'r')
+    corpus_fd = open(corpus_file, 'rb')
     fuzzer_proc = subprocess.Popen([fuzzer_bin], stdin=corpus_fd,
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True)
     (stdout, stderr) = fuzzer_proc.communicate()