Repository: freedomofpress/securedrop

Issue:
Remove `sh` dependency
*This is a good first issue for new contributors to take on, if you have any questions, please ask on the task or in our [Gitter room](https://gitter.im/freedomofpress/securedrop)!*

## Description

`sh` is a pretty cool project, it allows you to shell out to programs as if they were native Python programs. This comes with a downside, it obfuscates the underlying command being run, and it's an extra dependency.

Here are all the places we use it:
```
Targets
    Occurrences of 'from sh import' in Project
Found Occurrences in Project  (4 usages found)
    Unclassified  (4 usages found)
        securedrop  (4 usages found)
            securedrop  (1 usage found)
                i18n_tool.py  (1 usage found)
                    19 from sh import git, msgfmt, msgmerge, pybabel, sed, xgettext
            securedrop/tests  (3 usages found)
                test_i18n.py  (1 usage found)
                    34 from sh import pybabel, sed
                test_i18n_tool.py  (1 usage found)
                    13 from sh import git, msginit, pybabel, sed, touch
                test_template_filters.py  (1 usage found)
                    13 from sh import pybabel
```

Given how minimal its usage is (just `i18n_tool.py` plus tests), I propose we replace it with explicit `subprocess.check_output(...)` calls. The biggest advantage of this removal is that `pybabel` will be the only dependency needed to compile translations at package build time.


Discussion:
Hi @legoktm , I would love to work on this issue! Has it been assigned to anyone yet? Thanks!
Nope, go for it! Let me know if you have any questions.
Before seeing this issue, I had already started removing usages of `sh.sed()` in code that I was touching. I have opened a PR that removes calls to `sed()`: https://github.com/freedomofpress/securedrop/pull/6562

I will not work on other usages of `sh` so this issue is still available.
OK, thanks for the heads up! I won't touch those calls, then.

Resolving patch:
diff --git a/securedrop/i18n_tool.py b/securedrop/i18n_tool.py
--- a/securedrop/i18n_tool.py
+++ b/securedrop/i18n_tool.py
@@ -18,7 +18,6 @@
 from typing import Dict, List, Optional, Set
 
 import version
-from sh import git, msgfmt, msgmerge, pybabel, xgettext
 
 logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger(__name__)
@@ -59,10 +58,10 @@ def ensure_i18n_remote(self, args: argparse.Namespace) -> None:
         Make sure we have a git remote for the i18n repo.
         """
 
-        k = {"_cwd": args.root}
-        if b"i18n" not in git.remote(**k).stdout:
-            git.remote.add("i18n", args.url, **k)
-        git.fetch("i18n", **k)
+        k = {"cwd": args.root}
+        if "i18n" not in subprocess.check_output(["git", "remote"], encoding="utf-8", **k):
+            subprocess.check_call(["git", "remote", "add", "i18n", args.url], **k)
+        subprocess.check_call(["git", "fetch", "i18n"], **k)
 
     def translate_messages(self, args: argparse.Namespace) -> None:
         messages_file = Path(args.translations_dir).absolute() / "messages.pot"
@@ -71,22 +70,26 @@ def translate_messages(self, args: argparse.Namespace) -> None:
             if not os.path.exists(args.translations_dir):
                 os.makedirs(args.translations_dir)
             sources = args.sources.split(",")
-            pybabel.extract(
-                "--charset=utf-8",
-                "--mapping",
-                args.mapping,
-                "--output",
-                str(messages_file),
-                "--project=SecureDrop",
-                "--version",
-                args.version,
-                "--msgid-bugs-address=securedrop@freedom.press",
-                "--copyright-holder=Freedom of the Press Foundation",
-                "--add-comments=Translators:",
-                "--strip-comments",
-                "--add-location=never",
-                "--no-wrap",
-                *sources,
+            subprocess.check_call(
+                [
+                    "pybabel",
+                    "extract",
+                    "--charset=utf-8",
+                    "--mapping",
+                    args.mapping,
+                    "--output",
+                    messages_file,
+                    "--project=SecureDrop",
+                    "--version",
+                    args.version,
+                    "--msgid-bugs-address=securedrop@freedom.press",
+                    "--copyright-holder=Freedom of the Press Foundation",
+                    "--add-comments=Translators:",
+                    "--strip-comments",
+                    "--add-location=never",
+                    "--no-wrap",
+                    *sources,
+                ]
             )
 
             msg_file_content = messages_file.read_text()
@@ -101,31 +104,42 @@ def translate_messages(self, args: argparse.Namespace) -> None:
             ):
                 tglob = "{}/*/LC_MESSAGES/*.po".format(args.translations_dir)
                 for translation in glob.iglob(tglob):
-                    msgmerge("--previous", "--update", "--no-wrap", translation, messages_file)
+                    subprocess.check_call(
+                        [
+                            "msgmerge",
+                            "--previous",
+                            "--update",
+                            "--no-wrap",
+                            translation,
+                            messages_file,
+                        ]
+                    )
                 log.warning(f"messages translations updated in {messages_file}")
             else:
                 log.warning("messages translations are already up to date")
 
         if args.compile and len(os.listdir(args.translations_dir)) > 1:
-            pybabel.compile("--directory", args.translations_dir)
+            subprocess.check_call(["pybabel", "compile", "--directory", args.translations_dir])
 
     def translate_desktop(self, args: argparse.Namespace) -> None:
         messages_file = Path(args.translations_dir).absolute() / "desktop.pot"
 
         if args.extract_update:
             sources = args.sources.split(",")
-            k = {"_cwd": args.translations_dir}
-            xgettext(
-                "--output=desktop.pot",
-                "--language=Desktop",
-                "--keyword",
-                "--keyword=Name",
-                "--package-version",
-                args.version,
-                "--msgid-bugs-address=securedrop@freedom.press",
-                "--copyright-holder=Freedom of the Press Foundation",
-                *sources,
-                **k,
+            subprocess.check_call(
+                [
+                    "xgettext",
+                    "--output=desktop.pot",
+                    "--language=Desktop",
+                    "--keyword",
+                    "--keyword=Name",
+                    "--package-version",
+                    args.version,
+                    "--msgid-bugs-address=securedrop@freedom.press",
+                    "--copyright-holder=Freedom of the Press Foundation",
+                    *sources,
+                ],
+                cwd=args.translations_dir,
             )
             msg_file_content = messages_file.read_text()
             updated_content = _remove_from_content_line_with_text(
@@ -138,7 +152,7 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
                     if not f.endswith(".po"):
                         continue
                     po_file = os.path.join(args.translations_dir, f)
-                    msgmerge("--update", po_file, messages_file)
+                    subprocess.check_call(["msgmerge", "--update", po_file, messages_file])
                 log.warning(f"messages translations updated in {messages_file}")
             else:
                 log.warning("desktop translations are already up to date")
@@ -153,15 +167,18 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
 
                 for source in args.sources.split(","):
                     target = source.rstrip(".in")
-                    msgfmt(
-                        "--desktop",
-                        "--template",
-                        source,
-                        "-o",
-                        target,
-                        "-d",
-                        ".",
-                        _cwd=args.translations_dir,
+                    subprocess.check_call(
+                        [
+                            "msgfmt",
+                            "--desktop",
+                            "--template",
+                            source,
+                            "-o",
+                            target,
+                            "-d",
+                            ".",
+                        ],
+                        cwd=args.translations_dir,
                     )
                     self.sort_desktop_template(join(args.translations_dir, target))
             finally:
@@ -263,12 +280,11 @@ def update_docs(self, args: argparse.Namespace) -> None:
         io.open(l10n_txt, mode="w").write(l10n_content)
         self.require_git_email_name(includes)
         if self.file_is_modified(l10n_txt):
-            k = {"_cwd": includes}
-            git.add("l10n.txt", **k)
+            subprocess.check_call(["git", "add", "l10n.txt"], cwd=includes)
             msg = "docs: update the list of supported languages"
-            git.commit("-m", msg, "l10n.txt", **k)
+            subprocess.check_call(["git", "commit", "-m", msg, "l10n.txt"], cwd=includes)
             log.warning(l10n_txt + " updated")
-            git_show_out = git.show(**k)
+            git_show_out = subprocess.check_output(["git", "show"], encoding="utf-8", cwd=includes)
             log.warning(git_show_out)
         else:
             log.warning(l10n_txt + " already up to date")
@@ -289,13 +305,15 @@ def update_from_weblate(self, args: argparse.Namespace) -> None:
         self.ensure_i18n_remote(args)
 
         # Check out *all* remote changes to the LOCALE_DIRs.
-        git(
-            "-C",
-            args.root,
-            "checkout",
-            args.target,
-            "--",
-            *LOCALE_DIR.values(),
+        subprocess.check_call(
+            [
+                "git",
+                "checkout",
+                args.target,
+                "--",
+                *LOCALE_DIR.values(),
+            ],
+            cwd=args.root,
         )
 
         # Use the LOCALE_DIR corresponding to the "securedrop" component to
@@ -314,7 +332,7 @@ def add(path: str) -> None:
                 """
                 Add the file to the git index.
                 """
-                git("-C", args.root, "add", path)
+                subprocess.check_call(["git", "add", path], cwd=args.root)
                 paths.append(path)
 
             # Any translated locale may have changes that need to be staged from the
@@ -364,7 +382,7 @@ def translators(
 
         log_command.extend([args.target, "--", path])
 
-        log_lines = subprocess.check_output(log_command).decode("utf-8").strip().split("\n")
+        log_lines = subprocess.check_output(log_command, encoding="utf-8").strip().splitlines()
         path_changes = [c.split("\x1e") for c in log_lines]
         path_changes = [
             c
@@ -380,9 +398,8 @@ def get_path_commits(self, root: str, path: str) -> List[str]:
         """
         Returns the list of commit hashes involving the path, most recent first.
         """
-        return git(
-            "--no-pager", "-C", root, "log", "--format=%H", path, _encoding="utf-8"
-        ).splitlines()
+        cmd = ["git", "--no-pager", "log", "--format=%H", path]
+        return subprocess.check_output(cmd, encoding="utf-8", cwd=root).splitlines()
 
     def translated_locales(self, path: str) -> Dict[str, str]:
         """Return a dictionary of all locale directories present in `path`, where the keys
@@ -396,9 +413,8 @@ def commit_changes(
         """Check if any of the given paths have had changed staged.  If so, commit them."""
         self.require_git_email_name(args.root)
         authors = set()  # type: Set[str]
-        diffs = "{}".format(
-            git("--no-pager", "-C", args.root, "diff", "--name-only", "--cached", *paths)
-        )
+        cmd = ["git", "--no-pager", "diff", "--name-only", "--cached", *paths]
+        diffs = subprocess.check_output(cmd, cwd=args.root, encoding="utf-8")
 
         # If nothing was changed, "git commit" will return nonzero as a no-op.
         if len(diffs) == 0:
@@ -413,8 +429,10 @@ def commit_changes(
             path_commits = self.get_path_commits(args.root, path)
             since_commit = None
             for path_commit in path_commits:
-                commit_message = "{}".format(
-                    git("--no-pager", "-C", args.root, "show", path_commit, _encoding="utf-8")
+                commit_message = subprocess.check_output(
+                    ["git", "--no-pager", "show", path_commit],
+                    encoding="utf-8",
+                    cwd=args.root,
                 )
                 m = self.updated_commit_re.search(commit_message)
                 if m:
@@ -425,7 +443,9 @@ def commit_changes(
 
         authors_as_str = "\n  ".join(sorted(authors))
 
-        current = git("-C", args.root, "rev-parse", args.target)
+        current = subprocess.check_output(
+            ["git", "rev-parse", args.target], cwd=args.root, encoding="utf-8"
+        )
         message = textwrap.dedent(
             """
         l10n: updated {name} ({code})
@@ -438,7 +458,8 @@ def commit_changes(
           commit: {current}
         """
         ).format(remote=args.url, name=name, authors=authors_as_str, code=code, current=current)
-        git("-C", args.root, "commit", "-m", message, *paths)
+        subprocess.check_call(["git", "commit", "-m", message, *paths], cwd=args.root)
+        log.debug(f"Committing with this message: {message}")
 
     def set_update_from_weblate_parser(self, subps: _SubParsersAction) -> None:
         parser = subps.add_parser("update-from-weblate", help=("Import translations from weblate"))


Test patch:
diff --git a/securedrop/tests/test_i18n.py b/securedrop/tests/test_i18n.py
--- a/securedrop/tests/test_i18n.py
+++ b/securedrop/tests/test_i18n.py
@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 import re
+import subprocess
 from pathlib import Path
 from typing import List
 
@@ -31,7 +32,6 @@
 from flask_babel import gettext
 from i18n import parse_locale_set
 from sdconfig import FALLBACK_LOCALE
-from sh import pybabel
 from tests.functional.factories import SecureDropConfigFactory
 from tests.functional.sd_config_v2 import DEFAULT_SECUREDROP_ROOT, SecureDropConfig
 from werkzeug.datastructures import Headers
@@ -259,7 +259,7 @@ def test_i18n():
     )
 
     pot = translation_dirs / "messages.pot"
-    pybabel("init", "-i", pot, "-d", translation_dirs, "-l", "en_US")
+    subprocess.check_call(["pybabel", "init", "-i", pot, "-d", translation_dirs, "-l", "en_US"])
 
     for (locale, translated_msg) in (
         ("fr_FR", "code bonjour"),
@@ -268,7 +268,7 @@ def test_i18n():
         ("nb_NO", "code norwegian"),
         ("es_ES", "code spanish"),
     ):
-        pybabel("init", "-i", pot, "-d", translation_dirs, "-l", locale)
+        subprocess.check_call(["pybabel", "init", "-i", pot, "-d", translation_dirs, "-l", locale])
 
         # Populate the po file with a translation
         po_file = translation_dirs / locale / "LC_MESSAGES" / "messages.po"
diff --git a/securedrop/tests/test_i18n_tool.py b/securedrop/tests/test_i18n_tool.py
--- a/securedrop/tests/test_i18n_tool.py
+++ b/securedrop/tests/test_i18n_tool.py
@@ -4,6 +4,7 @@
 import os
 import shutil
 import signal
+import subprocess
 import time
 from os.path import abspath, dirname, exists, getmtime, join, realpath
 from pathlib import Path
@@ -11,7 +12,6 @@
 import i18n_tool
 import pytest
 from mock import patch
-from sh import git, msginit, pybabel, touch
 from tests.test_i18n import set_msg_translation_in_po_file
 
 
@@ -81,8 +81,17 @@ def test_translate_desktop_l10n(self, tmpdir):
             ("ro", "SOURCE RO"),
         ]:
             po_file = Path(tmpdir) / f"{locale}.po"
-            msginit(
-                "--no-translator", "--locale", locale, "--output", po_file, "--input", messages_file
+            subprocess.check_call(
+                [
+                    "msginit",
+                    "--no-translator",
+                    "--locale",
+                    locale,
+                    "--output",
+                    po_file,
+                    "--input",
+                    messages_file,
+                ]
             )
             set_msg_translation_in_po_file(
                 po_file=po_file,
@@ -139,7 +148,18 @@ def test_translate_messages_l10n(self, tmpdir):
 
         locale = "en_US"
         locale_dir = join(str(tmpdir), locale)
-        pybabel("init", "-i", messages_file, "-d", str(tmpdir), "-l", locale)
+        subprocess.check_call(
+            [
+                "pybabel",
+                "init",
+                "-i",
+                messages_file,
+                "-d",
+                str(tmpdir),
+                "-l",
+                locale,
+            ]
+        )
 
         # Add a dummy translation
         po_file = Path(locale_dir) / "LC_MESSAGES/messages.po"
@@ -185,7 +205,9 @@ def test_translate_messages_compile_arg(self, tmpdir):
         locale = "en_US"
         locale_dir = join(str(tmpdir), locale)
         po_file = join(locale_dir, "LC_MESSAGES/messages.po")
-        pybabel(["init", "-i", messages_file, "-d", str(tmpdir), "-l", locale])
+        subprocess.check_call(
+            ["pybabel", "init", "-i", messages_file, "-d", str(tmpdir), "-l", locale]
+        )
         assert exists(po_file)
         # pretend this happened a few seconds ago
         few_seconds_ago = time.time() - 60
@@ -241,25 +263,25 @@ def test_translate_messages_compile_arg(self, tmpdir):
             assert b"template hello i18n" not in mo
 
     def test_require_git_email_name(self, tmpdir):
-        k = {"_cwd": str(tmpdir)}
-        git("init", **k)
+        k = {"cwd": str(tmpdir)}
+        subprocess.check_call(["git", "init"], **k)
         with pytest.raises(Exception) as excinfo:
             i18n_tool.I18NTool.require_git_email_name(str(tmpdir))
         assert "please set name" in str(excinfo.value)
 
-        git.config("user.email", "you@example.com", **k)
-        git.config("user.name", "Your Name", **k)
+        subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k)
+        subprocess.check_call(["git", "config", "user.name", "Your Name"], **k)
         assert i18n_tool.I18NTool.require_git_email_name(str(tmpdir))
 
     def test_update_docs(self, tmpdir, caplog):
-        k = {"_cwd": str(tmpdir)}
-        git.init(**k)
-        git.config("user.email", "you@example.com", **k)
-        git.config("user.name", "Your Name", **k)
+        k = {"cwd": str(tmpdir)}
+        subprocess.check_call(["git", "init"], **k)
+        subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k)
+        subprocess.check_call(["git", "config", "user.name", "Your Name"], **k)
         os.makedirs(join(str(tmpdir), "docs/includes"))
-        touch("docs/includes/l10n.txt", **k)
-        git.add("docs/includes/l10n.txt", **k)
-        git.commit("-m", "init", **k)
+        subprocess.check_call(["touch", "docs/includes/l10n.txt"], **k)
+        subprocess.check_call(["git", "add", "docs/includes/l10n.txt"], **k)
+        subprocess.check_call(["git", "commit", "-m", "init"], **k)
 
         i18n_tool.I18NTool().main(["--verbose", "update-docs", "--docs-repo-dir", str(tmpdir)])
         assert "l10n.txt updated" in caplog.text
@@ -271,23 +293,23 @@ def test_update_from_weblate(self, tmpdir, caplog):
         d = str(tmpdir)
         for repo in ("i18n", "securedrop"):
             os.mkdir(join(d, repo))
-            k = {"_cwd": join(d, repo)}
-            git.init(**k)
-            git.config("user.email", "you@example.com", **k)
-            git.config("user.name", "Loïc Nordhøy", **k)
-            touch("README.md", **k)
-            git.add("README.md", **k)
-            git.commit("-m", "README", "README.md", **k)
+            k = {"cwd": join(d, repo)}
+            subprocess.check_call(["git", "init"], **k)
+            subprocess.check_call(["git", "config", "user.email", "you@example.com"], **k)
+            subprocess.check_call(["git", "config", "user.name", "Loïc Nordhøy"], **k)
+            subprocess.check_call(["touch", "README.md"], **k)
+            subprocess.check_call(["git", "add", "README.md"], **k)
+            subprocess.check_call(["git", "commit", "-m", "README", "README.md"], **k)
         for o in os.listdir(join(self.dir, "i18n")):
             f = join(self.dir, "i18n", o)
             if os.path.isfile(f):
                 shutil.copyfile(f, join(d, "i18n", o))
             else:
                 shutil.copytree(f, join(d, "i18n", o))
-        k = {"_cwd": join(d, "i18n")}
-        git.add("securedrop", "install_files", **k)
-        git.commit("-m", "init", "-a", **k)
-        git.checkout("-b", "i18n", "master", **k)
+        k = {"cwd": join(d, "i18n")}
+        subprocess.check_call(["git", "add", "securedrop", "install_files"], **k)
+        subprocess.check_call(["git", "commit", "-m", "init", "-a"], **k)
+        subprocess.check_call(["git", "checkout", "-b", "i18n", "master"], **k)
 
         def r():
             return "".join([str(l) for l in caplog.records])
@@ -350,7 +372,11 @@ def r():
         )
         assert "l10n: updated Dutch (nl)" not in r()
         assert "l10n: updated German (de_DE)" not in r()
-        message = str(git("--no-pager", "-C", "securedrop", "show", _cwd=d, _encoding="utf-8"))
+        message = subprocess.check_output(
+            ["git", "--no-pager", "-C", "securedrop", "show"],
+            cwd=d,
+            encoding="utf-8",
+        )
         assert "Loïc" in message
 
         # an update is done to nl in weblate
@@ -362,14 +388,15 @@ def r():
         updated_content = content.replace(text_to_update, "INACTIVITEIT")
         po_file.write_text(updated_content)
 
-        git.add(str(po_file), **k)
-        git.config("user.email", "somone@else.com", **k)
-        git.config("user.name", "Someone Else", **k)
-        git.commit("-m", "translation change", str(po_file), **k)
+        k = {"cwd": join(d, "i18n")}
+        subprocess.check_call(["git", "add", str(po_file)], **k)
+        subprocess.check_call(["git", "config", "user.email", "somone@else.com"], **k)
+        subprocess.check_call(["git", "config", "user.name", "Someone Else"], **k)
+        subprocess.check_call(["git", "commit", "-m", "translation change", str(po_file)], **k)
 
-        k = {"_cwd": join(d, "securedrop")}
-        git.config("user.email", "somone@else.com", **k)
-        git.config("user.name", "Someone Else", **k)
+        k = {"cwd": join(d, "securedrop")}
+        subprocess.check_call(["git", "config", "user.email", "somone@else.com"], **k)
+        subprocess.check_call(["git", "config", "user.name", "Someone Else"], **k)
 
         #
         # the nl translation update from weblate is copied
@@ -390,6 +417,10 @@ def r():
         )
         assert "l10n: updated Dutch (nl)" in r()
         assert "l10n: updated German (de_DE)" not in r()
-        message = str(git("--no-pager", "-C", "securedrop", "show", _cwd=d))
+        message = subprocess.check_output(
+            ["git", "--no-pager", "-C", "securedrop", "show"],
+            cwd=d,
+            encoding="utf-8",
+        )
         assert "Someone Else" in message
         assert "Loïc" not in message
diff --git a/securedrop/tests/test_template_filters.py b/securedrop/tests/test_template_filters.py
--- a/securedrop/tests/test_template_filters.py
+++ b/securedrop/tests/test_template_filters.py
@@ -1,3 +1,4 @@
+import subprocess
 from datetime import datetime, timedelta
 from pathlib import Path
 
@@ -8,7 +9,6 @@
 import template_filters
 from db import db
 from flask import session
-from sh import pybabel
 from tests.test_i18n import create_config_for_i18n_test
 
 
@@ -104,7 +104,7 @@ def do_test(create_app):
 
     for l in ("en_US", "fr_FR"):
         pot = Path(test_config.TEMP_DIR) / "messages.pot"
-        pybabel("init", "-i", pot, "-d", test_config.TEMP_DIR, "-l", l)
+        subprocess.check_call(["pybabel", "init", "-i", pot, "-d", test_config.TEMP_DIR, "-l", l])
 
     app = create_app(test_config)
     with app.app_context():