Repository: peterbe/hashin

Issue:
Add `--update-all` option and update all dependencies with their hashes?
Hey @peterbe!
I work with a lot of legacy projects that don't add hashes to their dependencies and I'd like to improve their security.
What do you think about an option which would just scan through all entries and add hashes to them?
Currently, I'm working around this with shell scripts but maybe it's a good idea to include it here.

I can even implement it if you think it will be useful.

Thanks in advance, `hashin` is awesome!

Discussion:
peterbe: Hey @jotes check it out! I deliberately left out the idea of `--interactive` which would ask the user `input("Wanna upgrade 'foo' to version 1.2.3 (from 1.2.2)?")` but that feature can easily be added after and separately. 

Please take a look at the patch and see if you can test it locally. 

Note, that if you use `hashin -u --dry-run` it'll just print to stdout and not update anything. 

What do you think about the idea of calling it "update" instead of "upgrade"?

Patch:
diff --git a/README.rst b/README.rst
index e22dd09..d7d6ae3 100644
--- a/README.rst
+++ b/README.rst
@@ -254,6 +254,10 @@ Version History
 
 next
 
+  * New flag ``--update-all`` (alias ``-u``) will parse the requirements file,
+    ignore the version, and update all packages that have new versions.
+    See https://github.com/peterbe/hashin/pull/88
+
   * Support for "extras syntax". E.g. ``hashin "requests[security]"``. Doesn't
     actually get hashes for ``security`` (in this case, that's not even a
     package) but allows that syntax into your ``requirements.txt`` file.
diff --git a/hashin.py b/hashin.py
index b74ccf2..cd471c1 100755
--- a/hashin.py
+++ b/hashin.py
@@ -40,10 +40,8 @@ DEFAULT_ALGORITHM = "sha256"
 parser = argparse.ArgumentParser()
 parser.add_argument(
     "packages",
-    help=(
-        "One or more package specifiers (e.g. some-package or " "some-package==1.2.3)"
-    ),
-    nargs="+",
+    help="One or more package specifiers (e.g. some-package or some-package==1.2.3)",
+    nargs="*",
 )
 parser.add_argument(
     "-r",
@@ -79,6 +77,13 @@ parser.add_argument(
     action="store_true",
     default=False,
 )
+parser.add_argument(
+    "-u",
+    "--update-all",
+    help="Update all mentioned packages in the requirements file.",
+    action="store_true",
+    default=False,
+)
 
 
 major_pip_version = int(pip_api.version().split(".")[0])
@@ -119,12 +124,22 @@ def _download(url, binary=False):
     return r.read().decode(encoding)
 
 
-def run(specs, *args, **kwargs):
+def run(specs, requirements_file, *args, **kwargs):
+    if not specs:  # then, assume all in the requirements file
+        regex = re.compile(r"(^|\n|\n\r).*==")
+        specs = []
+        with open(requirements_file) as f:
+            for line in f:
+                if regex.search(line):
+                    req = Requirement(line.split("\\")[0])
+                    # Deliberately strip the specifier (aka. the version)
+                    req.specifier = None
+                    specs.append(str(req))
     if isinstance(specs, str):
         specs = [specs]
 
     for spec in specs:
-        run_single_package(spec, *args, **kwargs)
+        run_single_package(spec, requirements_file, *args, **kwargs)
     return 0
 
 
@@ -198,12 +213,11 @@ def run_single_package(
 
 
 def amend_requirements_content(requirements, package, new_lines):
-    # if the package wasn't already there, add it to the bottom
     regex = re.compile(
         r"(^|\n|\n\r){0}==|(^|\n|\n\r){0}\[.*\]==".format(re.escape(package)),
         re.IGNORECASE,
     )
-
+    # if the package wasn't already there, add it to the bottom
     if not regex.search(requirements):
         # easy peasy
         if requirements:
@@ -493,6 +507,18 @@ def main():
 
     args = parser.parse_args()
 
+    if args.update_all:
+        if args.packages:
+            print(
+                "Can not combine the --update-all option with a list of packages.",
+                file=sys.stderr,
+            )
+            return 2
+    elif not args.packages:
+        print("If you don't use --update-all you must list packages.", file=sys.stderr)
+        parser.print_usage()
+        return 3
+
     try:
         return run(
             args.packages,


Test patch:
diff --git a/tests/test_arg_parse.py b/tests/test_arg_parse.py
index 4637366..a9718e0 100644
--- a/tests/test_arg_parse.py
+++ b/tests/test_arg_parse.py
@@ -27,6 +27,7 @@ def test_everything():
         version=False,
         include_prereleases=False,
         dry_run=True,
+        update_all=False,
     )
     assert args == (expected, [])
 
@@ -55,6 +56,7 @@ def test_everything_long():
         version=False,
         include_prereleases=False,
         dry_run=True,
+        update_all=False,
     )
     assert args == (expected, [])
 
@@ -70,5 +72,6 @@ def test_minimal():
         version=False,
         include_prereleases=False,
         dry_run=False,
+        update_all=False,
     )
     assert args == (expected, [])
diff --git a/tests/test_cli.py b/tests/test_cli.py
index a949b7e..cae9004 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -1,3 +1,4 @@
+import argparse
 import sys
 import json
 import os
@@ -21,6 +22,8 @@ if sys.version_info >= (3,):
 else:  # Python 2
     from StringIO import StringIO
 
+    FileNotFoundError = IOError  # ugly but necessary
+
 
 @contextmanager
 def redirect_stdout(stream):
@@ -187,11 +190,71 @@ class Tests(TestCase):
         # Doesn't matter so much what, just make sure it breaks
         mock_run.side_effect = hashin.PackageError("Some message here")
 
+        def mock_parse_args(*a, **k):
+            return argparse.Namespace(
+                packages=["something"],
+                requirements_file="requirements.txt",
+                algorithm="sha256",
+                python_version="3.8",
+                verbose=False,
+                include_prereleases=False,
+                dry_run=False,
+                update_all=False,
+            )
+
+        mock_parser.parse_args.side_effect = mock_parse_args
+
         error = hashin.main()
         self.assertEqual(error, 1)
         mock_sys.stderr.write.assert_any_call("Some message here")
         mock_sys.stderr.write.assert_any_call("\n")
 
+    @mock.patch("hashin.parser")
+    @mock.patch("hashin.sys")
+    def test_packages_and_update_all(self, mock_sys, mock_parser):
+        def mock_parse_args(*a, **k):
+            return argparse.Namespace(
+                packages=["something"],
+                requirements_file="requirements.txt",
+                algorithm="sha256",
+                python_version="3.8",
+                verbose=False,
+                include_prereleases=False,
+                dry_run=False,
+                update_all=True,  # Note!
+            )
+
+        mock_parser.parse_args.side_effect = mock_parse_args
+
+        error = hashin.main()
+        self.assertEqual(error, 2)
+        mock_sys.stderr.write.assert_any_call(
+            "Can not combine the --update-all option with a list of packages."
+        )
+
+    @mock.patch("hashin.parser")
+    @mock.patch("hashin.sys")
+    def test_no_packages_and_not_update_all(self, mock_sys, mock_parser):
+        def mock_parse_args(*a, **k):
+            return argparse.Namespace(
+                packages=[],  # Note!
+                requirements_file="requirements.txt",
+                algorithm="sha256",
+                python_version="3.8",
+                verbose=False,
+                include_prereleases=False,
+                dry_run=False,
+                update_all=False,
+            )
+
+        mock_parser.parse_args.side_effect = mock_parse_args
+
+        error = hashin.main()
+        self.assertEqual(error, 3)
+        mock_sys.stderr.write.assert_any_call(
+            "If you don't use --update-all you must list packages."
+        )
+
     @mock.patch("hashin.sys")
     def test_main_version(self, mock_sys):
         mock_sys.argv = [None, "--version"]
@@ -602,6 +665,178 @@ selenium==2.53.1 \
         it should find it and correct the cast typing per what it is
         inside the PyPI data."""
 
+        def mocked_get(url, **options):
+            if url == "https://pypi.org/pypi/hashin/json":
+                return _Response(
+                    {
+                        "info": {"version": "0.11", "name": "hashin"},
+                        "releases": {
+                            "0.11": [
+                                {
+                                    "url": "https://pypi.org/packages/source/p/hashin/hashin-0.11.tar.gz",
+                                    "digests": {"sha256": "bbbbb"},
+                                }
+                            ],
+                            "0.10": [
+                                {
+                                    "url": "https://pypi.org/packages/source/p/hashin/hashin-0.10.tar.gz",
+                                    "digests": {"sha256": "aaaaa"},
+                                }
+                            ],
+                        },
+                    }
+                )
+            elif url == "https://pypi.org/pypi/hashin/json":
+                return _Response(
+                    {
+                        "info": {"version": "0.11", "name": "hashin"},
+                        "releases": {
+                            "0.11": [
+                                {
+                                    "url": "https://pypi.org/packages/source/p/hashin/hashin-0.11.tar.gz",
+                                    "digests": {"sha256": "bbbbb"},
+                                }
+                            ],
+                            "0.10": [
+                                {
+                                    "url": "https://pypi.org/packages/source/p/hashin/hashin-0.10.tar.gz",
+                                    "digests": {"sha256": "aaaaa"},
+                                }
+                            ],
+                        },
+                    }
+                )
+            elif url == "https://pypi.org/pypi/requests/json":
+                return _Response(
+                    {
+                        "info": {"version": "1.2.4", "name": "requests"},
+                        "releases": {
+                            "1.2.4": [
+                                {
+                                    "url": "https://pypi.org/packages/source/p/requests/requests-1.2.4.tar.gz",
+                                    "digests": {"sha256": "dededede"},
+                                }
+                            ]
+                        },
+                    }
+                )
+            if url == "https://pypi.org/pypi/enum34/json":
+                return _Response(
+                    {
+                        "info": {"version": "1.1.6", "name": "enum34"},
+                        "releases": {
+                            "1.1.6": [
+                                {
+                                    "has_sig": False,
+                                    "upload_time": "2016-05-16T03:31:13",
+                                    "comment_text": "",
+                                    "python_version": "py2",
+                                    "url": "https://pypi.org/packages/c5/db/enum34-1.1.6-py2-none-any.whl",
+                                    "digests": {
+                                        "md5": "68f6982cc07dde78f4b500db829860bd",
+                                        "sha256": "aaaaa",
+                                    },
+                                    "md5_digest": "68f6982cc07dde78f4b500db829860bd",
+                                    "downloads": 4297423,
+                                    "filename": "enum34-1.1.6-py2-none-any.whl",
+                                    "packagetype": "bdist_wheel",
+                                    "path": "c5/db/enum34-1.1.6-py2-none-any.whl",
+                                    "size": 12427,
+                                },
+                                {
+                                    "has_sig": False,
+                                    "upload_time": "2016-05-16T03:31:19",
+                                    "comment_text": "",
+                                    "python_version": "py3",
+                                    "url": "https://pypi.org/packages/af/42/enum34-1.1.6-py3-none-any.whl",
+                                    "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43",
+                                    "digests": {
+                                        "md5": "a63ecb4f0b1b85fb69be64bdea999b43",
+                                        "sha256": "bbbbb",
+                                    },
+                                    "downloads": 98598,
+                                    "filename": "enum34-1.1.6-py3-none-any.whl",
+                                    "packagetype": "bdist_wheel",
+                                    "path": "af/42/enum34-1.1.6-py3-none-any.whl",
+                                    "size": 12428,
+                                },
+                                {
+                                    "has_sig": False,
+                                    "upload_time": "2016-05-16T03:31:30",
+                                    "comment_text": "",
+                                    "python_version": "source",
+                                    "url": "https://pypi.org/packages/bf/3e/enum34-1.1.6.tar.gz",
+                                    "md5_digest": "5f13a0841a61f7fc295c514490d120d0",
+                                    "digests": {
+                                        "md5": "5f13a0841a61f7fc295c514490d120d0",
+                                        "sha256": "ccccc",
+                                    },
+                                    "downloads": 188090,
+                                    "filename": "enum34-1.1.6.tar.gz",
+                                    "packagetype": "sdist",
+                                    "path": "bf/3e/enum34-1.1.6.tar.gz",
+                                    "size": 40048,
+                                },
+                                {
+                                    "has_sig": False,
+                                    "upload_time": "2016-05-16T03:31:48",
+                                    "comment_text": "",
+                                    "python_version": "source",
+                                    "url": "https://pypi.org/packages/e8/26/enum34-1.1.6.zip",
+                                    "md5_digest": "61ad7871532d4ce2d77fac2579237a9e",
+                                    "digests": {
+                                        "md5": "61ad7871532d4ce2d77fac2579237a9e",
+                                        "sha256": "dddddd",
+                                    },
+                                    "downloads": 775920,
+                                    "filename": "enum34-1.1.6.zip",
+                                    "packagetype": "sdist",
+                                    "path": "e8/26/enum34-1.1.6.zip",
+                                    "size": 44773,
+                                },
+                            ]
+                        },
+                    }
+                )
+
+            raise NotImplementedError(url)
+
+        murlopen.side_effect = mocked_get
+
+        with tmpfile() as filename:
+            with self.assertRaises(FileNotFoundError):
+                retcode = hashin.run(None, filename, "sha256")
+
+            with open(filename, "w") as f:
+                f.write("# This is comment. Ignore this.\n")
+                f.write("\n")
+                f.write("requests[security]==1.2.3 \\\n")
+                f.write("    --hash=sha256:99dcfdaae\n")
+                f.write("hashin==0.11 \\\n")
+                f.write("    --hash=sha256:a84b8c9ab623\n")
+                f.write("enum34==1.1.5; python_version <= '3.4' \\\n")
+                f.write("    --hash=sha256:12ce5c2ef718\n")
+                f.write("\n")
+
+            retcode = hashin.run(None, filename, "sha256", verbose=True)
+
+            self.assertEqual(retcode, 0)
+            with open(filename) as f:
+                output = f.read()
+
+            self.assertTrue("requests[security]==1.2.3" not in output)
+            self.assertTrue("requests[security]==1.2.4" in output)
+            # This one didn't need to be updated.
+            self.assertTrue("hashin==0.11" in output)
+            self.assertTrue('enum34==1.1.5; python_version <= "3.4"' not in output)
+            self.assertTrue('enum34==1.1.6; python_version <= "3.4"' in output)
+
+    @cleanup_tmpdir("hashin*")
+    @mock.patch("hashin.urlopen")
+    def test_run_update_all(self, murlopen):
+        """The --update-all flag will extra all the names from the existing
+        requirements file, and check with pypi.org if there's a new version."""
+
         def mocked_get(url, **options):
             if url == "https://pypi.org/pypi/HAShin/json":
                 return _Response(