Repository: pypi/warehouse

Issue:
Implement limit on Project.total_size
Now that we have the `Project.total_size` attribute, we should implement a limit on this value across all projects similar to per-file limit.

This would help catch projects that are a significant burden on our overall size on disk but remain under the 60MB per-file limit. It would also encourage projects that publish large nightlies to clean up older nightlies.

I think this limit should probably be something like 100GB, but happy to hear justification for different values. This should also be easy to adjust if need be.


Discussion:

Resolving patch:
diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py
--- a/warehouse/forklift/legacy.py
+++ b/warehouse/forklift/legacy.py
@@ -59,9 +59,13 @@
 )
 from warehouse.utils import http, readme
 
-MAX_FILESIZE = 60 * 1024 * 1024  # 60M
-MAX_SIGSIZE = 8 * 1024  # 8K
-MAX_PROJECT_SIZE = 10 * 1024 * 1024 * 1024  # 10GB
+ONE_MB = 1 * 1024 * 1024
+ONE_GB = 1 * 1024 * 1024 * 1024
+
+MAX_FILESIZE = 60 * ONE_MB
+MAX_SIGSIZE = 8 * 1024
+MAX_PROJECT_SIZE = 10 * ONE_GB
+
 PATH_HASHER = "blake2_256"
 
 
@@ -1185,6 +1189,7 @@ def file_upload(request):
     # it does then it may or may not be smaller or larger than our global file
     # size limits.
     file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
+    project_size_limit = max(filter(None, [MAX_PROJECT_SIZE, project.total_size_limit]))
 
     with tempfile.TemporaryDirectory() as tmpdir:
         temporary_filename = os.path.join(tmpdir, filename)
@@ -1205,12 +1210,22 @@ def file_upload(request):
                         HTTPBadRequest,
                         "File too large. "
                         + "Limit for project {name!r} is {limit} MB. ".format(
-                            name=project.name, limit=file_size_limit // (1024 * 1024)
+                            name=project.name, limit=file_size_limit // ONE_MB
                         )
                         + "See "
                         + request.help_url(_anchor="file-size-limit")
                         + " for more information.",
                     )
+                if file_size + project.total_size > project_size_limit:
+                    raise _exc_with_message(
+                        HTTPBadRequest,
+                        "Project size too large. Limit for "
+                        + "project {name!r} total size is {limit} GB. ".format(
+                            name=project.name, limit=project_size_limit // ONE_GB,
+                        )
+                        + "See "
+                        + request.help_url(_anchor="project-size-limit"),
+                    )
                 fp.write(chunk)
                 for hasher in file_hashes.values():
                     hasher.update(chunk)


Test patch:
diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py
--- a/tests/unit/forklift/test_legacy.py
+++ b/tests/unit/forklift/test_legacy.py
@@ -1887,6 +1887,208 @@ def test_upload_fails_with_too_large_file(self, pyramid_config, db_request):
             "See /the/help/url/ for more information."
         )
 
+    def test_upload_fails_with_too_large_project_size_default_limit(
+        self, pyramid_config, db_request
+    ):
+        pyramid_config.testing_securitypolicy(userid=1)
+
+        user = UserFactory.create()
+        db_request.user = user
+        EmailFactory.create(user=user)
+        project = ProjectFactory.create(
+            name="foobar",
+            upload_limit=legacy.MAX_FILESIZE,
+            total_size=legacy.MAX_PROJECT_SIZE - 1,
+        )
+        release = ReleaseFactory.create(project=project, version="1.0")
+        RoleFactory.create(user=user, project=project)
+
+        filename = "{}-{}.tar.gz".format(project.name, release.version)
+
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "1.2",
+                "name": project.name,
+                "version": release.version,
+                "filetype": "sdist",
+                "md5_digest": "nope!",
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(b"a" * 2),
+                    type="application/tar",
+                ),
+            }
+        )
+        db_request.help_url = pretend.call_recorder(lambda **kw: "/the/help/url/")
+
+        with pytest.raises(HTTPBadRequest) as excinfo:
+            legacy.file_upload(db_request)
+
+        resp = excinfo.value
+
+        assert db_request.help_url.calls == [pretend.call(_anchor="project-size-limit")]
+        assert resp.status_code == 400
+        assert resp.status == (
+            "400 Project size too large."
+            + " Limit for project 'foobar' total size is 10 GB. "
+            "See /the/help/url/"
+        )
+
+    def test_upload_fails_with_too_large_project_size_custom_limit(
+        self, pyramid_config, db_request
+    ):
+        pyramid_config.testing_securitypolicy(userid=1)
+
+        user = UserFactory.create()
+        db_request.user = user
+        EmailFactory.create(user=user)
+        one_megabyte = 1 * 1024 * 1024
+        project = ProjectFactory.create(
+            name="foobar",
+            upload_limit=legacy.MAX_FILESIZE,
+            total_size=legacy.MAX_PROJECT_SIZE,
+            total_size_limit=legacy.MAX_PROJECT_SIZE
+            + one_megabyte,  # Custom Limit for the project
+        )
+        release = ReleaseFactory.create(project=project, version="1.0")
+        RoleFactory.create(user=user, project=project)
+
+        filename = "{}-{}.tar.gz".format(project.name, release.version)
+
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "1.2",
+                "name": project.name,
+                "version": release.version,
+                "filetype": "sdist",
+                "md5_digest": "nope!",
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(b"a" * (one_megabyte + 1)),
+                    type="application/tar",
+                ),
+            }
+        )
+        db_request.help_url = pretend.call_recorder(lambda **kw: "/the/help/url/")
+
+        with pytest.raises(HTTPBadRequest) as excinfo:
+            legacy.file_upload(db_request)
+
+        resp = excinfo.value
+
+        assert db_request.help_url.calls == [pretend.call(_anchor="project-size-limit")]
+        assert resp.status_code == 400
+        assert resp.status == (
+            "400 Project size too large."
+            + " Limit for project 'foobar' total size is 10 GB. "
+            "See /the/help/url/"
+        )
+
+    def test_upload_succeeds_custom_project_size_limit(
+        self, pyramid_config, db_request, metrics
+    ):
+        pyramid_config.testing_securitypolicy(userid=1)
+
+        user = UserFactory.create()
+        db_request.user = user
+        EmailFactory.create(user=user)
+        one_megabyte = 1 * 1024 * 1024
+        project = ProjectFactory.create(
+            name="foobar",
+            upload_limit=legacy.MAX_FILESIZE,
+            total_size=legacy.MAX_PROJECT_SIZE,
+            total_size_limit=legacy.MAX_PROJECT_SIZE
+            + (one_megabyte * 60),  # Custom Limit for the project
+        )
+        release = ReleaseFactory.create(project=project, version="1.0")
+        RoleFactory.create(user=user, project=project)
+
+        filename = "{}-{}.tar.gz".format("example", "1.0")
+
+        db_request.POST = MultiDict(
+            {
+                "metadata_version": "1.2",
+                "name": "example",
+                "version": "1.0",
+                "filetype": "sdist",
+                "md5_digest": _TAR_GZ_PKG_MD5,
+                "content": pretend.stub(
+                    filename=filename,
+                    file=io.BytesIO(_TAR_GZ_PKG_TESTDATA),
+                    type="application/tar",
+                ),
+            }
+        )
+
+        storage_service = pretend.stub(store=lambda path, filepath, meta: None)
+        db_request.find_service = lambda svc, name=None, context=None: {
+            IFileStorage: storage_service,
+            IMetricsService: metrics,
+        }.get(svc)
+        db_request.remote_addr = "10.10.10.10"
+        db_request.user_agent = "warehouse-tests/6.6.6"
+
+        resp = legacy.file_upload(db_request)
+
+        assert resp.status_code == 200
+
+        # Ensure that a Project object has been created.
+        project = db_request.db.query(Project).filter(Project.name == "example").one()
+
+        # Ensure that a Role with the user as owner has been created.
+        role = (
+            db_request.db.query(Role)
+            .filter((Role.user == user) & (Role.project == project))
+            .one()
+        )
+        assert role.role_name == "Owner"
+
+        # Ensure that a Release object has been created.
+        release = (
+            db_request.db.query(Release)
+            .filter((Release.project == project) & (Release.version == "1.0"))
+            .one()
+        )
+
+        assert release.uploaded_via == "warehouse-tests/6.6.6"
+
+        # Ensure that a File object has been created.
+        db_request.db.query(File).filter(
+            (File.release == release) & (File.filename == filename)
+        ).one()
+
+        # Ensure that a Filename object has been created.
+        db_request.db.query(Filename).filter(Filename.filename == filename).one()
+
+        # Ensure that all of our journal entries have been created
+        journals = (
+            db_request.db.query(JournalEntry)
+            .options(joinedload("submitted_by"))
+            .order_by("submitted_date", "id")
+            .all()
+        )
+        assert [
+            (j.name, j.version, j.action, j.submitted_by, j.submitted_from)
+            for j in journals
+        ] == [
+            ("example", None, "create", user, "10.10.10.10"),
+            (
+                "example",
+                None,
+                "add Owner {}".format(user.username),
+                user,
+                "10.10.10.10",
+            ),
+            ("example", "1.0", "new release", user, "10.10.10.10"),
+            (
+                "example",
+                "1.0",
+                "add source file example-1.0.tar.gz",
+                user,
+                "10.10.10.10",
+            ),
+        ]
+
     def test_upload_fails_with_too_large_signature(self, pyramid_config, db_request):
         pyramid_config.testing_securitypolicy(userid=1)