Repository: pypi/warehouse

Issue:
Ability to increase file size limit on per-project basis?
Once in a while (three times in the years we've been hosting files) we get a request from a user to exceed the hard-coded file size limit in PyPI. Maybe there should be some way to cater for them? Maybe it's too much to ask to handle just their extremely rare use-case?



Discussion:
Hello, This issue effects me and I am adding a note here (many people probably encounter this issue, but they do not make noise about it).  I think it is also a chicken-and-egg (chicken-and-wheel? :-P) problem for getting more scientific Python packages on PyPI. A per-project increase would be helpful.  Thanks for improving PyPi :-).


Resolving patch:
diff --git a/warehouse/legacy/api/pypi.py b/warehouse/legacy/api/pypi.py
--- a/warehouse/legacy/api/pypi.py
+++ b/warehouse/legacy/api/pypi.py
@@ -38,7 +38,8 @@
 from warehouse.utils.http import require_POST
 
 
-MAXIMUM_FILESIZE = 60 * 1024 * 1024  # 60M
+MAX_FILESIZE = 60 * 1024 * 1024  # 60M
+MAX_SIGSIZE = 8 * 1024           # 8K
 
 
 ALLOWED_PLATFORMS = {
@@ -606,6 +607,11 @@ def file_upload(request):
             "different version.",
         )
 
+    # The project may or may not have a file size specified on the project, if
+    # it does then it may or may not be smaller or larger than our global file
+    # size limits.
+    file_size_limit = max(filter(None, [MAX_FILESIZE, project.upload_limit]))
+
     # Buffer the entire file into memory, checking the hash of the file as we
     # go along.
     file_content = io.BytesIO()
@@ -613,7 +619,7 @@ def file_upload(request):
     file_hash = hashlib.md5()
     for chunk in iter(lambda: request.POST["content"].file.read(8096), b""):
         file_size += len(chunk)
-        if file_size > MAXIMUM_FILESIZE:
+        if file_size > file_size_limit:
             raise _exc_with_message(HTTPBadRequest, "File too large.")
         file_content.write(chunk)
         file_hash.update(chunk)
@@ -626,7 +632,7 @@ def file_upload(request):
         for chunk in iter(
                 lambda: request.POST["gpg_signature"].file.read(8096), b""):
             signature_size += len(chunk)
-            if signature_size > MAXIMUM_FILESIZE:
+            if signature_size > MAX_SIGSIZE:
                 raise _exc_with_message(HTTPBadRequest, "Signature too large.")
             signature.write(chunk)
         signature.seek(0)
diff --git a/warehouse/migrations/versions/9177113533_add_a_column_to_specify_a_project_.py b/warehouse/migrations/versions/9177113533_add_a_column_to_specify_a_project_.py
new file mode 100644
--- /dev/null
+++ b/warehouse/migrations/versions/9177113533_add_a_column_to_specify_a_project_.py
@@ -0,0 +1,36 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Add a column to specify a project specific upload limit
+
+Revision ID: 9177113533
+Revises: 10cb17aea73
+Create Date: 2015-09-04 21:06:59.950947
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "9177113533"
+down_revision = "10cb17aea73"
+
+
+def upgrade():
+    op.add_column(
+        "packages",
+        sa.Column("upload_limit", sa.Integer(), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("packages", "upload_limit")
diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py
--- a/warehouse/packaging/models.py
+++ b/warehouse/packaging/models.py
@@ -94,6 +94,7 @@ class Project(db.ModelBase):
         server_default=sql.func.now(),
     )
     has_docs = Column(Boolean)
+    upload_limit = Column(Integer, nullable=True)
 
     releases = orm.relationship(
         "Release",


Test patch:
diff --git a/tests/unit/legacy/api/test_pypi.py b/tests/unit/legacy/api/test_pypi.py
--- a/tests/unit/legacy/api/test_pypi.py
+++ b/tests/unit/legacy/api/test_pypi.py
@@ -716,7 +716,7 @@ def test_upload_fails_with_too_large_file(self, pyramid_config,
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -751,7 +751,7 @@ def test_upload_fails_with_too_large_signature(self, pyramid_config,
             ),
             "gpg_signature": pretend.stub(
                 filename=filename + ".asc",
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -782,7 +782,7 @@ def test_upload_fails_with_previously_used_filename(self, pyramid_config,
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -817,7 +817,7 @@ def test_upload_fails_with_existing_file(self, pyramid_config, db_request):
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -850,7 +850,7 @@ def test_upload_fails_with_wrong_filename(self, pyramid_config,
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -886,7 +886,7 @@ def test_upload_fails_with_invalid_extension(self, pyramid_config,
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -921,7 +921,7 @@ def test_upload_fails_with_unsafe_filename(self, pyramid_config,
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })
 
@@ -950,7 +950,7 @@ def test_upload_fails_without_permission(self, pyramid_config, db_request):
             "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
-                file=io.BytesIO(b"a" * (pypi.MAXIMUM_FILESIZE + 1)),
+                file=io.BytesIO(b"a" * (pypi.MAX_FILESIZE + 1)),
             ),
         })