fancycode/pylzma — Issue #3: pylzma does not support large files (> 2GB)
Author: yongsun
Created: 2011-10-28 10:34:14+00:00
State: open

I'm trying to use pylzma to decode a very large file, met following error,

... ...
dec = pylzma.decompressobj(maxlength=self._start+self.size)
OverflowError: signed integer is greater than maximum

I made following patch, while I'm not sure if my understand is correct, please help to review

diff --git a/src/pylzma/pylzma_decompress.c b/src/pylzma/pylzma_decompress.c
index c94bf3b..a28df61 100644
--- a/src/pylzma/pylzma_decompress.c
+++ b/src/pylzma/pylzma_decompress.c
@@ -42,7 +42,7 @@ pylzma_decompress(PyObject *self, PyObject *args, PyObject *kwargs)
{
unsigned char *data;
Byte *tmp;
- int length, bufsize=BLOCK_SIZE, avail, totallength=-1;
+ SizeT length, bufsize=BLOCK_SIZE, avail, totallength=-1;
PyObject *result=NULL;
CLzmaDec state;
ELzmaStatus status;
@@ -51,8 +51,8 @@ pylzma_decompress(PyObject *self, PyObject *args, PyObject *kwargs)
CMemoryOutStream outStream;
// possible keywords for this function
static char *kwlist[] = {"data", "bufsize", "maxlength", NULL};
-
- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|ii", kwlist, &data, &length, &bufsize, &totallength))
+
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|ll", kwlist, &data, &length, &bufsize, &totallength))
return NULL;

if (totallength != -1) {
diff --git a/src/pylzma/pylzma_decompress_compat.c b/src/pylzma/pylzma_decompress_compat.c
index 1fd8e60..df54c42 100644
--- a/src/pylzma/pylzma_decompress_compat.c
+++ b/src/pylzma/pylzma_decompress_compat.c
@@ -46,13 +46,13 @@ const char doc_decompress_compat[] = \
PyObject *pylzma_decompress_compat(PyObject *self, PyObject *args)
{
char *data;
- int length, blocksize=BLOCK_SIZE;
+ SizeT length, blocksize=BLOCK_SIZE;
PyObject *result = NULL;
lzma_stream stream;
int res;
char *output;

- if (!PyArg_ParseTuple(args, "s#|i", &data, &length, &blocksize))
+ if (!PyArg_ParseTuple(args, "s#|l", &data, &length, &blocksize))
return NULL;

memset(&stream, 0, sizeof(stream));
diff --git a/src/pylzma/pylzma_decompressobj.c b/src/pylzma/pylzma_decompressobj.c
index b860daf..6d0e2bc 100644
--- a/src/pylzma/pylzma_decompressobj.c
+++ b/src/pylzma/pylzma_decompressobj.c
@@ -31,15 +31,14 @@
static int
pylzma_decomp_init(CDecompressionObject *self, PyObject *args, PyObject *kwargs)
{
- int max_length = -1;
+ SizeT max_length = -1;

// possible keywords for this function
static char *kwlist[] = {"maxlength", NULL};

- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &max_length))
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|l", kwlist, &max_length))
return -1;
-
- if (max_length == 0 || max_length < -1) {
+ if (max_length <= 0) {
PyErr_SetString(PyExc_ValueError, "the decompressed size must be greater than zero");
return -1;
}
@@ -64,12 +63,12 @@ pylzma_decomp_decompress(CDecompressionObject *self, PyObject *args)
PyObject *result=NULL;
unsigned char *data;
Byte *next_in, *next_out;
- int length, res, bufsize=BLOCK_SIZE;
+ SizeT length, res, bufsize=BLOCK_SIZE;
SizeT avail_in;
SizeT inProcessed, outProcessed;
ELzmaStatus status;

- if (!PyArg_ParseTuple(args, "s#|i", &data, &length, &bufsize)){
+ if (!PyArg_ParseTuple(args, "s#|l", &data, &length, &bufsize)){
return NULL;
}

@@ -297,12 +296,12 @@ doc_decomp_reset[] = \
static PyObject *
pylzma_decomp_reset(CDecompressionObject *self, PyObject *args, PyObject *kwargs)
{
- int max_length = -1;
+ SizeT max_length = -1;

// possible keywords for this function
static char *kwlist[] = {"maxlength", NULL};

- if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i", kwlist, &max_length))
+ if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|l", kwlist, &max_length))
return NULL;

LzmaDec_Free(&self->state, &allocator);
diff --git a/src/pylzma/pylzma_decompressobj_compat.c b/src/pylzma/pylzma_decompressobj_compat.c
index 6c9e876..048780a 100644
--- a/src/pylzma/pylzma_decompressobj_compat.c
+++ b/src/pylzma/pylzma_decompressobj_compat.c
@@ -38,9 +38,9 @@ static PyObject *pylzma_decomp_decompress(CCompatDecompressionObject *self, PyOb
{
PyObject *result=NULL;
char *data;
- int length, old_length, start_total_out, res, max_length=BLOCK_SIZE;
+ SizeT length, old_length, start_total_out, res, max_length=BLOCK_SIZE;

- if (!PyArg_ParseTuple(args, "s#|i", &data, &length, &max_length))
+ if (!PyArg_ParseTuple(args, "s#|l", &data, &length, &max_length))
return NULL;

if (max_length < 0)