Repository: jd-boyd/python-lzo (language: python)

Issue:
Support lzo1y_decompress_safe
Is there any plan to support _lzo1y_decompress_safe_ in decompress method?


Discussion / hints:
Thank you for your question.

There was no plan to do so prior to your inquiry.  I would not be opposed to that if someone wanted to make a pull request for it though.  Also, with more detail, I'd be willing to consider doing it myself when I have time.

I'm sad to confess that I'm not really sure what the difference between `lzo1y_decompress_safe` and `lzo1x_decompress_safe` is.  Reviewing what is currently implemented, I see so far this python module uses only lzo1x functions.

Would it suffice to just add an optional argument to the decompress method to specify whether 
LZO1, LZO1A, LZO1B, LZO1C, LZO1F, LZO1X, LZO1Y, LZO1Z is to be used?

Hi Joshua,

I would be glad if you add that optional argument to the decompress method.
Hi,

Wonder if you are still planning to implement this feature? I would need `lzo1z_decompress_safe` so what you have proposed would work for me too.

Thanks!


@sektoid0 I'd be happy to review and accept a pull request for it if you wanted to make one.  Also, I am available for paid work if people need it.  Otherwise, I don't have a lot of time to dedicate to new features here for a library I don't use much anymore.

Fix patch:
diff --git a/lzomodule.c b/lzomodule.c
index dfd465b..2eaec54 100644
--- a/lzomodule.c
+++ b/lzomodule.c
@@ -29,12 +29,21 @@
  */
 
 
-#define MODULE_VERSION  "1.15"
+#define MODULE_VERSION  "1.16"
 
 #define PY_SSIZE_T_CLEAN
 
 #include <Python.h>
+#include <string.h>
+#include <lzo/lzo1.h>
+#include <lzo/lzo1a.h>
+#include <lzo/lzo1b.h>
+#include <lzo/lzo1c.h>
+#include <lzo/lzo1f.h>
 #include <lzo/lzo1x.h>
+#include <lzo/lzo1y.h>
+#include <lzo/lzo1z.h>
+#include <lzo/lzo2a.h>
 
 /* Python 2x3 compatible macros */
 #if PY_VERSION_HEX >= 0x03000000
@@ -61,21 +70,26 @@
 
 static PyObject *LzoError;
 
+// custom function type definitions to allow compatibility of various algorithms
+typedef int (*lzo_compress_fn)(const lzo_bytep, lzo_uint, lzo_bytep, lzo_uintp, lzo_voidp);
+typedef int (*lzo_decompress_fn)(const lzo_bytep, lzo_uint, lzo_bytep, lzo_uintp, lzo_voidp /* NOT USED */);
 
 /***********************************************************************
 // compress
 ************************************************************************/
 
 static /* const */ char compress__doc__[] =
-"compress(string[,level[,header]]) -- Compress string, returning a string "
+"compress(string[,level[,header[,algorithm]]]) -- Compress string, returning a string "
 "containing compressed data.\n"
 "level  - Set compression level of either 1 (default) or 9.\n"
 "header - Include metadata header for decompression in the output "
 "(default: True).\n"
+"algorithm (keyword argument)  - can be either LZO1, LZO1A, LZO1B, LZO1C, LZO1F, LZO1X, LZO1Y, LZO1Z, LZO2A."
+"(default: LZO1X).\n"
 ;
 
 static PyObject *
-compress(PyObject *dummy, PyObject *args)
+compress(PyObject *dummy, PyObject *args, PyObject *kwds)
 {
     PyObject *result_str;
     lzo_voidp wrkmem = NULL;
@@ -90,9 +104,16 @@ compress(PyObject *dummy, PyObject *args)
     int header = 1;
     int err;
 
+    static char* argnames[] = {"", "", "", "algorithm", NULL};
+    char *algorithm = "LZO1X";
+    lzo_compress_fn compress_1_ptr;
+    lzo_compress_fn compress_999_ptr;
+    lzo_uint32_t MEM_COMPRESS_1;
+    lzo_uint32_t MEM_COMPRESS_999;
+
     /* init */
     UNUSED(dummy);
-    if (!PyArg_ParseTuple(args, "s#|ii", &in, &len, &level, &header))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|ii$s", argnames, &in, &len, &level, &header, &algorithm))
         return NULL;
     if (len < 0)
         return NULL;
@@ -107,6 +128,80 @@ compress(PyObject *dummy, PyObject *args)
       return NULL;
     }
 
+    /* init for different algorithms*/
+    if (strcmp(algorithm, "LZO1") == 0) {
+      // settings specific for lzo1
+      MEM_COMPRESS_1 = LZO1_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1_99_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1_compress;
+      compress_999_ptr = &lzo1_99_compress;
+    }
+    else if (strcmp(algorithm, "LZO1A") == 0){
+      // settings for LZO1A
+      MEM_COMPRESS_1 = LZO1A_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1A_99_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1a_compress;
+      compress_999_ptr = &lzo1a_99_compress;
+    }
+    else if (strcmp(algorithm, "LZO1B") == 0){
+      // settings for LZO1B
+      MEM_COMPRESS_1 = LZO1B_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1B_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1b_1_compress;
+      compress_999_ptr = &lzo1b_999_compress;
+    }
+    else if (strcmp(algorithm, "LZO1C") == 0){
+      // settings for LZO1C
+      MEM_COMPRESS_1 = LZO1C_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1C_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1c_1_compress;
+      compress_999_ptr = &lzo1c_999_compress;
+    }
+    else if (strcmp(algorithm, "LZO1F") == 0){
+      // settings for LZO1F
+      MEM_COMPRESS_1 = LZO1F_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1F_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1f_1_compress;
+      compress_999_ptr = &lzo1f_999_compress;
+    }
+    else if (strcmp(algorithm, "LZO1Y") == 0){
+      // settings for LZO1Y
+      MEM_COMPRESS_1 = LZO1Y_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1Y_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1y_1_compress;
+      compress_999_ptr = &lzo1y_999_compress;
+    }
+    else if (strcmp(algorithm, "LZO1Z") == 0){
+      // settings for LZO1Z
+      MEM_COMPRESS_1 = LZO1Z_999_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1Z_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1z_999_compress;
+      compress_999_ptr = &lzo1z_999_compress;
+    }
+    else if (strcmp(algorithm, "LZO2A") == 0){
+      // settings for LZO2A
+      MEM_COMPRESS_1 = LZO2A_999_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO2A_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo2a_999_compress;
+      compress_999_ptr = &lzo2a_999_compress;
+    }
+    else {
+      // settings for LZO1X
+      MEM_COMPRESS_1 = LZO1X_1_MEM_COMPRESS;
+      MEM_COMPRESS_999 = LZO1X_999_MEM_COMPRESS;
+
+      compress_1_ptr = &lzo1x_1_compress;
+      compress_999_ptr = &lzo1x_999_compress;
+    }
+
     in_len = len;
     out_len = in_len + in_len / 16 + 64 + 3;
 
@@ -119,9 +214,9 @@ compress(PyObject *dummy, PyObject *args)
     if (result_str == NULL)
         return PyErr_NoMemory();
     if (level == 1)
-        wrkmem = (lzo_voidp) PyMem_Malloc(LZO1X_1_MEM_COMPRESS);
+        wrkmem = (lzo_voidp) PyMem_Malloc(MEM_COMPRESS_1);
     else
-        wrkmem = (lzo_voidp) PyMem_Malloc(LZO1X_999_MEM_COMPRESS);
+        wrkmem = (lzo_voidp) PyMem_Malloc(MEM_COMPRESS_999);
     if (wrkmem == NULL)
     {
         Py_DECREF(result_str);
@@ -142,13 +237,13 @@ compress(PyObject *dummy, PyObject *args)
     {
         if (header)
             out[0] = 0xf0;
-        err = lzo1x_1_compress(in, in_len, outc, &new_len, wrkmem);
+        err = (*compress_1_ptr)(in, in_len, outc, &new_len, wrkmem);
     }
     else
     {
         if (header)
             out[0] = 0xf1;
-        err = lzo1x_999_compress(in, in_len, outc, &new_len, wrkmem);
+        err = (*compress_999_ptr)(in, in_len, outc, &new_len, wrkmem);
     }
     Py_END_ALLOW_THREADS
 
@@ -186,14 +281,16 @@ compress(PyObject *dummy, PyObject *args)
 ************************************************************************/
 
 static /* const */ char decompress__doc__[] =
-"decompress(string[,header[,buflen]]) -- Decompress the data in string, returning a string containing the decompressed data.\n"
+"decompress(string[,header[,buflen[,algorithm]]]) -- Decompress the data in string, returning a string containing the decompressed data.\n"
 "header - Metadata header is included in input (default: True).\n"
 "buflen - If header is False, a buffer length in bytes must be given that "
 "will fit the output.\n"
+"algorithm (keyword argument) - can be either LZO1, LZO1A, LZO1B, LZO1C, LZO1F, LZO1X, LZO1Y, LZO1Z, LZO2A."
+"(default: LZO1X).\n"
 ;
 
 static PyObject *
-decompress(PyObject *dummy, PyObject *args)
+decompress(PyObject *dummy, PyObject *args, PyObject *kwds)
 {
     PyObject *result_str;
     const lzo_bytep in;
@@ -206,9 +303,13 @@ decompress(PyObject *dummy, PyObject *args)
     int header = 1;
     int err;
 
+    static char* argnames[] = {"", "", "", "algorithm", NULL};
+    char *algorithm = "LZO1X";
+    lzo_decompress_fn decompress_ptr;
+
     /* init */
     UNUSED(dummy);
-    if (!PyArg_ParseTuple(args, "s#|ii", &in, &len, &header, &buflen))
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s#|ii$s", argnames, &in, &len, &header, &buflen, &algorithm))
         return NULL;
     if (header) {
         if (len < 5 + 3 || in[0] < 0xf0 || in[0] > 0xf1)
@@ -225,6 +326,44 @@ decompress(PyObject *dummy, PyObject *args)
         in_len = len;
     }
 
+    /* init for different algorithms*/
+    if (strcmp(algorithm, "LZO1") == 0) {
+      // settings specific for lzo1
+      decompress_ptr = &lzo1_decompress;
+    }
+    else if (strcmp(algorithm, "LZO1A") == 0){
+      // settings for LZO1A
+      decompress_ptr = &lzo1a_decompress;
+    }
+    else if (strcmp(algorithm, "LZO1B") == 0){
+      // settings for LZO1B
+      decompress_ptr = &lzo1b_decompress_safe;
+    }
+    else if (strcmp(algorithm, "LZO1C") == 0){
+      // settings for LZO1C
+      decompress_ptr = &lzo1c_decompress_safe;
+    }
+    else if (strcmp(algorithm, "LZO1F") == 0){
+      // settings for LZO1F
+      decompress_ptr = &lzo1f_decompress_safe;
+    }
+    else if (strcmp(algorithm, "LZO1Y") == 0){
+      // settings for LZO1Y
+      decompress_ptr = &lzo1y_decompress_safe;
+    }
+    else if (strcmp(algorithm, "LZO1Z") == 0){
+      // settings for LZO1Z
+      decompress_ptr = &lzo1z_decompress_safe;
+    }
+    else if (strcmp(algorithm, "LZO2A") == 0){
+      // settings for LZO2A
+      decompress_ptr = &lzo2a_decompress_safe;
+    }
+    else {
+      // settings for LZO1X
+      decompress_ptr = &lzo1x_decompress_safe;
+    }
+
     /* alloc buffers */
 #if PY_MAJOR_VERSION >= 3
     result_str = PyBytes_FromStringAndSize(NULL, out_len);
@@ -243,7 +382,7 @@ decompress(PyObject *dummy, PyObject *args)
 
     Py_BEGIN_ALLOW_THREADS
     new_len = out_len;
-    err = lzo1x_decompress_safe(in, in_len, out, &new_len, NULL);
+    err = (*decompress_ptr)(in, in_len, out, &new_len, NULL);
     Py_END_ALLOW_THREADS
 
     if (err != LZO_E_OK || (header && new_len != out_len) )
@@ -427,9 +566,9 @@ crc32(PyObject *dummy, PyObject *args)
 static /* const */ PyMethodDef methods[] =
 {
     {"adler32",    (PyCFunction)adler32,    METH_VARARGS, adler32__doc__},
-    {"compress",   (PyCFunction)compress,   METH_VARARGS, compress__doc__},
+    {"compress",   (PyCFunction)compress,   METH_VARARGS | METH_KEYWORDS, compress__doc__},
     {"crc32",      (PyCFunction)crc32,      METH_VARARGS, crc32__doc__},
-    {"decompress", (PyCFunction)decompress, METH_VARARGS, decompress__doc__},
+    {"decompress", (PyCFunction)decompress, METH_VARARGS | METH_KEYWORDS, decompress__doc__},
     {"optimize",   (PyCFunction)optimize,   METH_VARARGS, optimize__doc__},
     {NULL, NULL, 0, NULL}
 };
diff --git a/setup.py b/setup.py
index 8932f26..bb9ba8e 100644
--- a/setup.py
+++ b/setup.py
@@ -83,7 +83,7 @@ def get_kw(**kw): return kw
 
 setup_args = get_kw(
     name="python-lzo",
-    version="1.15",
+    version="1.16",
     description="Python bindings for the LZO data compression library",
     author="Markus F.X.J. Oberhumer",
     author_email="markus@oberhumer.com",


Test patch:
diff --git a/tests/test.py b/tests/test.py
index af761d9..f432e0b 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -31,7 +31,7 @@
 ##---------------------------------------------------------------------------##
 
 from __future__ import print_function
-
+import inspect
 import sys, string
 
 # update sys.path when running in the build directory
@@ -56,10 +56,10 @@ def print_modinfo():
 
 def gen(src, level=1):
     a0 = lzo.adler32(src)
-    c =  lzo.compress(src, level)
+    c  = lzo.compress(src, level)
     u1 = lzo.decompress(c)
     a1 = lzo.adler32(u1)
-    o =  lzo.optimize(c)
+    o  = lzo.optimize(c)
     u2 = lzo.decompress(o)
     a2 = lzo.adler32(u2)
     if src != u1:
@@ -70,13 +70,30 @@ def gen(src, level=1):
         raise lzo.error("internal error 2")
     print("compressed %6d -> %6d" % (len(src), len(c)))
 
+def gen_all(src):
+    all_algos = {"LZO1", "LZO1A", "LZO1B", "LZO1C", "LZO1F", "LZO1X", "LZO1Y", "LZO1Z", "LZO2A"}
+    if not src:
+        # LZO1 and LZO1A header exception for empty input
+        all_algos.remove("LZO1")
+        all_algos.remove("LZO1A")
+    for algo in all_algos:
+        a0 = lzo.adler32(src)
+        c  = lzo.compress(src, algorithm=algo)
+        u1 = lzo.decompress(c, algorithm=algo)
+        a1 = lzo.adler32(u1)
+        if src != u1:
+            raise lzo.error("internal error 1: %r %r", src, u1)
+        if a0 != a1:
+            raise lzo.error("internal error 2")
+        print(f"compressed using {algo} {len(src): 6} -> {len(c): 6}")
+
 
 def gen_raw(src, level=1):
     a0 = lzo.adler32(src)
-    c =  lzo.compress(src, level, False)
+    c  = lzo.compress(src, level, False)
     u1 = lzo.decompress(c, False, len(src))
     a1 = lzo.adler32(u1)
-    o =  lzo.optimize(c, False, len(src))
+    o  = lzo.optimize(c, False, len(src))
     u2 = lzo.decompress(o, False, len(src))
     a2 = lzo.adler32(u2)
     # make sure it still works when you overstate the output buffer length
@@ -99,6 +116,10 @@ def test_lzo():
     yield gen, b"abcabcabcabcabcabcabcabc"
     yield gen, b"abcabcabcabcabcabcabcabc", 9
 
+def test_lzo_all():
+    yield gen_all, b"aaaaaaaaaaaaaaaaaaaaaaaa"
+    yield gen_all, b"abcabcabcabcabcabcabcabc"
+    yield gen_all, b"abcabcabcabcabcabcabcabc"
 
 def test_lzo_raw():
     yield gen_raw, b"aaaaaaaaaaaaaaaaaaaaaaaa"
@@ -110,10 +131,12 @@ def test_lzo_empty():
     yield gen, b""
     yield gen_raw, b""
 
+def test_lzo_empty_all():
+    yield gen_all, b""
 
 def test_lzo_big():
     gen(b" " * 131072)
-
+    gen_all(b" " * 131072)
 
 def test_lzo_raw_big():
     gen_raw(b" " * 131072)
@@ -124,3 +147,14 @@ def test_lzo_raw_big():
     # this much data requires a 64-bit system.
     def test_lzo_compress_extremely_big():
         b = lzo.compress(bytes(bytearray((1024**3)*2)))
+
+if __name__ == "__main__":
+    all_tests = (test_lzo, test_lzo_raw, test_lzo_empty, test_lzo_big, test_lzo_raw_big, test_lzo_all, test_lzo_empty_all)
+    for test_func in all_tests:
+        if inspect.isgeneratorfunction(test_func) is True:
+            for test_case in test_func():
+                test = test_case[0]
+                args = test_case[1:]
+                test(*args)
+        else:
+            test_func()