Repository: pauldmccarthy/indexed_gzip (language: python)

Issue:
Perform CRC/size validation, and handle null padding bytes
Closes #69

The `_zran_inflate` function now calculates a CRC32 checksum, and keeps track of the total number of uncompressed bytes in each GZIP stream. When the end of a stream is reached, they are checked against the CRC32/size stored in the gzip stream footer.

The above validation check can be disabled by using the new `ZRAN_SKIP_CRC_CHECK` flag when calling `zran_init`.

As a by-product of the above changes, null padding bytes are now handled correctly. Currently any bytes at the end of a file, or between the end of one GZIP stream and the beginning of another, are ignored. This might be made more strict in the future by only allowing null bytes (i.e. with a value of `00`). 

Some of the unit tests written by @epicfaace in #70 - thanks!


Discussion / hints:
WIP - still needs some tweaking, and some more tests
> This might be made more strict in the future by only allowing null bytes (i.e. with a value of `00`).

Another option could be to add a warning if non-zero bytes are found in the padding (see https://github.com/libarchive/libarchive/issues/643).
True, although I'd want these warnings to be emitted from the Python code, so they can be easily suppressed via standard `logging` or `warnings` mechanisms. Unsolicited output to `stdout`/`stderr` from libraries that I have no control over is a bugbear of mine!

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6be31a9e..0eca518b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,11 @@
 
 
 * Python 2.7 wheels for Windows are no longer being built (#71, 73).
+* A backwards-compatible change to the index file format, to accommodate seek
+  points at stream boundaries. Index files created with older versions of
+  `indexed_gzip` can still be loaded, but index files created with
+  `indexed_gzip` 1.6.0 cannot be loaded by older versions of `indexed_gzip`
+  (#75).
 
 
 ## 1.5.3 (March 23rd 2021)
diff --git a/indexed_gzip/zran.c b/indexed_gzip/zran.c
index b2c38b54..3ab4082b 100644
--- a/indexed_gzip/zran.c
+++ b/indexed_gzip/zran.c
@@ -71,8 +71,13 @@ static double round(double val)
 #define zran_log(...)
 #endif
 
-/* Define magic bytes and version for export format. */
-const char zran_magic_bytes[] = {'G', 'Z', 'I', 'D', 'X', 0, 0};
+
+/*
+ * Identifier and version number for index files created by zran_export_index.
+ */
+const char    ZRAN_INDEX_FILE_ID[]    = {'G', 'Z', 'I', 'D', 'X'};
+const uint8_t ZRAN_INDEX_FILE_VERSION = 1;
+
 
 /*
  * Discards all points in the index which come after the specfiied
@@ -2466,9 +2471,10 @@ eof:         return ZRAN_READ_EOF;
     return ZRAN_READ_FAIL;
 }
 
+
 /*
- * Store checkpoint information from index to file fd. File should be opened in
- * binary write mode.
+ * Store checkpoint information from index to file fd. File should be opened
+ * in binary write mode.
  */
 int zran_export_index(zran_index_t *index,
                       FILE         *fd,
@@ -2486,6 +2492,9 @@ int zran_export_index(zran_index_t *index,
     zran_point_t *point;
     zran_point_t *list_end;
 
+    /* File flags, currently not used. Also used as a temporary variable. */
+    uint8_t flags = 0;
+
     zran_log("zran_export_index: (%lu, %lu, %u, %u, %u)\n",
              index->compressed_size,
              index->uncompressed_size,
@@ -2493,9 +2502,17 @@ int zran_export_index(zran_index_t *index,
              index->window_size,
              index->npoints);
 
-    /* Write magic bytes, and check for errors. */
-    f_ret = fwrite_(zran_magic_bytes, sizeof(zran_magic_bytes), 1, fd, f);
+    /* Write ID and version, and check for errors. */
+    f_ret = fwrite_(ZRAN_INDEX_FILE_ID, sizeof(ZRAN_INDEX_FILE_ID), 1, fd, f);
+    if (ferror_(fd, f)) goto fail;
+    if (f_ret != 1)     goto fail;
+
+    f_ret = fwrite_(&ZRAN_INDEX_FILE_VERSION, 1, 1, fd, f);
+    if (ferror_(fd, f)) goto fail;
+    if (f_ret != 1)     goto fail;
 
+    /* Write flags (currently unused) */
+    f_ret = fwrite_(&flags, 1, 1, fd, f);
     if (ferror_(fd, f)) goto fail;
     if (f_ret != 1)     goto fail;
 
@@ -2538,64 +2555,59 @@ int zran_export_index(zran_index_t *index,
      * together, which enables user to read all offset mappings in one pass.
      */
 
-    /*
-     * Initialize point to the first element of the list, and list_end to the
-     * end of the list.
-     */
+    /* Write all points iteratively for checkpoint offset mapping. */
     point    = index->list;
     list_end = index->list + index->npoints;
-
-    /* Write all points iteratively for checkpoint offset mapping. */
     while (point < list_end) {
 
-        /*
-         * Below, it's preferred to write members of points elementwise to
-         * avoid handling struct alignment and padding issues.
-         */
-
         /* Write compressed offset, and check for errors. */
         f_ret = fwrite_(&point->cmp_offset,
                         sizeof(point->cmp_offset), 1, fd, f);
-
         if (ferror_(fd, f)) goto fail;
         if (f_ret != 1)     goto fail;
 
         /* Write uncompressed offset, and check for errors. */
         f_ret = fwrite_(&point->uncmp_offset,
                         sizeof(point->uncmp_offset), 1, fd, f);
-
         if (ferror_(fd, f)) goto fail;
         if (f_ret != 1)     goto fail;
 
         /* Write bit offset, and check for errors. */
         f_ret = fwrite_(&point->bits, sizeof(point->bits), 1, fd, f);
+        if (ferror_(fd, f)) goto fail;
+        if (f_ret != 1)     goto fail;
 
+        /* Write data flag, and check for errors. */
+        flags = (point->data != NULL) ? 1 : 0;
+        f_ret = fwrite_(&flags, 1, 1, fd, f);
         if (ferror_(fd, f)) goto fail;
         if (f_ret != 1)     goto fail;
 
-        zran_log("zran_export_index: (%lu, %lu, %lu, %u)\n",
+        zran_log("zran_export_index: (p%lu, %lu, %lu, %u, %u)\n",
                  (index->npoints - (list_end - point)), // point index
                  point->cmp_offset,
                  point->uncmp_offset,
-                 point->bits);
+                 point->bits,
+                 flags);
 
-        /* Done with this point. Proceed to next one. */
         point++;
     }
 
     /*
-     * Initialize point to the second element of the list (as first element has
-     * no data), and list_end to the end of the list.
+     * Now write out the window data for every point. No data is written for
+     * points which don't have any data (e.g. at stream boundaries).
      */
-    point    = index->list + 1;
+    point    = index->list;
     list_end = index->list + index->npoints;
-
-    /* Write all points iteratively for writing checkpoint data. */
     while (point < list_end) {
 
+        if (point->data == NULL) {
+            point++;
+            continue;
+        }
+
         /* Write checkpoint data, and check for errors. */
         f_ret = fwrite_(point->data, index->window_size, 1, fd, f);
-
         if (ferror_(fd, f)) goto fail;
         if (f_ret != 1)     goto fail;
 
@@ -2621,7 +2633,6 @@ int zran_export_index(zran_index_t *index,
      * descriptor can be closed by Python code before having a chance to flush.
      */
     f_ret = fflush_(fd, f);
-
     if (ferror_(fd, f)) goto fail;
     if (f_ret != 0)     goto fail;
 
@@ -2646,11 +2657,24 @@ int zran_import_index(zran_index_t *index,
     int fail_ret;
 
     /* Used for iterating over elements of zran_index_t.list. */
+    uint64_t      i;
     zran_point_t *point;
     zran_point_t *list_end;
 
-    /* Used for checking magic bytes in the beginning of the file. */
-    char magic_bytes[sizeof(zran_magic_bytes)];
+    /*
+     * Used to store flags for each point - allocated once we
+     * know how many points there are.
+     */
+    uint8_t *dataflags = NULL;
+
+    /*
+     * Used for checking file ID, version, and
+     * flags at the beginning of the file. Flags
+     * is also used as a temporary variable.
+     */
+    char    file_id[sizeof(ZRAN_INDEX_FILE_ID)];
+    uint8_t version;
+    uint8_t flags;
 
     /*
      * Data fields that will be read from the file. They aren't stored directly
@@ -2667,36 +2691,50 @@ int zran_import_index(zran_index_t *index,
     /* Check if file is read only. */
     if (!is_readonly(fd, f)) goto fail;
 
-    /* Read magic bytes, and check for file errors and EOF. */
-    f_ret = fread_(magic_bytes, sizeof(magic_bytes), 1, fd, f);
-
+    /* Read ID, and check for file errors and EOF. */
+    f_ret = fread_(file_id, sizeof(file_id), 1, fd, f);
     if (feof_(fd, f, f_ret)) goto eof;
     if (ferror_(fd, f))      goto read_error;
     if (f_ret != 1)          goto read_error;
 
-    /* Verify magic bytes. */
-    if (memcmp(magic_bytes, zran_magic_bytes, sizeof(magic_bytes))) {
+    /* Verify file ID. */
+    if (memcmp(file_id, ZRAN_INDEX_FILE_ID, sizeof(file_id)))
         goto unknown_format;
-    }
+
+    /* Read file format version */
+    f_ret = fread_(&version, 1, 1, fd, f);
+    if (feof_(fd, f, f_ret)) goto eof;
+    if (ferror_(fd, f))      goto read_error;
+    if (f_ret != 1)          goto read_error;
+
+    /* This file is too new for us to cope */
+    if (version > ZRAN_INDEX_FILE_VERSION)
+        goto unsupported_version;
+
+    /* Read flags (currently unused) */
+    f_ret = fread_(&flags, 1, 1, fd, f);
+    if (feof_(fd, f, f_ret)) goto eof;
+    if (ferror_(fd, f))      goto read_error;
+    if (f_ret != 1)          goto read_error;
 
     /* Read compressed size, and check for file errors and EOF. */
     f_ret = fread_(&compressed_size, sizeof(compressed_size), 1, fd, f);
-
-    if (ferror_(fd, f)) goto read_error;
-    if (f_ret != 1)     goto read_error;
+    if (feof_(fd, f, f_ret)) goto eof;
+    if (ferror_(fd, f))      goto read_error;
+    if (f_ret != 1)          goto read_error;
 
     /*
-     * Compare compressed_size in the index file  to the existing size in
+     * Compare compressed_size in the index file to the existing size in
      * the current index (set in zran_init), if they don't match this means
      * this index file is not created for this compressed file.
      */
-    if (compressed_size != index->compressed_size) goto inconsistent;
+    if (compressed_size != index->compressed_size)
+        goto inconsistent;
 
     if (feof_(fd, f, f_ret)) goto eof;
 
     /* Read uncompressed size, and check for file errors and EOF. */
     f_ret = fread_(&uncompressed_size, sizeof(uncompressed_size), 1, fd, f);
-
     if (feof_(fd, f, f_ret)) goto eof;
     if (ferror_(fd, f))      goto read_error;
     if (f_ret != 1)          goto read_error;
@@ -2711,14 +2749,12 @@ int zran_import_index(zran_index_t *index,
 
     /* Read spacing, and check for file errors and EOF. */
     f_ret = fread_(&spacing, sizeof(spacing), 1, fd, f);
-
     if (feof_(fd, f, f_ret)) goto eof;
     if (ferror_(fd, f))      goto read_error;
     if (f_ret != 1)          goto read_error;
 
     /* Read window size, and check for file errors and EOF. */
     f_ret = fread_(&window_size, sizeof(window_size), 1, fd, f);
-
     if (feof_(fd, f, f_ret)) goto eof;
     if (ferror_(fd, f))      goto read_error;
     if (f_ret != 1)          goto read_error;
@@ -2737,7 +2773,8 @@ int zran_import_index(zran_index_t *index,
     if (ferror_(fd, f))      goto read_error;
     if (f_ret != 1)          goto read_error;
 
-    zran_log("zran_import_index: (%lu, %lu, %u, %u, %u)\n",
+    zran_log("zran_import_index: (%u, %lu, %lu, %u, %u, %u)\n",
+             version,
              compressed_size,
              uncompressed_size,
              spacing,
@@ -2753,24 +2790,23 @@ int zran_import_index(zran_index_t *index,
      * initialise the point list to 8 (same as in zran_init).
      */
     new_list = calloc(1, sizeof(zran_point_t) * max(npoints, 8));
-
-    if (new_list == NULL) goto memory_error;
+    if (new_list == NULL)
+        goto memory_error;
 
     /*
-     * Initialize point to the first element of the list, and list_end to the
-     * end of the list.
+     * Allocate space for the data flag for each point - whether or not
+     * there is data associated with it
      */
-    point    = new_list;
-    list_end = new_list + npoints;
+    dataflags = calloc(npoints, 1);
+    if (dataflags == NULL)
+        goto memory_error;
 
     /* Read new points iteratively for reading offset mapping. */
-    while (point < list_end)
-    {
+    for (i = 0, point = new_list; i < npoints; i++, point++) {
 
         /* Read compressed offset, and check for errors. */
         f_ret = fread_(&point->cmp_offset,
                        sizeof(point->cmp_offset), 1, fd, f);
-
         if (feof_(fd, f, f_ret)) goto eof;
         if (ferror_(fd, f))      goto read_error;
         if (f_ret != 1)          goto read_error;
@@ -2778,45 +2814,68 @@ int zran_import_index(zran_index_t *index,
         /* Read uncompressed offset, and check for errors. */
         f_ret = fread_(&point->uncmp_offset,
                        sizeof(point->uncmp_offset), 1, fd, f);
-
         if (feof_(fd, f, f_ret)) goto eof;
         if (ferror_(fd, f))      goto read_error;
         if (f_ret != 1)          goto read_error;
 
         /* Read bit offset, and check for errors. */
         f_ret = fread_(&point->bits, sizeof(point->bits), 1, fd, f);
-
         if (feof_(fd, f, f_ret)) goto eof;
         if (ferror_(fd, f))      goto read_error;
         if (f_ret != 1)          goto read_error;
 
-        zran_log("zran_import_index: (%lu, %lu, %lu, %u)\n",
-                 (npoints - (list_end - point)), // point index
+        /* Read data flag (added in version 1), and check for errors. */
+        if (version >= 1) {
+            f_ret = fread_(&flags, 1, 1, fd, f);
+            if (feof_(fd, f, f_ret)) goto eof;
+            if (ferror_(fd, f))      goto read_error;
+            if (f_ret != 1)          goto read_error;
+
+            /*
+             * The data flag determines whether or not any window data
+             * is associated with this point. We set point->data to 1
+             * to indicate to the loop below that this point has data
+             * to be loaded.
+             */
+        }
+        /*
+         * In index file version 0, the first point
+         * has no data, but all other points do.
+         */
+        else {
+            flags = (point == new_list) ? 0 : 1;
+        }
+
+        dataflags[i] = flags;
+
+        zran_log("zran_import_index: (p%lu, %lu, %lu, %u, %u)\n",
+                 i,
                  point->cmp_offset,
                  point->uncmp_offset,
-                 point->bits);
-
-        /* Done with this point. Proceed to the next one. */
-        point++;
+                 point->bits,
+                 flags);
     }
 
     /*
-     * Initialize point to the second element of the list (as the first element
-     * has no data), and list_end to the end of the list.
+     * Now loop through and load the window data for all index points.
      */
-    point    = new_list + 1;
-    list_end = new_list + npoints;
+    for (i = 0, point = new_list; i < npoints; i++, point++) {
 
-    /* Read new points iteratively for reading checkpoint data. */
-    while (point < list_end) {
+        /*
+         * There is no data associated with this point - it is either
+         * at the beginning of the file, or on a stream boundary.
+         */
+        if (dataflags[i] == 0) {
+            continue;
+        }
 
         /*
          * Allocate space for checkpoint data. These pointers in each point
          * should be cleaned up in case of any failures.
          */
         point->data = calloc(1, window_size);
-
-        if (point->data == NULL) goto memory_error;
+        if (point->data == NULL)
+            goto memory_error;
 
         /*
          * Read checkpoint data, and check for errors. End of file can be
@@ -2824,10 +2883,9 @@ int zran_import_index(zran_index_t *index,
          * the last element.
          */
         f_ret = fread_(point->data, window_size, 1, fd, f);
-
-        if (feof_(fd, f, f_ret) && point < list_end - 1) goto eof;
-        if (ferror_(fd, f))                              goto read_error;
-        if (f_ret != 1)                                  goto read_error;
+        if (feof_(fd, f, f_ret) && i < npoints - 1) goto eof;
+        if (ferror_(fd, f))                         goto read_error;
+        if (f_ret != 1)                             goto read_error;
 
         /*
          * TODO: If there are still more data after importing is done, it
@@ -2837,16 +2895,13 @@ int zran_import_index(zran_index_t *index,
         /* Print first and last three bytes of the checkpoint window. */
         zran_log("zran_import_index:"
                      "(%lu, [%02x %02x %02x...%02x %02x %02x])\n",
-                 (npoints - (list_end - point)), // point index
+                 i,
                  point->data[0],
                  point->data[1],
                  point->data[2],
                  point->data[window_size - 3],
                  point->data[window_size - 2],
                  point->data[window_size - 1]);
-
-        /* Done with this point. Proceed to the next one. */
-        point++;
     }
 
     /* There are no errors, it's safe to overwrite existing index data now. */
@@ -2900,6 +2955,8 @@ int zran_import_index(zran_index_t *index,
 
     zran_log("zran_import_index: done\n");
 
+    free(dataflags);
+
     return ZRAN_IMPORT_OK;
 
     /* For each failure case, we assign return value and then clean up. */
@@ -2927,6 +2984,10 @@ int zran_import_index(zran_index_t *index,
     fail_ret = ZRAN_IMPORT_UNKNOWN_FORMAT;
     goto cleanup;
 
+unsupported_version:
+    fail_ret = ZRAN_IMPORT_UNSUPPORTED_VERSION;
+    goto cleanup;
+
 cleanup:
     if (new_list != NULL) {
 
@@ -2951,5 +3012,9 @@ int zran_import_index(zran_index_t *index,
         free(new_list);
     }
 
+    if (dataflags != NULL) {
+        free(dataflags);
+    }
+
     return fail_ret;
 }
diff --git a/indexed_gzip/zran.h b/indexed_gzip/zran.h
index 0a3905ab..49b7a2d6 100644
--- a/indexed_gzip/zran.h
+++ b/indexed_gzip/zran.h
@@ -313,8 +313,12 @@ int64_t zran_read(
   uint64_t       len    /* Number of bytes to read   */
 );
 
-/* Magic bytes for exported index type. Value is defined in zran.c. */
-extern const char zran_magic_bytes[];
+/*
+ * Identifier and version number for index files created by zran_export_index,
+ * defined in zran.c.
+ */
+extern const char    ZRAN_INDEX_FILE_ID[];
+extern const uint8_t ZRAN_INDEX_FILE_VERSION;
 
 /* Return codes for zran_export_index. */
 enum {
@@ -332,13 +336,15 @@ enum {
  * structure. All fields are assumed to be stored with little-endian
  * ordering:
  *
- * | Offset | Length | Description                     |
- * | 0      | 7      | File header (GZIDX\00\00)       |
- * | 7      | 8      | Compressed file size  (uint64)  |
- * | 15     | 8      | Uncompressed file size (uint64) |
- * | 23     | 4      | Index point spacing (uint32)    |
- * | 27     | 4      | Index window size W (uint32)    |
- * | 31     | 4      | Number of index points (uint32) |
+ * | Offset | Length | Description                           |
+ * | 0      | 5      | File header (ascii, GZIDX)            |
+ * | 5      | 1      | Version (uint8, currently 1)          |
+ * | 6      | 1      | Reserved (uint8, currently must be 0) |
+ * | 7      | 8      | Compressed file size  (uint64)        |
+ * | 15     | 8      | Uncompressed file size (uint64)       |
+ * | 23     | 4      | Index point spacing (uint32)          |
+ * | 27     | 4      | Index window size W (uint32)          |
+ * | 31     | 4      | Number of index points (uint32)       |
  *
  * The header is followed by the offsets for each index point:
  *
@@ -346,17 +352,20 @@ enum {
  * | 0      | 8      | Compressed offset for point 0 (uint64)   |
  * | 8      | 8      | Uncompressed offset for point 0 (uint64) |
  * | 16     | 1      | Bit offset for point 0 (uint8)           |
+ * | 17     | 1      | Data flag - 1 if point has window data,  |
+ * |        |        | 0 otherwise (uint8, added in file format |
+ * |        |        | version 1)                               |
  * | ...    | ...    | ...                                      |
- * | N*17   | 8      | Compressed offset for point N (uint64)   |
+ * | N*18   | 8      | Compressed offset for point N (uint64)   |
  * | ...    | ...    | ...                                      |
  *
- * Finally the window data for every index point is concatenated
- * (W represents the index window size):
+ * Finally the window data for all index points that have data is
+ * concatenated (W represents the index window size):
  *
- * | Offset | Length | Description                   |
- * | 0      | W      | Window data for index point N |
- * | ...    | ...    | ...                           |
- * | N*W    | W      | Window data for index point N |
+ * | Offset | Length | Description                                 |
+ * | 0      | W      | Window data for first index point with data |
+ * | ...    | ...    | ...                                         |
+ * | N*W    | W      | Window data for Nth index point with data   |
  *
  * Returns:
  *   - ZRAN_EXPORT_OK for success.
@@ -370,17 +379,20 @@ int zran_export_index(
   PyObject      *f      /* Open handle to export file object */
 );
 
+
 /* Return codes for zran_import_index. */
 enum {
-    ZRAN_IMPORT_OK             =  0,
-    ZRAN_IMPORT_FAIL           = -1,
-    ZRAN_IMPORT_EOF            = -2,
-    ZRAN_IMPORT_READ_ERROR     = -3,
-    ZRAN_IMPORT_INCONSISTENT   = -4,
-    ZRAN_IMPORT_MEMORY_ERROR   = -5,
-    ZRAN_IMPORT_UNKNOWN_FORMAT = -6
+    ZRAN_IMPORT_OK                  =  0,
+    ZRAN_IMPORT_FAIL                = -1,
+    ZRAN_IMPORT_EOF                 = -2,
+    ZRAN_IMPORT_READ_ERROR          = -3,
+    ZRAN_IMPORT_INCONSISTENT        = -4,
+    ZRAN_IMPORT_MEMORY_ERROR        = -5,
+    ZRAN_IMPORT_UNKNOWN_FORMAT      = -6,
+    ZRAN_IMPORT_UNSUPPORTED_VERSION = -7
 };
 
+
 /*
  * Import current index from the given file.  index must have been initialized
  * by zran_init function before calling this function, as it is not supported
@@ -414,6 +426,9 @@ enum {
  *     index. This typically result from out-of-memory.
  *
  *   - ZRAN_IMPORT_UNKNOWN_FORMAT to indicate given file is of unknown format.
+ *
+ *   - ZRAN_IMPORT_UNSUPPORTED_VERSION to indicate that the file has a version
+ *     which is too new for this version of indexed_gzip to parse.
  */
 int zran_import_index(
   zran_index_t  *index, /* The index                         */


Test patch:
diff --git a/indexed_gzip/tests/ctest_zran.pyx b/indexed_gzip/tests/ctest_zran.pyx
index a19278bd..09a220a5 100644
--- a/indexed_gzip/tests/ctest_zran.pyx
+++ b/indexed_gzip/tests/ctest_zran.pyx
@@ -1018,13 +1018,41 @@ def test_readbuf_spacing_sizes(testfile, no_fds, nelems, niters, seed):
             print()
             zran.zran_free(&index)
 
-def test_export_then_import(testfile, no_fds):
 
-    cdef zran.zran_index_t  index1
-    cdef zran.zran_index_t  index2
+cdef _compare_indexes(zran.zran_index_t *index1,
+                      zran.zran_index_t *index2):
     cdef zran.zran_point_t *p1
     cdef zran.zran_point_t *p2
 
+    assert index2.compressed_size   == index1.compressed_size
+    assert index2.uncompressed_size == index1.uncompressed_size
+    assert index2.spacing           == index1.spacing
+    assert index2.window_size       == index1.window_size
+    assert index2.npoints           == index1.npoints
+
+    ws = index1.window_size
+
+    for i in range(index1.npoints):
+
+        p1 = &index1.list[i]
+        p2 = &index2.list[i]
+        msg = 'Error at point %d' % i
+
+        assert p2.cmp_offset   == p1.cmp_offset, msg
+        assert p2.uncmp_offset == p1.uncmp_offset, msg
+        assert p2.bits         == p1.bits, msg
+        if (not p1.data):
+            assert p1.data == p2.data, msg
+        else:
+            assert not memcmp(p2.data, p1.data, ws), msg
+
+
+
+def test_export_then_import(testfile, no_fds):
+
+    cdef zran.zran_index_t index1
+    cdef zran.zran_index_t index2
+
     indexSpacing = 1048576
     windowSize   = 32768
     readbufSize  = 131072
@@ -1063,30 +1091,7 @@ def test_export_then_import(testfile, no_fds):
             ret  = zran.zran_import_index(&index2, NULL if no_fds else cfid, <PyObject*>pyexportfid if no_fds else NULL)
             assert not ret, str(ret)
 
-        assert index2.compressed_size   == index1.compressed_size
-        assert index2.uncompressed_size == index1.uncompressed_size
-        assert index2.spacing           == index1.spacing
-        assert index2.window_size       == index1.window_size
-        assert index2.npoints           == index1.npoints
-
-        ws = index1.window_size
-
-        for i in range(index1.npoints):
-
-            p1 = &index1.list[i]
-            p2 = &index2.list[i]
-            msg = 'Error at point %d' % i
-
-            assert p2.cmp_offset   == p1.cmp_offset, msg
-            assert p2.uncmp_offset == p1.uncmp_offset, msg
-            assert p2.bits         == p1.bits, msg
-            if i == 0:
-                assert not p2.data, msg
-                assert not p1.data, msg
-            else:
-                assert p2.data, msg
-                assert p1.data, msg
-                assert not memcmp(p2.data, p1.data, ws), msg
+        _compare_indexes(&index1, &index2)
 
         zran.zran_free(&index1)
         zran.zran_free(&index2)
@@ -1144,3 +1149,63 @@ def test_export_import_no_points(no_fds):
             pybuf = <bytes>(<char *>buffer)[:100]
             assert np.all(np.frombuffer(pybuf, dtype=np.uint8) == data)
             zran.zran_free(&index)
+
+
+def test_export_import_format_v0():
+    """Test index export and import on a version 0 index file. """
+
+    cdef zran.zran_index_t index1
+    cdef zran.zran_index_t index2
+    cdef int               ret
+
+    data = np.random.randint(1, 255, 1000000, dtype=np.uint8)
+    with tempdir():
+
+        with gzip.open('data.gz', 'wb') as f:
+            f.write(data.tostring())
+
+        with open('data.gz', 'rb')  as pyfid:
+            cfid = fdopen(pyfid.fileno(), 'rb')
+            assert not zran.zran_init(
+                &index1, cfid, NULL, 50000, 32768, 131072, 0)
+
+            assert not zran.zran_build_index(&index1, 0, 0)
+            _write_index_file_v0(&index1, 'data.gz.index')
+
+        with open('data.gz', 'rb')  as pyfid:
+            cfid = fdopen(pyfid.fileno(), 'rb')
+            assert not zran.zran_init(
+                &index2, cfid, NULL, 50000, 32768, 131072, 0)
+
+            with open('data.gz.index', 'rb') as pyidxfid:
+                cidxfid = fdopen(pyidxfid.fileno(), 'rb')
+                ret = zran.zran_import_index(&index2, cidxfid,  NULL)
+                assert ret == 0, ret
+
+        _compare_indexes(&index1, &index2)
+        zran.zran_free(&index1)
+        zran.zran_free(&index2)
+
+
+cdef _write_index_file_v0(zran.zran_index_t *index, dest):
+
+    cdef zran.zran_point_t *point
+
+    with open(dest, 'wb') as f:
+        f.write(b'GZIDX\0\0')
+        f.write(<bytes>(<char *>(&index.compressed_size))[:8])
+        f.write(<bytes>(<char *>(&index.uncompressed_size))[:8])
+        f.write(<bytes>(<char *>(&index.spacing))[:4])
+        f.write(<bytes>(<char *>(&index.window_size))[:4])
+        f.write(<bytes>(<char *>(&index.npoints))[:4])
+
+        for i in range(index.npoints):
+            point = &index.list[i]
+            f.write(<bytes>(<char *>(&point.cmp_offset))[:8])
+            f.write(<bytes>(<char *>(&point.uncmp_offset))[:8])
+            f.write(<bytes>(<char *>(&point.bits))[:1])
+
+        for i in range(1, index.npoints):
+            point = &index.list[i]
+            data  = <bytes>point.data[:index.window_size]
+            f.write(data)
diff --git a/indexed_gzip/tests/test_zran.py b/indexed_gzip/tests/test_zran.py
index 96c6cdc6..815e07d2 100644
--- a/indexed_gzip/tests/test_zran.py
+++ b/indexed_gzip/tests/test_zran.py
@@ -118,3 +118,6 @@ def test_export_then_import(testfile):
     def test_export_import_no_points():
         for no_fds in (True, False):
             ctest_zran.test_export_import_no_points(no_fds)
+
+    def test_export_import_format_v0():
+        ctest_zran.test_export_import_format_v0()