Repository: hasindu2008/slow5lib (language: python)

Issue:
Idx



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml
index 534259a..83fb41c 100644
--- a/.github/workflows/c-cpp.yml
+++ b/.github/workflows/c-cpp.yml
@@ -107,6 +107,19 @@ jobs:
       run: make clean && make CC=icc -j8 zstd=1
     - name: test zstd
       run: make test zstd=1 CC=icc
+  ubuntu_22:
+    name: Ubuntu 22
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: sudo apt-get update && sudo apt-get install zlib1g-dev
+    - name: build-gcc
+      run: make -j8
+    - name: test-gcc
+      run: make test
+    - name: make mt
+      run: examples/mt/build.sh
   os_x_11:
     name: OSX 11
     runs-on: macos-11
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 3196a69..01bde46 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -82,7 +82,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: install packages
-      run: pip3 install numpy
+      run: pip3 install numpy setuptools
     - name: build
       run: make pyslow5
     - name: test
@@ -93,7 +93,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: install packages
-      run: pip3 install numpy
+      run: pip3 install numpy setuptools
     - name: build
       run: make pyslow5
     - name: test
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a3efccd..cd7b80b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,14 @@ if(SLOW5_USE_ZSTD)
 endif(SLOW5_USE_ZSTD)
 unset(SLOW5_USE_ZSTD CACHE)
 
+option(SLOW5_ENABLE_MT "Use SLOW5_ENABLE_MT" OFF) #OFF by default
+if(SLOW5_ENABLE_MT)
+    add_compile_options("-DSLOW5_ENABLE_MT")
+    link_libraries("-lpthread")
+    set(slow5_mt src/slow5_mt.c)
+endif(SLOW5_ENABLE_MT)
+unset(SLOW5_ENABLE_MT CACHE)
+
 # option(SLOW5_USE_OPENMP "Use OPENMP" OFF) #OFF by default
 # if(SLOW5_USE_OPENMP)
 #     add_compile_options("-fopenmp")
@@ -51,10 +59,10 @@ add_subdirectory(thirdparty/streamvbyte)
 option(SLOW5_LINK_STATIC "libslow5 will create a static lib" OFF) #OFF by default
 if(SLOW5_LINK_STATIC)
     message( STATUS "libslow5 will create a static lib" )
-    add_library(slow5 STATIC ${slow5_} ${slow5_idx} ${slow5_misc} ${slow5_press})
+    add_library(slow5 STATIC ${slow5_} ${slow5_idx} ${slow5_misc} ${slow5_press} ${slow5_mt})
 else()
     message( STATUS "libslow5 will create a shared lib" )
-    add_library(slow5 SHARED ${slow5_} ${slow5_idx} ${slow5_misc} ${slow5_press})
+    add_library(slow5 SHARED ${slow5_} ${slow5_idx} ${slow5_misc} ${slow5_press} ${slow5_mt})
 endif(SLOW5_LINK_STATIC)
 unset(SLOW5_LINK_STATIC CACHE)
 
diff --git a/docs/slow5_api/low_level_api/slow5_idx_load_with.md b/docs/slow5_api/low_level_api/slow5_idx_load_with.md
index 34d853e..f852f5a 100755
--- a/docs/slow5_api/low_level_api/slow5_idx_load_with.md
+++ b/docs/slow5_api/low_level_api/slow5_idx_load_with.md
@@ -1,15 +1,13 @@
 # slow5_idx_load_with
 
 ## NAME
-slow5_idx_load_with - loads the index file for a SLOW5 file given a file path for the index
+slow5_idx_load_with - loads the index file for a SLOW5 file for a user specified file path for the index
 
 ## SYNOPSYS
-`int slow5_idx_load_with(slow5_with_file_t *s5p, const char *pathname)`
+`int slow5_idx_load_with(slow5_file_t *s5p, const char *pathname)`
 
 ## DESCRIPTION
-`slow5_idx_load_with()` loads an index file for a SLOW5 file pointed by *s5p* into the memory from the disk and associates the index with *s5p*. If the index file is not found, the index is first created and written to the disk.
-
-`slow5_idx_load_with()` could be called instead of `slow5_idx_load_with()` when the index file is at a custom location.
+`slow5_idx_load_with()` loads an index file located at *pathname* for a SLOW5 file pointed by *s5p* into the memory from the disk and associates the index with *s5p*. If the index file is not found, it will error out.
 
 ## RETURN VALUE
 Upon successful completion, `slow5_idx_load_with()` returns a non-negative integer. Otherwise, a negative value is returned.
diff --git a/docs/slow5_api/mt_api/slow5_free_batch.md b/docs/slow5_api/mt_api/slow5_free_batch.md
new file mode 100755
index 0000000..aa7a967
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_free_batch.md
@@ -0,0 +1,64 @@
+# slow5_free_batch
+
+## NAME
+
+slow5_free_batch - frees up a slow5 record batch
+
+## SYNOPSYS
+
+`void slow5_free_batch(slow5_batch_t *read_batch)`
+
+## DESCRIPTION
+
+The `slow5_free_batch()` function frees the memory of a *slow5_batch_t* structure pointed by *read_batch*, which must have been returned by a previous call to `slow5_init_batch()`.
+
+## RETURN VALUE
+
+None
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+ slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int batch_size = 4096;
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
+    // ...
+
+    slow5_free_batch(read_batch);
+    slow5_free_mt(mt);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+## SEE ALSO
+
+[slow5_init_batch()](slow5_init_batch.md)
\ No newline at end of file
diff --git a/docs/slow5_api/mt_api/slow5_free_mt.md b/docs/slow5_api/mt_api/slow5_free_mt.md
new file mode 100755
index 0000000..3d68eab
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_free_mt.md
@@ -0,0 +1,57 @@
+# slow5_free_mt
+
+## NAME
+
+slow5_free_mt - frees up a slow5 multi-thread structure
+
+## SYNOPSYS
+
+`void slow5_free_mt(slow5_mt_t *mt)`
+
+## DESCRIPTION
+
+The `slow5_free_mt()` function frees the memory of a slow5_mt_t structure pointed by mt, which must have been returned by a previous call to `slow5_init_mt()`.
+
+## RETURN VALUE
+
+None
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+    slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //...
+
+    slow5_free_mt(mt);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+
+## SEE ALSO
+
+[slow5_init_mt()](slow5_init_mt.md)
\ No newline at end of file
diff --git a/docs/slow5_api/mt_api/slow5_get_batch.md b/docs/slow5_api/mt_api/slow5_get_batch.md
new file mode 100755
index 0000000..d7fce3f
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_get_batch.md
@@ -0,0 +1,102 @@
+# slow5_get_batch
+
+## NAME
+
+slow5_get_batch - fetches a batch of SLOW5 records corresponding to a given batch of read ID
+
+## SYNOPSYS
+
+`int slow5_get_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, char **rid, int num_rid)`
+
+## DESCRIPTION
+
+`slow5_get_batch()` fetches a batch of record from a SLOW5 file associated with *mt* for a specified list of read IDs. The list of read must be pointed by *rid* and the the size of the list must be specified by *num_rid*.
+
+## RETURN VALUE
+
+The number of records fetched.
+
+## NOTES
+
+*num_rid* must be less than or equal to the capacity of the read_batch.
+
+## ERRORS
+
+Currently, in case of error, this function will exit the programme. Proper error handling might be introduced in future, where a negative number return is indicative of error.
+
+## NOTES
+
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+    slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+       fprintf(stderr,"Error in opening file\n");
+       exit(EXIT_FAILURE);
+    }
+
+    slow5_rec_t **rec = NULL;
+    int ret=0;
+
+    ret = slow5_idx_load(sp);
+    if(ret<0){
+        fprintf(stderr,"Error in loading index\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int num_rid = 4;
+    int num_thread = 2;
+    int batch_size = 4096;
+    char *rid[num_rid];
+    rid[0]="r4";
+    rid[1]="r1",
+    rid[2]="r2";
+    rid[3]="r5";
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
+    ret = slow5_get_batch(mt, read_batch, rid, num_rid);
+    if(ret!=num_rid){
+        fprintf(stderr,"Error in fetching all records\n");
+        exit(EXIT_FAILURE);
+    }
+
+    for(int i=0;i<ret;i++){
+        rec = read_batch->slow5_rec;
+        uint64_t len_raw_signal = rec[i]->len_raw_signal;
+        printf("%s\t%ld\n",rec[i]->read_id,len_raw_signal);
+    }
+
+    slow5_free_batch(read_batch);
+    slow5_free_mt(mt);
+
+    slow5_idx_unload(sp);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+## SEE ALSO
+
+[slow5_get()](slow5_get_next_batch.md)
\ No newline at end of file
diff --git a/docs/slow5_api/mt_api/slow5_get_next_batch.md b/docs/slow5_api/mt_api/slow5_get_next_batch.md
new file mode 100755
index 0000000..a5d10c2
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_get_next_batch.md
@@ -0,0 +1,85 @@
+# slow5_get_next_batch
+
+## NAME
+
+slow5_get_next_batch - fetches the next batch of records at the current file pointer of a SLOW5 file
+
+## SYNOPSYS
+
+`int slow5_get_next_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads)`
+
+## DESCRIPTION
+
+`slow5_get_next_batch()` fetches the next *num_reads* number of records from a SLOW5 file associated with *mt* at the current file pointer into a *slow5_batch_t*.
+
+## RETURN VALUE
+
+The number of records fetched.
+
+## NOTES
+
+*num_reads* must be less than or equal to the capacity of the read_batch.
+
+## ERRORS
+
+Currently, in case of error, this function will exit the programme. Proper error handling might be introduced in future, where a negative number return is indicative of error.
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+    slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_rec_t **rec = NULL;
+    int ret=0;
+    int batch_size = 4096;
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
+    while((ret = slow5_get_next_batch(mt,read_batch,4096)) > 0){
+
+        for(int i=0;i<ret;i++){
+            rec = read_batch->slow5_rec;
+            uint64_t len_raw_signal = rec[i]->len_raw_signal;
+            printf("%s\t%ld\n",rec[i]->read_id,len_raw_signal);
+        }
+
+        if(ret<4096){ //this indicates nothing left to read //better handling of errors may be introduced in future
+            break;
+        }
+    }
+
+    slow5_free_batch(read_batch);
+    slow5_free_mt(mt);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+## SEE ALSO
+
+[slow5_get()](slow5_get_batch.md)
\ No newline at end of file
diff --git a/docs/slow5_api/mt_api/slow5_init_batch.md b/docs/slow5_api/mt_api/slow5_init_batch.md
new file mode 100755
index 0000000..5302167
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_init_batch.md
@@ -0,0 +1,69 @@
+# slow5_init_batch
+
+## NAME
+
+slow5_init_batch - initialises a slow5 record batch
+
+## SYNOPSYS
+
+`slow5_batch_t* slow5_init_batch(int batch_capacity)`
+
+## DESCRIPTION
+
+`slow5_init_batch()` function initialises and returns a batch of empty SLOW5 records. The number of empty records in the batch will be what is specified by *batch_capacity*. The returned SLOW5 record batch must be later freed by calling `slow5_free_batch()`.
+
+
+## RETURN VALUE
+
+Upon successful completion, `slow5_init_batch()` returns a *slow5_batch_t* pointer.
+
+## ERRORS
+
+Currently, in case of error, this function will exit the programme. Proper error handling might be introduced in future, where a NULL return is indicative of error.
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+ slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int batch_size = 4096;
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
+    // ...
+
+    slow5_free_batch(read_batch);
+    slow5_free_mt(mt);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+## SEE ALSO
+
+[slow5_free_batch()](slow5_free_batch.md)
diff --git a/docs/slow5_api/mt_api/slow5_init_mt.md b/docs/slow5_api/mt_api/slow5_init_mt.md
new file mode 100755
index 0000000..1fb7191
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_init_mt.md
@@ -0,0 +1,65 @@
+# slow5_init_mt
+
+## NAME
+
+slow5_init_mt - initialises a slow5 multi-thread structure
+
+## SYNOPSYS
+
+`slow5_mt_t *slow5_init_mt(int num_thread, slow5_file_t *s5p)`
+
+## DESCRIPTION
+
+`slow5_init_mt()` initialises and returns a slow5 multi-thread structure for the number of threads specified by *num_thread*. The argument *s5p* must to a *slow5_file_t* opened using `slow5_open()`. The returned slow5 multi-thread structure must be later freed by calling `slow5_free_mt()`.
+
+
+## RETURN VALUE
+
+Upon successful completion, `slow5_init_mt()` returns a slow5_mt_t pointer.
+
+
+## ERRORS
+
+Currently, in case of error, this function will exit the programme. Proper error handling might be introduced in future, where a NULL return is indicative of error.
+
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "examples/example.slow5"
+
+int main(){
+
+    slow5_file_t *sp = slow5_open(FILE_PATH,"r");
+    if(sp==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //...
+
+    slow5_free_mt(mt);
+    slow5_close(sp);
+
+    return 0;
+}
+```
+
+
+## SEE ALSO
+
+[slow5_free_mt()](slow5_free_mt.md)
+
diff --git a/docs/slow5_api/mt_api/slow5_write_batch.md b/docs/slow5_api/mt_api/slow5_write_batch.md
new file mode 100755
index 0000000..dddae2c
--- /dev/null
+++ b/docs/slow5_api/mt_api/slow5_write_batch.md
@@ -0,0 +1,128 @@
+# slow5_write_batch
+
+## NAME
+
+slow5_write_batch - writes a batch of SLOW5 records to a SLOW5 file
+
+## SYNOPSYS
+
+`int slow5_write_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads)`
+
+## DESCRIPTION
+
+`slow5_write_batch()` writes a batch of SLOW5 records pointed by *read_batch* to a SLOW5 file associated with *mt*.
+
+
+## RETURN VALUE
+
+The number of records written.
+
+## ERRORS
+
+Currently, in case of error, this function will exit the programme. Proper error handling might be introduced in future, where a negative number return is indicative of error.
+
+## EXAMPLES
+
+```
+#include <stdio.h>
+#include <stdlib.h>
+#include <slow5/slow5.h>
+#include <slow5/slow5_mt.h>
+
+#define FILE_PATH "test.blow5"
+
+int main(){
+
+    slow5_file_t *sf = slow5_open(FILE_PATH,"w");
+    if(sf==NULL){
+        fprintf(stderr,"Error in opening file\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_hdr_t *header=sf->header;
+
+    //add a header group attribute called run_id
+    if (slow5_hdr_add_attr("run_id", header) != 0){
+        fprintf(stderr,"Error adding run_id attribute\n");
+        exit(EXIT_FAILURE);
+    }
+
+    //set the run_id attribute to "run_0" for read group 0
+    if (slow5_hdr_set("run_id", "run_0", 0, header) != 0){
+        fprintf(stderr,"Error setting run_id attribute in read group 0\n");
+        exit(EXIT_FAILURE);
+    }
+
+    if(slow5_hdr_write(sf) < 0){
+        fprintf(stderr,"Error writing header!\n");
+        exit(EXIT_FAILURE);
+    }
+
+    int ret=0;
+    int batch_size = 16;
+    int num_thread = 8;
+
+    slow5_mt_t *mt = slow5_init_mt(num_thread,sf);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_rec_t **rec = read_batch->slow5_rec;
+
+    for(int i=0;i<batch_size;i++){
+
+        slow5_rec_t *slow5_record = rec[i] = slow5_rec_init();
+
+        if(slow5_record == NULL){
+            fprintf(stderr,"Could not allocate space for a slow5 record.");
+            exit(EXIT_FAILURE);
+        }
+
+        char tmp_read_id[100];
+        sprintf(tmp_read_id,"read_id_%d",i);
+        slow5_record -> read_id = strdup(tmp_read_id);
+        if(slow5_record->read_id == NULL){
+            fprintf(stderr,"Could not allocate space for strdup.");
+            exit(EXIT_FAILURE);
+        }
+        slow5_record -> read_id_len = strlen(slow5_record -> read_id);
+        slow5_record -> read_group = 0;
+        slow5_record -> digitisation = 4096.0;
+        slow5_record -> offset = 3.0+i;
+        slow5_record -> range = 10.0+i;
+        slow5_record -> sampling_rate = 4000.0;
+        slow5_record -> len_raw_signal = 10+i;
+        slow5_record -> raw_signal = malloc(sizeof(int16_t)*(10+i));
+        if(slow5_record->raw_signal == NULL){
+            fprintf(stderr,"Could not allocate space for raw signal.");
+            exit(EXIT_FAILURE);
+        }
+        for(int j=0;j<10+i;j++){
+            slow5_record->raw_signal[j] = j+i;
+        }
+
+    }
+
+    ret = slow5_write_batch(mt, read_batch,batch_size);
+
+    if(ret!=batch_size){
+        fprintf(stderr,"Writing failed\n");
+        exit(EXIT_FAILURE);
+    }
+
+    slow5_free_batch(read_batch);
+    slow5_free_mt(mt);
+    slow5_close(sf);
+
+    return 0;
+}
+```
+
+
diff --git a/docs/slow5_api/slow5.md b/docs/slow5_api/slow5.md
index d4ef72e..9e183e4 100755
--- a/docs/slow5_api/slow5.md
+++ b/docs/slow5_api/slow5.md
@@ -135,12 +135,15 @@ High-level API consists of following functions:
 * [slow5_set_press](slow5_set_press.md)<br/>
   &nbsp;&nbsp;&nbsp;&nbsp;sets the compression method for a BLOW5 file opened for writing
 
-
+You may also refer to examples [here](https://github.com/hasindu2008/slow5lib/tree/master/examples).
 
 ### Low-level API for reading and writing SLOW5 files
 
 Low-level API allows much more efficient access to BLOW5 files compared to the high-level API.  Low-level API is documented [here](slow5_low_level_api.md).
 
+### Easy multi-thread API
+
+The easy multi-thread (called *mt* API) API allows the user to access SLOW5 using multiple threads, simply by specifying the number of threads, rather than needing to write their own multi-threading code. mt API is documented [here](slow5_mt_api.md). Note that this API is for your convenience, rather than for efficiency.
 
 ## CITATION
 
diff --git a/docs/slow5_api/slow5_internal.md b/docs/slow5_api/slow5_internal.md
index ed70444..3090837 100755
--- a/docs/slow5_api/slow5_internal.md
+++ b/docs/slow5_api/slow5_internal.md
@@ -40,6 +40,9 @@ DO NOT USE.
 * slow5_hdr_set
 * slow5_hdr_add_rg_data
 
+* slow5_idx_init_empty (used in slow5curl)
+* slow5_idx_read (used in slow5curl)
+
 <!--
 int slow5_hdr_fwrite(FILE *fp, struct slow5_hdr *header, enum slow5_fmt format, slow5_press_method_t comp)
 int slow5_rec_fwrite(FILE *fp, struct slow5_rec *read, struct slow5_aux_meta *aux_meta, enum slow5_fmt format, struct slow5_press *compress)
diff --git a/docs/slow5_api/slow5_low_level_api.md b/docs/slow5_api/slow5_low_level_api.md
index 41897dc..4f58f95 100755
--- a/docs/slow5_api/slow5_low_level_api.md
+++ b/docs/slow5_api/slow5_low_level_api.md
@@ -12,7 +12,7 @@ Low-level API allows much more efficient access to BLOW5 files compared to the h
   &nbsp;&nbsp;&nbsp;&nbsp;sets the level log message verbosity
 * [slow5_set_exit_condition](low_level_api/slow5_set_exit_condition.md)<br/>
   &nbsp;&nbsp;&nbsp;&nbsp;sets if slow5lib should exit the programme on error
-* [slow5_idx_load_with](low_level_api/slow5_idx_load.md)<br/>
+* [slow5_idx_load_with](low_level_api/slow5_idx_load_with.md)<br/>
   &nbsp;&nbsp;&nbsp;&nbsp;loads the index file for a SLOW5 file given a file path for the index
 
 ### Reading and access
@@ -42,4 +42,6 @@ Low-level API allows much more efficient access to BLOW5 files compared to the h
   &nbsp;&nbsp;&nbsp;&nbsp;writes an encoded SLOW5 record to a SLOW5 file
 
 
+You may also refer to examples [here](https://github.com/hasindu2008/slow5lib/tree/master/examples/adv).
+
 **If you want any other functionality than what is described above, please open a GitHub issue so such functionality exposed, rather than trying to use internal functions by browsing the code. Such undocumented functions are subject to change.**
diff --git a/docs/slow5_api/slow5_mt_api.md b/docs/slow5_api/slow5_mt_api.md
new file mode 100755
index 0000000..6aacd4f
--- /dev/null
+++ b/docs/slow5_api/slow5_mt_api.md
@@ -0,0 +1,55 @@
+# slow5lib easy multi-thread API
+
+**NOTE: This API is in beta stage and the documentation is sparse. If something is not clear do not hesitate to open an [issue](https://github.com/hasindu2008/slow5lib/issues).**
+
+The easy multi-thread (called *mt* API) API allows the user to access SLOW5 files using multiple threads, simply by specifying the number of threads, rather than needing to write their own multi-threading code. This API is for programmer's convenience, rather than for efficiency. This API is not meant to be used by a programmer who has the expertise to write multi-threaded code and use the slow5 low-level API directly. While fetching using this API would be faster than using a single thread, it would not be as efficient as using low-level API functions directly from the programmer's own multi-threaded code.
+
+To use this API, include <slow5/slow5_mt.h> in your programme. Make sure to explicitly enable easy multi-thread API as `make slow5_mt=1`, during compile time of slow5lib.
+
+## Data structures
+
+The *slow5_mt_t* structure stores internal multi-threading related data. This structure has the following form:
+
+```
+typedef struct {
+    // private members that are not to be directly accessed*/
+} slow5_mt_t;
+```
+
+The *slow5_batch_t* structure stores a batch of SLOW5 records. This structure has the following form:
+
+```
+typedef struct {
+    // private members that are not to be directly accessed*/
+    slow5_rec_t **slow5_rec;
+} slow5_batch_t;
+```
+
+## API functions
+
+### Common
+
+* [slow5_init_mt](mt_api/slow5_init_mt.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;initialises a slow5 multi-thread structure
+* [slow5_free_mt](mt_api/slow5_free_mt.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;frees up a slow5 multi-thread structure
+* [slow5_init_batch](mt_api/slow5_init_batch.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;initialises a slow5 record batch
+* [slow5_free_batch](mt_api/slow5_free_batch.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;frees up a slow5 record batch
+
+### Reading and access
+
+* [slow5_get_next_batch](mt_api/slow5_get_next_batch.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;fetches the next batch of records at the current file pointer of a SLOW5 file
+* [slow5_get_batch](mt_api/slow5_get_batch.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;fetches a batch of SLOW5 records corresponding to a given batch of read IDs
+
+
+### Writing and editing
+
+* [slow5_write_batch](mt_api/slow5_write_batch.md)<br/>
+  &nbsp;&nbsp;&nbsp;&nbsp;writes a batch of SLOW5 records to a SLOW5 file
+
+
+You may also refer to the examples [here](https://github.com/hasindu2008/slow5lib/tree/master/examples/mt) on how these functions can be used.
diff --git a/examples/mt/README.md b/examples/mt/README.md
index 504b5e2..a08d764 100755
--- a/examples/mt/README.md
+++ b/examples/mt/README.md
@@ -1,12 +1,12 @@
 # slow5lib mt Examples
 
-**WARNING: This API is under development. There could be bugs**
+**WARNING: This API is in beta stage.**
 
 **This is not meant to be used by a programmer who has the expertise to write efficient multi-threaded code and use the slow5 low-level API directly. Such advanced examples are at [advanced examples](../adv/)**
 
-This directory contains examples that uses the built in multi-threaded API in slow5lib.  For these examples to work you must build slow5lib with multi-threaded API as `make slow5_mt=1`. To link against slow5lib built with multi-threaded API, make sure to append `-lpthread` flag.
+This directory contains examples that uses the easy multi-thread API in slow5lib.  For these examples to work you must build slow5lib with easy multi-thread API enabled as `make slow5_mt=1`. To link against slow5lib built with easy multi-thread API, make sure to append `-lpthread` flag.
 
-- *mt.c* demonstrates how to use multi-threaded API to first write and then fetch a batch of slow5/blow5 records in parallel.
+- *mt.c* demonstrates how to use the easy multi-thread API to first write and then fetch a batch of slow5/blow5 records in parallel.
 - *lazymt.c* demonstrates how to use the lazy function to do the same as above, which is meant for a lazy programmer.
 
 You can invoke [build.sh](build.sh) from slow5lib directory as `examples/mt/build.sh` to compile the example programmes. Have a look at the script to see the commands used for compiling and linking. Also make sure you get familiar with the basic examples first, before trying these.
diff --git a/examples/mt/mt.c b/examples/mt/mt.c
index 470d202..cf58ab2 100644
--- a/examples/mt/mt.c
+++ b/examples/mt/mt.c
@@ -48,7 +48,7 @@ void sequential_read_func(){
             printf("%s\t%ld\n",rec[i]->read_id,len_raw_signal);
         }
 
-        if(ret<batch_size){ //this indicates nothing left to read //need to handle errors
+        if(ret<batch_size){ //this indicates nothing left to read //better handling of errors may be introduced in future
             break;
         }
     }
@@ -68,9 +68,11 @@ void random_read_func(){
        fprintf(stderr,"Error in opening file\n");
        exit(EXIT_FAILURE);
     }
+
     slow5_rec_t **rec = NULL;
+    int ret=0;
 
-    int ret = slow5_idx_load(sp);
+    ret = slow5_idx_load(sp);
     if(ret<0){
         fprintf(stderr,"Error in loading index\n");
         exit(EXIT_FAILURE);
@@ -86,18 +88,30 @@ void random_read_func(){
     rid[3]="read_id_4";
 
     slow5_mt_t *mt = slow5_init_mt(num_thread,sp);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
 
     slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
 
     ret = slow5_get_batch(mt, read_batch, rid, num_rid);
-    assert(ret==num_rid);
+    if(ret!=num_rid){
+        fprintf(stderr,"Error in fetching all records\n");
+        exit(EXIT_FAILURE);
+    }
+
     for(int i=0;i<ret;i++){
         rec = read_batch->slow5_rec;
         uint64_t len_raw_signal = rec[i]->len_raw_signal;
         printf("%s\t%ld\n",rec[i]->read_id,len_raw_signal);
     }
-    slow5_free_batch(read_batch);
 
+    slow5_free_batch(read_batch);
     slow5_free_mt(mt);
 
     slow5_idx_unload(sp);
@@ -184,7 +198,17 @@ void write_func(){
     int num_thread = 8;
 
     slow5_mt_t *mt = slow5_init_mt(num_thread,sf);
+    if (mt==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising multi-thread struct\n");
+        exit(EXIT_FAILURE);
+    }
+
     slow5_batch_t *read_batch = slow5_init_batch(batch_size);
+    if (read_batch==NULL){ //currently not useful, but better have this for future proofing
+        fprintf(stderr,"Error in initialising slow5 record batch\n");
+        exit(EXIT_FAILURE);
+    }
+
     slow5_rec_t **rec = read_batch->slow5_rec;
 
     /******************* SLOW5 records ************************/
@@ -255,7 +279,7 @@ void write_func(){
 
     ret = slow5_write_batch(mt, read_batch,batch_size);
 
-    if(ret<batch_size){
+    if(ret!=batch_size){
         fprintf(stderr,"Writing failed\n");
         exit(EXIT_FAILURE);
     }
diff --git a/include/slow5/slow5_defs.h b/include/slow5/slow5_defs.h
index 9aef045..800722a 100644
--- a/include/slow5/slow5_defs.h
+++ b/include/slow5/slow5_defs.h
@@ -44,7 +44,7 @@ The API documentation is available at https://hasindu2008.github.io/slow5tools/
 */
 
 // library version
-#define SLOW5_LIB_VERSION "1.1.0"
+#define SLOW5_LIB_VERSION "1.1.0-dirty"
 
 // maximum file version supported by this library - independent of slow5 library version above
 // if updating change all 4 below
diff --git a/include/slow5/slow5_mt.h b/include/slow5/slow5_mt.h
index a444662..51c66be 100644
--- a/include/slow5/slow5_mt.h
+++ b/include/slow5/slow5_mt.h
@@ -4,19 +4,21 @@
 #ifndef SLOW5_MT_H
 #define SLOW5_MT_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
 /**************************************************************************************************
- ***  Multi-threading API *************************************************************************
+ ***  Easy Multi-thread API *************************************************************************
  **************************************************************************************************/
 
 /**************** This API is still in beta stage, there could be bugs *************************/
 
 /*
-This is a multi-threading API that can fetch a batch of slow5 records using multiple threads in parallel.
-This API uses a fork-join thread model.
-It is not meant to be used by a programmer who has the expertise to write multi-threaded code and use the slow5 low-level API directly.
+This is a easy multi-thread API that can fetch a batch of slow5 records using multiple threads in parallel.
+This API uses a fork-join thread model. It is not meant to be used by a programmer who has the expertise to write multi-threaded code and use the slow5 low-level API directly.
 While fetching using this API would be faster than using a single thread, it would not be as efficient as using low-level API functions
-directly from the programmer's own multi-threaded code.
-This API can be optimised using a thread pool, but that is not implemented yet.
+directly from the programmer's own multi-threaded code. This API can be optimised using a thread pool, but that is not implemented yet.
 */
 
 /* a batch of read data (dynamic data based on the reads) */
@@ -48,9 +50,10 @@ also each batch call will create and destruct threads rather than using a thread
 
 slow5_mt_t *slow5_init_mt(int num_thread, slow5_file_t *s5p);
 slow5_batch_t* slow5_init_batch(int batch_capacity);
-int slow5_get_next_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int batch_size);
+int slow5_get_next_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads);
 int slow5_get_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, char **rid, int num_rid);
-int slow5_write_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int batch_size);
+int slow5_write_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads);
+int slow5_encode_batch(slow5_mt_t *core, slow5_batch_t *db, int num_reads); //used by slow5curl
 void slow5_free_batch(slow5_batch_t *read_batch);
 void slow5_free_mt(slow5_mt_t *mt);
 
@@ -63,4 +66,8 @@ int slow5_get_batch_lazy(slow5_rec_t ***read, slow5_file_t *s5p, char **rid, int
 int slow5_write_batch_lazy(slow5_rec_t **read, slow5_file_t *s5p, int batch_size, int num_threads);
 void slow5_free_batch_lazy(slow5_rec_t ***read, int num_rec);
 
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
 #endif
diff --git a/setup.py b/setup.py
index 423ee2a..fd0ea49 100644
--- a/setup.py
+++ b/setup.py
@@ -89,7 +89,7 @@ def readme():
 
 setup(
     name = 'pyslow5',
-    version='1.1.0',
+    version='1.1.0-dev',
     url = 'https://github.com/hasindu2008/slow5lib',
     description='slow5lib python bindings',
     long_description=readme(),
diff --git a/src/slow5_idx.c b/src/slow5_idx.c
index 36f5979..d9aa270 100644
--- a/src/slow5_idx.c
+++ b/src/slow5_idx.c
@@ -14,11 +14,11 @@ extern enum slow5_exit_condition_opt  slow5_exit_condition;
 #define BUF_INIT_CAP (20*1024*1024)
 #define SLOW5_INDEX_BUF_INIT_CAP (64) // 2^6 TODO is this too little?
 
-static inline struct slow5_idx *slow5_idx_init_empty(void);
+
 static int slow5_idx_build(struct slow5_idx *index, struct slow5_file *s5p);
-static int slow5_idx_read(struct slow5_idx *index);
+int slow5_idx_read(struct slow5_idx *index);
 
-static inline struct slow5_idx *slow5_idx_init_empty(void) {
+struct slow5_idx *slow5_idx_init_empty(void) {
 
     struct slow5_idx *index = (struct slow5_idx *) calloc(1, sizeof *index);
     if(!index){
@@ -31,17 +31,23 @@ static inline struct slow5_idx *slow5_idx_init_empty(void) {
     return index;
 }
 
-static inline int slow5_idx_load_fp(struct slow5_idx *index, struct slow5_file *s5p, FILE *index_fp) {
+static inline int slow5_idx_load_fp(struct slow5_idx *index, struct slow5_file *s5p, FILE *index_fp, int8_t check_ts) {
 
     index->fp = index_fp;
-    int err;
-    if (slow5_filestamps_cmp(index->pathname, s5p->meta.pathname, &err) < 0.0) {
-        SLOW5_WARNING("Index file '%s' is older than slow5 file '%s'.",
-                index->pathname, s5p->meta.pathname);
-    }
-    if (err == -1) {
-        return -1;
+
+    if(check_ts){
+        int err;
+        if (slow5_filestamps_cmp(index->pathname, s5p->meta.pathname, &err) < 0.0) {
+            SLOW5_WARNING("Index file '%s' is older than slow5 file '%s'.",
+                    index->pathname, s5p->meta.pathname);
+        }
+        if (err == -1) {
+            return -1;
+        }
+    } else {
+        SLOW5_INFO("Custom index file '%s' is being used. Time stamps not checked.", index->pathname);
     }
+
     if (slow5_idx_read(index) != 0) {
         return -1;
     }
@@ -79,7 +85,7 @@ struct slow5_idx *slow5_idx_init_with(struct slow5_file *s5p, const char *pathna
         slow5_idx_free(index);
         return NULL;
     } else {
-        if (slow5_idx_load_fp(index, s5p, index_fp) != 0) {
+        if (slow5_idx_load_fp(index, s5p, index_fp, 0) != 0) {
             slow5_idx_free(index);
             return NULL;
         }
@@ -127,7 +133,7 @@ struct slow5_idx *slow5_idx_init(struct slow5_file *s5p) {
         fclose(index->fp);
         index->fp = NULL;
     } else {
-        if (slow5_idx_load_fp(index, s5p, index_fp) != 0) {
+        if (slow5_idx_load_fp(index, s5p, index_fp, 1) != 0) {
             slow5_idx_free(index);
             return NULL;
         }
@@ -398,7 +404,7 @@ int slow5_idx_write(struct slow5_idx *index, struct slow5_version version) {
     return 0;
 }
 
-static int slow5_idx_read(struct slow5_idx *index) {
+int slow5_idx_read(struct slow5_idx *index) {
 
     struct slow5_version max_supported = SLOW5_VERSION_ARRAY;
     const char magic[] = SLOW5_INDEX_MAGIC_NUMBER;
diff --git a/src/slow5_idx.h b/src/slow5_idx.h
index 740729e..05a328e 100644
--- a/src/slow5_idx.h
+++ b/src/slow5_idx.h
@@ -47,6 +47,8 @@ struct slow5_idx {
 
 struct slow5_idx *slow5_idx_init(struct slow5_file *s5p);
 struct slow5_idx *slow5_idx_init_with(struct slow5_file *s5p, const char *pathname);
+struct slow5_idx *slow5_idx_init_empty(void);
+int slow5_idx_read(struct slow5_idx *index);
 
 /**
  * Create the index file for slow5 file.
diff --git a/src/slow5_mt.c b/src/slow5_mt.c
index f7b9849..f0c4623 100644
--- a/src/slow5_mt.c
+++ b/src/slow5_mt.c
@@ -83,11 +83,9 @@ slow5_batch_t* slow5_init_batch(int batch_capacity){
 /* load a data batch from disk */
 static int slow5_load_db(slow5_mt_t* core, slow5_batch_t* db) {
 
-    db->n_rec = 0;
-
     int32_t i = 0;
-    while (db->n_rec < db->capacity_rec) {
-        i=db->n_rec;
+    while (i < db->n_rec) {
+
         db->mem_records[i] = (char *)slow5_get_next_mem(&(db->mem_bytes[i]), core->sf);
 
         if (db->mem_records[i] == NULL) {
@@ -101,25 +99,22 @@ static int slow5_load_db(slow5_mt_t* core, slow5_batch_t* db) {
             }
         }
         else {
-            db->n_rec++;
+            i++;
         }
     }
 
-    return db->n_rec;
+    return i;
 }
 
 
 static int slow5_write_db(slow5_mt_t* core, slow5_batch_t* db) {
 
-
     int32_t i = 0;
     for(i=0;i<db->n_rec;i++) {
-
         size_t n = fwrite(db->mem_records[i], db->mem_bytes[i], 1, core->sf->fp);
         if (n != 1) {
             SLOW5_ERROR("Writing failed for read id %s!\n", db->slow5_rec[i]->read_id);
         }
-
     }
 
     return i;
@@ -323,63 +318,54 @@ static void slow5_work_db(slow5_mt_t* core, slow5_batch_t* db, void (*func)(slow
 
 int slow5_get_batch(slow5_mt_t *core, slow5_batch_t *db, char **rid, int num_rid){
 
-    //slow5_mt_t *core = slow5_init_core(s5p,num_rid,num_threads);
-    //slow5_batch_t* db = slow5_init_db(core);
+    if(num_rid>db->capacity_rec){
+        SLOW5_ERROR("Requested %d is greater than the capacity %d",num_rid,db->capacity_rec);
+        exit(EXIT_FAILURE);
+    }
 
     db->rid = rid;
     db->n_rec = num_rid;
     slow5_work_db(core,db,slow5_work_per_single_read2);
     SLOW5_LOG_DEBUG("loaded and parsed %d recs\n",num_rid);
-
-    //*read = db->slow5_rec;
-
-    // slow5_free_db_tmp(db);
-    // slow5_free_db(db);
-    // slow5_free_core(core);
+    slow5_free_db_tmp(db);
 
     return num_rid;
 }
 
 
-int slow5_get_next_batch(slow5_mt_t *core, slow5_batch_t *db, int batch_size){
-
-    // slow5_mt_t *core = slow5_init_core(s5p,batch_size,num_threads);
-    // slow5_batch_t* db = slow5_init_db(core);
-    db->n_rec = batch_size;
+int slow5_get_next_batch(slow5_mt_t *core, slow5_batch_t *db, int num_reads){
 
-    int num_read=slow5_load_db(core,db);
-    SLOW5_LOG_DEBUG("Loaded %d recs\n",num_read);
+    if(num_reads>db->capacity_rec){
+        SLOW5_ERROR("Requested %d is greater than the capacity %d",num_reads,db->capacity_rec);
+        exit(EXIT_FAILURE);
+    }
+    db->n_rec = num_reads;
+    int n = slow5_load_db(core,db);
+    db->n_rec = n;
+    SLOW5_LOG_DEBUG("Loaded %d recs\n",n);
     slow5_work_db(core,db,slow5_work_per_single_read);
-    SLOW5_LOG_DEBUG("Parsed %d recs\n",num_read);
-
-    //*read = db->slow5_rec;
-
-    // slow5_free_db_tmp(db);
-    // slow5_free_db(db);
-    // slow5_free_core(core);
+    SLOW5_LOG_DEBUG("Parsed %d recs\n",n);
+    slow5_free_db_tmp(db);
 
-    return num_read;
+    return n;
 }
 
+int slow5_encode_batch(slow5_mt_t *core, slow5_batch_t *db, int num_reads){
+    db->n_rec = num_reads;
+    slow5_work_db(core,db,slow5_work_per_single_read3);
+    return db->n_rec;
+}
 
-int slow5_write_batch(slow5_mt_t *core, slow5_batch_t *db, int batch_size){
-
-    // slow5_mt_t *core = slow5_init_core(s5p,batch_size,num_threads);
-    // slow5_batch_t* db = slow5_init_db(core);
+int slow5_write_batch(slow5_mt_t *core, slow5_batch_t *db, int num_reads){
 
-    db->n_rec = batch_size;
-    // free(db->slow5_rec); //stupid lazy for now
-    // db->slow5_rec = read;
+    db->n_rec = num_reads;
     slow5_work_db(core,db,slow5_work_per_single_read3);
-    SLOW5_LOG_DEBUG("Processed %d recs\n",batch_size);
+    SLOW5_LOG_DEBUG("Processed %d recs\n",num_reads);
 
     int num_wr=slow5_write_db(core,db);
     SLOW5_LOG_DEBUG("Written %d recs\n",num_wr);
 
-    // db->slow5_rec = NULL;
-    // slow5_free_db_tmp(db);
-    // slow5_free_db(db);
-    // slow5_free_core(core);
+    slow5_free_db_tmp(db);
 
     return num_wr;
 }
@@ -396,7 +382,7 @@ void slow5_free_batch(slow5_batch_t *db){
     free(reads);
     //*read = NULL;
 
-    slow5_free_db_tmp(db);
+    //slow5_free_db_tmp(db);
     slow5_free_db(db);
 }
 
@@ -493,7 +479,7 @@ slow5_batch_t* slow5_init_batch(int batch_capacity){
     fprintf(stderr,"slow5lib has not been compiled with lazy multithreading support\n");
     exit(EXIT_FAILURE);
 }
-int slow5_get_next_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int batch_size){
+int slow5_get_next_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads){
     fprintf(stderr,"slow5lib has not been compiled with lazy multithreading support\n");
     exit(EXIT_FAILURE);
 }
@@ -501,7 +487,11 @@ int slow5_get_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, char **rid, int n
     fprintf(stderr,"slow5lib has not been compiled with lazy multithreading support\n");
     exit(EXIT_FAILURE);
 }
-int slow5_write_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int batch_size){
+int slow5_encode_batch(slow5_mt_t *core, slow5_batch_t *db, int num_reads){
+    fprintf(stderr,"slow5lib has not been compiled with lazy multithreading support\n");
+    exit(EXIT_FAILURE);
+}
+int slow5_write_batch(slow5_mt_t *mt, slow5_batch_t *read_batch, int num_reads){
     fprintf(stderr,"slow5lib has not been compiled with lazy multithreading support\n");
     exit(EXIT_FAILURE);
 }


Test patch:
diff --git a/test/.gitignore b/test/.gitignore
index a6c5aab..b67646c 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -35,8 +35,13 @@ data/test/same_diff_ids.slow5.idx
 data/exp/aux_array/exp_lossless_end_reason.slow5.idx
 test/data/exp/aux_array/exp_lossless_end_reason.slow5.idx
 
-bench/get_all_read_ids
-bench/get_all_samples
-bench/get_selected_read_ids_read_number
-bench/get_selected_read_ids_sample_count
-bench/get_selected_read_ids_samples
\ No newline at end of file
+bench/get_read_ids
+bench/get_samples
+bench/get_read_number_rand
+bench/get_sample_count_rand
+bench/get_samples_rand
+bench/get_samples_rand
+bench/convert_to_pa
+bench/convert_to_pa_rand
+bench/get_all
+bench/get_samples_no_pipelining
\ No newline at end of file
diff --git a/test/bench/build.sh b/test/bench/build.sh
index e66c559..c2182b5 100755
--- a/test/bench/build.sh
+++ b/test/bench/build.sh
@@ -3,8 +3,13 @@
 set -x
 set -e
 
-gcc -Wall -O2 -g -I include/ -o test/bench/get_all_read_ids test/bench/get_all_read_ids.c lib/libslow5.a -lm -lz -lzstd
-gcc -Wall -O2 -g -I include/ -o test/bench/get_all_samples test/bench/get_all_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_samples test/bench/get_selected_read_ids_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_sample_count test/bench/get_selected_read_ids_sample_count.c lib/libslow5.a -lm -lz -lzstd -lpthread
-gcc -Wall -O2 -g -I include/ -o test/bench/get_selected_read_ids_read_number test/bench/get_selected_read_ids_read_number.c lib/libslow5.a -lm -lz -lzstd -lpthread
+make clean && make -j zstd=1 slow5_mt=1
+gcc -Wall -O2 -g -I include/ -o test/bench/convert_to_pa test/bench/convert_to_pa.c lib/libslow5.a -lm -lz -lzstd -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/convert_to_pa_rand test/bench/convert_to_pa_rand.c lib/libslow5.a -lm -lz -lzstd -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_all test/bench/get_all.c lib/libslow5.a -lm -lz -lzstd -lpthread
+gcc -Wall -O2 -g -I include/ -o test/bench/get_read_ids test/bench/get_read_ids.c lib/libslow5.a -lm -lz -lzstd
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples test/bench/get_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples_no_pipelining test/bench/get_samples_no_pipelining.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_samples_rand test/bench/get_samples_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+gcc -Wall -O2 -g -I include/ -o test/bench/get_sample_count_rand test/bench/get_sample_count_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
+gcc -Wall -O2 -g -I include/ -o test/bench/get_read_number_rand test/bench/get_read_number_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
diff --git a/test/bench/convert_to_pa.c b/test/bench/convert_to_pa.c
index 32d8241..ec47b3b 100644
--- a/test/bench/convert_to_pa.c
+++ b/test/bench/convert_to_pa.c
@@ -64,7 +64,7 @@ void free_raw_batch(char **mem_records, size_t *mem_bytes, int batch_size) {
 int main(int argc, char *argv[]) {
 
     if(argc != 4) {
-        fprintf(stderr, "Usage: %s in_file.blow5 num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
diff --git a/test/bench/convert_to_pa_rand.c b/test/bench/convert_to_pa_rand.c
index c01bee8..973625c 100644
--- a/test/bench/convert_to_pa_rand.c
+++ b/test/bench/convert_to_pa_rand.c
@@ -23,7 +23,7 @@ static inline double realtime(void) {
 int main(int argc, char *argv[]) {
 
     if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 rid_list.txt num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 rid_list.txt num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
diff --git a/test/bench/get_all.c b/test/bench/get_all.c
index 867d99f..fd4b915 100644
--- a/test/bench/get_all.c
+++ b/test/bench/get_all.c
@@ -7,7 +7,7 @@
 #include <slow5/slow5.h>
 #include <omp.h>
 #include <sys/time.h>
-#include "../../python/slow5threads.h"
+#include <slow5/slow5_mt.h>
 
 static inline double realtime(void) {
     struct timeval tp;
@@ -20,7 +20,7 @@ static inline double realtime(void) {
 int main(int argc, char *argv[]) {
 
     if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv num_thread batch_size\n", argv[0]);
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
@@ -29,13 +29,8 @@ int main(int argc, char *argv[]) {
     int batch_size = atoi(argv[4]);
     int num_thread = atoi(argv[3]);
 
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fp);
+    FILE *fp = stdout;
+    fputs("#read_id\tsample_sum\n", fp);
 
     int64_t sum_samples = 0;
 
@@ -55,7 +50,7 @@ int main(int argc, char *argv[]) {
     while(ret > 0){
 
         t0 = realtime();
-        ret = slow5_get_next_batch(&rec,sp,batch_size,num_thread);
+        ret = slow5_get_next_batch_lazy(&rec,sp,batch_size,num_thread);
         tot_time += realtime() - t0;
         fprintf(stderr,"batch loaded with %d reads\n",ret);
 
@@ -64,7 +59,7 @@ int main(int argc, char *argv[]) {
         }
 
         t0 = realtime();
-        slow5_free_batch(&rec,ret);
+        slow5_free_batch_lazy(&rec,ret);
         tot_time += realtime() - t0;
 
         if(ret<batch_size){ //this indicates nothing left to read //need to handle errors
@@ -76,8 +71,6 @@ int main(int argc, char *argv[]) {
     slow5_close(sp);
     tot_time += realtime() - t0;
 
-    fclose(fp);
-
     fprintf(stderr,"Time for getting all records %f\n", tot_time);
     fprintf(stderr,"%f MSamples/s\n", (double)sum_samples/(tot_time*1000*1000));
 
diff --git a/test/bench/get_all_read_ids.c b/test/bench/get_read_ids.c
similarity index 70%
rename from test/bench/get_all_read_ids.c
rename to test/bench/get_read_ids.c
index e3625b5..aa942fd 100644
--- a/test/bench/get_all_read_ids.c
+++ b/test/bench/get_read_ids.c
@@ -1,6 +1,6 @@
 //get all the read ids and print them to stdout
 //make zstd=1
-//gcc -Wall -O2 -I include/ -o get_all_read_ids test/bench/get_all_read_ids.c lib/libslow5.a -lm -lz -lzstd
+//gcc -Wall -O2 -I include/ -o get_read_ids test/bench/get_read_ids.c lib/libslow5.a -lm -lz -lzstd
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -17,8 +17,8 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 3) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv\n", argv[0]);
+    if(argc != 2) {
+        fprintf(stderr, "Usage: %s reads.blow5 > out_file.tsv\n", argv[0]);
         return EXIT_FAILURE;
     }
 
@@ -44,20 +44,10 @@ int main(int argc, char *argv[]) {
 
     tot_time += realtime() - t0;
 
-
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-
-    fputs("read_id\n", fp);
     for(int i=0; i<num_reads; i++) {
-        fputs(read_ids[i],fp);
-        fputc('\n',fp);
+        fputs(read_ids[i],stdout);
+        fputc('\n',stdout);
     }
-    fclose(fp);
 
     t0 = realtime();
     slow5_idx_unload(sp);
diff --git a/test/bench/get_selected_read_ids_read_number.c b/test/bench/get_read_number_rand.c
similarity index 74%
rename from test/bench/get_selected_read_ids_read_number.c
rename to test/bench/get_read_number_rand.c
index 85e503f..19445b2 100644
--- a/test/bench/get_selected_read_ids_read_number.c
+++ b/test/bench/get_read_number_rand.c
@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1 slow5_mt=1
-//gcc -Wall -O2 -I include/ -o get_selected_read_ids_read_number test/bench/get_selected_read_ids_read_number.c lib/libslow5.a -lm -lz -lzstd -lpthread
+//gcc -Wall -O2 -I include/ -o get_read_number_rand test/bench/get_read_number_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -19,14 +19,14 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 6) {
-        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.csv out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 5) {
+        fprintf(stderr, "Usage: %s reads.blow5 inread_id.tsv num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     int ret=0;
-    int batch_size = atoi(argv[5]);
-    int num_thread = atoi(argv[4]);
+    int batch_size = atoi(argv[4]);
+    int num_thread = atoi(argv[3]);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
 
@@ -37,19 +37,9 @@ int main(int argc, char *argv[]) {
         perror("perr: ");;
         exit(EXIT_FAILURE);
     }
-    if(fscanf(fpr,"%s",tmp)<1 || strcmp(tmp,",read_id")!=0){
-        fprintf(stderr,"Bad input CSV file. CSV header is expected to have read_id\n");
-        exit(EXIT_FAILURE);
-    }
-
-    FILE *fpw = fopen(argv[3],"w");
-    if(fpw==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[3]);
-        perror("perr: ");;
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,read_number\n", fpw);
 
+    FILE *fpw = stdout;
+    fputs("#read_id\tread_number\n", fpw);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -75,8 +65,7 @@ int main(int argc, char *argv[]) {
 
         int i=0;
         for(i=0; i<batch_size; i++){
-            int crap;
-            if (fscanf(fpr,"%d,%s",&crap,tmp) < 2) {
+            if (fscanf(fpr,"%s",tmp) < 1) {
                 break;
             }
             rid[i] = strdup(tmp);
@@ -102,7 +91,7 @@ int main(int argc, char *argv[]) {
                 exit(EXIT_FAILURE);
             }
             tot_time += realtime() - t0;
-            fprintf(fpw,"%s,%d\n",rec[i]->read_id, rn);
+            fprintf(fpw,"%s\t%d\n",rec[i]->read_id, rn);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
@@ -127,7 +116,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
     fclose(fpr);
-    fclose(fpw);
 
     free(rid);
 
diff --git a/test/bench/get_selected_read_ids_sample_count.c b/test/bench/get_sample_count_rand.c
similarity index 70%
rename from test/bench/get_selected_read_ids_sample_count.c
rename to test/bench/get_sample_count_rand.c
index c657a85..6c10142 100644
--- a/test/bench/get_selected_read_ids_sample_count.c
+++ b/test/bench/get_sample_count_rand.c
@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1 slow5_mt=1
-//gcc -Wall -O2 -I include/ -o get_selected_read_ids_sample_count test/bench/get_selected_read_ids_sample_count.c lib/libslow5.a -lm -lz -lzstd -lpthread
+//gcc -Wall -O2 -I include/ -o get_sample_count_rand test/bench/get_sample_count_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -19,14 +19,14 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 6) {
-        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.csv out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 5) {
+        fprintf(stderr, "Usage: %s reads.blow5 inread_id.tsv num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     int ret=0;
-    int batch_size = atoi(argv[5]);
-    int num_thread = atoi(argv[4]);
+    int batch_size = atoi(argv[4]);
+    int num_thread = atoi(argv[3]);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
 
@@ -37,19 +37,9 @@ int main(int argc, char *argv[]) {
         perror("perr: ");;
         exit(EXIT_FAILURE);
     }
-    if(fscanf(fpr,"%s",tmp)<1 || strcmp(tmp,",read_id")!=0){
-        fprintf(stderr,"Bad input CSV file. CSV header is expected to have read_id\n");
-        exit(EXIT_FAILURE);
-    }
-
-    FILE *fpw = fopen(argv[3],"w");
-    if(fpw==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[3]);
-        perror("perr: ");;
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,sample_count\n", fpw);
 
+    FILE *fpw = stdout;
+    fputs("#read_id\tsample_count\n", fpw);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -75,8 +65,7 @@ int main(int argc, char *argv[]) {
 
         int i=0;
         for(i=0; i<batch_size; i++){
-            int crap;
-            if (fscanf(fpr,"%d,%s",&crap,tmp) < 2) {
+            if (fscanf(fpr,"%s",tmp) < 1) {
                 break;
             }
             rid[i] = strdup(tmp);
@@ -94,7 +83,7 @@ int main(int argc, char *argv[]) {
         fprintf(stderr,"batch loaded with %d reads\n",ret);
 
         for(int i=0;i<ret;i++){
-            fprintf(fpw,"%s,%ld\n",rec[i]->read_id,rec[i]->len_raw_signal);
+            fprintf(fpw,"%s\t%ld\n",rec[i]->read_id,rec[i]->len_raw_signal);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
@@ -119,7 +108,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
     fclose(fpr);
-    fclose(fpw);
 
     free(rid);
 
diff --git a/test/bench/get_all_samples.c b/test/bench/get_samples.c
similarity index 91%
rename from test/bench/get_all_samples.c
rename to test/bench/get_samples.c
index 05955ec..88b801d 100644
--- a/test/bench/get_all_samples.c
+++ b/test/bench/get_samples.c
@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1
-//gcc -Wall -O2 -I include/ -o get_all_samples test/bench/get_all_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+//gcc -Wall -O2 -I include/ -o get_samples test/bench/get_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -141,7 +141,7 @@ void* output_free_batch(void* voidargs){
 
 
     for(int i=0;i<batch_size;i++){
-        fprintf(fp,"%s,%ld\n",slow5_rec[i]->read_id,sums[i]);
+        fprintf(fp,"%s\t%ld\n",slow5_rec[i]->read_id,sums[i]);
     }
     //fprintf(stderr,"batch printed with %d reads\n",batch_size);
     free_raw_batch(mem_records, mem_bytes, batch_size);
@@ -162,25 +162,20 @@ void* output_free_batch(void* voidargs){
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 4) {
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
 
-    int batch_size = atoi(argv[4]);
-    int num_thread = atoi(argv[3]);
+    int batch_size = atoi(argv[3]);
+    int num_thread = atoi(argv[2]);
     int ret=batch_size;
     threads = num_thread;
 
 
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fp);
+    FILE *fp = stdout;
+    fputs("#read_id\tsample_sum\n", fp);
 
 
     double tot_time = 0;
@@ -272,8 +267,6 @@ int main(int argc, char *argv[]) {
     slow5_close(sp);
     tot_time += realtime() - t0;
 
-    fclose(fp);
-
     fprintf(stderr,"Time for getting raw bytes (exclude depress & parse) %f\n", tot_time);
 
     return 0;
diff --git a/test/bench/get_all_samples_no_pipelining.c b/test/bench/get_samples_no_pipelining.c
similarity index 68%
rename from test/bench/get_all_samples_no_pipelining.c
rename to test/bench/get_samples_no_pipelining.c
index 3fad05d..9d50e3d 100644
--- a/test/bench/get_all_samples_no_pipelining.c
+++ b/test/bench/get_samples_no_pipelining.c
@@ -1,13 +1,13 @@
 //get all the samples and sum them to stdout
 //make zstd=1
-//gcc -Wall -O2 -I include/ -o get_all_samples test/bench/get_all_samples.c lib/libslow5.a python/slow5threads.c -lm -lz -lzstd -lpthread  -fopenmp
+//gcc -Wall -O2 -I include/ -o get_samples test/bench/get_samples.c lib/libslow5.a python/slow5threads.c -lm -lz -lzstd -lpthread  -fopenmp
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <slow5/slow5.h>
 #include <omp.h>
 #include <sys/time.h>
-#include "../../python/slow5threads.h"
+#include <slow5/slow5_mt.h>
 
 static inline double realtime(void) {
     struct timeval tp;
@@ -19,27 +19,21 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 5) {
-        fprintf(stderr, "Usage: %s in_file.blow5 out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 4) {
+        fprintf(stderr, "Usage: %s reads.blow5 num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     slow5_rec_t **rec = NULL;
     int ret=1;
-    int batch_size = atoi(argv[4]);
-    int num_thread = atoi(argv[3]);
+    int batch_size = atoi(argv[3]);
+    int num_thread = atoi(argv[2]);
     omp_set_num_threads(num_thread);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
 
-    FILE *fp = fopen(argv[2],"w");
-    if(fp==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[2]);
-        perror("perr: ");
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fp);
-
+    FILE *fp = stdout;
+    fputs("#read_id\tsample_sum\n", fp);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -56,7 +50,7 @@ int main(int argc, char *argv[]) {
     while(ret > 0){
 
         t0 = realtime();
-        ret = slow5_get_next_batch(&rec,sp,batch_size,num_thread);
+        ret = slow5_get_next_batch_lazy(&rec,sp,batch_size,num_thread);
         tot_time += realtime() - t0;
         fprintf(stderr,"batch loaded with %d reads\n",ret);
 
@@ -72,12 +66,12 @@ int main(int argc, char *argv[]) {
         fprintf(stderr,"batch processed with %d reads\n",ret);
 
         for(int i=0;i<ret;i++){
-            fprintf(fp,"%s,%ld\n",rec[i]->read_id,sums[i]);
+            fprintf(fp,"%s\t%ld\n",rec[i]->read_id,sums[i]);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
         t0 = realtime();
-        slow5_free_batch(&rec,ret);
+        slow5_free_batch_lazy(&rec,ret);
         tot_time += realtime() - t0;
 
         if(ret<batch_size){ //this indicates nothing left to read //need to handle errors
@@ -91,8 +85,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
 
-    fclose(fp);
-
     fprintf(stderr,"Time for getting samples %f\n", tot_time);
 
     return 0;
diff --git a/test/bench/get_selected_read_ids_samples.c b/test/bench/get_samples_rand.c
similarity index 74%
rename from test/bench/get_selected_read_ids_samples.c
rename to test/bench/get_samples_rand.c
index 8469cef..921e271 100644
--- a/test/bench/get_selected_read_ids_samples.c
+++ b/test/bench/get_samples_rand.c
@@ -1,6 +1,6 @@
 //get all the samples and sum them to stdout
 //make zstd=1 slow5_mt=1
-//gcc -Wall -O2 -I include/ -o get_selected_read_ids_samples test/bench/get_selected_read_ids_samples.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
+//gcc -Wall -O2 -I include/ -o get_samples_rand test/bench/get_samples_rand.c lib/libslow5.a -lm -lz -lzstd -lpthread  -fopenmp
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -19,14 +19,14 @@ static inline double realtime(void) {
 
 int main(int argc, char *argv[]) {
 
-    if(argc != 6) {
-        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.csv out_file.csv num_thread batch_size\n", argv[0]);
+    if(argc != 5) {
+        fprintf(stderr, "Usage: %s in_file.blow5 inread_id.tsv num_thread batch_size\n", argv[0]);
         return EXIT_FAILURE;
     }
 
     int ret=0;
-    int batch_size = atoi(argv[5]);
-    int num_thread = atoi(argv[4]);
+    int batch_size = atoi(argv[4]);
+    int num_thread = atoi(argv[3]);
     omp_set_num_threads(num_thread);
 
     uint64_t *sums = malloc(sizeof(uint64_t)*batch_size);
@@ -38,19 +38,9 @@ int main(int argc, char *argv[]) {
         perror("perr: ");;
         exit(EXIT_FAILURE);
     }
-    if(fscanf(fpr,"%s",tmp)<1 || strcmp(tmp,",read_id")!=0){
-        fprintf(stderr,"Bad input CSV file. CSV header is expected to have read_id\n");
-        exit(EXIT_FAILURE);
-    }
-
-    FILE *fpw = fopen(argv[3],"w");
-    if(fpw==NULL){
-        fprintf(stderr,"Error in opening file %s for writing\n",argv[3]);
-        perror("perr: ");;
-        exit(EXIT_FAILURE);
-    }
-    fputs("read_id,samples\n", fpw);
 
+    FILE *fpw = stdout;
+    fputs("#read_id\tsamples\n", fpw);
 
     double tot_time = 0;
     double t0 = realtime();
@@ -76,8 +66,7 @@ int main(int argc, char *argv[]) {
 
         int i=0;
         for(i=0; i<batch_size; i++){
-            int crap;
-            if (fscanf(fpr,"%d,%s",&crap,tmp) < 2) {
+            if (fscanf(fpr,"%s",tmp) < 1) {
                 break;
             }
             rid[i] = strdup(tmp);
@@ -105,7 +94,7 @@ int main(int argc, char *argv[]) {
         fprintf(stderr,"batch processed with %d reads\n",ret);
 
         for(int i=0;i<ret;i++){
-            fprintf(fpw,"%s,%ld\n",rec[i]->read_id,sums[i]);
+            fprintf(fpw,"%s\t%ld\n",rec[i]->read_id,sums[i]);
         }
         fprintf(stderr,"batch printed with %d reads\n",ret);
 
@@ -130,7 +119,6 @@ int main(int argc, char *argv[]) {
 
     free(sums);
     fclose(fpr);
-    fclose(fpw);
 
     free(rid);
 
diff --git a/test/bench/readme.md b/test/bench/readme.md
new file mode 100755
index 0000000..a8e0226
--- /dev/null
+++ b/test/bench/readme.md
@@ -0,0 +1,15 @@
+# Miscellaneous Benchmarks
+
+From slow5lib directory, run `test/bench/build.sh` to compile the benchmarks.
+
+| benchmark                     | access type | threads      | description |
+| -----------                   | ----------- |------------  |-------------|
+|[convert_to_pa](convert_to_pa.c)                  |  sequential | openMP       | stream through the BLOW5 sequentially, while converting the signals to pA and summing up using openMP threads|
+|[convert_to_pa_rand](convert_to_pa_rand.c)             |  random     | openMP       | randomly access selected readIDs, while converting the signals to pA and summing up using openMP threads|
+|[get_all](get_all.c)                        |  sequential | slow_mt_lazy | stream through the BLOW5 sequentially, while sums up the signal samples. Uses inbuilt slow5_mt_lazy threads |
+|[get_read_ids](get_read_ids.c)                   |  -          | -            | get the list of read IDs |
+|[get_samples](get_samples.c)                    | sequential  | pthreads     | stream through the BLOW5 sequentially, sums up the signals using pthreads. I/O and processing pipelined.|
+|[get_samples_no_pipelining](get_samples_no_pipelining.c)      | sequential  | pthreads     | stream through the BLOW5 sequentially, sums up the signals using pthreads. I/O and processing are not pipelined. |
+|[get_read_number_rand](get_read_number_rand.c)           | random      | slow_mt_lazy | randomly access selected readIDs,  gets the read number field. Uses inbuilt slow5_mt_lazy threads. |
+|[get_sample_count_rand](get_sample_count_rand.c)          | random      | slow_mt_lazy | randomly access selected readIDs,  get the len_raw_signal field. Uses inbuilt slow5_mt_lazy threads. |
+|[get_samples_rand](get_samples_rand.c)               | random      | slow_mt_lazy | randomly access selected readIDs, sums up the signals. Uses inbuilt slow5_mt_lazy threads. |