Repository: hasindu2008/slow5lib (language: python)

Issue:
Pyslow5mac



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml
index 183c60d..534259a 100644
--- a/.github/workflows/c-cpp.yml
+++ b/.github/workflows/c-cpp.yml
@@ -129,19 +129,30 @@ jobs:
       run: make -j8 zstd=1
     - name: test
       run: make test zstd=1
-  # arm64:
-  #   name: ubuntu arm
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #   - run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
-  #   - uses: docker://multiarch/ubuntu-core:arm64-bionic
-  #     with:
-  #       args: 'uname -a'
-  #   - uses: actions/checkout@v1
-  #   - uses: docker://multiarch/ubuntu-core:arm64-bionic
-  #     with:
-  #       args: >
-  #         bash -c "apt-get update &&
-  #         apt-get install -y zlib1g-dev gcc make &&
-  #         ldd --version && gcc --version &&
-  #         make && make test
+  os_x_12:
+    name: OSX 12
+    runs-on: macos-12
+    steps:
+    - uses: actions/checkout@v2
+    - name: build
+      run: make -j8
+    - name: test
+      run: make test
+    - name: examples
+      run: make examples
+  arm64:
+    name: ubuntu arm
+    runs-on: ubuntu-latest
+    steps:
+    - run: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+    - uses: docker://multiarch/ubuntu-core:arm64-bionic
+      with:
+        args: 'uname -a'
+    - uses: actions/checkout@v1
+    - uses: docker://multiarch/ubuntu-core:arm64-bionic
+      with:
+        args: >
+          bash -c "apt-get update &&
+          apt-get install -y zlib1g-dev gcc make &&
+          ldd --version && gcc --version &&
+          make && make test"
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 39ab396..3196a69 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -74,5 +74,27 @@ jobs:
           apt-get install -y zlib1g-dev gcc make python3 python3-pip python3-numpy &&
           pip3 install setuptools cython &&
           ldd --version && gcc --version && python3 --version &&
-          make pyslow5
-          python3 -m unittest -v python/test.py
+          make pyslow5 &&
+          python3 -m unittest -v python/test.py"
+  os_x_11:
+    name: pyslow5 mac os
+    runs-on: macos-11
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: pip3 install numpy
+    - name: build
+      run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
+  os_x_12:
+    name: pyslow5 mac os
+    runs-on: macos-12
+    steps:
+    - uses: actions/checkout@v2
+    - name: install packages
+      run: pip3 install numpy
+    - name: build
+      run: make pyslow5
+    - name: test
+      run: python3 -m unittest -v python/test.py
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 2a945d0..c1913b5 100644
--- a/Makefile
+++ b/Makefile
@@ -90,6 +90,7 @@ pyslow5:
 	python3 setup.py build
 	cp build/lib.*/*.so  ./
 	python3 < python/example.py
+	rm -rf .eggs/
 	python3 setup.py sdist
 
 
diff --git a/docs/pyslow5_api/pyslow5.md b/docs/pyslow5_api/pyslow5.md
index c55d241..016e2fc 100644
--- a/docs/pyslow5_api/pyslow5.md
+++ b/docs/pyslow5_api/pyslow5.md
@@ -24,11 +24,13 @@ On OS X : brew install zstd
 
 BLOW5 files compressed with *zstd* offer smaller file size and better performance compared to the default *zlib*. However, *zlib* runtime library is available by default on almost all distributions unlike *zstd* and thus files compressed with *zlib* will be more 'portable'.
 
+### Install from pypi
+
 ```bash
 python3 -m venv path/to/slow5libvenv
 source path/to/slow5libvenv/bin/activate
 python3 -m pip install --upgrade pip
-python3 -m pip install setuptools cython numpy wheel
+
 # do this separately, after the libs above
 # zlib only build
 python3 -m pip install pyslow5
@@ -48,12 +50,12 @@ python3 -m pip install pyslow5
 # Then once activated, you can just use python3.
 
 # To install a specific version of python, the deadsnakes ppa is a good place to start
-# This is an example for installing python3.7
+# This is an example for installing python3.8
 # you can then call that specific python version
-# > python3.7 -m pip --version
+# > python3.8 -m pip --version
 sudo add-apt-repository ppa:deadsnakes/ppa
 sudo apt-get update
-sudo apt install python3.7 python3.7-dev python3.7-venv
+sudo apt install python3.8 python3.8-dev python3.8-venv
 
 
 # get zlib1g-dev
@@ -73,13 +75,28 @@ Building and installing the python library.
 python3 -m venv /path/to/slow5libvenv
 source /path/to/slow5libvenv/bin/activate
 python3 -m pip install --upgrade pip
-python3 -m pip install setuptools cython numpy wheel
 
 git clone git@github.com:hasindu2008/slow5lib.git
 cd slow5lib
-make
 
-# CHOOSE A OR B:  
+# New build method to work with setuptools deprication
+python3 -m pip install .
+
+# This should not require sudo if using a python virtual environment/venv
+# confirm installation, and find pyslow5==<version>
+python3 -m pip freeze
+
+# Ensure slow5 library is working by running the basic tests
+python3 ./python/example.py
+
+
+# To Remove the library
+python3 -m pip uninstall pyslow5
+
+
+
+# Legacy build methods - not recommended
+# CHOOSE A OR B:
 # (B is the cleanest method)
 # |=======================================================================|
 # |A. Install with pip if wheel is present, otherwise it uses setuptools  |
@@ -92,23 +109,13 @@ make
     python3 setup.py install
 # |=======================================================================|
 
-# This should not require sudo if using a python virtual environment/venv
-# confirm installation, and find pyslow5==<version>
-python3 -m pip freeze
-
-# Ensure slow5 library is working by running the basic tests
-python3 ./python/example.py
-
-
-# To Remove the library
-python3 -m pip uninstall pyslow5
 ```
 
 ## Usage
 
 ### Reading/writing a file
 
-#### `Open(FILE, mode, rec_press="zlib", sig_press="svb_zd", DEBUG=0)`:
+#### `Open(FILE, mode, rec_press="zlib", sig_press="svb-zd", DEBUG=0)`:
 
 The pyslow5 library has one main Class, `pyslow5.Open` which opens a slow5/blow5 (slow5 for easy reference) file for reading/writing.
 
@@ -131,7 +138,7 @@ Compression Options:
 
 `sig_press`:
 - "none"
-- "svb_zd" [default]
+- "svb-zd" [default]
 
 Example:
 
@@ -284,6 +291,10 @@ for r, read in zip(read_list, selected_reads):
         print(r, "read not found")
 ```
 
+#### `get_num_read_groups()`:
+**NEW: from version 1.1.0+**
+
+Return an int for the number of read_groups present in file
 
 #### `get_header_names()`:
 
@@ -325,6 +336,8 @@ end_reason_labels = s5.get_aux_enum_labels('end_reason')
 print(end_reason_labels)
 
 > ['unknown', 'partial', 'mux_change', 'unblock_mux_change', 'signal_positive', 'signal_negative']
+# or from newer datsets
+> ["unknown", "mux_change", "unblock_mux_change", "data_service_unblock_mux_change", "signal_positive", "signal_negative"]
 
 readID = "r1"
 read = s5.get_read(readID, aux='all')
@@ -529,4 +542,4 @@ Please cite the following in your publications when using *slow5lib/pyslow5*:
   year={2022},
   publisher={Nature Publishing Group}
 }
-```
\ No newline at end of file
+```
diff --git a/python/README.md b/python/README.md
index ef6acf7..016e2fc 100644
--- a/python/README.md
+++ b/python/README.md
@@ -24,11 +24,13 @@ On OS X : brew install zstd
 
 BLOW5 files compressed with *zstd* offer smaller file size and better performance compared to the default *zlib*. However, *zlib* runtime library is available by default on almost all distributions unlike *zstd* and thus files compressed with *zlib* will be more 'portable'.
 
+### Install from pypi
+
 ```bash
 python3 -m venv path/to/slow5libvenv
 source path/to/slow5libvenv/bin/activate
 python3 -m pip install --upgrade pip
-python3 -m pip install setuptools cython numpy wheel
+
 # do this separately, after the libs above
 # zlib only build
 python3 -m pip install pyslow5
@@ -48,12 +50,12 @@ python3 -m pip install pyslow5
 # Then once activated, you can just use python3.
 
 # To install a specific version of python, the deadsnakes ppa is a good place to start
-# This is an example for installing python3.7
+# This is an example for installing python3.8
 # you can then call that specific python version
-# > python3.7 -m pip --version
+# > python3.8 -m pip --version
 sudo add-apt-repository ppa:deadsnakes/ppa
 sudo apt-get update
-sudo apt install python3.7 python3.7-dev python3.7-venv
+sudo apt install python3.8 python3.8-dev python3.8-venv
 
 
 # get zlib1g-dev
@@ -73,12 +75,27 @@ Building and installing the python library.
 python3 -m venv /path/to/slow5libvenv
 source /path/to/slow5libvenv/bin/activate
 python3 -m pip install --upgrade pip
-python3 -m pip install setuptools cython numpy wheel
 
 git clone git@github.com:hasindu2008/slow5lib.git
 cd slow5lib
-make
 
+# New build method to work with setuptools deprication
+python3 -m pip install .
+
+# This should not require sudo if using a python virtual environment/venv
+# confirm installation, and find pyslow5==<version>
+python3 -m pip freeze
+
+# Ensure slow5 library is working by running the basic tests
+python3 ./python/example.py
+
+
+# To Remove the library
+python3 -m pip uninstall pyslow5
+
+
+
+# Legacy build methods - not recommended
 # CHOOSE A OR B:
 # (B is the cleanest method)
 # |=======================================================================|
@@ -92,23 +109,13 @@ make
     python3 setup.py install
 # |=======================================================================|
 
-# This should not require sudo if using a python virtual environment/venv
-# confirm installation, and find pyslow5==<version>
-python3 -m pip freeze
-
-# Ensure slow5 library is working by running the basic tests
-python3 ./python/example.py
-
-
-# To Remove the library
-python3 -m pip uninstall pyslow5
 ```
 
 ## Usage
 
 ### Reading/writing a file
 
-#### `Open(FILE, mode, rec_press="zlib", sig_press="svb_zd", DEBUG=0)`:
+#### `Open(FILE, mode, rec_press="zlib", sig_press="svb-zd", DEBUG=0)`:
 
 The pyslow5 library has one main Class, `pyslow5.Open` which opens a slow5/blow5 (slow5 for easy reference) file for reading/writing.
 
@@ -131,7 +138,7 @@ Compression Options:
 
 `sig_press`:
 - "none"
-- "svb_zd" [default]
+- "svb-zd" [default]
 
 Example:
 
@@ -284,6 +291,10 @@ for r, read in zip(read_list, selected_reads):
         print(r, "read not found")
 ```
 
+#### `get_num_read_groups()`:
+**NEW: from version 1.1.0+**
+
+Return an int for the number of read_groups present in file
 
 #### `get_header_names()`:
 
@@ -325,6 +336,8 @@ end_reason_labels = s5.get_aux_enum_labels('end_reason')
 print(end_reason_labels)
 
 > ['unknown', 'partial', 'mux_change', 'unblock_mux_change', 'signal_positive', 'signal_negative']
+# or from newer datsets
+> ["unknown", "mux_change", "unblock_mux_change", "data_service_unblock_mux_change", "signal_positive", "signal_negative"]
 
 readID = "r1"
 read = s5.get_read(readID, aux='all')
diff --git a/python/example.py b/python/example.py
index 1f5434b..ff365b3 100644
--- a/python/example.py
+++ b/python/example.py
@@ -130,6 +130,8 @@
 s52 = slow5.Open('examples/example2.slow5','r', DEBUG=debug)
 print("get_read check, 0d624d4b-671f-40b8-9798-84f2ccc4d7fc")
 read1 = s52.get_read("0d624d4b-671f-40b8-9798-84f2ccc4d7fc", aux=["read_number", "start_mux", "blah"])
+num_read_groups = s52.get_num_read_groups()
+print("number of read groups: {}".format(num_read_groups))
 # print all fields
 print("read_id:", read1['read_id'])
 print("read_group:", read1['read_group'])
@@ -512,6 +514,59 @@
 
 s58.close()
 F.close()
+print("==============================================")
+print("write reads with new aux fields")
+
+F = slow5.Open('examples/example_write_new_aux.blow5','w', DEBUG=debug)
+header, end_reason_labels = F.get_empty_header(aux=True)
+
+counter = 0
+for i in header:
+    header[i] = "test_{}".format(counter)
+    counter += 1
+
+ret = F.write_header(header, end_reason_labels=end_reason_labels)
+print("ret: write_header(): {}".format(ret))
+
+s58 = slow5.Open('examples/adv/example3.blow5','r', DEBUG=debug)
+reads = s58.seq_reads(aux='all')
+
+for read in reads:
+    record, aux = F.get_empty_record(aux=True)
+    for i in read:
+        if i in record:
+            record[i] = read[i]
+        if i in aux:
+            aux[i] = read[i]
+    aux["tracked_scaling_shift"] = 10.0
+    aux["tracked_scaling_scale"] = 20.0
+    aux["predicted_scaling_shift"] = 30.0
+    aux["predicted_scaling_scale"] = 40.0
+    aux["num_reads_since_mux_change"] = 60
+    aux["time_since_mux_change"] = 50.0
+    aux["num_minknow_events"] = 700
+    ret = F.write_record(record, aux)
+    print("ret: write_record(): {}".format(ret))
+
+s58.close()
+F.close()
+print("==============================================")
+print("read reads with new aux fields")
+
+F = slow5.Open('examples/example_write_new_aux.blow5','r', DEBUG=debug)
+
+reads = F.seq_reads(aux='all')
+
+for read in reads:
+    print("tracked_scaling_shift:", read["tracked_scaling_shift"])
+    print("tracked_scaling_scale:", read["tracked_scaling_scale"])
+    print("predicted_scaling_shift:", read["predicted_scaling_shift"])
+    print("predicted_scaling_scale:", read["predicted_scaling_scale"])
+    print("num_reads_since_mux_change:", read["num_reads_since_mux_change"])
+    print("time_since_mux_change:", read["time_since_mux_change"])
+    print("num_minknow_events:", read["num_minknow_events"])
+
+F.close()
 
 print("==============================================")
 print("write reads with aux multi")
diff --git a/python/pyslow5.pyx b/python/pyslow5.pyx
index 0a3140d..c219fe2 100644
--- a/python/pyslow5.pyx
+++ b/python/pyslow5.pyx
@@ -6,8 +6,10 @@ import time
 import logging
 import copy
 from itertools import chain
+# from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
+# I should replace these with the cpython methods above
 from libc.stdlib cimport malloc, free
-from libc.string cimport strdup
+from libc.string cimport strdup, memcpy
 cimport pyslow5
 # Import the Python-level symbols of numpy
 import numpy as np
@@ -22,6 +24,7 @@ np.import_array()
 # m attribute sets the read/write state and file extension of p sets the type
 #
 
+
 cdef class Open:
     cdef pyslow5.slow5_file_t *s5
     cdef pyslow5.slow5_rec_t *rec
@@ -52,6 +55,7 @@ cdef class Open:
     cdef str rec_press
     cdef str sig_press
     cdef int state
+    cdef int num_read_groups
     cdef int V
     cdef object logger
     cdef list aux_names
@@ -87,6 +91,13 @@ cdef class Open:
     cdef char *start_mux
     cdef char *start_time
     cdef char *end_reason
+    cdef char *tracked_scaling_shift
+    cdef char *tracked_scaling_scale
+    cdef char *predicted_scaling_shift
+    cdef char *predicted_scaling_scale
+    cdef char *num_reads_since_mux_change
+    cdef char *time_since_mux_change
+    cdef char *num_minknow_events
     cdef char **end_reason_labels
     cdef pyslow5.uint8_t end_reason_labels_len
     cdef char *channel_number_val
@@ -95,12 +106,28 @@ cdef class Open:
     cdef pyslow5.uint8_t start_mux_val
     cdef pyslow5.uint64_t start_time_val
     cdef pyslow5.uint8_t end_reason_val
+    cdef float tracked_scaling_shift_val
+    cdef float tracked_scaling_scale_val
+    cdef float predicted_scaling_shift_val
+    cdef float predicted_scaling_scale_val
+    cdef pyslow5.uint32_t num_reads_since_mux_change_val
+    cdef float time_since_mux_change_val
+    cdef pyslow5.uint64_t num_minknow_events_val
     cdef char **channel_number_val_array
     cdef double *median_before_val_array
     cdef pyslow5.int32_t *read_number_val_array
     cdef pyslow5.uint8_t *start_mux_val_array
     cdef pyslow5.uint64_t *start_time_val_array
     cdef pyslow5.uint8_t *end_reason_val_array
+    cdef float *tracked_scaling_shift_val_array
+    cdef float *tracked_scaling_scale_val_array
+    cdef float *predicted_scaling_shift_val_array
+    cdef float *predicted_scaling_scale_val_array
+    cdef pyslow5.uint32_t *num_reads_since_mux_change_val_array
+    cdef float *time_since_mux_change_val_array
+    cdef pyslow5.uint64_t *num_minknow_events_val_array
+
+    cdef np.int16_t[:] temp_array 
 
     cdef pyslow5.float total_time_slow5_get_next
     cdef pyslow5.float total_time_yield_reads
@@ -108,6 +135,7 @@ cdef class Open:
     cdef pyslow5.float total_multi_write_signal_time
     cdef pyslow5.float total_multi_write_time
 
+
     def __cinit__(self, pathname, mode, rec_press="zlib", sig_press="svb_zd", DEBUG=0):
         # Set to default NULL type
         self.s5 = NULL
@@ -123,6 +151,7 @@ cdef class Open:
         self.sig_press = ""
         # state for read/write. -1=null, 0=read, 1=write, 2=append
         self.state = -1
+        self.num_read_groups = 0
         self.trec = NULL
         self.rid = NULL
         self.rids = NULL
@@ -133,23 +162,23 @@ cdef class Open:
         self.close_state = False
         self.s5_aux_type = NULL
         self.s5_aux_enum = NULL
-        self.aux_get_err = 1
+        self.aux_get_err = 0
         self.aux_get_len = 0
         self.head_len = 0
         self.aux_len = 0
         self.enum_len = 0
         self.shape_seq[0] = 0
         self.shape_get[0] = 0
-        self.e0 = -1
-        self.e1 = -1
-        self.e2 = -1
-        self.e3 = -1
-        self.e4 = -1
-        self.e5 = -1
-        self.e6 = -1
-        self.e7 = -1
-        self.e8 = -1.0
-        self.e9 = -1.0
+        self.e0 = 0
+        self.e1 = 0
+        self.e2 = 0
+        self.e3 = 0
+        self.e4 = 0
+        self.e5 = 0
+        self.e6 = 0
+        self.e7 = 0
+        self.e8 = 0.0
+        self.e9 = 0.0
         self.e10 = 0
         self.e11 = 0
         self.e12 = NULL
@@ -169,20 +198,41 @@ cdef class Open:
         self.start_mux = strdup("start_mux")
         self.start_time = strdup("start_time")
         self.end_reason = strdup("end_reason")
+        self.tracked_scaling_shift = strdup("tracked_scaling_shift")
+        self.tracked_scaling_scale = strdup("tracked_scaling_scale")
+        self.predicted_scaling_shift = strdup("predicted_scaling_shift")
+        self.predicted_scaling_scale = strdup("predicted_scaling_scale")
+        self.num_reads_since_mux_change = strdup("num_reads_since_mux_change")
+        self.time_since_mux_change = strdup("time_since_mux_change")
+        self.num_minknow_events = strdup("num_minknow_events")
         self.end_reason_labels = NULL
         self.end_reason_labels_len = 0
-        channel_number_val = NULL
-        median_before_val = -1.0
-        read_number_val = -1
-        start_mux_val = -1
-        start_time_val = -1
-        end_reason_val = -1
-        channel_number_val_array = NULL
-        median_before_val_array = NULL
-        read_number_val_array = NULL
-        start_mux_val_array = NULL
-        start_time_val_array = NULL
-        end_reason_val_array = NULL
+        self.channel_number_val = NULL
+        self.median_before_val = 0.0
+        self.read_number_val = 0
+        self.start_mux_val = 0
+        self.start_time_val = 0
+        self.end_reason_val = 0
+        self.tracked_scaling_shift_val = 0.0
+        self.tracked_scaling_scale_val = 0.0
+        self.predicted_scaling_shift_val = 0.0
+        self.predicted_scaling_scale_val = 0.0
+        self.num_reads_since_mux_change_val = 0
+        self.time_since_mux_change_val = 0.0
+        self.num_minknow_events_val = 0
+        self.channel_number_val_array = NULL
+        self.median_before_val_array = NULL
+        self.read_number_val_array = NULL
+        self.start_mux_val_array = NULL
+        self.start_time_val_array = NULL
+        self.end_reason_val_array = NULL
+        self.tracked_scaling_shift_val_array = NULL
+        self.tracked_scaling_scale_val_array = NULL
+        self.predicted_scaling_shift_val_array = NULL
+        self.predicted_scaling_scale_val_array = NULL
+        self.num_reads_since_mux_change_val_array = NULL
+        self.time_since_mux_change_val_array = NULL
+        self.num_minknow_events_val_array = NULL
 
 
         self.total_time_slow5_get_next = 0.0
@@ -193,14 +243,22 @@ cdef class Open:
 
         # sets up logging level/verbosity
         self.V = DEBUG
-        self.logger = logging.getLogger(__name__)
+        self.logger = logging.getLogger("pyslow5")
+        loghandler = logging.StreamHandler()
+        if self.V == 1:
+            loghandler.setLevel(logging.DEBUG)
+        else:
+            loghandler.setLevel(logging.WARNING)
+        formatter = logging.Formatter('%(asctime)s - %(name)s - [%(levelname)s]: %(message)s', datefmt='%d-%b-%y %H:%M:%S')
+        loghandler.setFormatter(formatter)
+        self.logger.addHandler(loghandler)
         if self.V == 1:
-            lev = logging.DEBUG
+            self.logger.setLevel(logging.DEBUG)
         else:
-            lev = logging.WARNING
+            self.logger.setLevel(logging.WARNING)
 
-        logging.basicConfig(format='%(asctime)s - [%(levelname)s]: %(message)s',
-                            datefmt='%d-%b-%y %H:%M:%S', level=lev)
+        # logging.basicConfig(format='%(asctime)s - %(name)s - [%(levelname)s]: %(message)s',
+        #                     datefmt='%d-%b-%y %H:%M:%S', level=lev)
 
         # This should match slow5_defs.h error codes
         self.error_codes = {-1: ["SLOW5_ERR_EOF", "End Of File reached"],
@@ -230,6 +288,7 @@ cdef class Open:
         self.slow5_press_method = {"none": 0,
                                    "zlib": 1,
                                    "svb_zd": 2,
+                                   "svb-zd": 2,
                                    "zstd": 3}
 
         p = str.encode(pathname)
@@ -256,6 +315,7 @@ cdef class Open:
         if self.state == 0:
             self.s5 = pyslow5.slow5_open(self.p, self.m)
             self.logger.debug("Number of read_groups: {}".format(self.s5.header.num_read_groups))
+            self.num_read_groups = self.s5.header.num_read_groups
         elif self.state == 1:
             self.s5 = pyslow5.slow5_open(self.p, self.m)
             if self.s5 is NULL:
@@ -321,6 +381,14 @@ cdef class Open:
         free(self.start_mux)
         free(self.start_time)
         free(self.end_reason)
+        free(self.tracked_scaling_shift)
+        free(self.tracked_scaling_scale)
+        free(self.predicted_scaling_shift)
+        free(self.predicted_scaling_scale)
+        free(self.num_reads_since_mux_change)
+        free(self.time_since_mux_change)
+        free(self.num_minknow_events)
+        
         if self.end_reason_labels is not NULL:
             for i in range(self.end_reason_labels_len):
                 free(self.end_reason_labels[i])
@@ -1025,6 +1093,11 @@ cdef class Open:
 
         return dic
 
+    def get_num_read_groups(self):
+        '''
+        return an int for the number of read_groups present in file
+        '''
+        return self.num_read_groups
 
     def get_read_ids(self):
         '''
@@ -1451,7 +1524,15 @@ cdef class Open:
                "read_number": None,
                "start_mux": None,
                "start_time": None,
-               "end_reason": None}
+               "end_reason": None,
+               "tracked_scaling_shift": None,
+               "tracked_scaling_scale": None,
+               "predicted_scaling_shift": None,
+               "predicted_scaling_scale": None,
+               "num_reads_since_mux_change": None,
+               "time_since_mux_change": None,
+               "num_minknow_events": None,
+               }
 
         if aux:
             return record, aux_rec
@@ -1466,12 +1547,12 @@ cdef class Open:
             if header[h] is not None:
                 t = type(header[h])
                 if t is not type("string"):
-                    self.logger.warning("_header_type_validation {}: {} is not a string type, attempting to convert".format(h, header[h]))
+                    self.logger.debug("_header_type_validation {}: {} is not a string type, attempting to convert".format(h, header[h]))
                     #try and convert
                     try:
                         s = str(header[h])
                         header[h] = s
-                        self.logger.warning("_header_type_validation {}: {} conversion successful".format(h, s))
+                        self.logger.debug("_header_type_validation {}: {} conversion successful".format(h, s))
                     except:
                         self.logger.error("_header_type_validation {}: {} could not convert value to string".format(h, header[h]))
                         raise
@@ -1489,14 +1570,30 @@ cdef class Open:
                      "read_number": type(10),
                      "start_mux": type(1),
                      "start_time": type(100),
-                     "end_reason": type(["a", "b", "c"])}
+                     "end_reason": type(["a", "b", "c"]),
+                     "tracked_scaling_shift": type(1.0),
+                     "tracked_scaling_scale": type(1.0),
+                     "predicted_scaling_shift": type(1.0),
+                     "predicted_scaling_scale": type(1.0),
+                     "num_reads_since_mux_change": type(1),
+                     "time_since_mux_change": type(1.0),
+                     "num_minknow_events": type(1),
+                     }
 
         C_aux_types = {"channel_number": SLOW5_STRING,
                      "median_before":SLOW5_DOUBLE,
                      "read_number": SLOW5_INT32_T,
                      "start_mux": SLOW5_UINT8_T,
                      "start_time": SLOW5_UINT64_T,
-                     "end_reason": SLOW5_ENUM}
+                     "end_reason": SLOW5_ENUM,
+                     "tracked_scaling_shift": SLOW5_FLOAT,
+                     "tracked_scaling_scale": SLOW5_FLOAT,
+                     "predicted_scaling_shift": SLOW5_FLOAT,
+                     "predicted_scaling_scale": SLOW5_FLOAT,
+                     "num_reads_since_mux_change": SLOW5_UINT32_T,
+                     "time_since_mux_change": SLOW5_FLOAT,
+                     "num_minknow_events": SLOW5_UINT64_T,
+                     }
 
         for a in user_aux_types:
             if user_aux_types[a] is None:
@@ -1532,7 +1629,15 @@ cdef class Open:
                         "read_number": type(10),
                         "start_mux": type(1),
                         "start_time": type(100),
-                        "end_reason": type(1)}
+                        "end_reason": type(1),
+                        "tracked_scaling_shift": type(1.0),
+                        "tracked_scaling_scale": type(1.0),
+                        "predicted_scaling_shift": type(1.0),
+                        "predicted_scaling_scale": type(1.0),
+                        "num_reads_since_mux_change": type(1),
+                        "time_since_mux_change": type(1.0),
+                        "num_minknow_events": type(1),
+                        }
 
         new_aux = {}
 
@@ -1579,6 +1684,27 @@ cdef class Open:
                     elif a == "end_reason":
                         self.end_reason_val = <uint8_t>aux[a]
                         new_aux[a] = aux[a]
+                    elif a == "tracked_scaling_shift":
+                        self.tracked_scaling_shift_val = <float>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "tracked_scaling_scale":
+                        self.tracked_scaling_scale_val = <float>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "predicted_scaling_shift":
+                        self.predicted_scaling_shift_val = <float>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "predicted_scaling_scale":
+                        self.predicted_scaling_scale_val = <float>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "num_reads_since_mux_change":
+                        self.num_reads_since_mux_change_val = <uint32_t>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "time_since_mux_change":
+                        self.time_since_mux_change_val = <float>aux[a]
+                        new_aux[a] = aux[a]
+                    elif a == "num_minknow_events":
+                        self.num_minknow_events_val = <uint64_t>aux[a]
+                        new_aux[a] = aux[a]
                     else:
                          self.logger.error("_record_type_validation {}: {} user aux field unknown?".format(a, aux[a]))
 
@@ -1606,7 +1732,15 @@ cdef class Open:
                         "read_number": type(10),
                         "start_mux": type(1),
                         "start_time": type(100),
-                        "end_reason": type(1)}
+                        "end_reason": type(1),
+                        "tracked_scaling_shift": type(1.0),
+                        "tracked_scaling_scale": type(1.0),
+                        "predicted_scaling_shift": type(1.0),
+                        "predicted_scaling_scale": type(1.0),
+                        "num_reads_since_mux_change": type(1),
+                        "time_since_mux_change": type(1.0),
+                        "num_minknow_events": type(1),
+                        }
 
         new_aux = {}
 
@@ -1647,6 +1781,20 @@ cdef class Open:
                         new_aux[a] = aux[a]
                     elif a == "end_reason":
                         new_aux[a] = aux[a]
+                    elif a == "tracked_scaling_shift":
+                        new_aux[a] = aux[a]
+                    elif a == "tracked_scaling_scale":
+                        new_aux[a] = aux[a]
+                    elif a == "predicted_scaling_shift":
+                        new_aux[a] = aux[a]
+                    elif a == "predicted_scaling_scale":
+                        new_aux[a] = aux[a]
+                    elif a == "num_reads_since_mux_change":
+                        new_aux[a] = aux[a]
+                    elif a == "time_since_mux_change":
+                        new_aux[a] = aux[a]
+                    elif a == "num_minknow_events":
+                        new_aux[a] = aux[a]
                     else:
                         self.logger.error("_record_type_validation {}: {} user aux field unknown".format(a, aux[a]))
 
@@ -1704,7 +1852,7 @@ cdef class Open:
                     try:
                         s = str(i)
                         erl.append(s)
-                        self.logger.warning("write_header end_reason_labels: {} conversion successful".format(s))
+                        self.logger.debug("write_header end_reason_labels: {} conversion successful".format(s))
                     except:
                         self.logger.error("write_header end_reason_labels: {} could not convert value to string".format(i))
                         errors = True
@@ -1735,7 +1883,31 @@ cdef class Open:
                      "read_number": type(10),
                      "start_mux": type(1),
                      "start_time": type(100),
-                     "end_reason": type(["a", "b", "c"])}
+                     "end_reason": type(["a", "b", "c"]),
+                     "tracked_scaling_shift": type(1.0),
+                     "tracked_scaling_scale": type(1.0),
+                     "predicted_scaling_shift": type(1.0),
+                     "predicted_scaling_scale": type(1.0),
+                     "num_reads_since_mux_change": type(1),
+                     "time_since_mux_change": type(1.0),
+                     "num_minknow_events": type(1),
+                     }
+        
+        aux_types_keys = [
+            "channel_number",
+            "median_before",
+            "read_number",
+            "start_mux",
+            "start_time",
+            "end_reason",
+            "tracked_scaling_shift",
+            "tracked_scaling_scale",
+            "predicted_scaling_shift",
+            "predicted_scaling_scale",
+            "num_reads_since_mux_change",
+            "time_since_mux_change",
+            "num_minknow_events",
+        ]
 
         self.logger.debug("write_record: _record_type_validation running")
         checked_record, checked_aux = self._record_type_validation(record, aux)
@@ -1753,7 +1925,7 @@ cdef class Open:
             if aux is not None:
                 if checked_aux is not None:
                     slow5_aux_types = self._aux_header_type_validation(aux_types)
-                    for a in slow5_aux_types:
+                    for a in aux_types_keys:
                         if a not in checked_aux:
                             continue
                         elif checked_aux[a] is None:
@@ -1808,10 +1980,23 @@ cdef class Open:
 
         self.logger.debug("write_record: self.write processing raw_signal")
         start_write_copy_signal = time.time()
-        # grabs buffer of numby array so the for loop operats in C not python = super fast
-        memview = memoryview(checked_record["signal"])
-        for i in range(checked_record["len_raw_signal"]):
-            self.write.raw_signal[i] = memview[i]
+
+
+        if checked_record["signal"].data.contiguous:
+            self.temp_array = checked_record["signal"]
+            num_elements = checked_record["signal"].size
+            memcpy(self.write.raw_signal, &self.temp_array[0], num_elements * sizeof(int16_t))
+        else:
+            self.logger.warning("write_record: numpy array of signal is not contiguous, please check your numpy array with np.info(array)")
+            self.logger.warning("write_record: falling back to old memory view element by element contruction. This is ~57 times slower...")
+            memview = memoryview(checked_record["signal"])
+            for i in range(checked_record["len_raw_signal"]):
+                self.write.raw_signal[i] = memview[i]
+       
+        # memview = memoryview(checked_record["signal"])
+        # for i in range(checked_record["len_raw_signal"]):
+        #     self.write.raw_signal[i] = memview[i]
+
         # for i in range(checked_record["len_raw_signal"]):
         #     self.write.raw_signal[i] = checked_record["signal"][i]
         end_write_copy_signal = (time.time() - start_write_copy_signal)
@@ -1848,6 +2033,20 @@ cdef class Open:
                     ret = slow5_aux_set(self.write, self.start_time, <const void *>&self.start_time_val, self.s5.header)
                 elif a == "end_reason":
                     ret = slow5_aux_set(self.write, self.end_reason, <const void *>&self.end_reason_val, self.s5.header)
+                elif a == "tracked_scaling_shift":
+                    ret = slow5_aux_set(self.write, self.tracked_scaling_shift, <const void *>&self.tracked_scaling_shift_val, self.s5.header)
+                elif a == "tracked_scaling_scale":
+                    ret = slow5_aux_set(self.write, self.tracked_scaling_scale, <const void *>&self.tracked_scaling_scale_val, self.s5.header)
+                elif a == "predicted_scaling_shift":
+                    ret = slow5_aux_set(self.write, self.predicted_scaling_shift, <const void *>&self.predicted_scaling_shift_val, self.s5.header)
+                elif a == "predicted_scaling_scale":
+                    ret = slow5_aux_set(self.write, self.predicted_scaling_scale, <const void *>&self.predicted_scaling_scale_val, self.s5.header)
+                elif a == "num_reads_since_mux_change":
+                    ret = slow5_aux_set(self.write, self.num_reads_since_mux_change, <const void *>&self.num_reads_since_mux_change_val, self.s5.header)
+                elif a == "time_since_mux_change":
+                    ret = slow5_aux_set(self.write, self.time_since_mux_change, <const void *>&self.time_since_mux_change_val, self.s5.header)
+                elif a == "num_minknow_events":
+                    ret = slow5_aux_set(self.write, self.num_minknow_events, <const void *>&self.num_minknow_events_val, self.s5.header)
                 else:
                     ret = -1
                 if ret < 0:
@@ -1865,11 +2064,18 @@ cdef class Open:
         if aux is not None:
             free(self.channel_number_val)
             self.channel_number_val = NULL
-            self.median_before_val = -1.0
-            self.read_number_val = -1
-            self.start_mux_val = -1
-            self.start_time_val = -1
-            self.end_reason_val = -1
+            self.median_before_val = 0.0
+            self.read_number_val = 0
+            self.start_mux_val = 0
+            self.start_time_val = 0
+            self.end_reason_val = 0
+            self.tracked_scaling_shift_val = 0.0
+            self.tracked_scaling_scale_val = 0.0
+            self.predicted_scaling_shift_val = 0.0
+            self.predicted_scaling_scale_val = 0.0
+            self.num_reads_since_mux_change_val = 0
+            self.time_since_mux_change_val = 0.0
+            self.num_minknow_events_val = 0
 
 
         # free memory
@@ -1895,7 +2101,32 @@ cdef class Open:
                      "read_number": type(10),
                      "start_mux": type(1),
                      "start_time": type(100),
-                     "end_reason": type(["a", "b", "c"])}
+                     "end_reason": type(["a", "b", "c"]),
+                     "tracked_scaling_shift": type(1.0),
+                     "tracked_scaling_scale": type(1.0),
+                     "predicted_scaling_shift": type(1.0),
+                     "predicted_scaling_scale": type(1.0),
+                     "num_reads_since_mux_change": type(1),
+                     "time_since_mux_change": type(1.0),
+                     "num_minknow_events": type(1),
+                     }
+        
+        aux_types_keys = [
+            "channel_number",
+            "median_before",
+            "read_number",
+            "start_mux",
+            "start_time",
+            "end_reason",
+            "tracked_scaling_shift",
+            "tracked_scaling_scale",
+            "predicted_scaling_shift",
+            "predicted_scaling_scale",
+            "num_reads_since_mux_change",
+            "time_since_mux_change",
+            "num_minknow_events",
+        ]
+
         # check if empty dic was given
         if aux is not None:
             if len(aux) == 0:
@@ -1922,6 +2153,13 @@ cdef class Open:
             self.start_mux_val_array = <uint8_t *> malloc(sizeof(uint8_t)*batch_len)
             self.start_time_val_array = <uint64_t *> malloc(sizeof(uint64_t)*batch_len)
             self.end_reason_val_array = <uint8_t *> malloc(sizeof(uint8_t)*batch_len)
+            self.tracked_scaling_shift_val_array = <float *> malloc(sizeof(float)*batch_len)
+            self.tracked_scaling_scale_val_array = <float *> malloc(sizeof(float)*batch_len)
+            self.predicted_scaling_shift_val_array = <float *> malloc(sizeof(float)*batch_len)
+            self.predicted_scaling_scale_val_array = <float *> malloc(sizeof(float)*batch_len)
+            self.num_reads_since_mux_change_val_array = <uint32_t *> malloc(sizeof(uint32_t)*batch_len)
+            self.time_since_mux_change_val_array = <float *> malloc(sizeof(float)*batch_len)
+            self.num_minknow_events_val_array = <uint64_t *> malloc(sizeof(uint64_t)*batch_len)
             for i, idx in enumerate(batch):
                 if aux is not None:
                     checked_record, checked_aux = self._multi_record_type_validation(records[idx], aux[idx])
@@ -1939,6 +2177,20 @@ cdef class Open:
                             self.start_time_val_array[i] = <uint64_t>checked_aux[a]
                         elif a == "end_reason":
                             self.end_reason_val_array[i] = <uint8_t>checked_aux[a]
+                        elif a == "tracked_scaling_shift":
+                            self.tracked_scaling_shift_val_array[i] = <float>checked_aux[a]
+                        elif a == "tracked_scaling_scale":
+                            self.tracked_scaling_scale_val_array[i] = <float>checked_aux[a]
+                        elif a == "predicted_scaling_shift":
+                            self.predicted_scaling_shift_val_array[i] = <float>checked_aux[a]
+                        elif a == "predicted_scaling_scale":
+                            self.predicted_scaling_scale_val_array[i] = <float>checked_aux[a]
+                        elif a == "num_reads_since_mux_change":
+                            self.num_reads_since_mux_change_val_array[i] = <uint32_t>checked_aux[a]
+                        elif a == "time_since_mux_change":
+                            self.time_since_mux_change_val_array[i] = <float>checked_aux[a]
+                        elif a == "num_minknow_events":
+                            self.num_minknow_events_val_array[i] = <uint64_t>checked_aux[a]
                     checked_auxs[idx] = checked_aux
                 else:
                     checked_record, checked_aux = self._record_type_validation(records[idx], aux)
@@ -1960,7 +2212,7 @@ cdef class Open:
                     if aux is not None:
                         if checked_aux is not None:
                             slow5_aux_types = self._aux_header_type_validation(aux_types)
-                            for a in slow5_aux_types:
+                            for a in aux_types_keys:
                                 if a not in checked_aux:
                                     continue
                                 elif checked_aux[a] is None:
@@ -2016,12 +2268,29 @@ cdef class Open:
 
                 self.logger.debug("write_record_batch: self.write processing raw_signal")
                 start_write_copy_signal = time.time()
-                # grabs buffer of numby array so the for loop operats in C not python = super fast
-                memview = memoryview(checked_records[batch[idx]]["signal"])
-                for i in range(checked_records[batch[idx]]["len_raw_signal"]):
-                    self.twrite[idx].raw_signal[i] = memview[i]
+
+                # check that data in numpy array is contiguous in memory, otherwise error out for now.
+                # need to know how often this actually happens.
+                # I think it shouldn't ever happen given we allocate the numpy arary all at once but...
+                # users could do some weird stuff, so better to check.
+
+
+                if checked_records[batch[idx]]["signal"].data.contiguous:
+                    self.temp_array = checked_records[batch[idx]]["signal"]
+                    num_elements = checked_records[batch[idx]]["signal"].size
+                    memcpy(self.twrite[idx].raw_signal, &self.temp_array[0], num_elements * sizeof(int16_t))
+                else:
+                    self.logger.warning("write_record_batch: numpy array of signal is not contiguous, please check your numpy array with np.info(array)")
+                    self.logger.warning("write_record_batch: falling back to old memory view element by element contruction. This is ~57 times slower...")
+                    memview = memoryview(checked_records[batch[idx]]["signal"])
+                    for i in range(checked_records[batch[idx]]["len_raw_signal"]):
+                        self.twrite[idx].raw_signal[i] = memview[i]
+                
+
                 # for i in range(checked_records[batch[idx]]["len_raw_signal"]):
                 #     self.twrite[idx].raw_signal[i] = checked_records[batch[idx]]["signal"][i]
+
+
                 end_write_copy_signal = (time.time() - start_write_copy_signal)
                 self.total_multi_write_signal_time = self.total_multi_write_signal_time + end_write_copy_signal
 
@@ -2057,6 +2326,20 @@ cdef class Open:
                             ret = slow5_aux_set(self.twrite[idx], self.start_time, <const void *>&self.start_time_val_array[idx], self.s5.header)
                         elif a == "end_reason":
                             ret = slow5_aux_set(self.twrite[idx], self.end_reason, <const void *>&self.end_reason_val_array[idx], self.s5.header)
+                        elif a == "tracked_scaling_shift":
+                            ret = slow5_aux_set(self.twrite[idx], self.tracked_scaling_shift, <const void *>&self.tracked_scaling_shift_val_array[idx], self.s5.header)
+                        elif a == "tracked_scaling_scale":
+                            ret = slow5_aux_set(self.twrite[idx], self.tracked_scaling_scale, <const void *>&self.tracked_scaling_scale_val_array[idx], self.s5.header)
+                        elif a == "predicted_scaling_shift":
+                            ret = slow5_aux_set(self.twrite[idx], self.predicted_scaling_shift, <const void *>&self.predicted_scaling_shift_val_array[idx], self.s5.header)
+                        elif a == "predicted_scaling_scale":
+                            ret = slow5_aux_set(self.twrite[idx], self.predicted_scaling_scale, <const void *>&self.predicted_scaling_scale_val_array[idx], self.s5.header)
+                        elif a == "num_reads_since_mux_change":
+                            ret = slow5_aux_set(self.twrite[idx], self.num_reads_since_mux_change, <const void *>&self.num_reads_since_mux_change_val_array[idx], self.s5.header)
+                        elif a == "time_since_mux_change":
+                            ret = slow5_aux_set(self.twrite[idx], self.time_since_mux_change, <const void *>&self.time_since_mux_change_val_array[idx], self.s5.header)
+                        elif a == "num_minknow_events":
+                            ret = slow5_aux_set(self.twrite[idx], self.num_minknow_events, <const void *>&self.num_minknow_events_val_array[idx], self.s5.header)
                         else:
                             ret = -1
                         if ret < 0:
@@ -2094,6 +2377,13 @@ cdef class Open:
                 free(self.start_mux_val_array)
                 free(self.start_time_val_array)
                 free(self.end_reason_val_array)
+                free(self.tracked_scaling_shift_val_array)
+                free(self.tracked_scaling_scale_val_array)
+                free(self.predicted_scaling_shift_val_array)
+                free(self.predicted_scaling_scale_val_array)
+                free(self.num_reads_since_mux_change_val_array)
+                free(self.time_since_mux_change_val_array)
+                free(self.num_minknow_events_val_array)
 
         end_multi_write = time.time() - start_multi_write
         self.total_multi_write_time = self.total_multi_write_time + end_multi_write
diff --git a/setup.py b/setup.py
index 0e85a81..7cde272 100644
--- a/setup.py
+++ b/setup.py
@@ -48,14 +48,15 @@ def build_ext(*args, ** kwargs ):
 # extra_compile_args = []
 # os.environ["CFLAGS"] = '-g -Wall -O2 -std=c99'
 
-arch=platform.machine()
-if arch in ["aarch64", "arm64"]:
-    extra_compile_args.append('-D__ARM_NEON__')
-elif arch in ["aarch64"]:
-	extra_compile_args.append('-mfpu=neon')
-elif arch in ["x86_64"]:
-    extra_compile_args.extend(['-DSTREAMVBYTE_SSSE3=1', '-mssse3'])   # WARNING: ancient x86_64 CPUs don't have SSSE3
-
+if platform.system() != 'Darwin':
+    arch=platform.machine()
+    if arch in ["aarch64"]:
+        extra_compile_args.append('-D__ARM_NEON__')
+    elif arch in ["armv7l"]:
+        extra_compile_args.append('-mfpu=neon')
+    elif arch in ["x86_64"]:
+        extra_compile_args.extend(['-DSTREAMVBYTE_SSSE3=1', '-mssse3'])   # WARNING: ancient x86_64 CPUs don't have SSSE3
+# note that on macos, for now we just let it compile without simd
 
 # include_dirs = ['include/', np.get_include(), 'thirdparty/streamvbyte/include']
 libraries = ['m', 'z']
@@ -88,7 +89,7 @@ def readme():
 
 setup(
     name = 'pyslow5',
-    version='1.0.0-dirty',
+    version='1.0.0-dev',
     url = 'https://github.com/hasindu2008/slow5lib',
     description='slow5lib python bindings',
     long_description=readme(),
diff --git a/thirdparty/streamvbyte/Makefile b/thirdparty/streamvbyte/Makefile
index 4e52fa4..b9ff8ee 100644
--- a/thirdparty/streamvbyte/Makefile
+++ b/thirdparty/streamvbyte/Makefile
@@ -24,7 +24,7 @@ else
 CFLAGS = -fPIC -std=c99 -O3 -Wall -Wextra -pedantic -Wshadow
 endif
 endif
-
+# note that on macos, it is arm64 instead of aarch64. however, for now we just let it compile without simd
 
 #LDFLAGS = -shared
 LIBNAME=libstreamvbyte.so.0.0.1


Test patch:
diff --git a/python/test.py b/python/test.py
index 91d529b..00b6d93 100644
--- a/python/test.py
+++ b/python/test.py
@@ -402,7 +402,14 @@ def test_get_empty_aux(self):
                "read_number": None,
                "start_mux": None,
                "start_time": None,
-               "end_reason": None}
+               "end_reason": None,
+               "tracked_scaling_shift": None,
+               "tracked_scaling_scale": None,
+               "predicted_scaling_shift": None,
+               "predicted_scaling_scale": None,
+               "num_reads_since_mux_change": None,
+               "time_since_mux_change": None,
+               "num_minknow_events": None}
         _, aux = self.F.get_empty_record(aux=True)
         for i, rec in enumerate(aux):
             with self.subTest(i=i, attr=rec):