Repository: cryoem-uoft/cryosparc-tools (language: python)

Issue:
pip install -e ".[dev]" only works up to python <= 3.10
When running

`pip install -e ".[dev]"`

with python 3.11 it will fail with following error (3.10 is fine). Can be fixed with installing python-snappy with conda.

`conda install python=3.11 python-snappy  -c conda-forge`

which will install python-snappy 0.6.1. Not sure that it is a valid fix though.

cheers
Kilian

```
Building wheels for collected packages: python-snappy, cryosparc-tools
  Building wheel for python-snappy (setup.py) ... error
  error: subprocess-exited-with-error
  
  × python setup.py bdist_wheel did not run successfully.
  │ exit code: 1
  ╰─> [27 lines of output]
      /sbdata/home/kischnel/.conda/envs/ctools/lib/python3.11/site-packages/setuptools/_distutils/dist.py:265: UserWarning: Unknown distribution option: 'cffi_modules'
        warnings.warn(msg)
      running bdist_wheel
      running build
      running build_py
      creating build
      creating build/lib.linux-x86_64-cpython-311
      creating build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/__init__.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_cffi_builder.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_formats.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_cffi.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/__main__.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/hadoop_snappy.py -> build/lib.linux-x86_64-cpython-311/snappy
      running build_ext
      building 'snappy._snappy' extension
      creating build/temp.linux-x86_64-cpython-311
      creating build/temp.linux-x86_64-cpython-311/src
      creating build/temp.linux-x86_64-cpython-311/src/snappy
      gcc -pthread -B /sbdata/home/kischnel/.conda/envs/ctools/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -I/sbdata/home/kischnel/.conda/envs/ctools/include/python3.11 -c src/snappy/crc32c.c -o build/temp.linux-x86_64-cpython-311/src/snappy/crc32c.o
      gcc -pthread -B /sbdata/home/kischnel/.conda/envs/ctools/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -I/sbdata/home/kischnel/.conda/envs/ctools/include/python3.11 -c src/snappy/snappymodule.cc -o build/temp.linux-x86_64-cpython-311/src/snappy/snappymodule.o
      src/snappy/snappymodule.cc:33:10: fatal error: snappy-c.h: No such file or directory
         33 | #include <snappy-c.h>
            |          ^~~~~~~~~~~~
      compilation terminated.
      error: command '/usr/bin/gcc' failed with exit code 1
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
  ERROR: Failed building wheel for python-snappy
  Running setup.py clean for python-snappy
  Building editable for cryosparc-tools (pyproject.toml) ... done
  Created wheel for cryosparc-tools: filename=cryosparc_tools-4.1.1-0.editable-cp311-cp311-linux_x86_64.whl size=17730 sha256=06b15e7d5cb50bcded190f26d9b23f783594c0c2c9d019860592b0f2d8d1d40f
  Stored in directory: /tmp/pip-ephem-wheel-cache-f5q1jv4x/wheels/a4/39/9c/6445fa63949882abccd53287e59d5e255529d31e757bb1927b
Successfully built cryosparc-tools
Failed to build python-snappy
Installing collected packages: python-snappy, py-cpuinfo, mypy-extensions, typing-extensions, ruff, psutil, pluggy, platformdirs, pep517, pathspec, packaging, numpy, nodeenv, iniconfig, httpretty, execnet, cython, click, attrs, pytest, pyright, cryosparc-tools, build, black, pytest-xdist, pytest-benchmark
  Running setup.py install for python-snappy ... error
  error: subprocess-exited-with-error
  
  × Running setup.py install for python-snappy did not run successfully.
  │ exit code: 1
  ╰─> [29 lines of output]
      /sbdata/home/kischnel/.conda/envs/ctools/lib/python3.11/site-packages/setuptools/_distutils/dist.py:265: UserWarning: Unknown distribution option: 'cffi_modules'
        warnings.warn(msg)
      running install
      /sbdata/home/kischnel/.conda/envs/ctools/lib/python3.11/site-packages/setuptools/command/install.py:34: SetuptoolsDeprecationWarning: setup.py install is deprecated. Use build and pip and other standards-based tools.
        warnings.warn(
      running build
      running build_py
      creating build
      creating build/lib.linux-x86_64-cpython-311
      creating build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/__init__.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_cffi_builder.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_formats.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/snappy_cffi.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/__main__.py -> build/lib.linux-x86_64-cpython-311/snappy
      copying src/snappy/hadoop_snappy.py -> build/lib.linux-x86_64-cpython-311/snappy
      running build_ext
      building 'snappy._snappy' extension
      creating build/temp.linux-x86_64-cpython-311
      creating build/temp.linux-x86_64-cpython-311/src
      creating build/temp.linux-x86_64-cpython-311/src/snappy
      gcc -pthread -B /sbdata/home/kischnel/.conda/envs/ctools/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -I/sbdata/home/kischnel/.conda/envs/ctools/include/python3.11 -c src/snappy/crc32c.c -o build/temp.linux-x86_64-cpython-311/src/snappy/crc32c.o
      gcc -pthread -B /sbdata/home/kischnel/.conda/envs/ctools/compiler_compat -DNDEBUG -fwrapv -O2 -Wall -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -O2 -isystem /sbdata/home/kischnel/.conda/envs/ctools/include -fPIC -I/sbdata/home/kischnel/.conda/envs/ctools/include/python3.11 -c src/snappy/snappymodule.cc -o build/temp.linux-x86_64-cpython-311/src/snappy/snappymodule.o
      src/snappy/snappymodule.cc:33:10: fatal error: snappy-c.h: No such file or directory
         33 | #include <snappy-c.h>
            |          ^~~~~~~~~~~~
      compilation terminated.
      error: command '/usr/bin/gcc' failed with exit code 1
      [end of output]
  
  note: This error originates from a subprocess, and is likely not a problem with pip.
error: legacy-install-failure

× Encountered error while trying to install package.
╰─> python-snappy

note: This is an issue with the package mentioned above, not pip.
hint: See above for output from the failure.
```


Discussion / hints:
Hi @KiSchnelle, thanks for reporting. We're already tracking this issue with Python 3.11 and hope to address in a future revision. Your workaround of installing python-snappy with conda is valid, you may proceed with that if you don't run into further issues.

Fix patch:
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 4e7f07e4..77322fdc 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -25,6 +25,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - uses: actions/setup-python@v4
         with:
           python-version: "3.10"
@@ -60,6 +62,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -89,6 +93,8 @@ jobs:
         os: ["ubuntu-20.04", "macos-11", "windows-2019"]
     steps:
       - uses: actions/checkout@v3
+        with:
+          submodules: true
       - uses: pypa/cibuildwheel@v2.11.2
         env:
           CIBW_SKIP: cp36-* pp*-win* pp*-macosx* *_i686 cp3{8,9,10,11}-macosx_x86_64
diff --git a/.gitignore b/.gitignore
index 10a91ba6..cfd8b2bd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,4 +67,3 @@ ipython_config.py
 .vercel
 cryosparc/core.c
 *.dSYM
-Python-*
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 00000000..bcdb7fca
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "cryosparc/include/lz4"]
+	path = cryosparc/include/lz4
+	url = https://github.com/lz4/lz4.git
diff --git a/MANIFEST.in b/MANIFEST.in
index b5819a3c..6438bc9e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,8 @@
 include cryosparc/core.pyx
 include cryosparc/dataset.c
 include cryosparc/dataset.pxd
+include cryosparc/lz4.pxd
 include cryosparc/include/cryosparc-tools/dataset.h
+include cryosparc/include/lz4/lib/lz4.h
+include cryosparc/include/lz4/lib/lz4.c
+include cryosparc/include/lz4/lib/LICENSE
diff --git a/Makefile b/Makefile
index 3fc9958e..739133bb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,5 @@
 TARGET=cryosparc/core.cpython-310-x86_64-linux-gnu.so  # for CryoSPARC
 
-# For vercel build
-PYTHON=python3.9
-PYTHON_VERSION=3.9.16
-
 all: $(TARGET)
 
 # -----------------------------------------------------------------------------
@@ -17,39 +13,22 @@ $(TARGET): cryosparc/include/cryosparc-tools/*.h cryosparc/dataset.c cryosparc/*
 #    Vercel deployment-related targets
 # -----------------------------------------------------------------------------
 
-Python-$(PYTHON_VERSION).tgz:
-	curl -L https://www.python.org/ftp/python/$(PYTHON_VERSION)/Python-$(PYTHON_VERSION).tgz -o Python-$(PYTHON_VERSION).tgz
-
-Python-$(PYTHON_VERSION): Python-$(PYTHON_VERSION).tgz
-	tar -xzf Python-${PYTHON_VERSION}.tgz
-
-python: Python-$(PYTHON_VERSION)
-	cd Python-$(PYTHON_VERSION) && ./configure && cd .. || cd ..
-	make -C Python-$(PYTHON_VERSION)
-	make -C Python-$(PYTHON_VERSION) altinstall
+/usr/local/bin/micromamba:
+	curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj -C /usr/local bin/micromamba
 
-.venv/bin/python:
-	$(PYTHON) -m venv .venv
-
-.venv/bin/pip: .venv/bin/python
-	.venv/bin/python -m pip install -U pip
-
-.venv/bin/jupyter-book: .venv/bin/pip
-	.venv/bin/pip install -e ".[build]"
+.venv:
+	micromamba create -p ./.venv -y -c conda-forge python=3.10 pip wheel cython numpy jupyter-book autodocsumm
+	micromamba run -p ./.venv pip install -e ".[build]"
 
 .vercel/output/config.json:
 	mkdir -p .vercel/output
-	echo '{"version":3,"cache":[".venv/**","build/**","docs/_build/**","Python-*.tgz","Python-*/**"]}' > .vercel/output/config.json
-
-verceldeps:
-	yum update -y
-	yum install bzip2-devel libffi-devel openssl-devel sqlite-devel -y
+	echo '{"version":3,"cache":["/usr/local/bin/micromamba", ".venv/**","build/**","docs/_build/**"]}' > .vercel/output/config.json
 
-vercelinstall: verceldeps python .venv/bin/python
+vercelinstall: /usr/local/bin/micromamba .venv
 	echo "Install complete"
 
-vercelbuild: .vercel/output/config.json .venv/bin/jupyter-book
-	.venv/bin/jupyter-book build docs
+vercelbuild: .vercel/output/config.json .venv
+	micromamba run -p ./.venv jupyter-book build docs
 	rm -rf .vercel/output/static && cp -R docs/_build/html .vercel/output/static
 
 # -----------------------------------------------------------------------------
@@ -63,6 +42,6 @@ clean:
 	rm -rf build
 	rm -rf dist
 	rm -rf *.egg-info
-	rm -rf Python-$(PYTHON_VERSION)
+	rm -rf .venv
 
-.PHONY: clean all python verceldeps vercelinstall vercelbuild
+.PHONY: clean all vercelinstall vercelbuild
diff --git a/README.md b/README.md
index c7b9e4fc..44b10efb 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Toolkit for interfacing with CryoSPARC. Read the documentation at
 
 1. Clone this repository
    ```sh
-   git clone https://github.com/cryoem-uoft/cryosparc-tools.git
+   git clone --recursive https://github.com/cryoem-uoft/cryosparc-tools.git
    cd cryosparc-tools
    git lfs pull
    ```
diff --git a/cryosparc/core.pyx b/cryosparc/core.pyx
index a65304f9..bc34e422 100644
--- a/cryosparc/core.pyx
+++ b/cryosparc/core.pyx
@@ -1,6 +1,10 @@
 from . cimport dataset
-from cython.view cimport array
-from cpython.ref cimport PyObject
+from . cimport lz4
+from libc.stdint cimport uint64_t
+from cpython.ref cimport PyObject, Py_XINCREF, Py_XDECREF
+from cpython.mem cimport PyMem_Realloc, PyMem_Free
+
+cdef int LZ4_ACCELERATION = 1000
 
 
 # Mirror of equivalent C-datatype enumeration
@@ -23,7 +27,6 @@ cpdef enum DsetType:
 
 cdef class Data:
     cdef dataset.Dset _handle
-    cdef dict _strcache
 
     def __cinit__(self, other = None):
         cdef Data othr
@@ -32,6 +35,7 @@ cdef class Data:
             # copy constructor
             othr = <Data> other
             self._handle = dataset.dset_copy(othr._handle)
+            othr._increfs()
         elif other:
             # Initialize with a numeric handle
             self._handle = <dataset.Dset> other
@@ -41,14 +45,53 @@ cdef class Data:
         if self._handle == 0:
             raise MemoryError()
 
-        self._strcache = dict()
-
     def __dealloc__(self):
-        if self._handle:
-            dataset.dset_del(self._handle)
+        cdef int nrow
+        cdef int coltype
+        cdef char *colkey
+        cdef PyObject **mem
+        cdef size_t size
+        cdef size_t itemsize
+        if not self._handle:
+            return
+
+        self._decrefs()
+        dataset.dset_del(self._handle)
+
+    def _increfs(self):
+        # Increment reference counts for all Python object fields.
+        # Call this after making a copy of this dataset.
+        nrow = dataset.dset_nrow(self._handle)
+        for i in xrange(dataset.dset_ncol(self._handle)):
+            with nogil:
+                colkey = dataset.dset_key(self._handle, i)
+                coltype = dataset.dset_type(self._handle, colkey)
+                if coltype != DsetType.T_OBJ:
+                    continue
+                mem = <PyObject **> (dataset.dset_get(self._handle, colkey))
+
+            for j in xrange(nrow):
+                Py_XINCREF(mem[j])
+
+    def _decrefs(self):
+        # Decrement reference counts for all Python object fields.
+        # Call this after destroying this dataset.
+        nrow = dataset.dset_nrow(self._handle)
+        for i in xrange(dataset.dset_ncol(self._handle)):
+            with nogil:
+                colkey = dataset.dset_key(self._handle, i)
+                coltype = dataset.dset_type(self._handle, colkey)
+                if coltype != DsetType.T_OBJ:
+                    continue
+                mem = <PyObject **> (dataset.dset_get(self._handle, colkey))
+
+            for j in xrange(nrow):
+                Py_XDECREF(mem[j])
 
     def innerjoin(self, str key, Data other):
-        return type(self)(dataset.dset_innerjoin(key.encode(), self._handle, other._handle))
+        cdef Data data = Data(dataset.dset_innerjoin(key.encode(), self._handle, other._handle))
+        data._increfs()
+        return data
 
     def totalsz(self):
         return dataset.dset_totalsz(self._handle)
@@ -84,7 +127,7 @@ cdef class Data:
 
     def getbuf(self, str colkey):
         cdef void *mem
-        cdef Py_ssize_t size
+        cdef size_t size
         cdef bytes colkey_b = colkey.encode()
         cdef const char *colkey_c = colkey_b
         with nogil:
@@ -95,7 +138,7 @@ cdef class Data:
         else:
             return <unsigned char [:size]> mem
 
-    def getstr(self, str colkey, Py_ssize_t index):
+    def getstr(self, str colkey, size_t index):
         return dataset.dset_getstr(self._handle, colkey.encode(), index)  # returns bytes
 
     def tocstrs(self, str colkey):
@@ -103,17 +146,20 @@ cdef class Data:
         cdef bytes colkey_b = colkey.encode()
         cdef const char *colkey_c = colkey_b
         cdef int prevtype = dataset.dset_type(self._handle, colkey_c)
-        cdef Py_ssize_t nrow = dataset.dset_nrow(self._handle)
+        cdef size_t nrow = dataset.dset_nrow(self._handle)
         cdef PyObject **pycol = <PyObject **> dataset.dset_get(self._handle, colkey_c)
-        cdef str pybytes
+        cdef str pystr
+        cdef bytes pybytes
 
         if prevtype != T_OBJ or not dataset.dset_changecol(self._handle, colkey_c, T_STR):
             return False
 
-        for i in range(nrow):
-            pybytes = <str> (pycol[i])
-            pycol[i] = NULL  # so string is not deallocated
-            dataset.dset_setstr(self._handle, colkey_c, i, pybytes.encode())
+        for i in xrange(nrow):
+            pystr = <str> pycol[i]
+            pybytes = pystr.encode()
+            Py_XDECREF(pycol[i])  # so string is deallocated
+            pycol[i] = NULL  # so that strfree not attempted
+            dataset.dset_setstr(self._handle, colkey_c, i, pybytes, len(pybytes))
 
         return True
 
@@ -122,9 +168,9 @@ cdef class Data:
         cdef bytes colkey_b = colkey.encode()
         cdef const char *colkey_c = colkey_b
         cdef int prevtype = dataset.dset_type(self._handle, colkey_c)
-        cdef Py_ssize_t nrow = dataset.dset_nrow(self._handle)
-        cdef Py_ssize_t *pycol = <Py_ssize_t *> dataset.dset_get(self._handle, colkey_c)
-        cdef void **pystrcol = <void **> pycol
+        cdef size_t nrow = dataset.dset_nrow(self._handle)
+        cdef size_t *pycol = <size_t *> dataset.dset_get(self._handle, colkey_c)
+        cdef PyObject **pystrcol = <PyObject **> pycol
         cdef dict strcache = dict()
         cdef char *cstr
         cdef bytes cbytes
@@ -133,11 +179,7 @@ cdef class Data:
         if prevtype != T_STR:
             return False
 
-        # Save computed strcache to prevent string handles from getting garbage
-        # collected (since no other Python reference to them is kept)
-        self._strcache[colkey] = strcache
-
-        for i in range(nrow):
+        for i in xrange(nrow):
             if pycol[i] in strcache:
                 pystr = strcache[pycol[i]]
             else:
@@ -145,18 +187,225 @@ cdef class Data:
                 pystr = cbytes.decode()
                 strcache[pycol[i]] = pystr
 
-            pystrcol[i] = <void *> pystr
+            pystrcol[i] = <PyObject *> pystr
+            Py_XINCREF(<PyObject *> pystr)
 
         if not dataset.dset_changecol(self._handle, colkey_c, T_OBJ):
             return False
 
         return True
 
+    def stralloc(self, str val):
+        cdef uint64_t idx
+        cdef bytes pybytes = val.encode()
+        if not dataset.dset_stralloc(self._handle, pybytes, len(pybytes), &idx):
+            raise MemoryError()
+        return <int> idx
+
+    def dump(self):
+        cdef void *mem
+        cdef size_t size
+        with nogil:
+            mem = dataset.dset_dump(self._handle)
+            size = dataset.dset_totalsz(self._handle)
+        return <unsigned char [:size]> mem
+
+    def dumpstrheap(self):
+        cdef void *mem
+        cdef size_t size
+        with nogil:
+            mem = dataset.dset_strheap(self._handle)
+            size = dataset.dset_strheapsz(self._handle)
+        return <unsigned char [:size]> mem
+
+    def setstrheap(self, bytes heap):
+        if not dataset.dset_setstrheap(self._handle, <const char *> heap, len(heap)):
+            raise MemoryError()
+
     def defrag(self, bint realloc_smaller):
         return dataset.dset_defrag(self._handle, realloc_smaller)
 
-    def dumptxt(self):
-        dataset.dset_dumptxt(self._handle)
+    def dumptxt(self, bint dump_data = 0):
+        dataset.dset_dumptxt(self._handle, dump_data)
 
     def handle(self):
         return self._handle
+
+
+cdef class Stream:
+    # Helper class for initializing a dataset from a compressed stream or
+    # generating a compressed stream for that dataset.
+    #
+    # WARNING: Methods here recycle result buffers to minimize allocations. If
+    # results are not consumed prior to future calls to this class, the contents
+    # will be overwritten.
+    cdef Data data
+    cdef char *buf  # internal compression buffer
+    cdef uint64_t *aux  # internal string index array
+    cdef size_t bufsz
+    cdef size_t auxsz
+
+    def __cinit__(self, Data data = None):
+        self.data = data
+        self.buf = NULL
+        self.aux = NULL
+        self.bufsz = 0
+        self.auxsz = 0
+
+    def __dealloc__(self):
+        if self.buf != NULL:
+            PyMem_Free(self.buf)
+            self.buf = NULL
+        if self.aux != NULL:
+            PyMem_Free(self.aux)
+            self.aux = NULL
+
+    def _ensure_buf(self, int min_len):
+        cdef size_t sz = min_len
+        if self.bufsz < sz:
+            self.buf = <char *> PyMem_Realloc(<void *> self.buf, sz)
+            self.bufsz = sz
+        return <int> self.bufsz
+
+    def _ensure_aux(self, int min_sz):
+        cdef size_t sz = min_sz
+        if self.auxsz < sz:
+            self.aux = <uint64_t *> PyMem_Realloc(<void *> self.aux, sz)
+            self.auxsz = sz
+        return <int> self.auxsz
+
+    def cast_objs_to_strs(self):
+        # change all T_OBJ column types to T_STR (underlying data unaffected)
+        cdef dataset.Dset handle = self.data._handle
+        cdef int ncol = self.data.ncol()
+        cdef int coltype
+        cdef const char *colkey
+
+        with nogil:
+            for i in xrange(ncol):
+                colkey = dataset.dset_key(handle, i)
+                coltype = dataset.dset_type(handle, colkey)
+                if coltype == T_OBJ:
+                    dataset.dset_changecol(handle, colkey, T_STR)
+
+    def stralloc_col(self, str col):
+        # Allocate C strings for every Python string in the given object column
+        # Returns 0 if the column does not have object type.
+        cdef int coltype = self.data.type(col)
+        if coltype != T_OBJ:
+            return 0  # invalid column
+
+        cdef PyObject **pycol
+        cdef unsigned char [:] data = self.data.getbuf(col)
+        cdef size_t sz = data.size
+
+        # Convert to C strings and compress index array instead
+        self._ensure_aux(sz)
+        pycol = <PyObject **> &data[0]
+
+        cdef uint64_t idx
+        cdef str pystr
+        cdef bytes pybytes
+        for i in xrange(sz // sizeof(uint64_t)):
+            pystr = <str> pycol[i]
+            pybytes = pystr.encode()
+            if not dataset.dset_stralloc(self.data._handle, pybytes, len(pybytes), &idx):
+                raise MemoryError()
+            self.aux[i] = idx
+
+        return <unsigned char [:sz]> (<unsigned char *> self.aux)
+
+    def compress_col(self, str col):
+        # Use when multiple compression calls are required to minimize total
+        # number of allocations.
+        #
+        # Overwrites (and potentially re-allocates) internal buffer each time
+        # it's called so only the latest return value is valid memory (until the
+        # instance is garbage collected).
+        #
+        # If the column is has type T_OBJ, allocates each python string as
+        # a C string in the string heap and compresses the resulting array of
+        # indexes into the C string.
+        #
+        # Returns array memoryview with compressed data
+        cdef int coltype = self.data.type(col)
+
+        if coltype == 0:
+            return 0  # invalid column
+
+        cdef PyObject **pycol
+        cdef unsigned char [:] data = self.data.getbuf(col)
+        cdef size_t sz = data.size
+        if coltype == T_OBJ:
+            # Convert to C strings and compress index array instead
+            self._ensure_aux(sz)
+            pycol = <PyObject **> &data[0]
+
+            for i in xrange(sz // sizeof(uint64_t)):
+                self.aux[i] = self.data.stralloc(<str> pycol[i])
+
+            data = <unsigned char [:sz]> (<unsigned char *> self.aux)
+
+        return self.compress(data)
+
+    def compress_numpy(self, arr):
+        cdef size_t sz = arr.size * arr.itemsize
+        cdef size_t arr_ptr_val = arr.ctypes.data
+        cdef void *arr_ptr = <void *> arr_ptr_val
+        return self.compress(<unsigned char [:sz]> arr_ptr)
+
+    def compress(self, unsigned char [:] data):
+        cdef int sz = data.size
+        cdef int max_compressed_sz = lz4.LZ4_compressBound(sz)
+
+        self._ensure_buf(max_compressed_sz)
+
+         # Call compress. Write final length into compressed_sz
+        cdef int compressed_sz = lz4.LZ4_compress_fast(
+            <const char *> &data[0],
+            self.buf,
+            sz,
+            max_compressed_sz,
+            LZ4_ACCELERATION
+        )
+        if compressed_sz > 0:
+            return <char [:compressed_sz]> self.buf
+
+        raise ValueError(f"Could not compress (error {compressed_sz})")
+
+    def decompress_col(self, str col, bytes data):
+        cdef void *mem
+        cdef size_t size
+        cdef bytes colkey_b = col.encode()
+        cdef const char *colkey_c = colkey_b
+        with nogil:
+            mem = dataset.dset_get(self.data._handle, colkey_c)
+            size = dataset.dset_getsz(self.data._handle, colkey_c)
+        if mem == NULL:
+            raise ValueError(f"Invalid column {col}")
+        else:
+            return self.decompress(data, <size_t> mem, size)
+
+    def decompress_numpy(self, bytes data, arr):
+        cdef size_t dstptr = arr.ctypes.data
+        cdef int size = arr.size
+        cdef int itemsize = arr.itemsize
+        return self.decompress(data, dstptr, size * itemsize)
+
+    def decompress(self, bytes data, size_t dstptr = 0, int dstsz = 0):
+        cdef const char *compressed = data
+        cdef int compressed_sz = len(data)
+        cdef char *uncompressed = <char *> dstptr
+        if dstsz <= 0:
+            raise ValueError("Decompression buffer size must be > 0")
+
+        if dstptr == 0:
+            self._ensure_buf(dstsz)
+            uncompressed = self.buf
+
+        cdef int uncompressed_sz = lz4.LZ4_decompress_safe(compressed, uncompressed, compressed_sz, dstsz)
+        if uncompressed_sz >= 0:
+            return <char [:uncompressed_sz]> uncompressed
+
+        raise ValueError(f"Could not decompress (error {uncompressed_sz})")
+
diff --git a/cryosparc/dataset.c b/cryosparc/dataset.c
index 959c42e8..ddf22b0d 100644
--- a/cryosparc/dataset.c
+++ b/cryosparc/dataset.c
@@ -1,2 +1,5 @@
+#define DSREALLOC PyMem_Realloc
+#define DSFREE PyMem_Free
 #define DATASET_IMPLEMENTATION
+#include <Python.h>
 #include <cryosparc-tools/dataset.h>
diff --git a/cryosparc/dataset.pxd b/cryosparc/dataset.pxd
index 7b585de4..2b037e7c 100644
--- a/cryosparc/dataset.pxd
+++ b/cryosparc/dataset.pxd
@@ -15,7 +15,7 @@ cdef extern from "cryosparc-tools/dataset.h":
     int dset_type(Dset dset, const char *colkey) nogil
     void *dset_get(Dset dset, const char *colkey) nogil
     uint64_t dset_getsz(Dset dset, const char *colkey) nogil
-    bint dset_setstr(Dset dset, const char *colkey, uint64_t index, const char *value) nogil
+    bint dset_setstr(Dset dset, const char *colkey, uint64_t index, const char *value, size_t length) nogil
     const char *dset_getstr(Dset dset, const char *colkey, uint64_t index) nogil
     uint32_t dset_getshp(Dset dset, const char *colkey) nogil
 
@@ -23,6 +23,12 @@ cdef extern from "cryosparc-tools/dataset.h":
     bint dset_addcol_scalar(Dset dset, const char *key, int type) nogil
     bint dset_addcol_array(Dset dset, const char *key, int type, int shape0, int shape1, int shape2) nogil
     bint dset_changecol(Dset dset, const char *key, int type) nogil
+
     bint dset_defrag(Dset dset, bint realloc_smaller) nogil
+    void *dset_dump(Dset dset) nogil
+    void dset_dumptxt(Dset dset, bint dump_data) nogil
 
-    void dset_dumptxt(Dset dset) nogil
+    uint64_t dset_strheapsz(Dset dset) nogil
+    char *dset_strheap(Dset dset) nogil
+    bint dset_setstrheap(Dset dset, const char *heap, size_t size) nogil
+    bint dset_stralloc(Dset dset, const char *value, size_t length, uint64_t *index) nogil
diff --git a/cryosparc/dataset.py b/cryosparc/dataset.py
index aa3bcf28..fb3b564f 100644
--- a/cryosparc/dataset.py
+++ b/cryosparc/dataset.py
@@ -33,6 +33,7 @@
     Collection,
     Dict,
     Generic,
+    Iterable,
     List,
     Mapping,
     MutableMapping,
@@ -47,11 +48,7 @@
 import numpy as n
 import numpy.core.records
 
-
-if TYPE_CHECKING:
-    from numpy.typing import NDArray, ArrayLike, DTypeLike
-
-from .core import Data, DsetType
+from .core import Data, DsetType, Stream
 from .dtype import (
     NEVER_COMPRESS_FIELDS,
     TYPE_TO_DSET_MAP,
@@ -71,6 +68,11 @@
 from .row import Row, Spool, R
 from .util import bopen, default_rng, hashcache, random_integers, u32bytesle, u32intle
 
+if TYPE_CHECKING:
+    from numpy.typing import NDArray, ArrayLike, DTypeLike
+    from .core import MemoryView
+
+
 # Save format options
 NUMPY_FORMAT = 1
 """
@@ -94,7 +96,7 @@
 
 FORMAT_MAGIC_PREFIXES = {
     NUMPY_FORMAT: b"\x93NUMPY",  # .npy file format
-    CSDAT_FORMAT: b"\x94CSDAT",  # .csl binary format
+    CSDAT_FORMAT: b"\x95CSDAT",  # .csl compressed stream format
 }
 MAGIC_PREFIX_FORMATS = {v: k for k, v in FORMAT_MAGIC_PREFIXES.items()}  # inverse dict
 
@@ -162,6 +164,54 @@ def allocate(cls, size: int = 0, fields: List[Field] = []):
         dset.add_fields(fields)
         return dset
 
+    def extend(self, *others: "Dataset", repeat_allowed=False):
+        """
+        Add the given dataset(s) to the end of the current dataset. Other
+        datasets must have at least the same fields of the current dataset.
+
+        Args:
+            repeat_allowed (bool, optional): If True, does not fail when there
+                are duplicate UIDs. Defaults to False.
+
+        Returns:
+            Dataset: current dataset with others appended
+
+        Examples:
+
+            >>> len(d1), len(d2), len(d3)
+            (42, 3, 5)
+            >>> d1.extend(d2, d3)
+            Dataset(...)
+            >>> len(d1)
+            50
+        """
+        datasets = tuple(d for d in others if len(d) > 0)  # skip empty datasets
+        if not datasets:
+            return self
+
+        if not repeat_allowed:
+            all_uids = n.concatenate([dset["uid"] for dset in datasets])
+            assert len(all_uids) == len(n.unique(all_uids)), "Cannot append datasets that contain the same UIDs."
+
+        required_fields = self.descr()
+        common_fields = self.common_fields(self, *others)
+        assert set(common_fields) == set(required_fields), (
+            "Cannot extend, some datasets are missing required target fields. "
+            f"Required fields: {required_fields}. "
+            f"Missing fields: {set(required_fields).difference(common_fields)}"
+        )
+
+        extra_size = sum(len(d) for d in datasets)
+        startidx = len(self)
+        self._data.addrows(extra_size)
+        for dset in datasets:
+            num = len(dset)
+            for key, *_ in required_fields:
+                self[key][startidx : startidx + num] = dset[key]
+            startidx += num
+
+        return self
+
     def append(self, *others: "Dataset", assert_same_fields=False, repeat_allowed=False):
         """
         Concatenate many datasets together into one new one.
@@ -472,7 +522,7 @@ def common_fields(cls, *datasets: "Dataset", assert_same_fields=False) -> List[F
         return [f for f in datasets[0].descr() if f in fields]
 
     @classmethod
-    def load(cls, file: Union[str, PurePath, IO[bytes]]):
+    def load(cls, file: Union[str, PurePath, IO[bytes]], cstrs: bool = False):
         """
         Read a dataset from path or file handle.
 
@@ -484,6 +534,8 @@ def load(cls, file: Union[str, PurePath, IO[bytes]]):
             file (str | Path | IO): Readable file path or handle. Must be
                 seekable if loading a dataset saved in the default
                 ``NUMPY_FORMAT``
+            cstrs (bool): If True, load internal string columns as C strings
+                instead of Python strings. Defaults to False.
 
         Raises:
             TypeError: If cannot determine type of dataset file.
@@ -497,37 +549,68 @@ def load(cls, file: Union[str, PurePath, IO[bytes]]):
             if prefix == FORMAT_MAGIC_PREFIXES[NUMPY_FORMAT]:
                 f.seek(0)
                 indata = n.load(f, allow_pickle=False)
-                return cls(indata)
+                dset = cls(indata)
+                if cstrs:
+                    dset.to_cstrs()
+                return dset
             elif prefix == FORMAT_MAGIC_PREFIXES[CSDAT_FORMAT]:
-                import snappy
-
                 headersize = u32intle(f.read(4))
                 header = decode_dataset_header(f.read(headersize))
-                cols = {}
+
+                # Calling addrows separately to minimizes column-based
+                # allocations, improves performance by ~20%
+                dset = cls.allocate(0, header["dtype"])
+                data = dset._data
+                data.addrows(header["length"])
+                loader = Stream(data)
                 for field in header["dtype"]:
                     colsize = u32intle(f.read(4))
                     buffer = f.read(colsize)
                     if field[0] in header["compressed_fields"]:
-                        buffer = snappy.uncompress(buffer)
-                    cols[field[0]] = n.frombuffer(buffer, dtype=fielddtype(field))
-                return cls(cols)
+                        loader.decompress_col(field[0], buffer)
+                    else:
+                        data.getbuf(field[0])[:] = buffer
+
+                # Read in the string heap (rest of stream)
+                # NOTE: There will be a bug here for long column keys that are
+                # added when there's already an allocated string in a T_STR
+                # column in the saved dataset (should be rare).
+                heap = f.read()
+                data.setstrheap(heap)
+
+                # Convert C strings to Python strings
+                loader.cast_objs_to_strs()  # dtype may be T_OBJ but actually all are T_STR
+                if not cstrs:
+                    dset.to_pystrs()
+                return dset
 
         raise TypeError(f"Could not determine dataset format for file {file} (prefix is {prefix})")
 
     @classmethod
     async def from_async_stream(cls, stream: AsyncBinaryIO):
-        import snappy
-
         headersize = u32intle(await stream.read(4))
         header = decode_dataset_header(await stream.read(headersize))
-        cols = {}
+
+        # Calling addrows separately to minimizes column-based allocations
+        dset = cls.allocate(0, header["dtype"])
+        data = dset._data
+        data.addrows(header["length"])
+        loader = Stream(data)
         for field in header["dtype"]:
             colsize = u32intle(await stream.read(4))
             buffer = await stream.read(colsize)
             if field[0] in header["compressed_fields"]:
-                buffer = snappy.uncompress(buffer)
-            cols[field[0]] = n.frombuffer(buffer, dtype=fielddtype(field))
-        return cls(cols)
+                loader.decompress_col(field[0], buffer)
+            else:
+                data.getbuf(field[0])[:] = buffer
+
+        heap = stream.read()
+        data.setstrheap(heap)
+
+        # Convert C strings to Python strings
+        loader.cast_objs_to_strs()  # dtype may be T_OBJ but actually all are T_STR
+        dset.to_pystrs()
+        return dset
 
     def save(self, file: Union[str, PurePath, IO[bytes]], format: int = DEFAULT_FORMAT):
         """
@@ -552,12 +635,12 @@ def save(self, file: Union[str, PurePath, IO[bytes]], format: int = DEFAULT_FORM
                 n.save(f, outdata, allow_pickle=False)
         elif format == CSDAT_FORMAT:
             with bopen(file, "wb") as f:
-                for chunk in self.stream():
+                for chunk in self.stream(compression="lz4"):
                     f.write(chunk)
         else:
             raise TypeError(f"Invalid dataset save format for {file}: {format}")
 
-    def stream(self):
+    def stream(self, compression: Literal["lz4", None] = None) -> Iterable[Union[bytes, memoryview, "MemoryView"]]:
         """
         Generate a binary representation for this dataset. Results may be
         written to a file or buffer to be sent over the network.
@@ -569,28 +652,31 @@ def stream(self):
         Yields:
             bytes: Dataset file chunks
         """
-        import snappy
-
-        cols = self.cols()
-        arrays = [cols[c].to_fixed() for c in cols]
-        descr = [makefield(f, arraydtype(a)) for f, a in zip(cols, arrays)]
+        data = self._data
+        stream = Stream(data)
 
         yield FORMAT_MAGIC_PREFIXES[CSDAT_FORMAT]
 
-        compressed_fields = [col for col in cols if col not in NEVER_COMPRESS_FIELDS]
+        compressed_fields = [f for f in self if compression and f not in NEVER_COMPRESS_FIELDS]
         header = encode_dataset_header(
-            DatasetHeader(dtype=descr, compression="snap", compressed_fields=compressed_fields)
+            DatasetHeader(
+                length=len(self), dtype=self.descr(), compression=compression, compressed_fields=compressed_fields
+            )
         )
         yield u32bytesle(len(header))
         yield header
 
-        for f, arr in zip(cols, arrays):
-            if f in NEVER_COMPRESS_FIELDS:
-                fielddata = arr.data.tobytes()
+        for f in self:
+            fielddata: "MemoryView"
+            if f in compressed_fields:
+                # obj columns added to strheap and loaded as indexes
+                fielddata = stream.compress_col(f)
             else:
-                fielddata: bytes = snappy.compress(arr.data)
+                fielddata = stream.stralloc_col(f) or data.getbuf(f)
             yield u32bytesle(len(fielddata))
-            yield fielddata
+            yield fielddata.memview
+
+        yield data.dumpstrheap().memview
 
     def __init__(
         self,
@@ -653,9 +739,10 @@ def __init__(
             populate.insert(0, (("uid", "<u8"), generate_uids(nrows)))
 
         self.add_fields([entry[0] for entry in populate])
-        self._data.addrows(nrows)
-        for field, data in populate:
-            self[field[0]] = data
+        if nrows > 0:
+            self._data.addrows(nrows)
+            for field, data in populate:
+                self[field[0]] = data
 
     def __len__(self):
         """
@@ -914,7 +1001,7 @@ def add_fields(
                 ('uid', [14727850622008419978 309606388100339041 15935837513913527085]),
                 ('foo', [0 0 0]),
                 ('bar', [[0. 0.] [0. 0.] [0. 0.]]),
-                ('baz', ["", "", ",]),
+                ('baz', ["" "" ""]),
             ])
         """
         if len(fields) == 0:
diff --git a/cryosparc/dtype.py b/cryosparc/dtype.py
index 9e5eef2d..c87b5454 100644
--- a/cryosparc/dtype.py
+++ b/cryosparc/dtype.py
@@ -43,8 +43,9 @@ class DatasetHeader(TypedDict):
     Dataset header description when saving in CSDAT format.
     """
 
+    length: int
     dtype: List[Field]
-    compression: Literal["snap"]
+    compression: Literal["lz4", None]
     compressed_fields: List[str]
 
 
@@ -130,20 +131,25 @@ def decode_dataset_header(data: Union[bytes, dict]) -> DatasetHeader:
     try:
         header = json.loads(data) if isinstance(data, bytes) else data
         assert isinstance(header, dict), f"Incorrect decoded header type (expected dict, got {type(header)})"
+        assert "length" in header and isinstance(
+            header["length"], int
+        ), 'Dataset header "length" key missing or has incorrect type'
         assert "dtype" in header and isinstance(
             header["dtype"], list
         ), 'Dataset header "dtype" key missing or has incorrect type'
-        assert (
-            "compression" in header and header["compression"] == "snap"
-        ), 'Dataset header "compression" key missing or has incorrect type'
+        assert "compression" in header and header["compression"] in {
+            None,
+            "lz4",
+        }, 'Dataset header "compression" key missing or has incorrect type'
         assert (
             "compressed_fields" and header or isinstance(header["compressed_fields"], list)
         ), 'Dataset header "compressed_fields" key missing or has incorrect type'
 
+        length: int = header["length"]
         dtype: List[Field] = [(f, d, tuple(rest[0])) if rest else (f, d) for f, d, *rest in header["dtype"]]
-        compression: Literal["snap"] = header["compression"]
+        compression: Literal["lz4"] = header["compression"]
         compressed_fields: List[str] = header["compressed_fields"]
 
-        return DatasetHeader(dtype=dtype, compression=compression, compressed_fields=compressed_fields)
+        return DatasetHeader(length=length, dtype=dtype, compression=compression, compressed_fields=compressed_fields)
     except Exception as e:
         raise ValueError(f"Incorrect dataset field format: {data.decode() if isinstance(data, bytes) else data}") from e
diff --git a/cryosparc/include/cryosparc-tools/dataset.h b/cryosparc/include/cryosparc-tools/dataset.h
index 1b5e53b4..18c81ecd 100755
--- a/cryosparc/include/cryosparc-tools/dataset.h
+++ b/cryosparc/include/cryosparc-tools/dataset.h
@@ -5,14 +5,14 @@
 #include <stdint.h>    // fixed width integer types
 #include <inttypes.h>  // printf specifiers for fixed width integer types
 
-#define repr(x,a,b,val) val 
+#define repr(x,a,b,val) val
 
 #ifdef _WIN32
 typedef _Fcomplex ds_float_complex_t;
 typedef _Dcomplex ds_double_complex_t;
 #else
-typedef float complex ds_float_complex_t;
-typedef double complex ds_double_complex_t;
+typedef float complex ds_float_complex_t; // NOLINT
+typedef double complex ds_double_complex_t; // NOLINT
 #endif
 
 #define DSET_TYPELIST(X) \
@@ -61,7 +61,7 @@ const char* dset_key    (uint64_t dset, uint64_t index);
 int         dset_type   (uint64_t dset, const char * colkey);
 void *      dset_get    (uint64_t dset, const char * colkey);
 uint64_t    dset_getsz  (uint64_t dset, const char * colkey);
-int         dset_setstr (uint64_t dset, const char * colkey, uint64_t index, const char * value);
+int         dset_setstr (uint64_t dset, const char * colkey, uint64_t index, const char * value, size_t length);
 const char* dset_getstr (uint64_t dset, const char * colkey, uint64_t index);
 uint32_t    dset_getshp (uint64_t dset, const char * colkey);
 
@@ -71,10 +71,16 @@ int        dset_addcol_array  (uint64_t dset, const char * key, int type, int sh
 int        dset_changecol     (uint64_t dset, const char * key, int type);
 
 int        dset_defrag (uint64_t dset, int realloc_smaller);
-void       dset_dumptxt (uint64_t dset);
+void       dset_dumptxt (uint64_t dset, int dump_data);
 void *     dset_dump (uint64_t dset);
 
+uint64_t   dset_strheapsz (uint64_t dset);
+char *     dset_strheap (uint64_t dset);
+int        dset_setstrheap (uint64_t dset, const char *heap, size_t length);
+int        dset_stralloc (uint64_t dset, const char *value, size_t length, uint64_t *index);
+
 #endif
+
 /*
 ===============================================================================
 
@@ -149,14 +155,14 @@ typedef	int ds_mutex_lock_t;
 
 #ifndef DSPRINTERR
 #define DSPRINTERR(str) do{fputs(str, stderr);fflush(stderr);}while(0);
-#endif 
+#endif
 
 
 
 /*
 	Default error message logging actions
 */
-static void 
+static void
 #if defined(__clang__) || defined(__GNUC__)
 __attribute__ ((format (printf, 1, 2)))
 #endif
@@ -216,9 +222,9 @@ typedef struct {
 		char shortkey[SHORTKEYSZ];
 		uint64_t longkey;
 	};
-	int8_t type; // magnitude matches enum dset_type. negative means use longkey  
+	int8_t type; // magnitude matches enum dset_type. negative means use longkey
 	uint8_t shape[3]; // safe to leave as zero for scalars
-	uint64_t offset;  // relative to start of respective heap 
+	uint64_t offset;  // relative to start of respective heap
 
 } ds_column;
 
@@ -236,29 +242,26 @@ typedef struct {
 	uint64_t   nrow;  // actual number of rows
 	uint64_t   arrheap_start; // offset where column data begins
 	uint64_t   strheap_start; // offset where string (and other data structure) heap begins
-	uint64_t   strheap_sz; // 
-	struct {
-		// lets track the number of times the more expensive operations occurr
-		// in the future we can improve the implementation based on what actually happens a lot
-		uint32_t nrealloc;   
-		uint32_t nreassign_arroffsets;
-		uint32_t nshift_strhandles;
-		uint32_t nmore_arrheap;
-		uint32_t nmore_strheap;
-		uint32_t nmore_colspace;
-	} stats;
+	uint64_t   strheap_sz; //
 	ds_column  columns[];
-
 } ds;
 
 
+typedef uint64_t ds_ht64_row[2];
+
+typedef struct ds_ht64 {
+	ds_ht64_row *ht;
+	int32_t len;
+	int32_t exp;
+} ds_ht64;
+
 /*
 	We'll be managing datasets via integer handles instead of via pointers.
 	The handle is a composite value containing an index and a "generation" counter.
 	The latter is in the upper few bits (15 by default).
 
-	The index is just an index into an array in memory. 
-	The generation counter is a way to make sure that if an index gets re-used, 
+	The index is just an index into an array in memory.
+	The generation counter is a way to make sure that if an index gets re-used,
 	we can tell if an old handle that points to this index does indeed refer to this dset.
 	This detects the case the user accidentally kept a handle around after deleting a dset.
 */
@@ -266,8 +269,22 @@ typedef struct {
 typedef struct {
 
 	ds         *memory;
+	ds_ht64    ht;  // hash table for fast string lookups. Keys are string
+					// hashes and values are an index into the dataset string
+					// heap. note that values are not removed until the whole
+					// dataset slot is deallocated
 	uint16_t   generation;
-
+	struct {
+		// lets track the number of times the more expensive operations occurr
+		// in the future we can improve the implementation based on what actually happens a lot
+		uint32_t nrealloc;
+		uint32_t nreassign_arroffsets;
+		uint32_t nshift_strhandles;
+		uint32_t nmore_arrheap;
+		uint32_t nmore_strheap;
+		uint32_t nmore_colspace;
+		uint32_t htnrealloc; // hash table reallocation count
+	} stats;
 } ds_slot;
 
 
@@ -351,7 +368,7 @@ unlock (void) {
 }
 
 
-static void 
+static void
 moreslots (void) {
 	static const size_t chunk = 4096;
 	void * mem = DSREALLOC(ds_module.slots, sizeof(ds_module.slots[0]) * (ds_module.nslots+chunk));
@@ -372,8 +389,10 @@ roundup(uint64_t value, uint64_t to)
 	return value+to-(value%to);
 }
 
-static uint64_t 
-dset_new_(size_t newsize, ds **allocation) 
+static int ht64_realloc(ds_ht64 *t, uint32_t sz); // implementation is below
+
+static uint64_t
+dset_new_(size_t newsize, ds **allocation)
 {
 	module_init();
 	lock();
@@ -392,7 +411,7 @@ dset_new_(size_t newsize, ds **allocation)
 	if (i == ds_module.nslots)
 		moreslots();
 
-	if (i == ds_module.nslots) 
+	if (i == ds_module.nslots)
 		goto out_of_memory;
 
 
@@ -406,6 +425,12 @@ dset_new_(size_t newsize, ds **allocation)
 	unlock();
 
 	memset(s->memory, 0, newsize);
+	memset(&(s->stats), 0, sizeof(s->stats));
+
+	// Alloc hash table for fast field name and C string lookup
+	s->ht.ht = 0;
+	ht64_realloc(&(s->ht), 255);
+
 	if (s->generation >= MAX_GEN) {
 		// Generation limit reached, trigger overflow so that handle != 0
 		s->generation = 0;
@@ -413,7 +438,7 @@ dset_new_(size_t newsize, ds **allocation)
 	gen = ++s->generation;
 
 	return i | (gen << SHIFT_GEN);
-	 
+
 	out_of_memory:
 	unlock();
 	nonfatal("out of memory");
@@ -422,8 +447,8 @@ dset_new_(size_t newsize, ds **allocation)
 
 
 
-static ds* 
-handle_lookup (uint64_t h, const char * msg_fragment, uint16_t * gen, uint64_t * idx) 
+static ds*
+handle_lookup (uint64_t h, const char * msg_fragment, uint16_t * gen, uint64_t * idx)
 {
 	uint16_t gen_ = 0;
 	uint64_t idx_ = 0;
@@ -440,14 +465,14 @@ handle_lookup (uint64_t h, const char * msg_fragment, uint16_t * gen, uint64_t *
 		return 0;
 	}
 
-	if (!ds_module.slots[*idx].memory) { 
+	if (!ds_module.slots[*idx].memory) {
 		nonfatal("%s: invalid handle %" PRIu64 ", no heap at index %" PRIu64, msg_fragment, h, *idx);
 		return 0;
 	}
 
 	if (ds_module.slots[*idx].generation != *gen) {
 		nonfatal("%s: invalid handle %" PRIu64 ", wrong generation counter"
-				" (given %" PRIu16 ", expected %" PRIu16")", 
+				" (given %" PRIu16 ", expected %" PRIu16")",
 				msg_fragment, h, *gen, ds_module.slots[*idx].generation);
 		return 0;
 	}
@@ -466,14 +491,14 @@ handle_lookup (uint64_t h, const char * msg_fragment, uint16_t * gen, uint64_t *
 
 
 #define EMIT_TYPEENUM_ONLY(typeenum,a,b,c,d,e) typeenum,
-static const 
+static const
 int valid_types[] = { DSET_TYPELIST(EMIT_TYPEENUM_ONLY) };
 
-static const 
+static const
 size_t Ntypes = sizeof(valid_types)/sizeof(valid_types[0]);
 
 #define EMIT_SZ_ARRAY_ENTRY(typeenum,a,ctype,b,c,d) [typeenum]=sizeof(ctype),
-static const 
+static const
 size_t type_size[] = { DSET_TYPELIST(EMIT_SZ_ARRAY_ENTRY) };
 
 #define EMIT_ALIGNCHECK(a,name,ctype,b,c,d) static_assert(DSALIGNOF(ctype) <= 16, "platform incompatible");
@@ -491,9 +516,9 @@ static int
 tcheck(int8_t type)
 {
 	const int t  = abs_i8(type);
-	
-	for (int i = 0; i < Ntypes; i++) 
-		if (t == valid_types[i]) 
+
+	for (int i = 0; i < Ntypes; i++)
+		if (t == valid_types[i])
 			return 1;
 
 	return 0;
@@ -512,24 +537,26 @@ stride (const ds_column *c)
 }
 
 static const char *
-getkey(const ds *d, const ds_column *c) 
+getkey(const ds *d, const ds_column *c)
 {
 	char * ptr = (char *)d;
 	const char * key = (c->type < 0) ? ptr + d->strheap_start + c->longkey : c->shortkey;
 	return key;
 }
 
-static ds_column * 
-column_lookup(ds * d, const char * colkey)
+static ds_column *
+column_lookup(ds * d, const char *colkey, size_t *idx)
 {
 	if(!d) return 0;
 
 	char * ptr = (char *)d;
 	ds_column *c = d->columns;
-	long i = 0;
-	for (;  i < d->ncol;  i++,c++) {
+	for (size_t i = 0; i < d->ncol; i++, c++) {
 		const char * key = getkey(d, c);
-		if (!strcmp(key, colkey)) return c;
+		if (!strcmp(key, colkey)) {
+			if (idx != NULL) *idx = i;
+			return c;
+		}
 	}
 
 	// nonfatal("key error: %s", colkey);
@@ -537,7 +564,7 @@ column_lookup(ds * d, const char * colkey)
 }
 
 
-static inline uint64_t 
+static inline uint64_t
 arrheap_capacity (const ds *d) {
 	return d->strheap_start - d->arrheap_start;
 }
@@ -559,8 +586,9 @@ more_memory (uint64_t idx, uint64_t nbytes_more) {
 
 	const uint64_t more = roundup(nbytes_more, 1<<15); // 32 kB at a time (too little? too much?)
 
-	ds *d = ds_module.slots[idx].memory;
-	d->stats.nrealloc++;
+	ds_slot *slot = &ds_module.slots[idx];
+	ds *d = slot->memory;
+	slot->stats.nrealloc++;
 
 	ds * newptr = DSREALLOC(d, d->total_sz + more);
 	if (!newptr) {
@@ -568,7 +596,7 @@ more_memory (uint64_t idx, uint64_t nbytes_more) {
 		return 0;
 	}
 
-	ds_module.slots[idx].memory = d = newptr;
+	slot->memory = d = newptr;
 
 	char * ptr = (char *) newptr;
 	memset(ptr + d->total_sz, 0, more);
@@ -578,11 +606,12 @@ more_memory (uint64_t idx, uint64_t nbytes_more) {
 	return d;
 }
 
-static ds* 
-more_strheap (uint64_t idx, uint64_t nbytes_more) {
+static ds*
+more_strheap (uint64_t dsetidx, uint64_t nbytes_more) {
 
-	ds *d = ds_module.slots[idx].memory;
-	d->stats.nmore_strheap++;
+	ds_slot *slot = &ds_module.slots[dsetidx];
+	ds *d = slot->memory;
+	slot->stats.nmore_strheap++;
 
 	uint64_t arrheap_reqdsize = 0;
 	if (d->ncol > 0) {
@@ -597,7 +626,7 @@ more_strheap (uint64_t idx, uint64_t nbytes_more) {
 
 		char * ptr = (char *) d;
 		char * move_src = ptr + d->strheap_start;
-		char * move_dst = move_dst - nbytes_more;
+		char * move_dst = move_src - nbytes_more;
 
 		memmove (move_dst, move_src, d->strheap_sz);
 		memset  (move_dst + d->strheap_sz, 0, d->strheap_sz);
@@ -607,14 +636,15 @@ more_strheap (uint64_t idx, uint64_t nbytes_more) {
 	}
 
 	// otherwise we need more memory
-	return more_memory(idx, nbytes_more);
+	return more_memory(dsetidx, nbytes_more);
 }
 
 static ds*
-more_arrheap (uint64_t idx, uint64_t nbytes_more) {
+more_arrheap (uint64_t dsetidx, uint64_t nbytes_more) {
 
-	ds *d = ds_module.slots[idx].memory;
-	d->stats.nmore_arrheap++;
+	ds_slot *slot = &ds_module.slots[dsetidx];
+	ds *d = slot->memory;
+	slot->stats.nmore_arrheap++;
 
 	do {
 		// if we can find the space just by shrinking the string heap, do that.
@@ -632,18 +662,19 @@ more_arrheap (uint64_t idx, uint64_t nbytes_more) {
 		}
 
 		// otherwise we need more memory
-		d = more_memory(idx, nbytes_more);
+		d = more_memory(dsetidx, nbytes_more);
 	} while(d);
 	return 0;
 }
 
 static ds*
-more_columndescr_space (uint64_t idx, uint64_t ncolumns_more) {
+more_columndescr_space (uint64_t dsetidx, uint64_t ncolumns_more) {
 
 	uint64_t nbytes_more = ncolumns_more * sizeof(ds_column);
 
-	ds *d = ds_module.slots[idx].memory;
-	d->stats.nmore_colspace++;
+	ds_slot *slot = &ds_module.slots[dsetidx];
+	ds *d = slot->memory;
+	slot->stats.nmore_colspace++;
 
 	// for simplicity, let's not steal from the array heap, just from the string heap
 	do {
@@ -665,7 +696,7 @@ more_columndescr_space (uint64_t idx, uint64_t ncolumns_more) {
 		}
 
 		// otherwise we need more memory
-		d = more_memory(idx, nbytes_more);
+		d = more_memory(dsetidx, nbytes_more);
 	} while(d);
 	return 0;
 
@@ -680,6 +711,25 @@ actual_arrheap_sz (ds *d) {
 	return 0;
 }
 
+static inline uint64_t hash(const char *s, size_t len) {
+    uint64_t h = 0x100;
+    for (int32_t i = 0; i < len; i++) {
+        h ^= s[i] & 255;
+        h *= 1111111111111111111;
+    }
+    return h ^ h>>32;
+}
+
+// https://nullprogram.com/blog/2018/07/31/
+static inline uint64_t hash64(uint64_t x) {
+    x ^= x >> 32;
+    x *= 0xd6e8feb86659fd93U;
+    x ^= x >> 32;
+    x *= 0xd6e8feb86659fd93U;
+    x ^= x >> 32;
+    return x;
+}
+
 /**
  * Definitions for hash table where every key and value is 64 bits
  * Based on https://nullprogram.com/blog/2022/08/08/
@@ -689,45 +739,74 @@ actual_arrheap_sz (ds *d) {
 // Invalid or unset hashtable entry (64 bit all ones)
 #define DSHT64_INVALID 0xffffffffffffffffU
 
-typedef uint64_t ds_ht64_row[2];
-
 /**
- * Numeric hashtable where each key and value is a 64 bit integer
+ * Numeric hashtable definitions
+ * for ds_ht64, entries are indexes into a string heap.
+ * for ds_ht64, each key and value is a 64 bit integer
  */
-typedef struct ds_ht64 {
-	ds_ht64_row *ht; // array of key/value pairs
-	int32_t len;
-	int32_t exp; // exponent that denotes total hashtable capacity
-} ds_ht64;
 
-// https://nullprogram.com/blog/2018/07/31/
-static inline uint64_t hash64(uint64_t x) {
-    x ^= x >> 32;
-    x *= 0xd6e8feb86659fd93U;
-    x ^= x >> 32;
-    x *= 0xd6e8feb86659fd93U;
-    x ^= x >> 32;
-    return x;
+// Compute the next candidate index. Initialize idx to the hash.
+static inline int32_t ht_lookup(uint64_t hash, int exp, int32_t idx) {
+	uint32_t mask = ((uint32_t)1 << exp) - 1;
+	uint32_t step = (hash >> (64 - exp)) | 1;
+	return (idx + step) & mask;
 }
 
-// Allocate a hashtable with the given size
-static void ht64_new(ds_ht64 *t, uint32_t sz) {
-	// Allocate the given size for the hash table t
-	uint32_t exp = 0;
-	do { exp++; } while ((1 << exp) < sz);
-	exp += 1; // at least double the required size for fast lookups
+static inline uint32_t ht64_len(ds_ht64 *t) {
+	return t->len;
+}
+
+static inline uint32_t ht64_capacity(ds_ht64 *t) {
+	return (1 << (uint32_t) t->exp);
+}
 
+// Allocate a hash table with at least double the required size for fast
+// lookups. Initializes memory to all bits 1 (everything is invalid) since zero
+// is a valid entry. Note that if the hash table has already been allocated,
+// this wipes its contents.
+static int ht64_realloc(ds_ht64 *t, uint32_t sz) {
+	uint32_t exp = 0;
+	do { exp++; } while ((1 << exp) <= sz);
+	exp += 1;
 	size_t totalsz = sizeof(ds_ht64_row) * (1 << exp);
-	void *mem = DSREALLOC(0, totalsz);
-	// Initialize memory to -1 (all ones)
+	void *mem = DSREALLOC(t->ht, totalsz);
+	if (!mem) {
+		nonfatal("could not alloc hash table; out of memory");
+		return 1;
+	}
 	memset(mem, -1, totalsz);
 	t->ht = mem;
-	xassert(t->ht[0][0] == DSHT64_INVALID) // verify that memset worked
-	t->len =  0;
+	t->len = 0;
 	t->exp = exp;
+	return 0;
+}
+
+// Total number of bytes used by the hash table's `ht` memory field
+static inline size_t ht64_memsize(ds_ht64 *t) {
+	return sizeof(ds_ht64_row) * (1 << t->exp);
+}
+
+// Make hash table dst into an exact copy of src
+static void ht64_copy(ds_ht64 *dst, ds_ht64 *src) {
+	if (dst->exp != src->exp) {
+		uint32_t sz = (1 << (uint32_t) (src->exp) - 1) - 1; // half of capacity - 1
+		ht64_realloc(dst, sz);
+	}
+	memcpy(dst->ht, src->ht, ht64_memsize(dst));
+	dst->len = src->len;
+	dst->exp = src->exp;
+}
+
+// Note that this wipes the existing contents of the hash table
+static inline int ht64_double_capacity(ds_ht64 *t) {
+	return ht64_realloc(t, (1 << (uint32_t) t->exp) - 1);
+}
+
+static inline void ht64_clear(ds_ht64 *t) {
+	memset(t->ht, -1, ht64_memsize(t));
+	t->len = 0;
 }
 
-// Free an allocated hash table
 static void ht64_del(ds_ht64 *t) {
 	if (t->ht) {
 	DSFREE(t->ht);
@@ -737,17 +816,10 @@ static void ht64_del(ds_ht64 *t) {
 	t->exp = 0;
 }
 
-// Compute the next candidate index. Initialize idx to the hash.
-static inline int32_t ht64_lookup(uint64_t hash, int exp, int32_t idx) {
-	uint32_t mask = ((uint32_t)1 << exp) - 1;
-	uint32_t step = (hash >> (64 - exp)) | 1;
-	return (idx + step) & mask;
-}
-
-static inline int ht64_has(ds_ht64 *t, const uint64_t key) {
+static inline int ht64_has(ds_ht64 *t, uint64_t key) {
 	uint64_t h = hash64(key);
 	for (int32_t i = h;;) {
-		i = ht64_lookup(h, t->exp, i);
+		i = ht_lookup(h, t->exp, i);
 		if (t->ht[i][0] == DSHT64_INVALID) return 0; // empty, does not exist
 		else if (t->ht[i][0] == key) return 1; // found
 		// otherwise keep looking
@@ -757,10 +829,10 @@ static inline int ht64_has(ds_ht64 *t, const uint64_t key) {
 
 // Find the value of key in the hash table. Put the result in val. Returns 1
 // (true) if located, 0 otherwise.
-static int ht64_find(ds_ht64 *t, const uint64_t key, uint64_t *val) {
+static int ht64_find(ds_ht64 *t, uint64_t key, uint64_t *val) {
 	uint64_t h = hash64(key);
 	for (int32_t i = h;;) {
-		i = ht64_lookup(h, t->exp, i);
+		i = ht_lookup(h, t->exp, i);
 		if (t->ht[i][0] == DSHT64_INVALID) {
 			// empty
 			return 0;
@@ -776,15 +848,13 @@ static int ht64_find(ds_ht64 *t, const uint64_t key, uint64_t *val) {
 
 // Insert a value into the hash table
 // Will overwrite val if the key already exists
-static int ht64_insert(ds_ht64 *t, const uint64_t key, const uint64_t val) {
-	if ((uint32_t) t->len == (uint32_t) (1 << t->exp)) {
-		// hash table is full
-		return 0;
-	}
+static int ht64_insert(ds_ht64 *t, uint64_t key, uint64_t val) {
+	// there are external guards that check, assume table never full
+	xassert((uint32_t) t->len < ht64_capacity(t))  // fail if the table if full
 
 	uint64_t h = hash64(key);
 	for (int32_t i = h;;) {
-		i = ht64_lookup(h, t->exp, i);
+		i = ht_lookup(h, t->exp, i);
 		if (t->ht[i][0] == DSHT64_INVALID || t->ht[i][0] == key) {
 			// empty or existing key, insert here
 			t->len++;
@@ -797,97 +867,121 @@ static int ht64_insert(ds_ht64 *t, const uint64_t key, const uint64_t val) {
 	return 0;
 }
 
+// Check to see if the given hash is in the table. Use the hash as the key.
+//
+// If the key exists, populate target with the value address for the external
+// caller to retrieve and return 1.
+//
+// If the key does not exist, populate target with an address the caller can
+// write a new value into and return 0.
+static int ht64_intern_raw(ds_ht64 *t, uint64_t hash, uint64_t **target) {
+	xassert((uint32_t) t->len < (uint32_t) (1 << t->exp))  // fail if the table if full
+	for (int32_t i = hash;;) {
+		i = ht_lookup(hash, t->exp, i);
+		if (t->ht[i][0] == DSHT64_INVALID) {
+			// not found, insert here
+			t->len++;
+			t->ht[i][0] = hash;
+			*target = &(t->ht[i][1]);
+			return 0;
+		} else if (t->ht[i][0] == hash) {
+			// found key, retrieve from here or insert here if invalid
+			*target = &(t->ht[i][1]);
+			return t->ht[i][1] != DSHT64_INVALID; // value might be invalid
+		}
+		// Otherwise keep looking for a spot to insert
+	}
+	*target = NULL; // cannot write, likely full
+	return 0;
+}
 
-static uint64_t
-stralloc(ds **d, uint64_t idx, const char * str)
-{
-	const size_t sz = 1 + strlen(str);
-	char * base = (char*)*d;
+// Allocate a string in the given dataset's heap. Writes heap string index into
+// index param. Returns dataset pointer.
+static ds *
+stralloc(uint64_t dsetidx, const char *str, size_t len, uint64_t *index) {
 
-	// do we already have this string?
-	{
-		char * strheap = base + (*d)->strheap_start;
-		char * strheap_end = strheap + (*d)->strheap_sz;
+	ds_slot *slot = &ds_module.slots[dsetidx];
+	ds *d = slot->memory;
 
-		char * p = strheap;
-		while (p < strheap_end) {
-			if(!strcmp(str,p)) return p-strheap;
-			p += strlen(p) + 1;
-		}
-	} // guess not...
-
-	// do we need more space?
-	if ((*d)->total_sz - (*d)->strheap_start < (*d)->strheap_sz + sz) {
-		*d = more_strheap(idx, sz);
-		if (!*d) return 0;
+	if (len == 0) { // empty string
+		*index = 0;
+		return d;
 	}
 
-	uint64_t newstr   = (*d)->strheap_sz;
-	(*d)->strheap_sz += sz;
+	size_t sz = 1 + len;
 
-	memcpy(base + (*d)->strheap_start + newstr, str, sz);
-	return newstr; 
-}
+	char *base = (char *) d;
+	char *strheap = base + d->strheap_start;
+	char *strheap_end = strheap + d->strheap_sz;
 
-static void 
-shift_all_string_handles(ds *d, int64_t shift, uint64_t shift_greater_than)
-{
-	char * ptr = (char *) d;
-	for (uint32_t i = 0; i < d->ncol; i++) {
-		if (d->columns[i].type < 0   &&   d->columns[i].longkey > shift_greater_than) {
-			d->columns[i].longkey += shift;
-		}
+	// setup hash table for fast strheap index lookup
+	ds_ht64 *ht = &(slot->ht);
+	uint64_t hsh;
+	uint64_t *htidx; // address of hash table address storage
 
-		if (tcheck_s(d->columns[i].type)) {
-			uint64_t *handles = ptr + d->arrheap_start + d->columns[i].offset;
-			for (uint64_t j = 0 ; j < d->nrow; j++) {
-			
-				if (handles[j] > shift_greater_than)
-					handles[j] += shift;
-			}
+	// double hashtable capacity if the table is half full (for fast lookups)
+	if (ht->len >= ht64_capacity(ht) / 2) {
+		ht64_double_capacity(ht);
+		slot->stats.htnrealloc += 1;
+		// rehash string heap
+		char *p = strheap;
+		while (p < strheap_end) {
+			size_t len = strlen(p);
+			hsh = hash(p, len);
+			ht64_intern_raw(ht, hsh, &htidx);
+			*htidx = p - strheap;
+			p += len + 1;
 		}
 	}
-	d->stats.nshift_strhandles++;
-}
 
-static void
-strfree (uint64_t oldstr, ds *d)
-{
-	// TODO: This should check that no one else is using this string before freeing it
-	if (!oldstr) return;
-	char * strheap = ((char *)d) + d->strheap_start;
-	char * s = strheap + oldstr;
-	int64_t sz = 1 + strlen(s);
-	memmove(s, s+sz, (strheap+d->strheap_sz) - (s+sz));
-	shift_all_string_handles(d, -sz, oldstr);
-	d->strheap_sz -= sz;
+	// do we already have this string?
+	hsh = hash(str, len);
+	if (ht64_intern_raw(ht, hsh, &htidx)) {
+		// Index already exists, record and return
+		*index = *htidx;
+		return d;
+	}
+
+	// guess not...
+
+	// do we need more space?
+	if (d->total_sz - d->strheap_start < d->strheap_sz + sz) {
+		slot->memory = d = more_strheap(dsetidx, sz);
+		if (!d) return 0;
+	}
+
+	*index = *htidx = d->strheap_sz;
+	d->strheap_sz += sz;
+
+	memcpy(base + d->strheap_start + (*index), str, sz);
+	return d;
 }
 
 static inline char *
-getstr(ds *d, ds_column *c, uint64_t index) {
-	char * ptr = (char *) d;
-	uint64_t *handles = ptr + d->arrheap_start + c->offset;
+getstr(ds *d, uint64_t col, uint64_t index) {
+	char *ptr = (char *) d;
+	uint64_t *handles = ptr + d->arrheap_start + d->columns[col].offset;
 	return ptr + d->strheap_start + handles[index];
 }
 
 // Set string helper that returns dataset pointer with string assigned (may be
 // the same dataset pointer or different if required reallocation)
-static ds *
-setstr (ds *d, ds_column *c, uint64_t index, const char *value) {
-	uint64_t idx;
-	char *ptr = (char *) d;
-	uint64_t *handles = ptr + d->arrheap_start + c->offset;
-	strfree(handles[index], d);
-	uint64_t newstr = stralloc(&d, idx, value);
+static inline ds *setstr(uint64_t dsetidx, uint64_t col, uint64_t index, const char *value, size_t length) {
+	// Allocate str and retrieve updated dataset
+	uint64_t stridx;
+	ds *d = stralloc(dsetidx, value, length, &stridx);
 	if (!d) return 0; // Could not allocate string
 
-	handles[index] = newstr;
+	uint64_t *handles = (char *) d + d->arrheap_start + d->columns[col].offset;
+	handles[index] = stridx;
 	return d;
 }
 
 static void
-reassign_arrayoffsets (ds *d,  uint32_t new_crow)
+reassign_arrayoffsets (uint64_t idx, uint32_t new_crow)
 {
+	ds_slot *slot = &ds_module.slots[idx];
+	ds *d = slot->memory;
 	uint64_t cur_arrheap_used_sz = actual_arrheap_sz(d);
 
 	char * arrheap = ((char *)d) + d->arrheap_start;
@@ -915,27 +1009,28 @@ reassign_arrayoffsets (ds *d,  uint32_t new_crow)
 		d->columns[i].offset = mov_dst-arrheap;
 	}
 
-	d->stats.nreassign_arroffsets++;
+	slot->stats.nreassign_arroffsets++;
 }
 
 static inline void
 copyval(
-	ds *dst_ds, ds_column *dst_col, uint64_t dst_idx,
-	ds *src_ds, ds_column *src_col, uint64_t src_idx,
+	ds *dst_ds, uint64_t dst_col, uint64_t dst_idx,
+	ds *src_ds, uint64_t src_col, uint64_t src_idx,
 	size_t itemsize // assume same stride on each column
 ) {
-	char *dst_ptr = (char *) dst_ds + dst_ds->arrheap_start + dst_col->offset + (dst_idx * itemsize);
-	char *src_ptr = (char *) src_ds + src_ds->arrheap_start + src_col->offset + (src_idx * itemsize);
+	char *dst_ptr = (char *) dst_ds + dst_ds->arrheap_start + dst_ds->columns[dst_col].offset + (dst_idx * itemsize);
+	char *src_ptr = (char *) src_ds + src_ds->arrheap_start + src_ds->columns[src_col].offset + (src_idx * itemsize);
 	memcpy(dst_ptr, src_ptr, itemsize);
 }
 
+// target dataset must be passed in as index in case it needs to get copied
 static inline ds *
 copystr(
-	ds *dst_ds, ds_column *dst_col, uint64_t dst_idx,
-	ds *src_ds, ds_column *src_col, uint64_t src_idx
+	uint64_t dst_dsetidx, uint64_t dst_col, uint64_t dst_idx,
+	ds *src_ds, uint64_t src_col, uint64_t src_idx
 ) {
-	char *str = getstr(src_ds, src_col, src_idx);
-	return setstr(dst_ds, dst_col, dst_idx, str);
+	const char *str = getstr(src_ds, src_col, src_idx);
+	return setstr(dst_dsetidx, dst_col, dst_idx, str, strlen(str));
 }
 
 /*
@@ -951,7 +1046,7 @@ uint64_t dset_new(void) {
 	ds * d = 0;
 
 	uint64_t handle = dset_new_(DS_INITIAL_SZ, &d);
-	if(handle == UINT64_MAX) 
+	if(handle == UINT64_MAX)
 		return handle;
 
 	*d = (ds) {
@@ -961,8 +1056,8 @@ uint64_t dset_new(void) {
 		.strheap_sz    = 1, // the null string is the string with index zero.
 	};
 
-	// 0x94 CSDAT
-	d->magic[0] = 0x94;
+	// 0x95 CSDAT
+	d->magic[0] = 0x95;
 	d->magic[1] = 0x43;
 	d->magic[2] = 0x53;
 	d->magic[3] = 0x44;
@@ -984,13 +1079,17 @@ uint64_t dset_copy(uint64_t dset)
 
 	ds* newds = 0;
 	uint64_t newhandle = dset_new_(oldds->total_sz, &newds);
+	uint64_t newidx = MASK_IDX & newhandle;
 
-	if (newhandle != UINT64_MAX) 
-		memcpy(newds,oldds,oldds->total_sz); 
+	if (newhandle != UINT64_MAX) {
+		memcpy(newds,oldds,oldds->total_sz);
+		ht64_copy(&ds_module.slots[newidx].ht, &ds_module.slots[idx].ht);
+	}
 
 	return newhandle;
 }
 
+
 // Compute the inner join of two Datasets R and S by matching values in the
 // column with the given key. Currently only 64-bit columns (e.g., T_U64) with
 // zero shape may be specified as keys.
@@ -1001,7 +1100,7 @@ uint64_t dset_copy(uint64_t dset)
 // Implements Classic Hash Join algorithm
 // https://en.wikipedia.org/wiki/Hash_join#Classic_hash_join
 typedef struct ds_innerjoin_coldata {
-	ds_column *col;
+	uint64_t col;
 	int itemsize;
 	int is_str;
 } ds_innerjoin_coldata;
@@ -1020,15 +1119,15 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 	// Look up the two datasets and columns to join
 	if (!(ds_r = handle_lookup(dset_r, "dset_innerjoin", &generation_r, &idx_r))) return UINT64_MAX;
 	if (!(ds_s = handle_lookup(dset_s, "dset_innerjoin", &generation_s, &idx_s))) return UINT64_MAX;
-	keycol_r = column_lookup(ds_r, key);
-	keycol_s = column_lookup(ds_s, key);
+	keycol_r = column_lookup(ds_r, key, NULL);
+	keycol_s = column_lookup(ds_s, key, NULL);
 
 	if (!keycol_r || !keycol_s) {
 		nonfatal("dset_innerjoin: input dataset does not contain %s column", key);
 		return UINT64_MAX;
 	}
-	if (keycol_r->type != keycol_s->type) {
-		nonfatal("dset_innerjoin: input %s column types do match (%d, %d)", key, keycol_r->type, keycol_s->type);
+	if (abs_i8(keycol_r->type) != abs_i8(keycol_s->type)) {
+		nonfatal("dset_innerjoin: input %s column types do match (%d, %d)", key, abs_i8(keycol_r->type), abs_i8(keycol_s->type));
 		return UINT64_MAX;
 	}
 	if (keycol_r->shape[0] != 0 || keycol_s->shape[0] != 0) {
@@ -1036,9 +1135,9 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 		return UINT64_MAX;
 	}
 
-	if (keycol_r->type != T_U64 && keycol_r->type != T_I64 && keycol_r->type != T_F64 && keycol_r->type != T_C32) {
+	if (abs_i8(keycol_r->type) != T_U64 && abs_i8(keycol_r->type) != T_I64 && abs_i8(keycol_r->type) != T_F64 && abs_i8(keycol_r->type) != T_C32) {
 		// TODO: Allow innerjoining any type (or least numeric types)
-		nonfatal("dset_innerjoin: cannot innerjoin column %s with non-64bit type %d", key, keycol_r->type);
+		nonfatal("dset_innerjoin: cannot innerjoin column %s with non-64bit type %d", key, abs_i8(keycol_r->type));
 		return UINT64_MAX;
 	}
 
@@ -1048,6 +1147,7 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 	// Populate fields from the first dataset R. Declare a stack-allocated
 	// dynamic array of structs which memoize the required column data
 	ds_column *col;
+	size_t colidx;
 	char *colkey;
 
 	// Cache source column details (try to use stack version if possible)
@@ -1060,20 +1160,20 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 	uint32_t nrcol = 0, nscol = 0; // number of columns used from R and S
 	for (uint32_t c = 0; c < ds_r->ncol; c++) {
 		colkey = dset_key(dset_r, c);
-		if (strcmp(key, colkey) == 0 || !column_lookup(ds_s, colkey)) {
+		if (strcmp(key, colkey) == 0 || !column_lookup(ds_s, colkey, NULL)) {
 			// key is either target join key or not in other dataset, add now
 			// with correct type details
-			col = column_lookup(ds_r, colkey);
+			col = column_lookup(ds_r, colkey, &colidx);
 			if (!dset_addcol_array(
-				dset, colkey, col->type,
+				dset, colkey, abs_i8(col->type),
 				col->shape[0], col->shape[1], col->shape[2]
 			)) {
 				nonfatal("dset_innerjoin: cannot add column %s to result dataset", colkey);
 				goto fail;
 			}
-			src_coldata[nrcol].col = col;
+			src_coldata[nrcol].col = colidx;
 			src_coldata[nrcol].itemsize = type_size[abs_i8(col->type)] * stride(col);
-			src_coldata[nrcol].is_str = col->type == T_STR;
+			src_coldata[nrcol].is_str = abs_i8(col->type) == T_STR;
 			nrcol++;
 		} // otherwise defer to dataset S for this column
 	}
@@ -1084,17 +1184,17 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 		if (strcmp(key, colkey) == 0) {
 			continue; // already added in previous loop
 		}
-		col = column_lookup(ds_s, colkey);
+		col = column_lookup(ds_s, colkey, &colidx);
 		if (!dset_addcol_array(
-			dset, colkey, col->type,
+			dset, colkey, abs_i8(col->type),
 			col->shape[0], col->shape[1], col->shape[2]
 		)) {
 			nonfatal("dset_innerjoin: cannot add column %s to result dataset", colkey);
 			goto fail;
 		}
-		src_coldata[nrcol + nscol].col = col;
+		src_coldata[nrcol + nscol].col = colidx;
 		src_coldata[nrcol + nscol].itemsize = type_size[abs_i8(col->type)] * stride(col);
-		src_coldata[nrcol + nscol].is_str = col->type == T_STR;
+		src_coldata[nrcol + nscol].is_str = abs_i8(col->type) == T_STR;
 		nscol++;
 	}
 
@@ -1104,7 +1204,7 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 
 	// Create a hash table of values which store the index of each column value
 	// in dataset S
-	ht64_new(&idx_lookup, ds_s->nrow);
+	ht64_realloc(&idx_lookup, ds_s->nrow);
 	for (uint64_t j = 0; j < ds_s->nrow; j++) {
 		if (!ht64_insert(&idx_lookup, keydata_s[j], j)) {
 			nonfatal("dset_innerjoin: hash table full?? cannot proceed");
@@ -1137,18 +1237,18 @@ uint64_t dset_innerjoin(const char *key, uint64_t dset_r, uint64_t dset_s)
 		// Copy row values from Dataset R
 		for (c = 0; c < nrcol; c++) {
 			if (src_coldata[c].is_str) {
-				d = copystr(d, &d->columns[c], k, ds_r, src_coldata[c].col, i);
+				d = copystr(idx, c, k, ds_r, src_coldata[c].col, i);
 			} else {
-				copyval(d, &d->columns[c], k, ds_r, src_coldata[c].col, i, (size_t) src_coldata[c].itemsize);
+				copyval(d, c, k, ds_r, src_coldata[c].col, i, (size_t) src_coldata[c].itemsize);
 			}
 		}
 
 		// Copy row values from Dataset S
 		for (c = nrcol; c < nrcol + nscol; c++) {
 			if (src_coldata[c].is_str) {
-				d = copystr(d, &d->columns[c], k, ds_s, src_coldata[c].col, j);
+				d = copystr(idx, c, k, ds_s, src_coldata[c].col, j);
 			} else {
-				copyval(d, &d->columns[c], k, ds_s, src_coldata[c].col, j, (size_t) src_coldata[c].itemsize);
+				copyval(d, c, k, ds_s, src_coldata[c].col, j, (size_t) src_coldata[c].itemsize);
 			}
 		}
 
@@ -1182,6 +1282,8 @@ void dset_del(uint64_t dset)
 	if (handle_lookup(dset, "dset_del", &generation, &idx)) {
 
 		DSFREE(ds_module.slots[idx].memory);
+		ht64_del(&(ds_module.slots[idx].ht));
+
 		ds_module.slots[idx].memory = 0;
 	}
 	unlock();
@@ -1224,7 +1326,7 @@ const char *dset_key(uint64_t dset, uint64_t index)
 int dset_type (uint64_t dset, const char * colkey)
 {
 	const ds        *d  = handle_lookup(dset, colkey, 0, 0);
-	const ds_column *c  = column_lookup(d, colkey);
+	const ds_column *c  = column_lookup(d, colkey, NULL);
 
 	if(!(d && c)) return 0;
 	return abs_i8(c->type);
@@ -1235,7 +1337,7 @@ void *dset_get (uint64_t dset, const char * colkey)
 	// Caution: T_STR columns cannot be used directly, actual strings must be
 	// retrieved through dset_getstr
 	const ds        *d  = handle_lookup(dset, colkey, 0, 0);
-	const ds_column *c  = column_lookup(d, colkey);
+	const ds_column *c  = column_lookup(d, colkey, NULL);
 	char * ptr = (char *) d;
 
 	if(!(d && c)) return 0;
@@ -1246,17 +1348,17 @@ void *dset_get (uint64_t dset, const char * colkey)
 uint64_t dset_getsz(uint64_t dset, const char * colkey)
 {
 	const ds        *d  = handle_lookup(dset, colkey, 0, 0);
-	const ds_column *c  = column_lookup(d, colkey);
+	const ds_column *c  = column_lookup(d, colkey, NULL);
 
 	if(!(d && c)) return 0;
 
-	return d->nrow * abs_i8(type_size[c->type]) * stride(c);
+	return d->nrow * type_size[abs_i8(c->type)] * stride(c);
 }
 
 uint32_t dset_getshp (uint64_t dset, const char * colkey)
 {
 	const ds        *d  = handle_lookup(dset, colkey, 0, 0);
-	const ds_column *c  = column_lookup(d, colkey);
+	const ds_column *c  = column_lookup(d, colkey, NULL);
 
 	if(!(d && c)) return 0;
 
@@ -1273,13 +1375,16 @@ int dset_addcol_scalar (uint64_t dset, const char * key, int type) {
 int dset_addcol_array (uint64_t dset, const char * key, int type, int shape0, int shape1, int shape2) {
 
 	if(!tcheck(type)) {
-		nonfatal("invalid column data type: %i", type);
+		nonfatal("invalid column data type: %i (key %s)", type, key);
 		return 0;
 	}
 
-	uint64_t idx; 
+	uint64_t idx;
 	ds *d = handle_lookup(dset, "add column", 0, &idx);
-	if(!d) return 0;
+	if (!d) {
+		nonfatal("could not find dataset with handle %lu (adding column %s)", dset, key);
+		return 0;
+	}
 
 	const size_t ksz = 1 + strlen(key);
 	const int8_t t   = abs_i8(type);
@@ -1295,12 +1400,15 @@ int dset_addcol_array (uint64_t dset, const char * key, int type, int shape0, in
 
 		// need more space for column descriptors
 		d = more_columndescr_space(idx, 30);
-		if (!d) return 0;
+		if (!d) {
+			nonfatal("could not allocate more column descr space (adding column %s)", key);
+			return 0;
+		}
 	}
 
 	// compute the offset for the new column
 	col.offset = actual_arrheap_sz(d);
-	
+
 	// compute the required space on the array heap for the  new column
 	const uint64_t reqd_arrheap_space = compute_col_reserved_space(d->crow, &col);
 
@@ -1311,14 +1419,21 @@ int dset_addcol_array (uint64_t dset, const char * key, int type, int shape0, in
 		// need more space on the array heap
 		const size_t howmuch_need = col.offset + reqd_arrheap_space - arrheap_sz;
 		d = more_arrheap(idx, howmuch_need);
-		if (!d) return 0;
+		if (!d) {
+			nonfatal("could not allocate more array heap space (adding column %s)", key);
+			return 0;
+		}
 	}
 
 	if (col.type < 0) {
-		// key is long, so allocate a string for it	
+		// key is long, so allocate a string for it
 
-		uint64_t newstr = stralloc(&d, idx, key);
-		if (!d) return 0;
+		uint64_t newstr;
+		d = stralloc(idx, key, ksz - 1, &newstr);
+		if (!d) {
+			nonfatal("could not allocate more string space (adding column %s)", key);
+			return 0;
+		}
 		col.longkey    = newstr;
 
 	} else {
@@ -1339,24 +1454,24 @@ int dset_changecol (uint64_t dset, const char *key, int type) {
 	}
 
 	const ds  *d  = handle_lookup(dset, key, 0, 0);
-	ds_column *c  = column_lookup(d, key);
+	ds_column *c  = column_lookup(d, key, NULL);
 
 	if (!(d && c)) return 0;
 
-	int8_t current_size = abs_i8(type_size[c->type]);
-	int8_t proposed_size = abs_i8(type_size[type]);
+	int8_t current_size = type_size[abs_i8(c->type)];
+	int8_t proposed_size = type_size[type];
 
 	if (current_size != proposed_size) {
-		nonfatal("cannot change column with type %i to incompatible type %i", c->type, type);
+		nonfatal("cannot change column with type %i to incompatible type %i", abs_i8(c->type), type);
 		return 0;
 	}
 
-	c->type = type;
+	c->type = c->type < 0 ? -type : type;
 	return 1;
 }
 
 int dset_addrows (uint64_t dset, uint32_t num) {
-	uint64_t idx; 
+	uint64_t idx;
 
 	ds *d = handle_lookup(dset, "dset_addrows", 0, &idx);
 	if (!d) return 0;
@@ -1401,7 +1516,7 @@ int dset_addrows (uint64_t dset, uint32_t num) {
 
 
 	// now we have enough space, we just need to reassign the offsets and do some memmoves
-	reassign_arrayoffsets(d, new_crow);
+	reassign_arrayoffsets(idx, new_crow);
 
 	d->crow  = new_crow;
 	d->nrow += num;
@@ -1412,7 +1527,8 @@ int dset_addrows (uint64_t dset, uint32_t num) {
 
 int dset_defrag (uint64_t dset, int realloc_smaller)
 {
-	ds *d = handle_lookup(dset, "dset_compress", 0, 0);
+    uint64_t idx;
+	ds *d = handle_lookup(dset, "dset_compress", 0, &idx);
 	if(!d) return 0;
 	char * pd = (char *) d;
 
@@ -1427,7 +1543,7 @@ int dset_defrag (uint64_t dset, int realloc_smaller)
 	}
 
 	if (d->crow > d->nrow) {
-		reassign_arrayoffsets(d, d->nrow);
+		reassign_arrayoffsets(idx, d->nrow);
 		d->crow = d->nrow;
 	}
 
@@ -1439,7 +1555,7 @@ int dset_defrag (uint64_t dset, int realloc_smaller)
 	}
 
 	if (realloc_smaller) {
-		d->stats.nrealloc++;
+		ds_module.slots[idx].stats.nrealloc++;
 		d = DSREALLOC(d, d->strheap_start + d->strheap_sz);
 		if (!d) return 0;
 	}
@@ -1449,10 +1565,11 @@ int dset_defrag (uint64_t dset, int realloc_smaller)
 
 
 
-const char *dset_getstr (uint64_t dset, const char * colkey, uint64_t index) 
+const char *dset_getstr (uint64_t dset, const char * colkey, uint64_t index)
 {
+	uint64_t colidx;
 	ds        *d = handle_lookup(dset, colkey, 0, 0);
-	ds_column *c = column_lookup(d, colkey);
+	ds_column *c = column_lookup(d, colkey, &colidx);
 
 	if(!(d && c)) return 0;
 
@@ -1460,17 +1577,17 @@ const char *dset_getstr (uint64_t dset, const char * colkey, uint64_t index)
 		nonfatal("dset_getstr: column '%s' is not a string", colkey);
 		return 0;
 	} else {
-		return getstr(d, c, index);
+		return getstr(d, colidx, index);
 	}
 }
 
 
-int dset_setstr (uint64_t dset, const char * colkey, uint64_t index, const char * value)
+int dset_setstr (uint64_t dset, const char * colkey, uint64_t index, const char *value, size_t length)
 {
-	uint64_t idx;
+	uint64_t idx, colidx;
 
 	ds        *d = handle_lookup(dset, colkey, 0, &idx);
-	ds_column *c = column_lookup(d, colkey);
+	ds_column *c = column_lookup(d, colkey, &colidx);
 
 	if(!(d && c)) return 0;
 
@@ -1484,10 +1601,10 @@ int dset_setstr (uint64_t dset, const char * colkey, uint64_t index, const char
 		return 0;
 	}
 
-	return setstr(d, c, index, value) ? 1 : 0;
+	return setstr(idx, colidx, index, value, length) ? 1 : 0;
 }
 
-static char* 
+static char*
 repr_cfloat (uint64_t ds, int sz, char * buf, ds_float_complex_t fc)
 {
 	snprintf(buf,sz,"(%f,%f)", crealf(fc), cimagf(fc));
@@ -1511,11 +1628,14 @@ repr_str (uint64_t dset, int sz, char * buf, uint64_t handle)
 }
 
 
-void dset_dumptxt (uint64_t dset) {
+// Specify dump_data = 1 to also output column data
+void dset_dumptxt (uint64_t dset, int dump_data) {
 
-	ds *d = handle_lookup(dset, "dset_dumptxt", 0, 0);
+	uint64_t idx;
+	ds *d = handle_lookup(dset, "dset_dumptxt", 0, &idx);
 	xassert(d);
 
+	ds_slot *slot = &ds_module.slots[idx];
 
 	printf ("dataset %"PRIu64"\n"
 		"\ttotal size:            %"PRIu64"\n"
@@ -1523,22 +1643,28 @@ void dset_dumptxt (uint64_t dset) {
 		"\trows (capacity)        %"PRIu64"\n"
 		"\tcols (actual)          %"PRIu32"\n"
 		"\tcols (capacity)        %"PRIu32"\n\n"
-		"\tnrealloc:              %"PRIu32"\n" 
-		"\tnreassign_arroffsets:  %"PRIu32"\n" 
-		"\tnshift_strhandles:     %"PRIu32"\n" 
-		"\tnmore_arrheap:         %"PRIu32"\n" 
-		"\tnmore_strheap:         %"PRIu32"\n" 
-		"\tnmore_colspace:        %"PRIu32"\n" ,
+		"\tnrealloc:              %"PRIu32"\n"
+		"\tnreassign_arroffsets:  %"PRIu32"\n"
+		"\tnshift_strhandles:     %"PRIu32"\n"
+		"\tnmore_arrheap:         %"PRIu32"\n"
+		"\tnmore_strheap:         %"PRIu32"\n"
+		"\tnmore_colspace:        %"PRIu32"\n\n"
+		"\thtnrealloc:            %"PRIu32"\n"
+		"\thtlen                  %"PRIu32"\n"
+		"\thtcapacity             %"PRIu32"\n\n",
 		d,
 		d->total_sz,
 		d->nrow, d->crow,
 		d->ncol, d->ccol,
-		d->stats.nrealloc,
-		d->stats.nreassign_arroffsets,
-		d->stats.nshift_strhandles,
-		d->stats.nmore_arrheap,
-		d->stats.nmore_strheap,
-		d->stats.nmore_colspace
+		slot->stats.nrealloc,
+		slot->stats.nreassign_arroffsets,
+		slot->stats.nshift_strhandles,
+		slot->stats.nmore_arrheap,
+		slot->stats.nmore_strheap,
+		slot->stats.nmore_colspace,
+		slot->stats.htnrealloc,
+		slot->ht.len,
+		ht64_capacity(&slot->ht)
 		);
 
 	char *sep = "";
@@ -1550,6 +1676,8 @@ void dset_dumptxt (uint64_t dset) {
 	}
 	fputc('\n',stdout);
 
+	if (!dump_data) return;
+
 	for (unsigned j = 0; j < d->nrow; j++) {
 		c = d->columns;
 		for (unsigned i = 0; i < d->ncol; i++,c++) {
@@ -1562,7 +1690,7 @@ void dset_dumptxt (uint64_t dset) {
 
 			#define REPR(sym,_a,type,_c,spec,reprfn) \
 				case sym: printf("%s" spec, sep, reprfn(dset, sizeof(buf), buf, ((type*)data)[j])); break;
-				
+
 			switch (abs_i8(c->type)) {
 				DSET_TYPELIST(REPR);
 				default:
@@ -1579,4 +1707,46 @@ void *dset_dump(uint64_t dset) {
 	return (void *) handle_lookup(dset, "dset_dump", 0, 0);
 }
 
+uint64_t dset_strheapsz(uint64_t dset) {
+	ds *d = handle_lookup(dset, "dset_strheapsz", 0, 0);
+	return d->strheap_sz;
+}
+
+// Returns pointer to first item in the string heap
+char *dset_strheap(uint64_t dset) {
+	ds *d = handle_lookup(dset, "dset_strheap", 0, 0);
+	return (char *) d + d->strheap_start;
+}
+
+int dset_setstrheap(uint64_t dset, const char *heap, size_t size) {
+	uint64_t dsetidx;
+	ds *d = handle_lookup(dset, "dset_setstrheap", 0, &dsetidx);
+	ds_slot *slot = &ds_module.slots[dsetidx];
+
+	// erase current strings
+	d->strheap_sz = 1; // 1 for empty string
+	ht64_clear(&slot->ht);
+
+	char *s = heap;
+	size_t len;
+	uint64_t idx;
+	while (d && s < heap + size) {
+		len = strlen(s);
+		d = stralloc(dsetidx, s, len, &idx);
+		if (!d) return 0;
+		s += len + 1;
+	}
+
+	return d != 0;
+}
+
+// Raw string allocation for the dataset without assigning to any column
+// Returns 1 if given index was successfully assigned a value. Should not
+// normally be used.
+int dset_stralloc(uint64_t dset, const char *value, size_t length, uint64_t *index) {
+	uint64_t dsetidx;
+	ds *d = handle_lookup(dset, "dset_stralloc", 0, &dsetidx);
+	return stralloc(dsetidx, value, length, index) != 0;
+}
+
 #endif
diff --git a/cryosparc/include/lz4 b/cryosparc/include/lz4
new file mode 160000
index 00000000..82435a85
--- /dev/null
+++ b/cryosparc/include/lz4
@@ -0,0 +1,1 @@
+Subproject commit 82435a85b7964da88aee957083d13b7f5c7bf870
diff --git a/cryosparc/lz4.pxd b/cryosparc/lz4.pxd
new file mode 100644
index 00000000..c24743f3
--- /dev/null
+++ b/cryosparc/lz4.pxd
@@ -0,0 +1,5 @@
+cdef extern from "lz4/lib/lz4.h":
+    int LZ4_compress_default(const char *src, char *dst, int srcSize, int dstCapacity) nogil
+    int LZ4_decompress_safe (const char *src, char *dst, int compressedSize, int dstCapacity) nogil
+    int LZ4_compressBound(int inputSize) nogil
+    int LZ4_compress_fast (const char *src, char *dst, int srcSize, int dstCapacity, int acceleration) nogil
diff --git a/cryosparc/stream.py b/cryosparc/stream.py
index 5d69bf1a..71146590 100644
--- a/cryosparc/stream.py
+++ b/cryosparc/stream.py
@@ -18,6 +18,7 @@
 
 if TYPE_CHECKING:
     from typing_extensions import Self  # not present in typing-extensions=3.7
+    from .core import MemoryView
 
 
 class AsyncBinaryIO(Protocol):
@@ -153,7 +154,7 @@ async def from_async_iterator(cls, iterator: Union[AsyncIterator[bytes], AsyncGe
         return await cls.from_async_stream(AsyncBinaryIteratorIO(iterator))
 
     @abstractmethod
-    def stream(self) -> Generator[bytes, None, None]:
+    def stream(self) -> Generator[Union[bytes, memoryview, "MemoryView"], None, None]:
         ...
 
     async def astream(self):
diff --git a/cryosparc/util.py b/cryosparc/util.py
index 6dc56401..0d6855ff 100644
--- a/cryosparc/util.py
+++ b/cryosparc/util.py
@@ -41,7 +41,7 @@
 K = TypeVar("K")
 V = TypeVar("V")
 E = TypeVar("E")  # error
-INT = TypeVar("INT", bound=n.integer, covariant=True)
+INT = TypeVar("INT", bound=n.integer)
 
 Result = Union[Tuple[T, Literal[None]], Tuple[Literal[None], E]]
 """
diff --git a/docs/intro.md b/docs/intro.md
index 4647893d..ba00dc5e 100644
--- a/docs/intro.md
+++ b/docs/intro.md
@@ -19,7 +19,7 @@ Source code is [available on GitHub](https://github.com/cryoem-uoft/cryosparc-to
 
 ## Pre-requisites
 
-- [Python ≥ 3.7, ≤ 3.10](https://www.python.org/downloads/) (3.11 support is experimental)
+- [Python ≥ 3.7](https://www.python.org/downloads/)
 - [CryoSPARC ≥ v4.1](https://cryosparc.com/download)
 
 CryoSPARC installation must be accessible via one of the following methods:
diff --git a/pyproject.toml b/pyproject.toml
index b524cc27..3b26a79d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,7 +21,6 @@ classifiers = [
 ]
 dependencies = [
     "numpy ~= 1.15",
-    "python-snappy ~= 0.6",
     "typing-extensions >= 3.7",
 ]
 
@@ -66,7 +65,10 @@ include = ["cryosparc", "tests", "setup.py"]
 
 [tool.black]
 line-length = 120
+extend-exclude = """
+^/cryosparc/include/.*\\.py
+"""
 
 [tool.ruff]
 line-length = 120
-exclude = ["docs", ".venv*"]
+exclude = ["docs", ".venv*", "cryosparc/include"]
diff --git a/pyrightconfig.json b/pyrightconfig.json
new file mode 100644
index 00000000..03baaa21
--- /dev/null
+++ b/pyrightconfig.json
@@ -0,0 +1,9 @@
+{
+    "exclude": [
+        "**/node_modules",
+        "**/__pycache__",
+        "**/.*",
+        "cryosparc/include/**/*.py",
+        "docs/_build/**/*.py"
+    ]
+}
diff --git a/setup.py b/setup.py
index 7d6f3901..c3aa4831 100644
--- a/setup.py
+++ b/setup.py
@@ -11,6 +11,7 @@
 
 if sys.platform == "win32":
     extra_compile_args += ["/std:c11"]
+    define_macros += [("__WIN32__", 1)]
 else:
     libraries.append("pthread")
 
@@ -29,14 +30,19 @@
     ext_modules=cythonize(
         Extension(
             name="cryosparc.core",
-            sources=["cryosparc/dataset.c", "cryosparc/core.pyx"],
+            sources=["cryosparc/include/lz4/lib/lz4.c", "cryosparc/dataset.c", "cryosparc/core.pyx"],
             include_dirs=["cryosparc/include/"],
             libraries=libraries,
             define_macros=define_macros,
             undef_macros=undef_macros,
             extra_compile_args=extra_compile_args,
             extra_link_args=extra_link_args,
-            depends=["cryosparc/include/cryosparc-tools/dataset.h", "cryosparc/dataset.pxd"],
+            depends=[
+                "cryosparc/include/lz4/lib/lz4.h",
+                "cryosparc/include/cryosparc-tools/dataset.h",
+                "cryosparc/lz4.pxd",
+                "cryosparc/dataset.pxd",
+            ],
         ),
         language_level=3,
         gdb_debug=DEBUG,
diff --git a/typings/cryosparc/core.pyi b/typings/cryosparc/core.pyi
index 1bdb469c..03f746aa 100644
--- a/typings/cryosparc/core.pyi
+++ b/typings/cryosparc/core.pyi
@@ -2,9 +2,43 @@
 This type stub file was generated by pyright.
 """
 from enum import Enum
+from typing import Optional, Tuple
+from numpy.typing import NDArray
+
+__all__ = ["DsetType", "Stream", "Data"]
+
+class MemoryView:  # Note: Supports buffer protocol.
+    base: "Array"
+    size: int
+    itemsize: int
+    nbytes: int
+    ndim: int
+    shape: Tuple[int, ...]
+    strides: Tuple[int, ...]
+    suboffsets: Tuple[int, ...]
+    T: "MemoryView"
+
+    def copy(self) -> "MemoryView": ...
+    def copy_fortran(self) -> "MemoryView": ...
+    def is_c_contig(self) -> bool: ...
+    def is_f_contig(self) -> bool: ...
+
+class Array:
+    memview: MemoryView
 
 class DsetType(int, Enum):
     pass
 
 class Data:
     pass
+
+class Stream:
+    def __init__(self, data: Data) -> None: ...
+    def cast_objs_to_strs(self) -> None: ...
+    def stralloc_col(self, col: str) -> Optional[Array]: ...
+    def compress_col(self, col: str) -> Array: ...
+    def compress_numpy(self, arr: NDArray) -> Array: ...
+    def compress(self, arr: Array) -> Array: ...
+    def decompress_col(self, col: str, data: bytes) -> Array: ...
+    def decompress_numpy(self, data: bytes, arr: NDArray) -> Array: ...
+    def decompress(self, data: bytes, outptr: int = 0) -> Array: ...


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index 9a8d962e..64cec81d 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -182,17 +182,18 @@ def test_from_data_none():
 
 
 def test_streaming_bytes():
+    field3 = "fieldwithsuperduperlongcolumnnamethatislongandtestable"
     dset = Dataset.allocate(
         4,
         fields=[
             ("field1", "u8"),
             ("field2", "f4"),
-            ("field3", "O"),
+            (field3, "O"),
         ],
     )
     dset["field1"] = 42
     dset["field2"] = n.array([3.14, 2.73, 1.62, 3.14], dtype="f8")
-    dset["field3"][:] = n.array(["Hello", "World", "!", "!"])
+    dset[field3][:] = n.array(["Hello", "World", "!", "!"])
 
     stream = BytesIO()
     for dat in dset.stream():
@@ -234,27 +235,28 @@ def test_row_array_type(t20s_dset):
 
 
 def test_innerjoin_bigger():
+    dat2 = "dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2"
     d1 = Dataset([("uid", [1, 2, 3]), ("dat1", ["Hello", "World", "!"])])
-    d2 = Dataset([("uid", [0, 1, 2, 3, 4]), ("dat2", ["(", "Hello", "World", "!", ")"])])
-
+    d2 = Dataset([("uid", [0, 1, 2, 3, 4]), (dat2, ["(", "Hello", "World", "!", ")"])])
     assert d1.innerjoin(d2) == Dataset(
         [
             ("uid", [1, 2, 3]),
             ("dat1", ["Hello", "World", "!"]),
-            ("dat2", ["Hello", "World", "!"]),
+            (dat2, ["Hello", "World", "!"]),
         ]
     )
 
 
 def test_innerjoin_smaller():
+    dat2 = "dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2dat2"
     d1 = Dataset([("uid", [1, 2, 3]), ("dat1", ["Hello", "World", "!"])])
-    d2 = Dataset([("uid", [3, 1]), ("dat2", ["Hello", "World"])])
+    d2 = Dataset([("uid", [3, 1]), (dat2, ["Hello", "World"])])
 
     assert d1.innerjoin(d2) == Dataset(
         [
             ("uid", [1, 3]),
             ("dat1", ["Hello", "!"]),
-            ("dat2", ["World", "Hello"]),
+            (dat2, ["World", "Hello"]),
         ]
     )
 
diff --git a/tests/test_dataset_bench.py b/tests/test_dataset_bench.py
index 12923b34..9a630667 100644
--- a/tests/test_dataset_bench.py
+++ b/tests/test_dataset_bench.py
@@ -1,5 +1,4 @@
 from io import BytesIO
-import os
 import pytest
 import numpy as n
 import cryosparc.dataset as ds
@@ -379,7 +378,7 @@ def test_streaming_bytes(benchmark, dset: Dataset):
     @benchmark
     def _():
         total_bytes = 0
-        for dat in dset.stream():
+        for dat in dset.stream(compression="lz4"):
             stream.write(dat)
             total_bytes += len(dat)
         stream.seek(0)
@@ -390,26 +389,23 @@ def _():
 
 def test_from_streaming_bytes(benchmark, big_dset: Dataset):
     stream = BytesIO()
-    for dat in big_dset.stream():
+    for dat in big_dset.stream(compression="lz4"):
         stream.write(dat)
-    stream.seek(0)
 
     def load():
-        result = Dataset.load(stream)
         stream.seek(0)
+        result = Dataset.load(stream)
         return result
 
     result = benchmark(load)
     assert len(result) == len(big_dset)
 
 
-@pytest.mark.skipif(os.getenv("CI") == "true", reason="Too slow at the moment")
 def test_to_cstrs(benchmark, dset: Dataset):
     result: Dataset = benchmark(dset.to_cstrs, copy=True)
     assert result["ctf/type"].dtype.type == n.uint64
 
 
-@pytest.mark.skipif(os.getenv("CI") == "true", reason="Too slow at the moment")
 def test_to_pystrs(benchmark, dset_cstrs: Dataset):
     result: Dataset = benchmark(dset_cstrs.to_pystrs, copy=True)
     assert result["ctf/type"].dtype.type == n.object_