Repository: cctbx/cctbx_project (language: python)

Issue:
Diffbragg fhkl and save
Lots of changes to the hopper_utils Python API, and extension of the C++/CUDA API. Original stage2 script pertaining to exascale work should not be affected by this. 

It was my intention that current codes are unaffected by this merge... However, if your code uses the output of the `hopper_utils.refine` method, then maybe slight changes will be needed. 

I plan to squash these commits before merging.  I think the most effective way to get this merged is if everyone can test their codes, in particular exaFEL. I'm happy and eager to help!

Laue diffuse
Mike's and David's PR for better diffuse scattering models in diffBragg


Discussion / hints:

Fix patch:
diff --git a/.azure-pipelines/xfel/unix-conda-build.yml b/.azure-pipelines/xfel/unix-conda-build.yml
index 6c1c874793..4ecabfd080 100644
--- a/.azure-pipelines/xfel/unix-conda-build.yml
+++ b/.azure-pipelines/xfel/unix-conda-build.yml
@@ -80,6 +80,9 @@ steps:
     rm -rf modules/boost
     export KOKKOS_ARCH=HSW
     export KOKKOS_DEVICES=OpenMP
+    if [ "$(OS)" = "macos" ]; then
+      export KOKKOS_DEVICES=Serial
+    fi
     python bootstrap.py \
       --builder=xfel \
       --use-conda $CONDA_PREFIX \
diff --git a/kokkostbx/.clang-format b/kokkostbx/.clang-format
new file mode 100644
index 0000000000..7ecc221c5c
--- /dev/null
+++ b/kokkostbx/.clang-format
@@ -0,0 +1,12 @@
+---
+BasedOnStyle: Chromium
+---
+AlignAfterOpenBracket: AlwaysBreak
+BreakStringLiterals: false
+ColumnLimit: 100
+CompactNamespaces: true
+IndentPPDirectives: BeforeHash
+IndentWidth: '4'
+Language: Cpp
+
+...
diff --git a/kokkostbx/Legacy/kokkos_matrix.h b/kokkostbx/Legacy/kokkos_matrix.h
new file mode 100644
index 0000000000..140d69e01d
--- /dev/null
+++ b/kokkostbx/Legacy/kokkos_matrix.h
@@ -0,0 +1,194 @@
+#ifndef KOKKOSTBX_MATRIX_H
+#define KOKKOSTBX_MATRIX_H
+
+#include <cmath>
+#include <iostream>
+#include <Kokkos_Core.hpp>
+
+namespace kokkostbx {
+
+    template <typename NumType, size_t size>
+    struct matrix {
+
+        NumType data[size] = { };
+
+        // CONSTRUCTOR
+        KOKKOS_FUNCTION matrix() = default;
+
+        KOKKOS_FUNCTION matrix(NumType a) : {
+            for (NumType& d : data) {
+                d = a;
+            }
+        }
+
+        KOKKOS_FUNCTION matrix(NumType a[size]) {
+            for (int i=0; i<size; ++i) {
+                data[i] = a[i];
+            }
+        }
+
+        KOKKOS_FUNCTION matrix(NumType a[9]) : data(a[0]), y(a[1]), z(a[2]) {}
+
+        KOKKOS_FUNCTION matrix(NumType a, NumType b, NumType c) : x(a), y(b), z(c) {}
+
+        // OPERATORS
+        // streaming
+        friend std::ostream& operator<< (std::ostream &os, const matrix<NumType>& v) {
+            for (int i=0; i<size; ++i) {
+                if (i>0) { os << ", "; }
+                os << v.data[i];
+            }
+            return os;
+        }
+
+        // access
+        KOKKOS_INLINE_FUNCTION NumType& operator[](const int index) {
+            switch(index) {
+                case 0: return x;
+                case 1: return y;
+                case 2: return z;
+                default:
+                    ::Kokkos::abort("Can't access kokkos::matrix, index outside [0,1,2].");
+            }
+        }
+
+        // addition
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator+(const matrix<NumType>& lhs, const matrix<NumType>& rhs) {
+            return matrix<NumType>(lhs.x+rhs.x, lhs.y+rhs.y, lhs.z+rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator+(const matrix<NumType>& lhs, NumType& rhs) {
+            return matrix<NumType>(lhs.x+rhs, lhs.y+rhs, lhs.z+rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator+(NumType lhs, const matrix<NumType>& rhs) {
+            return rhs + lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const matrix<NumType>& v) {
+            x += v.x;
+            y += v.y;
+            z += v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const NumType& v) {
+            x += v;
+            y += v;
+            z += v;
+        }
+
+        // subtraction
+        KOKKOS_INLINE_FUNCTION matrix<NumType> operator-() const {
+            return matrix<NumType>(-x, -y, -z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix<NumType> operator-(const matrix<NumType>& v) const {
+            return matrix<NumType>(x-v.x, y-v.y, z-v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix<NumType> operator-(const NumType& v) const {
+            return matrix<NumType>(x-v, y-v, z-v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const matrix<NumType>& v) {
+            x -= v.x;
+            y -= v.y;
+            z -= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const NumType& v) {
+            x -= v;
+            y -= v;
+            z -= v;
+        }
+
+        // multiplication
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator*(const matrix<NumType>& lhs, const matrix<NumType>& rhs) {
+            return matrix<NumType>(lhs.x*rhs.x, lhs.y*rhs.y, lhs.z*rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator*(const matrix<NumType>& lhs, NumType& rhs) {
+            return matrix<NumType>(lhs.x*rhs, lhs.y*rhs, lhs.z*rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix<NumType> operator*(NumType lhs, const matrix<NumType>& rhs) {
+            return rhs * lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const matrix<NumType>& v) {
+            x *= v.x;
+            y *= v.y;
+            z *= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const NumType& v) {
+            x *= v;
+            y *= v;
+            z *= v;
+        }
+
+        // division
+        KOKKOS_INLINE_FUNCTION matrix<NumType> operator/(const matrix<NumType>& v) const {
+            return matrix<NumType>(x/v.x, y/v.y, z/v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix<NumType> operator/(const NumType& v) const {
+            return matrix<NumType>(x/v, y/v, z/v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const matrix<NumType>& v) {
+            x /= v.x;
+            y /= v.y;
+            z /= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const NumType& v) {
+            x /= v;
+            y /= v;
+            z /= v;
+        }
+
+        // METHODS
+        KOKKOS_INLINE_FUNCTION void zero() {
+            x = 0;
+            z = 0;
+            y = 0;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length_sqr() const {
+            return x*x + y*y + z*z;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length() const {
+            return ::Kokkos::Experimental::sqrt(length_sqr());
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType dot(const matrix<NumType>& v) const {
+            return x*v.x + y*v.y + z*v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix<NumType> cross(const matrix<NumType>& v) const {
+            return matrix<NumType>(y*v.z - z*v.y,
+                                    z*v.x - x*v.z,
+                                    x*v.y - y*v.x);
+        }
+
+        KOKKOS_INLINE_FUNCTION void normalize() {
+            NumType l = length();
+            if (l>0) {
+                x /= l;
+                y /= l;
+                z /= l;
+            }
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix<NumType> get_unit_vector() const {
+            NumType l = length();
+            if (l>0) {
+                return matrix<NumType>(x/l, y/l, z/l);
+            } else {
+                return matrix<NumType>(0, 0, 0);
+            }
+        }
+
+
+
+}
+
+#endif
diff --git a/kokkostbx/Legacy/kokkos_matrix3.h b/kokkostbx/Legacy/kokkos_matrix3.h
new file mode 100644
index 0000000000..96105beb3a
--- /dev/null
+++ b/kokkostbx/Legacy/kokkos_matrix3.h
@@ -0,0 +1,185 @@
+#ifndef KOKKOSTBX_MATRIX3_H
+#define KOKKOSTBX_MATRIX3_H
+
+#include <cmath>
+#include <iostream>
+#include <Kokkos_Core.hpp>
+
+namespace kokkostbx {
+
+    template <typename NumType>
+    struct matrix3 {
+
+        NumType data[9] = { };
+
+        // CONSTRUCTOR
+        KOKKOS_FUNCTION matrix3() = default;
+
+        KOKKOS_FUNCTION matrix3(NumType a) : {
+            for (NumType& d : data) {
+                d = a;
+            }
+        }
+
+        KOKKOS_FUNCTION matrix3(NumType a[9]) : data(a[0]), y(a[1]), z(a[2]) {}
+
+        KOKKOS_FUNCTION matrix3(NumType a, NumType b, NumType c) : x(a), y(b), z(c) {}
+
+        // OPERATORS
+        // streaming
+        friend std::ostream& operator<< (std::ostream &os, const matrix3<NumType>& v) {
+            os << v.x << ", " << v.y << ", " << v.z;
+            return os;
+        }
+
+        // access
+        KOKKOS_INLINE_FUNCTION NumType& operator[](const int index) {
+            switch(index) {
+                case 0: return x;
+                case 1: return y;
+                case 2: return z;
+                default:
+                    ::Kokkos::abort("Can't access kokkos::matrix3, index outside [0,1,2].");
+            }
+        }
+
+        // addition
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator+(const matrix3<NumType>& lhs, const matrix3<NumType>& rhs) {
+            return matrix3<NumType>(lhs.x+rhs.x, lhs.y+rhs.y, lhs.z+rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator+(const matrix3<NumType>& lhs, NumType& rhs) {
+            return matrix3<NumType>(lhs.x+rhs, lhs.y+rhs, lhs.z+rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator+(NumType lhs, const matrix3<NumType>& rhs) {
+            return rhs + lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const matrix3<NumType>& v) {
+            x += v.x;
+            y += v.y;
+            z += v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const NumType& v) {
+            x += v;
+            y += v;
+            z += v;
+        }
+
+        // subtraction
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> operator-() const {
+            return matrix3<NumType>(-x, -y, -z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> operator-(const matrix3<NumType>& v) const {
+            return matrix3<NumType>(x-v.x, y-v.y, z-v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> operator-(const NumType& v) const {
+            return matrix3<NumType>(x-v, y-v, z-v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const matrix3<NumType>& v) {
+            x -= v.x;
+            y -= v.y;
+            z -= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const NumType& v) {
+            x -= v;
+            y -= v;
+            z -= v;
+        }
+
+        // multiplication
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator*(const matrix3<NumType>& lhs, const matrix3<NumType>& rhs) {
+            return matrix3<NumType>(lhs.x*rhs.x, lhs.y*rhs.y, lhs.z*rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator*(const matrix3<NumType>& lhs, NumType& rhs) {
+            return matrix3<NumType>(lhs.x*rhs, lhs.y*rhs, lhs.z*rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend matrix3<NumType> operator*(NumType lhs, const matrix3<NumType>& rhs) {
+            return rhs * lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const matrix3<NumType>& v) {
+            x *= v.x;
+            y *= v.y;
+            z *= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const NumType& v) {
+            x *= v;
+            y *= v;
+            z *= v;
+        }
+
+        // division
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> operator/(const matrix3<NumType>& v) const {
+            return matrix3<NumType>(x/v.x, y/v.y, z/v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> operator/(const NumType& v) const {
+            return matrix3<NumType>(x/v, y/v, z/v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const matrix3<NumType>& v) {
+            x /= v.x;
+            y /= v.y;
+            z /= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const NumType& v) {
+            x /= v;
+            y /= v;
+            z /= v;
+        }
+
+        // METHODS
+        KOKKOS_INLINE_FUNCTION void zero() {
+            x = 0;
+            z = 0;
+            y = 0;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length_sqr() const {
+            return x*x + y*y + z*z;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length() const {
+            return ::Kokkos::Experimental::sqrt(length_sqr());
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType dot(const matrix3<NumType>& v) const {
+            return x*v.x + y*v.y + z*v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> cross(const matrix3<NumType>& v) const {
+            return matrix3<NumType>(y*v.z - z*v.y,
+                                    z*v.x - x*v.z,
+                                    x*v.y - y*v.x);
+        }
+
+        KOKKOS_INLINE_FUNCTION void normalize() {
+            NumType l = length();
+            if (l>0) {
+                x /= l;
+                y /= l;
+                z /= l;
+            }
+        }
+
+        KOKKOS_INLINE_FUNCTION matrix3<NumType> get_unit_vector() const {
+            NumType l = length();
+            if (l>0) {
+                return matrix3<NumType>(x/l, y/l, z/l);
+            } else {
+                return matrix3<NumType>(0, 0, 0);
+            }
+        }
+
+
+
+}
+
+#endif
diff --git a/kokkostbx/Legacy/kokkos_vector.h b/kokkostbx/Legacy/kokkos_vector.h
new file mode 100644
index 0000000000..76beaaa245
--- /dev/null
+++ b/kokkostbx/Legacy/kokkos_vector.h
@@ -0,0 +1,284 @@
+#ifndef KOKKOSTBX_VECTOR_H
+#define KOKKOSTBX_VECTOR_H
+
+#include <iostream>
+
+namespace kokkostbx {
+
+    template <typename Derived, typename NumType, size_t size>
+    struct vector_base {
+
+        NumType data[size] = {};
+
+        // CONSTRUCTOR
+        KOKKOS_FUNCTION vector_base() = default;
+
+        KOKKOS_FUNCTION vector_base(NumType val) {
+            for (NumType& d : data) { d = val; }
+        }
+
+        KOKKOS_FUNCTION vector_base(NumType arr[]) {
+            for (int i=0; i<size; ++i) {
+                data[i] = arr[i];
+            }
+        }
+
+        // OPERATORS
+        // streaming
+        friend std::ostream& operator<< (std::ostream &os, const vector_base<Derived, NumType, size>& v) {
+            for (int i=0; i<size: ++i ) {
+                if (i>0) { os << ", "; }
+                os << data[i];
+            }
+            return os;
+        }
+
+        // access
+        KOKKOS_INLINE_FUNCTION NumType& operator[](const int index) {
+            return data[index];
+        }
+
+        // addition
+        KOKKOS_INLINE_FUNCTION friend vector_base<Derived, NumType, size> operator+(
+            const vector_base<Derived, NumType, size>& lhs,
+            const vector_base<Derived, NumType, size>& rhs) {
+
+            vector_base<Derived, NumType, size> sum = lhs;
+            sum += rhs;
+            return sum;
+        }
+        KOKKOS_INLINE_FUNCTION friend vector_base<Derived, NumType, size> operator+(
+            const vector_base<Derived, NumType, size>& lhs,
+            NumType& rhs) {
+
+            vector_base<Derived, NumType, size> sum = lhs;
+            sum += rhs;
+            return sum;
+        }
+        KOKKOS_INLINE_FUNCTION friend vector_base<Derived, NumType, size> operator+(
+            NumType lhs,
+            const vector_base<Derived, NumType, size>& rhs) {
+            return rhs + lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const vector_base<Derived, NumType, size>& v) {
+            for (int i=0; i<size; ++i) {
+                data[i] += v[i];
+            }
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const NumType& v) {
+            for (int i=0; i<size; ++i) {
+                data[i] += v;
+            }
+        }
+    }
+
+    };
+
+
+
+    template <typename NumType>
+    struct vector3 {
+        NumType x = 0;
+        NumType y = 0;
+        NumType z = 0;
+
+        // CONSTRUCTOR
+        KOKKOS_FUNCTION vector3() = default;
+
+        KOKKOS_FUNCTION vector3(NumType a) : x(a), y(a), z(a) {}
+
+        KOKKOS_FUNCTION vector3(NumType a[3]) : x(a[0]), y(a[1]), z(a[2]) {}
+
+        KOKKOS_FUNCTION vector3(NumType a, NumType b, NumType c) : x(a), y(b), z(c) {}
+
+        // OPERATORS
+        // streaming
+        friend std::ostream& operator<< (std::ostream &os, const vector3<NumType>& v) {
+            os << v.x << ", " << v.y << ", " << v.z;
+            return os;
+        }
+
+        // access
+        KOKKOS_INLINE_FUNCTION NumType& operator[](const int index) {
+            switch(index) {
+                case 0: return x;
+                case 1: return y;
+                case 2: return z;
+                default:
+                    ::Kokkos::abort("Can't access kokkos::vector3, index outside [0,1,2].");
+            }
+        }
+
+        // addition
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(const vector3<NumType>& lhs, const vector3<NumType>& rhs) {
+            return vector3<NumType>(lhs.x+rhs.x, lhs.y+rhs.y, lhs.z+rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(const vector3<NumType>& lhs, NumType& rhs) {
+            return vector3<NumType>(lhs.x+rhs, lhs.y+rhs, lhs.z+rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(NumType lhs, const vector3<NumType>& rhs) {
+            return rhs + lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const vector3<NumType>& v) {
+            x += v.x;
+            y += v.y;
+            z += v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const NumType& v) {
+            x += v;
+            y += v;
+            z += v;
+        }
+
+        // subtraction
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-() const {
+            return vector3<NumType>(-x, -y, -z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-(const vector3<NumType>& v) const {
+            return vector3<NumType>(x-v.x, y-v.y, z-v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-(const NumType& v) const {
+            return vector3<NumType>(x-v, y-v, z-v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const vector3<NumType>& v) {
+            x -= v.x;
+            y -= v.y;
+            z -= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const NumType& v) {
+            x -= v;
+            y -= v;
+            z -= v;
+        }
+
+        // multiplication
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(const vector3<NumType>& lhs, const vector3<NumType>& rhs) {
+            return vector3<NumType>(lhs.x*rhs.x, lhs.y*rhs.y, lhs.z*rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(const vector3<NumType>& lhs, NumType& rhs) {
+            return vector3<NumType>(lhs.x*rhs, lhs.y*rhs, lhs.z*rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(NumType lhs, const vector3<NumType>& rhs) {
+            return rhs * lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const vector3<NumType>& v) {
+            x *= v.x;
+            y *= v.y;
+            z *= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const NumType& v) {
+            x *= v;
+            y *= v;
+            z *= v;
+        }
+
+        // division
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator/(const vector3<NumType>& v) const {
+            return vector3<NumType>(x/v.x, y/v.y, z/v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator/(const NumType& v) const {
+            return vector3<NumType>(x/v, y/v, z/v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const vector3<NumType>& v) {
+            x /= v.x;
+            y /= v.y;
+            z /= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const NumType& v) {
+            x /= v;
+            y /= v;
+            z /= v;
+        }
+
+        // METHODS
+        KOKKOS_INLINE_FUNCTION void zero() {
+            x = 0;
+            z = 0;
+            y = 0;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length_sqr() const {
+            return x*x + y*y + z*z;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length() const {
+            return ::Kokkos::Experimental::sqrt(length_sqr());
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType dot(const vector3<NumType>& v) const {
+            return x*v.x + y*v.y + z*v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> cross(const vector3<NumType>& v) const {
+            return vector3<NumType>(y*v.z - z*v.y,
+                                    z*v.x - x*v.z,
+                                    x*v.y - y*v.x);
+        }
+
+        KOKKOS_INLINE_FUNCTION void normalize() {
+            NumType l = length();
+            if (l>0) {
+                x /= l;
+                y /= l;
+                z /= l;
+            }
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> get_unit_vector() const {
+            NumType l = length();
+            if (l>0) {
+                return vector3<NumType>(x/l, y/l, z/l);
+            } else {
+                return vector3<NumType>(0, 0, 0);
+            }
+        }
+
+        // rotate a point about a unit vector3 axis
+        KOKKOS_INLINE_FUNCTION vector3<NumType> rotate_around_axis(const vector3<NumType>& axis, NumType angle) const {
+            NumType sinphi = ::Kokkos::Experimental::sin(angle);
+            NumType cosphi = ::Kokkos::Experimental::cos(angle);
+            NumType dot_factor = axis.dot(*this) * (1.0-cosphi);
+
+            vector3<NumType> vector_rot = axis.cross(*this) * sinphi;
+            vector_rot += axis * dot_factor;
+            vector_rot += (*this) * cosphi;
+            return vector_rot;
+        }
+
+        // rotate a vector using a 9-element unitary matrix
+        KOKKOS_INLINE_FUNCTION vector3<NumType> rotate_matrix(const NumType * __restrict__ umat) {
+            // for convenience, assign matrix x-y coordinate
+            vector3<NumType> ux = {umat[0], umat[1], umat[2]};
+            vector3<NumType> uy = {umat[3], umat[4], umat[5]};
+            vector3<NumType> uz = {umat[6], umat[7], umat[8]};
+
+            // rotate the vector
+            NumType newx = ux.dot(*this);
+            NumType newy = uy.dot(*this);
+            NumType newz = uz.dot(*this);
+
+            return vector3<NumType>(newx, newy, newz);
+        }
+    };
+
+}
+
+#endif
+
+
+
+
+#endif
diff --git a/kokkostbx/Legacy/kokkos_vector3.h b/kokkostbx/Legacy/kokkos_vector3.h
new file mode 100644
index 0000000000..802a3c8e26
--- /dev/null
+++ b/kokkostbx/Legacy/kokkos_vector3.h
@@ -0,0 +1,208 @@
+#ifndef KOKKOSTBX_VECTOR3_H
+#define KOKKOSTBX_VECTOR3_H
+
+#include <cmath>
+#include <iostream>
+#include <Kokkos_Core.hpp>
+
+namespace kokkostbx {
+
+    template <typename NumType>
+    struct vector3 {
+        NumType x = 0;
+        NumType y = 0;
+        NumType z = 0;
+
+        // CONSTRUCTOR
+        KOKKOS_FUNCTION vector3() = default;
+
+        KOKKOS_FUNCTION vector3(NumType a) : x(a), y(a), z(a) {}
+
+        KOKKOS_FUNCTION vector3(NumType a[3]) : x(a[0]), y(a[1]), z(a[2]) {}
+
+        KOKKOS_FUNCTION vector3(NumType a, NumType b, NumType c) : x(a), y(b), z(c) {}
+
+        // OPERATORS
+        // streaming
+        friend std::ostream& operator<< (std::ostream &os, const vector3<NumType>& v) {
+            os << v.x << ", " << v.y << ", " << v.z;
+            return os;
+        }
+
+        // access
+        KOKKOS_INLINE_FUNCTION NumType& operator[](const int index) {
+            switch(index) {
+                case 0: return x;
+                case 1: return y;
+                case 2: return z;
+                default:
+                    ::Kokkos::abort("Can't access kokkos::vector3, index outside [0,1,2].");
+            }
+        }
+
+        // addition
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(const vector3<NumType>& lhs, const vector3<NumType>& rhs) {
+            return vector3<NumType>(lhs.x+rhs.x, lhs.y+rhs.y, lhs.z+rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(const vector3<NumType>& lhs, NumType& rhs) {
+            return vector3<NumType>(lhs.x+rhs, lhs.y+rhs, lhs.z+rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator+(NumType lhs, const vector3<NumType>& rhs) {
+            return rhs + lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const vector3<NumType>& v) {
+            x += v.x;
+            y += v.y;
+            z += v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator+=(const NumType& v) {
+            x += v;
+            y += v;
+            z += v;
+        }
+
+        // subtraction
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-() const {
+            return vector3<NumType>(-x, -y, -z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-(const vector3<NumType>& v) const {
+            return vector3<NumType>(x-v.x, y-v.y, z-v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator-(const NumType& v) const {
+            return vector3<NumType>(x-v, y-v, z-v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const vector3<NumType>& v) {
+            x -= v.x;
+            y -= v.y;
+            z -= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator-=(const NumType& v) {
+            x -= v;
+            y -= v;
+            z -= v;
+        }
+
+        // multiplication
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(const vector3<NumType>& lhs, const vector3<NumType>& rhs) {
+            return vector3<NumType>(lhs.x*rhs.x, lhs.y*rhs.y, lhs.z*rhs.z);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(const vector3<NumType>& lhs, NumType& rhs) {
+            return vector3<NumType>(lhs.x*rhs, lhs.y*rhs, lhs.z*rhs);
+        }
+        KOKKOS_INLINE_FUNCTION friend vector3<NumType> operator*(NumType lhs, const vector3<NumType>& rhs) {
+            return rhs * lhs;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const vector3<NumType>& v) {
+            x *= v.x;
+            y *= v.y;
+            z *= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator*=(const NumType& v) {
+            x *= v;
+            y *= v;
+            z *= v;
+        }
+
+        // division
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator/(const vector3<NumType>& v) const {
+            return vector3<NumType>(x/v.x, y/v.y, z/v.z);
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> operator/(const NumType& v) const {
+            return vector3<NumType>(x/v, y/v, z/v);
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const vector3<NumType>& v) {
+            x /= v.x;
+            y /= v.y;
+            z /= v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION void operator/=(const NumType& v) {
+            x /= v;
+            y /= v;
+            z /= v;
+        }
+
+        // METHODS
+        KOKKOS_INLINE_FUNCTION void zero() {
+            x = 0;
+            z = 0;
+            y = 0;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length_sqr() const {
+            return x*x + y*y + z*z;
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType length() const {
+            return ::Kokkos::Experimental::sqrt(length_sqr());
+        }
+
+        KOKKOS_INLINE_FUNCTION NumType dot(const vector3<NumType>& v) const {
+            return x*v.x + y*v.y + z*v.z;
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> cross(const vector3<NumType>& v) const {
+            return vector3<NumType>(y*v.z - z*v.y,
+                                    z*v.x - x*v.z,
+                                    x*v.y - y*v.x);
+        }
+
+        KOKKOS_INLINE_FUNCTION void normalize() {
+            NumType l = length();
+            if (l>0) {
+                x /= l;
+                y /= l;
+                z /= l;
+            }
+        }
+
+        KOKKOS_INLINE_FUNCTION vector3<NumType> get_unit_vector() const {
+            NumType l = length();
+            if (l>0) {
+                return vector3<NumType>(x/l, y/l, z/l);
+            } else {
+                return vector3<NumType>(0, 0, 0);
+            }
+        }
+
+        // rotate a point about a unit vector3 axis
+        KOKKOS_INLINE_FUNCTION vector3<NumType> rotate_around_axis(const vector3<NumType>& axis, NumType angle) const {
+            NumType sinphi = ::Kokkos::Experimental::sin(angle);
+            NumType cosphi = ::Kokkos::Experimental::cos(angle);
+            NumType dot_factor = axis.dot(*this) * (1.0-cosphi);
+
+            vector3<NumType> vector_rot = axis.cross(*this) * sinphi;
+            vector_rot += axis * dot_factor;
+            vector_rot += (*this) * cosphi;
+            return vector_rot;
+        }
+
+        // rotate a vector using a 9-element unitary matrix
+        KOKKOS_INLINE_FUNCTION vector3<NumType> rotate_matrix(const NumType * __restrict__ umat) {
+            // for convenience, assign matrix x-y coordinate
+            vector3<NumType> ux = {umat[0], umat[1], umat[2]};
+            vector3<NumType> uy = {umat[3], umat[4], umat[5]};
+            vector3<NumType> uz = {umat[6], umat[7], umat[8]};
+
+            // rotate the vector
+            NumType newx = ux.dot(*this);
+            NumType newy = uy.dot(*this);
+            NumType newz = uz.dot(*this);
+
+            return vector3<NumType>(newx, newy, newz);
+        }
+    };
+
+}
+
+#endif
diff --git a/kokkostbx/SConscript b/kokkostbx/SConscript
new file mode 100644
index 0000000000..840dcdb9d2
--- /dev/null
+++ b/kokkostbx/SConscript
@@ -0,0 +1,192 @@
+import libtbx.load_env
+import os
+import subprocess
+Import("env_etc")
+from shutil import copy, which
+
+if env_etc.enable_kokkos:
+
+  # libkokkos.a
+  # call kokkos build system directly
+  # set environment variable defaults if necessary
+  if os.getenv('KOKKOS_DEVICES') is None:
+    os.environ['KOKKOS_DEVICES'] = "OpenMP"
+  use_openmp = "OpenMP" in os.getenv('KOKKOS_DEVICES')  
+  use_cuda = "Cuda" in os.getenv('KOKKOS_DEVICES')
+  use_hip = "HIP" in os.getenv('KOKKOS_DEVICES')
+  use_sycl = "SYCL" in os.getenv('KOKKOS_DEVICES')
+  if os.getenv('KOKKOS_PATH') is None:
+    os.environ['KOKKOS_PATH'] = libtbx.env.under_dist('simtbx', '../../kokkos')
+  if os.getenv('KOKKOSKERNELS_PATH') is None:
+    os.environ['KOKKOSKERNELS_PATH'] = libtbx.env.under_dist('simtbx', '../../kokkos-kernels')
+  if os.getenv('KOKKOS_ARCH') is None:
+    os.environ['KOKKOS_ARCH'] = "HSW"
+  if use_cuda and os.getenv('KOKKOS_CUDA_OPTIONS') is None:
+    os.environ['KOKKOS_CUDA_OPTIONS'] = "enable_lambda,force_uvm"
+  os.environ['CXXFLAGS'] = '-O3 -fPIC -DCUDAREAL=double'
+
+  library_flags = "-Llib"
+  if use_cuda:
+    library_flags += " -L$(CUDA_HOME)/lib64 -L$(CUDA_HOME)/lib64/stubs"
+    if os.getenv('CUDA_COMPATIBILITY'):
+      library_flags += " -L$(CUDA_HOME)/compat"
+  os.environ['LDFLAGS'] = library_flags
+
+  linked_libraries = "-lkokkoscontainers -lkokkoscore -ldl"
+  if use_cuda:
+    linked_libraries += " -lcudart -lcuda"
+  os.environ['LDLIBS'] = linked_libraries
+
+  cxx_standard = '14'
+  if use_sycl:
+    cxx_standard = '17'
+
+  original_cxx = None
+  kokkos_lib = 'libkokkos.a'
+  kokkos_cxxflags = None
+
+  if os.getenv('CXX') is not None:
+    original_cxx = os.environ['CXX']
+  if use_cuda:
+    os.environ['CXX'] = os.path.join(os.environ['KOKKOS_PATH'], 'bin', 'nvcc_wrapper')
+  elif use_hip:
+    os.environ['CXX'] = 'hipcc'
+  else:
+    os.environ['CXX'] = os.environ.get('CXX', 'g++')
+  print('='*79)
+  print('Building Kokkos')
+  print('-'*79)
+  returncode = subprocess.call(['make', '-f', 'Makefile.kokkos', kokkos_lib],
+                                cwd=os.environ['KOKKOS_PATH'])
+  print()
+
+  print('Copying Kokkos library')
+  print('-'*79)
+  src = os.path.join(os.environ['KOKKOS_PATH'], kokkos_lib)
+  dst = os.path.join(libtbx.env.under_build('lib'), kokkos_lib)
+  if os.path.isfile(src):
+    copy(src, dst)
+    print('Copied')
+    print('  source:     ', src)
+    print('  destination:', dst)
+  else:
+    print('Error: {src} does not exist'.format(src=src))
+  print()
+
+  # =============================================================================
+  # Build kokkos with CMake
+  # The build needs to be in a directory not in the build directory, otherwise
+  # kokkos-kernels will find that build directory instead of build/lib/cmake/Kokkos
+  # The error will look like
+  #
+  #   -- The project name is: KokkosKernels
+  #   CMake Error at /dev/shm/bkpoon/software/xfel/build/kokkos/KokkosConfig.cmake:48 (INCLUDE):
+  #     INCLUDE could not find requested file:
+
+  #       /dev/shm/bkpoon/software/xfel/build/kokkos/KokkosTargets.cmake
+  #   Call Stack (most recent call first):
+  #     CMakeLists.txt:107 (FIND_PACKAGE)
+  #
+  # kokkos will be installed in build with the libraries in lib, not lib64
+  #
+  # TODO:
+  # - Change over from Makefile version to cmake version
+  # - Change system configuration classes to select backends with cmake
+  #   flags instead of environment variables
+  # - Verify that libraries from cmake version works the same as libkokkos.a
+
+  OnOff = {True:'ON', False:'OFF'}
+  supported_architectures = ['Native', 'HSW', 'Zen', 'Zen2', 'Zen3', 'Volta70', 'Ampere80', 'Vega908', 'Vega90A', 'XeHP', 'PVC']
+  architectures = {}
+  for arch in supported_architectures:
+    architectures[arch] = OnOff[ arch in os.getenv('KOKKOS_ARCH') ]
+
+  cmake_is_available = which('cmake')
+  if cmake_is_available:
+    print('='*79)
+    print('Building kokkos with cmake')
+    print('-'*79)
+    kokkos_build_dir = libtbx.env.under_dist('simtbx', '../../kokkos_build')
+    if not os.path.isdir(kokkos_build_dir):
+      os.mkdir(kokkos_build_dir)
+    returncode = subprocess.call([
+        'cmake',
+        os.environ['KOKKOS_PATH'],
+        '-DCMAKE_CXX_STANDARD={}'.format(cxx_standard),
+        '-DCMAKE_INSTALL_PREFIX={}'.format(libtbx.env.under_build('.')),
+        '-DCMAKE_INSTALL_LIBDIR=lib',
+        '-DBUILD_SHARED_LIBS={}'.format(OnOff[use_sycl]),
+        '-DKokkos_ARCH_NATIVE={}'.format(architectures['Native']),
+        '-DKokkos_ARCH_HSW={}'.format(architectures['HSW']),
+        '-DKokkos_ARCH_ZEN={}'.format(architectures['Zen']),
+        '-DKokkos_ARCH_ZEN2={}'.format(architectures['Zen2']),
+        '-DKokkos_ARCH_ZEN3={}'.format(architectures['Zen3']),
+        '-DKokkos_ARCH_VOLTA70={}'.format(architectures['Volta70']),
+        '-DKokkos_ARCH_AMPERE80={}'.format(architectures['Ampere80']),
+        '-DKokkos_ARCH_VEGA908={}'.format(architectures['Vega908']),
+        '-DKokkos_ARCH_VEGA90A={}'.format(architectures['Vega90A']),
+        '-DKokkos_ARCH_INTEL_XEHP={}'.format(architectures['XeHP']),
+        '-DKokkos_ARCH_INTEL_PVC={}'.format(architectures['PVC']),
+        '-DKokkos_ENABLE_SERIAL=ON',
+        '-DKokkos_ENABLE_OPENMP={}'.format(OnOff[use_openmp]),
+        '-DKokkos_ENABLE_CUDA={}'.format(OnOff[use_cuda]),
+        '-DKokkos_ENABLE_CUDA_UVM={}'.format(OnOff[use_cuda]),
+        '-DKokkos_ENABLE_HIP={}'.format(OnOff[use_hip]),
+        '-DKokkos_ENABLE_SYCL={}'.format(OnOff[use_sycl])
+      ],
+      cwd=kokkos_build_dir)
+
+    returncode = subprocess.call(['make', '-j', '4', 'install'], cwd=kokkos_build_dir)
+
+  # -----------------------------------------------------------------------------
+  # Build kokkos-kernels with CMake.
+  # Turn off all ETI builds for now, until needed, for maximum machine compatibility
+    print('='*79)
+    print('Building kokkos_kernels')
+    print('-'*79)
+    kokkos_kernels_build_dir = libtbx.env.under_dist('simtbx', '../../kokkos-kernels/build')
+    if not os.path.isdir(kokkos_kernels_build_dir):
+      os.mkdir(kokkos_kernels_build_dir)
+    returncode = subprocess.call([
+        'cmake',
+        os.environ['KOKKOSKERNELS_PATH'],
+        '-DCMAKE_INSTALL_PREFIX={}'.format(libtbx.env.under_build('.')),
+        '-DCMAKE_INSTALL_LIBDIR=lib',
+        '-DKokkos_ROOT={}'.format(libtbx.env.under_build('.')),
+        '-DKokkosKernels_ADD_DEFAULT_ETI={}'.format(OnOff[False]),
+        '-DKokkosKernels_INST_LAYOUTLEFT:BOOL={}'.format(OnOff[False]),
+        '-DKokkosKernels_INST_LAYOUTRIGHT:BOOL={}'.format(OnOff[False]),
+        '-DKokkosKernels_ENABLE_TPL_CUBLAS={}'.format(OnOff[False]),
+        '-DKokkosKernels_ENABLE_TPL_CUSPARSE={}'.format(OnOff[False])
+      ],
+      cwd=kokkos_kernels_build_dir)
+    returncode = subprocess.call(['make', '-j', '4'], cwd=kokkos_kernels_build_dir)
+    returncode = subprocess.call(['make', '-j', '4', 'install'], cwd=kokkos_kernels_build_dir)
+  else:
+    print('*'*79)
+    print('cmake was not found')
+    print('Skipping builds of kokkos and kokkos-kernels')
+    print('*'*79)
+
+  # =============================================================================
+  print('Getting environment variables')
+  print('-'*79)
+  kokkos_cxxflags = subprocess.check_output(
+    ['make', '-f', 'Makefile.kokkos', 'print-cxx-flags'],
+  cwd=os.environ['KOKKOS_PATH'])
+  kokkos_cxxflags = kokkos_cxxflags.split(b'\n')
+  kokkos_cxxflags = kokkos_cxxflags[1].decode('utf8')
+  os.environ['KOKKOS_CXXFLAGS'] = kokkos_cxxflags
+  kokkos_cxxflags = kokkos_cxxflags.split()
+  if kokkos_cxxflags and kokkos_cxxflags[0] == 'echo':
+    kokkos_cxxflags = [f.strip('"') for f in kokkos_cxxflags[1:]]
+  print('KOKKOS_CXXFLAGS:', kokkos_cxxflags)
+  print('='*79)
+
+
+  # reset CXX
+  if original_cxx is not None:
+    print("RESTORE CXX")
+    print('   old:', os.environ['CXX'])
+    print('   new:', original_cxx)
+    os.environ['CXX'] = original_cxx
diff --git a/kokkostbx/kokkos_matrix.h b/kokkostbx/kokkos_matrix.h
new file mode 100644
index 0000000000..2db8462505
--- /dev/null
+++ b/kokkostbx/kokkos_matrix.h
@@ -0,0 +1,152 @@
+#ifndef KOKKOSTBX_MATRIX_H
+#define KOKKOSTBX_MATRIX_H
+
+#include "kokkos_vector.h"
+
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+namespace kokkostbx {
+
+template <typename Derived, typename NumType, size_t rank>
+struct matrix_base : public vector_base<matrix_base<Derived, NumType, rank>, NumType, rank * rank> {
+    static constexpr size_t size = rank * rank;
+
+    // data stored in row-major format!
+    using vector_base =
+        kokkostbx::vector_base<matrix_base<Derived, NumType, rank>, NumType, rank * rank>;
+    using VectorType = kokkostbx::vector<NumType, rank>;
+
+    matrix_base() = default;
+    KOKKOS_FUNCTION matrix_base(NumType val) : vector_base(val) {};
+    KOKKOS_FUNCTION matrix_base(NumType arr[]) : vector_base(arr) {};
+
+    static Derived KOKKOS_FUNCTION diagonal_matrix(NumType arr[]) {
+        Derived diagonal;
+        for (size_t i = 0; i < rank; ++i) {
+            diagonal.data[i * (rank + 1)] = arr[i];
+        }
+        return diagonal;
+    };
+
+    // access
+    KOKKOS_FUNCTION NumType& get(size_t row, size_t col) {
+        assert(row < rank);
+        assert(col < rank);
+        return vector_base::data[row * rank + col];
+    }
+
+    KOKKOS_FUNCTION NumType get(size_t row, size_t col) const {
+        assert(row < rank);
+        assert(col < rank);
+        return vector_base::data[row * rank + col];
+    }
+
+    KOKKOS_FUNCTION NumType& operator()(size_t row, size_t col) {
+        return get(row, col);
+    }
+
+    KOKKOS_FUNCTION NumType operator()(size_t row, size_t col) const {
+        return get(row, col);
+    }
+
+    // Override multiplication
+    KOKKOS_FUNCTION friend Derived operator*(const Derived& lhs, const Derived& rhs) {
+        Derived prod = lhs;
+        prod *= rhs;
+        return prod;
+    }
+
+    KOKKOS_FUNCTION friend VectorType operator*(const Derived& lhs, const VectorType& rhs) {
+        return lhs.dot(rhs);
+    }
+
+    KOKKOS_FUNCTION friend Derived operator*(const Derived& lhs, const NumType& rhs) {
+        Derived prod = lhs;
+        prod *= rhs;
+        return prod;
+    }
+
+    KOKKOS_FUNCTION friend Derived operator*(const NumType& lhs, const Derived& rhs) {
+        return rhs * lhs;
+    }
+
+    KOKKOS_FUNCTION void operator*=(const Derived& rhs) {
+        Derived prod = this->dot(rhs);
+        for (size_t i = 0; i < size; ++i) {
+            vector_base::data[i] = prod[i];
+        }
+    }
+
+    KOKKOS_FUNCTION void operator*=(const NumType& rhs) {
+        vector_base::operator*=(rhs);
+    }
+
+    KOKKOS_FUNCTION VectorType dot(const VectorType v) const {
+        static_assert(v.get_size() == get_rank(), "Vector length must be equal to matrix rank!");
+        VectorType result;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                result[i] += get(i, j) * v.data[j];
+            }
+        }
+        return result;
+    }
+
+    KOKKOS_FUNCTION Derived dot(const Derived v) const {
+        Derived result;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                for (size_t k = 0; k < rank; ++k) {
+                    result(i, j) += get(i, k) * v(k, j);
+                }
+            }
+        }
+        return result;
+    }
+
+    static constexpr KOKKOS_FUNCTION size_t get_rank() { return rank; }
+
+    KOKKOS_FUNCTION NumType trace() const {
+        NumType result = 0;
+        for (size_t i = 0; i < rank; ++i) {
+            result += get(i, i);
+        }
+        return result;
+    }
+
+    KOKKOS_FUNCTION void print(const char name[]) const {
+        printf("%s: ", name);
+        for (size_t i = 0; i < size; ++i) {
+            if (i % rank == 0) {
+                printf("\n");
+            }
+            print_num(vector_base::data[i]);
+        }
+        printf("\n");
+    }
+
+    KOKKOS_FUNCTION Derived transpose() const {
+        Derived new_mat;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                new_mat(j, i) = get(i, j);
+            }
+        }
+        return new_mat;
+    }
+};
+
+template <typename NumType, size_t rank>
+struct matrix : public matrix_base<matrix<NumType, rank>, NumType, rank> {
+    using matrix_base = kokkostbx::matrix_base<matrix<NumType, rank>, NumType, rank>;
+
+    matrix() = default;
+    KOKKOS_FUNCTION matrix(NumType val) : matrix_base(val){};
+    KOKKOS_FUNCTION matrix(NumType arr[]) : matrix_base(arr){};
+};
+
+}  // namespace kokkostbx
+
+#endif
diff --git a/kokkostbx/kokkos_matrix3.h b/kokkostbx/kokkos_matrix3.h
new file mode 100644
index 0000000000..681c0f898e
--- /dev/null
+++ b/kokkostbx/kokkos_matrix3.h
@@ -0,0 +1,87 @@
+#ifndef KOKKOSTBX_MATRIX3_H
+#define KOKKOSTBX_MATRIX3_H
+
+#include "kokkos_matrix.h"
+
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+namespace kokkostbx {
+
+template <typename NumType>
+struct matrix3 : public matrix_base<matrix3<NumType>, NumType, 3> {
+    using matrix_base = kokkostbx::matrix_base<matrix3<NumType>, NumType, 3>;
+
+    matrix3() = default;
+    KOKKOS_FUNCTION matrix3(NumType val) : matrix_base(val){};
+    KOKKOS_FUNCTION matrix3(NumType arr[]) : matrix_base(arr){};
+    KOKKOS_FUNCTION matrix3(const matrix_base& mat) : matrix_base(mat){};
+
+    KOKKOS_FUNCTION matrix3(NumType x, NumType y, NumType z) : matrix_base() {
+        matrix_base::get(0, 0) = x;
+        matrix_base::get(1, 1) = y;
+        matrix_base::get(2, 2) = z;
+    }
+
+    KOKKOS_FUNCTION matrix3(
+        NumType a, NumType b, NumType c,
+        NumType d, NumType e, NumType f,
+        NumType g, NumType h, NumType i)
+        : matrix_base() {
+        matrix_base::get(0, 0) = a;
+        matrix_base::get(0, 1) = b;
+        matrix_base::get(0, 2) = c;
+        matrix_base::get(1, 0) = d;
+        matrix_base::get(1, 1) = e;
+        matrix_base::get(1, 2) = f;
+        matrix_base::get(2, 0) = g;
+        matrix_base::get(2, 1) = h;
+        matrix_base::get(2, 2) = i;
+    }
+
+    KOKKOS_FUNCTION matrix3 inverse() const {
+        matrix3 result;
+        // Full matrix:
+        //  a b c
+        //  d e f
+        //  g h i
+        const NumType a = matrix_base::get(0, 0);
+        const NumType b = matrix_base::get(0, 1);
+        const NumType c = matrix_base::get(0, 2);
+        const NumType d = matrix_base::get(1, 0);
+        const NumType e = matrix_base::get(1, 1);
+        const NumType f = matrix_base::get(1, 2);
+        const NumType g = matrix_base::get(2, 0);
+        const NumType h = matrix_base::get(2, 1);
+        const NumType i = matrix_base::get(2, 2);
+
+        result(0, 0) = e * i - f * h;
+        result(0, 1) = c * h - b * i;
+        result(0, 2) = b * f - c * e;
+        result(1, 0) = f * g - d * i;
+        result(1, 1) = a * i - c * g;
+        result(1, 2) = c * d - a * f;
+        result(2, 0) = d * h - e * g;
+        result(2, 1) = b * g - a * h;
+        result(2, 2) = a * e - b * d;
+
+        result /= determinant();
+        return result;
+    }
+
+    KOKKOS_FUNCTION NumType determinant() const {
+        NumType result = 0;
+
+        result += matrix_base::get(0, 0) * matrix_base::get(1, 1) * matrix_base::get(2, 2);
+        result -= matrix_base::get(0, 2) * matrix_base::get(1, 1) * matrix_base::get(2, 0);
+        result += matrix_base::get(0, 1) * matrix_base::get(1, 2) * matrix_base::get(2, 0);
+        result -= matrix_base::get(0, 0) * matrix_base::get(1, 2) * matrix_base::get(2, 1);
+        result += matrix_base::get(0, 2) * matrix_base::get(1, 0) * matrix_base::get(2, 1);
+        result -= matrix_base::get(0, 1) * matrix_base::get(1, 0) * matrix_base::get(2, 2);
+        return result;
+    }
+};
+}  // namespace kokkostbx
+
+#endif
diff --git a/kokkostbx/kokkos_matrix_old.h b/kokkostbx/kokkos_matrix_old.h
new file mode 100644
index 0000000000..3401b18d31
--- /dev/null
+++ b/kokkostbx/kokkos_matrix_old.h
@@ -0,0 +1,122 @@
+#ifndef KOKKOSTBX_MATRIX_H
+#define KOKKOSTBX_MATRIX_H
+
+#include "kokkos_vector.h"
+
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+namespace kokkostbx {
+
+template <typename Derived, typename NumType, size_t rank>
+struct matrix_base : public vector_base<matrix_base<Derived, NumType, rank>, NumType, rank * rank> {
+    static constexpr size_t size = rank * rank;
+
+    // data stored in row-major format!
+    using vector_base =
+        kokkostbx::vector_base<matrix_base<Derived, NumType, rank>, NumType, rank * rank>;
+    using vector = kokkostbx::vector<NumType, rank>;
+
+    matrix_base() = default;
+    KOKKOS_FUNCTION matrix_base(NumType val) : vector_base(val){};
+    KOKKOS_FUNCTION matrix_base(NumType arr[]) : vector_base(arr){};
+
+    static Derived KOKKOS_FUNCTION diagonal_matrix(NumType arr[]) {
+        Derived diagonal;
+        for (size_t i = 0; i < rank; ++i) {
+            diagonal.data[i * (rank + 1)] = arr[i];
+        }
+        return diagonal;
+    };
+
+    KOKKOS_FUNCTION NumType& operator()(size_t row, size_t col) {
+        assert(row < rank);
+        assert(col < rank);
+        return vector_base::data[row * rank + col];
+    }
+
+    KOKKOS_FUNCTION NumType operator()(size_t row, size_t col) const {
+        assert(row < rank);
+        assert(col < rank);
+        return vector_base::data[row * rank + col];
+    }
+
+    template <typename VectorType>
+    KOKKOS_FUNCTION VectorType dot(const VectorType v) const {
+        static_assert(v.get_size() == get_rank(), "Vector length must be equal to matrix rank!");
+        VectorType result;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                result[i] += vector_base::data[i * rank + j] * v.data[j];
+            }
+        }
+        return result;
+    }
+
+    KOKKOS_FUNCTION Derived dot(const Derived v) const {
+        Derived result;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                for (size_t k = 0; k < rank; ++k) {
+                    result[i * rank + j] += vector_base::data[i * rank + k] * v.data[k * rank + j];
+                }
+            }
+        }
+        return result;
+    }
+
+    static constexpr KOKKOS_FUNCTION size_t get_rank() { return rank; }
+
+    KOKKOS_FUNCTION NumType trace() const {
+        NumType result = 0;
+        for (size_t i = 0; i < rank; ++i) {
+            result += vector_base::data[i * rank + i];
+        }
+        return result;
+    }
+
+    KOKKOS_FUNCTION void print(const char name[]) const {
+        printf("%s: ", name);
+        for (size_t i = 0; i < size; ++i) {
+            if (i % rank == 0) {
+                printf("\n");
+            }
+            print_num(vector_base::data[i]);
+        }
+        printf("\n");
+    }
+
+    // KOKKOS_FUNCTION void transpose() {
+    //     for (size_t i=0; i<(rank-1); ++i) {
+    //         for (size_t j=(i+1); j<rank; ++j) {
+    //             NumType temp = vector_base::data[i*rank+j];
+    //             vector_base::data[i*rank+j] = vector_base::data[j*rank+i];
+    //             vector_base::data[j*rank+i] = temp;
+    //         }
+    //     }
+    // }
+
+    KOKKOS_FUNCTION Derived transpose() const {
+        Derived new_mat;
+        for (size_t i = 0; i < rank; ++i) {
+            for (size_t j = 0; j < rank; ++j) {
+                new_mat[j * rank + i] = vector_base::data[i * rank + j];
+            }
+        }
+        return new_mat;
+    }
+};
+
+template <typename NumType, size_t rank>
+struct matrix : public matrix_base<matrix<NumType, rank>, NumType, rank> {
+    using matrix_base = kokkostbx::matrix_base<matrix<NumType, rank>, NumType, rank>;
+
+    matrix() = default;
+    KOKKOS_FUNCTION matrix(NumType val) : matrix_base(val){};
+    KOKKOS_FUNCTION matrix(NumType arr[]) : matrix_base(arr){};
+};
+
+}  // namespace kokkostbx
+
+#endif
diff --git a/kokkostbx/kokkos_types.h b/kokkostbx/kokkos_types.h
new file mode 100644
index 0000000000..619fa20568
--- /dev/null
+++ b/kokkostbx/kokkos_types.h
@@ -0,0 +1,50 @@
+#ifndef KOKKOS_TYPES_H
+#define KOKKOS_TYPES_H
+#include <Kokkos_Core.hpp>
+
+#ifdef KOKKOS_ENABLE_CUDA
+    #define MemSpace Kokkos::CudaUVMSpace
+#endif
+#ifdef KOKKOS_ENABLE_HIP
+    #define MemSpace Kokkos::Experimental::HIPSpace
+#endif
+#ifdef KOKKOS_ENABLE_OPENMPTARGET
+    #define MemSpace Kokkos::OpenMPTargetSpace
+#endif
+
+#ifndef MemSpace
+    #define MemSpace Kokkos::HostSpace
+#endif
+
+using ExecSpace = MemSpace::execution_space;
+using range_policy = Kokkos::RangePolicy<ExecSpace>;
+
+template <typename T>
+using view_1d_t = Kokkos::View<T*, MemSpace>;
+
+using vector_bool_t = view_1d_t<bool>;
+using vector_double_t = view_1d_t<double>;
+using vector_float_t = view_1d_t<float>;
+using vector_cudareal_t = view_1d_t<CUDAREAL>;
+using vector_int_t = view_1d_t<int>;
+using vector_size_t = view_1d_t<std::size_t>;
+using vector_uint_t = view_1d_t<unsigned int>;
+using vector_ushort_t = view_1d_t<unsigned int short>;
+
+template <typename T>
+void print_view(const view_1d_t<T>& arg_view, size_t arg_first, size_t arg_last) {
+    std::string label = arg_view.label();
+    printf("print_view '%s'\n", label.c_str());
+
+    Kokkos::parallel_for(
+        "print_view", range_policy(arg_first, arg_last),
+        KOKKOS_LAMBDA(const int i) { printf(" P[ %d ] = '%f'\n", i, (double)arg_view(i)); });
+    Kokkos::fence();
+}
+
+template <typename T>
+void print_view(const view_1d_t<T>& arg_view) {
+    print_view(arg_view, 0, arg_view.span());
+}
+
+#endif  // KOKKOS_TYPES_H
diff --git a/kokkostbx/kokkos_utils.cpp b/kokkostbx/kokkos_utils.cpp
new file mode 100644
index 0000000000..ee46cc35d5
--- /dev/null
+++ b/kokkostbx/kokkos_utils.cpp
@@ -0,0 +1,21 @@
+#include "kokkostbx/kokkos_utils.h"
+
+namespace kokkostbx {
+void transfer_double2kokkos(vector_cudareal_t& dst, const double* src, const size_t length) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", dst.label().c_str(), (void*) dst.data());
+        // printf(" - size src|dst: %d|%d\n", length, dst.span() );
+    }
+    if (dst.span() < length) {
+        resize(dst, length);
+        // printf(" - size changed, new size: %d\n", dst.span() );
+    }
+
+    transfer_X2kokkos(dst, src, length);
+
+    if (length < 20) {
+        // print_view(dst);
+    }
+}
+
+}  // namespace kokkostbx
diff --git a/kokkostbx/kokkos_utils.h b/kokkostbx/kokkos_utils.h
new file mode 100644
index 0000000000..e82a776ebc
--- /dev/null
+++ b/kokkostbx/kokkos_utils.h
@@ -0,0 +1,129 @@
+#ifndef KOKKOS_UTILS_H
+#define KOKKOS_UTILS_H
+
+#include <vector>
+#include <algorithm>
+
+#include "kokkostbx/kokkos_types.h"
+#include "scitbx/array_family/flex_types.h"
+#include "scitbx/array_family/shared.h"
+
+//*******************************************************************
+// Transfer data
+//*******************************************************************
+namespace {
+template <typename T, typename U>
+void transfer_X2kokkos(view_1d_t<T>& dst, const U& src) {
+    auto host_view = Kokkos::create_mirror_view(dst);
+    auto src_ptr = src.begin();
+    for (int i = 0; i < src.size(); ++i) {
+        host_view(i) = src_ptr[i];
+    }
+    Kokkos::deep_copy(dst, host_view);
+}
+
+template <typename T, typename U>
+void transfer_X2kokkos(view_1d_t<T>& dst, const U* src, const size_t length) {
+    auto host_view = Kokkos::create_mirror_view(dst);
+    for (int i = 0; i < length; ++i) {
+        host_view(i) = src[i];
+    }
+    Kokkos::deep_copy(dst, host_view);
+}
+
+template <typename T, typename U>
+void transfer_kokkos2X(T& dst, const view_1d_t<U>& src) {
+    auto host_view = Kokkos::create_mirror_view(src);
+    Kokkos::deep_copy(host_view, src);
+    auto dst_ptr = dst.begin();
+    for (int i = 0; i < host_view.span(); ++i) {
+        dst_ptr[i] = host_view(i);
+    }
+}
+}  // namespace
+
+//*******************************************************************
+// math functions
+//*******************************************************************
+
+namespace kokkostbx {
+
+namespace af = scitbx::af;
+
+void transfer_double2kokkos(vector_cudareal_t& dst, const double* src, const size_t length);
+
+template <typename T>
+void transfer_shared2kokkos(view_1d_t<T>& dst, const af::shared<T>& src) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", dst.label().c_str(), (void*) dst.data());
+        // printf(" - size src|dst: %d|%d\n", src.size(), dst.span() );
+    }
+    if (dst.span() < src.size()) {
+        resize(dst, src.size());
+        // printf(" - size changed, new size: %d\n", dst.span() );
+    }
+
+    transfer_X2kokkos(dst, src);
+    if (src.size() < 20) {
+        // print_view(dst);
+    }
+}
+
+template <typename T>
+void transfer_kokkos2shared(af::shared<T>& dst, const view_1d_t<T>& src) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", src.label().c_str(), (void*)
+        // src.data());
+    }
+    transfer_kokkos2X(dst, src);
+}
+
+template <typename T>
+void transfer_vector2kokkos(view_1d_t<T>& dst, const std::vector<T>& src) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", dst.label().c_str(), (void*) dst.data());
+        // printf(" - size src|dst: %d|%d\n", src.size(), dst.span() );
+    }
+    if (dst.span() < src.size()) {
+        resize(dst, src.size());
+        // printf(" - size changed, new size: %d\n", dst.span() );
+    }
+    auto host_view = Kokkos::create_mirror_view(dst);
+    for (int i = 0; i < src.size(); ++i) {
+        host_view(i) = src[i];
+    }
+    Kokkos::deep_copy(dst, host_view);
+}
+
+template <typename T>
+void transfer_kokkos2vector(std::vector<T>& dst, const view_1d_t<T>& src) {
+    auto host_view = Kokkos::create_mirror_view(src);
+    // printf(" - size src|dst: %d|%d\n", src.span(), dst.size() );
+    const int length = std::min(dst.size(), src.span());
+    Kokkos::deep_copy(host_view, src);
+    for (int i = 0; i < length; ++i) {
+        dst[i] = host_view(i);
+    }
+}
+
+template <typename T>
+void transfer_kokkos2flex(af::flex_double& dst, const view_1d_t<T>& src) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", src.label().c_str(), (void*)
+        // src.data());
+    }
+    transfer_kokkos2X(dst, src);
+}
+
+template <typename T>
+void transfer_kokkos2flex(af::flex_int& dst, const view_1d_t<T>& src) {
+    if (true) {
+        // printf("== Transfer %s from %p\n", src.label().c_str(), (void*)
+        // src.data());
+    }
+    transfer_kokkos2X(dst, src);
+}
+
+}  // namespace kokkostbx
+
+#endif  // KOKKOS_UTILS_H
diff --git a/kokkostbx/kokkos_vector.h b/kokkostbx/kokkos_vector.h
new file mode 100644
index 0000000000..3e1da2d905
--- /dev/null
+++ b/kokkostbx/kokkos_vector.h
@@ -0,0 +1,299 @@
+#ifndef KOKKOSTBX_VECTOR_H
+#define KOKKOSTBX_VECTOR_H
+
+#include <Kokkos_Core.hpp>
+#include <iostream>
+#include <type_traits>
+
+// Bandaid since parts of diffBragg don't support Kokkos yet
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+// #ifdef KOKKOS_CORE_HPP
+//     template <typename T> T KOKKOS_FUNCTION sqrt_func(T x) { return
+//     ::Kokkos::Experimental::sqrt(x); }
+// #else
+//     #include <cmath>
+//     template <typename T> T KOKKOS_FUNCTION sqrt_func(T x) { return sqrt(x); }
+// #endif
+
+namespace {
+template <typename T>
+KOKKOS_FUNCTION typename std::enable_if<std::is_integral<T>::value, void>::type print_num(
+    const T& x) {
+    printf("%d ", x);
+}
+
+template <typename T>
+KOKKOS_FUNCTION typename std::enable_if<std::is_floating_point<T>::value, void>::type print_num(
+    const T& x) {
+    printf("%f ", x);
+}
+}  // namespace
+
+namespace kokkostbx {
+
+template<typename Derived, typename NumType>
+struct stream_initializer {
+    Derived& vector;
+    int pointer;
+
+    KOKKOS_INLINE_FUNCTION stream_initializer(Derived& v, NumType val) : vector(v), pointer(0) {
+        vector[pointer] = val;
+    }
+
+    KOKKOS_INLINE_FUNCTION stream_initializer(stream_initializer&& other) = default;
+
+    KOKKOS_FUNCTION stream_initializer& operator,(NumType val) {
+        assert(pointer<(int)vector.get_size() &&
+               "Too many coefficients to initialize with '<<'-operator'!");
+        pointer += 1;
+        vector[pointer] = val;
+        return *this;
+    }
+
+    ~stream_initializer() {
+        assert((pointer+1)==vector.get_size() &&
+                "Too few coefficients to initialize with '<<'-operator!");
+    }
+};
+
+template <typename Derived, typename NumType, size_t size>
+struct vector_base {
+    NumType data[size] = {};
+
+    // CONSTRUCTOR
+    vector_base() = default;
+
+    KOKKOS_FUNCTION vector_base(NumType val) {
+        for (NumType& d : data) {
+            d = val;
+        }
+    }
+
+    KOKKOS_FUNCTION vector_base(NumType arr[]) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] = arr[i];
+        }
+    }
+
+    KOKKOS_FUNCTION vector_base(Derived v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] = v[i];
+        }
+    }
+
+    // OPERATORS
+    // streaming
+    friend std::ostream& operator<<(
+        std::ostream& os,
+        const vector_base<Derived, NumType, size>& v) {
+        for (size_t i = 0; i < size; ++i) {
+            if (i > 0) {
+                os << " ";
+            }
+            os << v.data[i];
+        }
+        return os;
+    }
+
+    // access
+    friend KOKKOS_FUNCTION stream_initializer<Derived, NumType> operator<<(Derived& v, NumType val) {
+        return stream_initializer<Derived, NumType>(v, val);
+    }
+
+
+    KOKKOS_FUNCTION NumType& operator[](const int index) { return data[index]; }
+
+    KOKKOS_FUNCTION NumType operator[](const int index) const { return data[index]; }
+
+    // addition
+    KOKKOS_FUNCTION friend Derived operator+(const Derived& lhs, const Derived& rhs) {
+        Derived sum = lhs;
+        sum += rhs;
+        return sum;
+    }
+
+    // KOKKOS_FUNCTION friend Derived operator+(const Derived& lhs, NumType& rhs) {
+    //     Derived sum = lhs;
+    //     sum += rhs;
+    //     return sum;
+    // }
+
+    // KOKKOS_FUNCTION friend Derived operator+(NumType lhs, const Derived& rhs) {
+    //     return rhs + lhs;
+    // }
+
+    KOKKOS_FUNCTION void operator+=(const Derived& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] += v.data[i];
+        }
+    }
+
+    KOKKOS_FUNCTION void operator+=(const NumType& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] += v;
+        }
+    }
+
+    // subtraction
+    KOKKOS_FUNCTION friend Derived operator-(const Derived& vec) {
+        Derived negative = vec;
+        for (size_t i = 0; i < size; ++i) {
+            negative[i] *= -1;
+        }
+        return negative;
+    }
+
+    KOKKOS_FUNCTION friend Derived operator-(const Derived& lhs, const Derived& rhs) {
+        Derived sum = lhs;
+        sum -= rhs;
+        return sum;
+    }
+
+    KOKKOS_FUNCTION void operator-=(const Derived& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] -= v.data[i];
+        }
+    }
+
+    KOKKOS_FUNCTION void operator-=(const NumType& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] -= v;
+        }
+    }
+
+    // multiplication
+    KOKKOS_FUNCTION friend Derived operator*(const Derived& lhs, const Derived& rhs) {
+        Derived prod = lhs;
+        prod *= rhs;
+        return prod;
+    }
+
+    // KOKKOS_FUNCTION friend Derived operator*(const Derived& lhs, NumType& rhs) {
+    //     Derived prod = lhs;
+    //     prod *= rhs;
+    //     return prod;
+    // }
+
+    // KOKKOS_FUNCTION friend Derived operator*(NumType lhs, const Derived& rhs) {
+    //     return rhs * lhs;
+    // }
+
+    KOKKOS_FUNCTION void operator*=(const Derived& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] *= v.data[i];
+        }
+    }
+
+    KOKKOS_FUNCTION void operator*=(const NumType& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] *= v;
+        }
+    }
+
+    // division
+    KOKKOS_FUNCTION friend Derived operator/(const Derived& lhs, const Derived& rhs) {
+        Derived quot = lhs;
+        quot /= rhs;
+        return quot;
+    }
+
+    KOKKOS_FUNCTION void operator/=(const Derived& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] /= v.data[i];
+        }
+    }
+
+    KOKKOS_FUNCTION void operator/=(const NumType& v) {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] /= v;
+        }
+    }
+
+    // METHODS
+    KOKKOS_FUNCTION void print(const char name[]) const {
+        printf("%s: ", name);
+        for (size_t i = 0; i < size; ++i) {
+            print_num(data[i]);
+        }
+        printf("\n");
+    }
+
+    static constexpr KOKKOS_FUNCTION size_t get_size() { return size; }
+
+    KOKKOS_FUNCTION void zero() {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] = 0;
+        }
+    }
+
+    KOKKOS_FUNCTION void ones() {
+        for (size_t i = 0; i < size; ++i) {
+            data[i] = 1;
+        }
+    }
+
+    KOKKOS_FUNCTION bool is_zero() const {
+        for (size_t i = 0; i < size; ++i) {
+            if (data[i] != 0)
+                return false;
+        }
+        return true;
+    }
+
+    KOKKOS_FUNCTION NumType dot(const Derived& v) const {
+        NumType sum = 0;
+        for (size_t i = 0; i < size; ++i) {
+            sum += data[i] * v.data[i];
+        }
+        return sum;
+    }
+
+    KOKKOS_FUNCTION NumType length_sqr() const {
+        NumType sum = 0;
+        for (size_t i = 0; i < size; ++i) {
+            sum += data[i] * data[i];
+        }
+        return sum;
+    }
+
+    KOKKOS_FUNCTION NumType length() const {
+        // return sqrt_func(length_sqr());
+        return ::Kokkos::Experimental::sqrt(length_sqr());
+    }
+
+    KOKKOS_FUNCTION void normalize() {
+        NumType l = length();
+        if (l > 0) {
+            for (size_t i = 0; i < size; ++i) {
+                data[i] /= l;
+            }
+        }
+    }
+
+    KOKKOS_FUNCTION Derived get_unit_vector() const {
+        NumType l = length();
+        Derived unit_vector{};
+        if (l > 0) {
+            for (size_t i = 0; i < size; ++i) {
+                unit_vector[i] = data[i] / l;
+            }
+        }
+        return unit_vector;
+    }
+};
+
+template <typename NumType, size_t size>
+struct vector : public vector_base<vector<NumType, size>, NumType, size> {
+    using vector_base = kokkostbx::vector_base<vector<NumType, size>, NumType, size>;
+
+    vector() = default;
+    KOKKOS_FUNCTION vector(NumType val) : vector_base(val){};
+    KOKKOS_FUNCTION vector(NumType arr[]) : vector_base(arr){};
+};
+
+}  // namespace kokkostbx
+
+#endif
diff --git a/kokkostbx/kokkos_vector2.h b/kokkostbx/kokkos_vector2.h
new file mode 100644
index 0000000000..1c409ee510
--- /dev/null
+++ b/kokkostbx/kokkos_vector2.h
@@ -0,0 +1,36 @@
+#ifndef KOKKOSTBX_VECTOR2_H
+#define KOKKOSTBX_VECTOR2_H
+
+#include "kokkos_vector.h"
+
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+namespace kokkostbx {
+
+template <typename NumType>
+struct vector2 : public vector_base<vector2<NumType>, NumType, 2> {
+    using vector_base = kokkostbx::vector_base<vector2<NumType>, NumType, 2>;
+
+    vector2() = default;
+    KOKKOS_FUNCTION vector2(NumType val) : vector_base(val){};
+    KOKKOS_FUNCTION vector2(NumType arr[]) : vector_base(arr){};
+    KOKKOS_FUNCTION vector2(const vector_base& vec) : vector_base(vec){};
+
+    KOKKOS_FUNCTION vector2(NumType x, NumType y) : vector_base() {
+        vector_base::data[0] = x;
+        vector_base::data[1] = y;
+    }
+
+    // decided against using properties, as this would increase the size of the class
+    KOKKOS_FUNCTION NumType& x_val() { return vector_base::data[0]; }
+    KOKKOS_FUNCTION NumType& y_val() { return vector_base::data[1]; }
+
+    KOKKOS_FUNCTION NumType x_val() const { return vector_base::data[0]; }
+    KOKKOS_FUNCTION NumType y_val() const { return vector_base::data[1]; }
+};
+
+}  // namespace kokkostbx
+
+#endif
diff --git a/kokkostbx/kokkos_vector3.h b/kokkostbx/kokkos_vector3.h
new file mode 100644
index 0000000000..98d46ee7dc
--- /dev/null
+++ b/kokkostbx/kokkos_vector3.h
@@ -0,0 +1,77 @@
+#ifndef KOKKOSTBX_VECTOR3_H
+#define KOKKOSTBX_VECTOR3_H
+
+#include "kokkos_vector.h"
+
+// Bandaid since parts of diffBragg don't support Kokkos yet
+// #ifndef KOKKOS_FUNCTION
+// #define KOKKOS_FUNCTION
+// #endif
+
+// #ifdef KOKKOS_CORE_HPP
+//     template <typename T> KOKKOS_FUNCTION T sin_func(T x) { return
+//     ::Kokkos::Experimental::sin(x); } template <typename T> KOKKOS_FUNCTION T cos_func(T x) {
+//     return ::Kokkos::Experimental::cos(x); }
+// #else
+//     #include <cmath>
+//     template <typename T> KOKKOS_FUNCTION T sin_func(T x) { return sin(x); }
+//     template <typename T> KOKKOS_FUNCTION T cos_func(T x) { return cos(x); }
+// #endif
+
+namespace kokkostbx {
+
+template <typename NumType>
+//struct vector3 : public vector_base<vector3<NumType>, NumType, 3> {
+struct vector3 : public vector<NumType, 3> {
+    //using vector_base = kokkostbx::vector_base<vector<NumType>, NumType, 3>;
+    using vector_base = kokkostbx::vector<NumType, 3>;
+
+    vector3() = default;
+    KOKKOS_FUNCTION vector3(NumType val) : vector_base(val){};
+    KOKKOS_FUNCTION vector3(NumType arr[]) : vector_base(arr){};
+    KOKKOS_FUNCTION vector3(const vector_base& vec) : vector_base(vec){};
+
+    KOKKOS_FUNCTION vector3(NumType x, NumType y, NumType z) : vector_base() {
+        vector_base::data[0] = x;
+        vector_base::data[1] = y;
+        vector_base::data[2] = z;
+    }
+
+    // decided against using properties, as this would increase the size of the class
+    KOKKOS_FUNCTION NumType& x_val() { return vector_base::data[0]; }
+    KOKKOS_FUNCTION NumType& y_val() { return vector_base::data[1]; }
+    KOKKOS_FUNCTION NumType& z_val() { return vector_base::data[2]; }
+
+    KOKKOS_FUNCTION NumType x_val() const { return vector_base::data[0]; }
+    KOKKOS_FUNCTION NumType y_val() const { return vector_base::data[1]; }
+    KOKKOS_FUNCTION NumType z_val() const { return vector_base::data[2]; }
+
+    KOKKOS_FUNCTION vector3<NumType> cross(const vector3<NumType>& v) const {
+        vector3<NumType> cross_vector{};
+        cross_vector.x_val() = y_val() * v.z_val() - z_val() * v.y_val();
+        cross_vector.y_val() = z_val() * v.x_val() - x_val() * v.z_val();
+        cross_vector.z_val() = x_val() * v.y_val() - y_val() * v.x_val();
+
+        return cross_vector;
+    }
+
+    // rotate a point around a unit vector3 axis
+    KOKKOS_FUNCTION vector3<NumType> rotate_around_axis(const vector3<NumType>& axis, NumType angle)
+        const {
+        // NumType sinphi = sin_func(angle);
+        // NumType cosphi = cos_func(angle);
+        NumType sinphi = ::Kokkos::Experimental::sin(angle);
+        NumType cosphi = ::Kokkos::Experimental::cos(angle);
+
+        NumType dot_factor = axis.dot(*this) * (1.0 - cosphi);
+
+        vector3<NumType> vector_rot = axis.cross(*this) * sinphi;
+        vector_rot += axis * dot_factor;
+        vector_rot += (*this) * cosphi;
+        return vector_rot;
+    }
+};
+
+}  // namespace kokkostbx
+
+#endif
diff --git a/libtbx/SConscript b/libtbx/SConscript
index f6ed1af70c..9c117af4a7 100644
--- a/libtbx/SConscript
+++ b/libtbx/SConscript
@@ -1497,6 +1497,8 @@ def enable_cuda_if_possible():
 def enable_kokkos_if_possible():
   env_etc.enable_kokkos = False
   if (libtbx.env.build_options.enable_kokkos):
+    if sys.version_info.major == 2:
+      raise RuntimeError('Kokkos is not available on Python 2')
     env_etc.enable_kokkos = True
 
 enable_openmp_if_possible()
diff --git a/libtbx/auto_build/bootstrap.py b/libtbx/auto_build/bootstrap.py
index d33dd98d8d..91119ef38d 100644
--- a/libtbx/auto_build/bootstrap.py
+++ b/libtbx/auto_build/bootstrap.py
@@ -1002,17 +1002,17 @@ class xia2_module(SourceModule):
 
 class kokkos_module(SourceModule):
   module = 'kokkos'
-  anonymous = ['git',
+  anonymous = ['git', '-b 3.7.01',
                'git@github.com:kokkos/kokkos.git',
                'https://github.com/kokkos/kokkos.git',
-               'https://github.com/kokkos/kokkos/archive/master.zip']
+               'https://github.com/kokkos/kokkos/archive/refs/tags/3.7.01.zip']
 
 class kokkos_kernels_module(SourceModule):
   module = 'kokkos-kernels'
-  anonymous = ['git',
+  anonymous = ['git', '-b 3.7.01',
                'git@github.com:kokkos/kokkos-kernels.git',
                'https://github.com/kokkos/kokkos-kernels.git',
-               'https://github.com/kokkos/kokkos-kernels/archive/master.zip']
+               'https://github.com/kokkos/kokkos-kernels/archive/refs/tags/3.7.01.zip']
 
 # Duke repositories
 class probe_module(SourceModule):
diff --git a/simtbx/SConscript b/simtbx/SConscript
index a588386feb..37219bc5d8 100644
--- a/simtbx/SConscript
+++ b/simtbx/SConscript
@@ -29,6 +29,9 @@ if not env_etc.no_boost_python:
      "cctbx"])
   env_etc.enable_more_warnings(env=env_simtbx)
   env_simtbx.SConscript("nanoBragg/SConscript",exports={ 'env' : env_simtbx })
+  # only build if kokkos is enabled
+  if env_etc.enable_kokkos:
+    env_simtbx.SConscript("../kokkostbx/SConscript")
   if (
       libtbx.env.build_options.enable_cxx11
       or libtbx.env.build_options.cxxstd == "c++14"
@@ -37,6 +40,6 @@ if not env_etc.no_boost_python:
   env_simtbx.SConscript("gpu/SConscript",exports={ 'env' : env_simtbx })
   env_simtbx.SConscript("pixel/SConscript",exports={ 'env' : env_simtbx })
 
-  # only build kokkos on linux and if kokkos is enabled
-  if sys.platform.startswith('linux') and env_etc.enable_kokkos:
+  # only build if kokkos is enabled
+  if env_etc.enable_kokkos:
     env_simtbx.SConscript("kokkos/SConscript",exports={ 'env' : env_simtbx })
diff --git a/simtbx/command_line/geometry_refiner.py b/simtbx/command_line/geometry_refiner.py
index aaa3a74321..4cf4f82809 100644
--- a/simtbx/command_line/geometry_refiner.py
+++ b/simtbx/command_line/geometry_refiner.py
@@ -653,7 +653,6 @@ def update_detector(x, ref_params, SIM, save=None):
         #print("Saved detector model to %s (took %.4f sec)" % (save, t), flush=True )
 
 
-
 def target_and_grad(x, ref_params, data_modelers, SIM, params):
     """
     Returns the target functional and the gradients
diff --git a/simtbx/command_line/hopper.py b/simtbx/command_line/hopper.py
index e7796a441c..20ef0cf6c8 100644
--- a/simtbx/command_line/hopper.py
+++ b/simtbx/command_line/hopper.py
@@ -16,6 +16,8 @@
 except ImportError:
     LineProfiler = None
 
+from simtbx.diffBragg.device import DeviceWrapper
+
 ROTX_ID = 0
 ROTY_ID = 1
 ROTZ_ID = 2
@@ -79,6 +81,9 @@ def __init__(self):
                 o.write(self.parser.diff_phil.as_str())
         self.params = COMM.bcast(self.params)
 
+        self.dev = COMM.rank % self.params.refiner.num_devices
+        logging.info("Rank %d will use device %d on host %s" % (COMM.rank, self.dev, socket.gethostname()))
+
         if self.params.logging.logname is None:
             self.params.logging.logname = "main_stage1.log"
         if self.params.profile_name is None:
@@ -237,14 +242,7 @@ def run(self):
                 Modeler.all_data = Modeler.all_data.astype(np.float32)
                 Modeler.all_background = Modeler.all_background.astype(np.float32)
 
-            if self.params.refiner.randomize_devices:
-                dev = np.random.choice(self.params.refiner.num_devices)
-                logging.info("Rank %d will use randomly chosen device %d on host %s" % (COMM.rank, dev, socket.gethostname()))
-            else:
-                dev = COMM.rank % self.params.refiner.num_devices
-                logging.info("Rank %d will use device %d on host %s" % (COMM.rank, dev, socket.gethostname()))
-
-            SIM.D.device_Id = dev
+            SIM.D.device_Id = self.dev
 
             nparam = len(Modeler.P)
             if SIM.refining_Fhkl:
@@ -263,6 +261,7 @@ def run(self):
                 utils.show_diffBragg_state(SIM.D, Modeler.params.refiner.debug_pixel_panelfastslow)
 
             Modeler.clean_up(SIM)
+            del SIM.D  # TODO: is this necessary ?
 
         if self.params.dump_gathers and self.params.gathered_output_file is not None:
             exp_gatheredRef_spec = COMM.reduce(exp_gatheredRef_spec)
@@ -291,7 +290,8 @@ def run(self):
         elif script.params.profile:
             print("Install line_profiler in order to use logging: libtbx.python -m pip install line_profiler")
 
-        RUN()
+        with DeviceWrapper(script.dev) as _:
+            RUN()
 
         if lp is not None:
             stats = lp.get_stats()
diff --git a/simtbx/command_line/hopper_process.py b/simtbx/command_line/hopper_process.py
index 02979f9241..f170c89018 100644
--- a/simtbx/command_line/hopper_process.py
+++ b/simtbx/command_line/hopper_process.py
@@ -132,12 +132,8 @@ def _create_modeler_dir(self):
 
     @property
     def device_id(self):
-        if self.params.diffBragg.refiner.randomize_devices:
-            dev = np.random.choice(self.params.diffBragg.refiner.num_devices)
-            print("Rank %d will use random device %d on host %s" % (COMM.rank, dev, socket.gethostname()), flush=True)
-        else:
-            dev = COMM.rank % self.params.diffBragg.refiner.num_devices
-            print("Rank %d will use fixed device %d on host %s" % (COMM.rank, dev, socket.gethostname()), flush=True)
+        dev = COMM.rank % self.params.diffBragg.refiner.num_devices
+        print("Rank %d will use fixed device %d on host %s" % (COMM.rank, dev, socket.gethostname()), flush=True)
         return dev
 
     def find_spots(self, experiments):
diff --git a/simtbx/command_line/stage_two.py b/simtbx/command_line/stage_two.py
index c186ea8fe8..bb72c9a541 100644
--- a/simtbx/command_line/stage_two.py
+++ b/simtbx/command_line/stage_two.py
@@ -3,6 +3,8 @@
 # LIBTBX_SET_DISPATCHER_NAME simtbx.diffBragg.stage_two
 
 from libtbx.mpi4py import MPI
+from simtbx.kokkos import gpu_instance
+kokkos_run = gpu_instance(deviceId = 0)
 from simtbx.command_line.hopper import hopper_phil
 import time
 import logging
@@ -62,8 +64,7 @@ def run(self):
             raise ValueError("Pandas table input required")
 
         refine_starttime = time.time()
-        if not self.params.refiner.randomize_devices:
-            self.params.simulator.device_id = COMM.rank % self.params.refiner.num_devices
+        self.params.simulator.device_id = COMM.rank % self.params.refiner.num_devices
         refiner = ensemble_refine_launcher.global_refiner_from_parameters(self.params)
         print("Time to refine experiment: %f" % (time.time()- refine_starttime))
 
diff --git a/simtbx/diffBragg/SConscript b/simtbx/diffBragg/SConscript
index 3362cf9dba..806c2d4c23 100644
--- a/simtbx/diffBragg/SConscript
+++ b/simtbx/diffBragg/SConscript
@@ -3,6 +3,9 @@ import libtbx.load_env
 Import("env_etc env_simtbx nanoBragg_obj nanoBragg_env")
 
 diffBragg_env = nanoBragg_env.Clone()
+if env_etc.enable_kokkos:
+  diffBragg_env.Prepend(CPPDEFINES=["DIFFBRAGG_HAVE_KOKKOS"])
+
 
 env_etc.eigen_dist = os.path.abspath(os.path.join(libtbx.env.dist_path("simtbx"),"../../eigen"))
 if not os.path.isdir(env_etc.eigen_dist) and hasattr(env_etc, "conda_cpppath"):
@@ -22,13 +25,6 @@ if not show_all_warnings:
     warn_suppress_flags="-Xcudafe --diag_suppress=esa_on_defaulted_function_ignored --expt-relaxed-constexpr"
     diffBragg_env.Prepend(NVCCFLAGS=warn_suppress_flags.split())
 
-diffBragg_obj = diffBragg_env.SharedObject(
-  source=[
-    "src/diffBragg_ext.cpp",
-    "src/diffBragg.cpp",
-    "src/diffBragg_cpu_kernel.cpp"
-  ])
-
 if (env_etc.enable_cuda):
   diffBragg_env.cudaSharedLibrary(
     target="#lib/libsimtbx_diffBraggCUDA",
@@ -37,9 +33,86 @@ if (env_etc.enable_cuda):
   diffBragg_env.Prepend(NVCCFLAGS=["-DDIFFBRAGG_HAVE_CUDA"])
   env_simtbx.Prepend(LIBS=["simtbx_diffBraggCUDA"])
 
+diffBragg_obj = diffBragg_env.SharedObject(
+  source=[
+    "src/diffBragg_ext.cpp",
+    "src/diffBragg.cpp",
+    "src/diffBragg_cpu_kernel.cpp"
+  ])
+
+libs = []
+libpath = []
+
+if (env_etc.enable_kokkos):
+  print('Getting kokkos build variables')
+  print('-'*79)
+  kokkos_cxxflags = os.environ['KOKKOS_CXXFLAGS'].split()
+  if kokkos_cxxflags and kokkos_cxxflags[0] == 'echo':
+    kokkos_cxxflags = [f.strip('"') for f in kokkos_cxxflags[1:]]
+  print('KOKKOS_CXXFLAGS:', kokkos_cxxflags)
+  print('='*79)
+
+  '''
+  kokkos_cxxflags = ['-std=c++14',
+                     '-Xcudafe', 
+                     '--diag_suppress=esa_on_defaulted_function_ignored', 
+                     '-expt-extended-lambda', 
+                     '-arch=sm_80', 
+                     '-I./', 
+                     '-I/pscratch/sd/f/fwittwer/alcc-recipes/cctbx/modules/kokkos/core/src', 
+                     '-I/pscratch/sd/f/fwittwer/alcc-recipes/cctbx/modules/kokkos/containers/src', 
+                     '-I/pscratch/sd/f/fwittwer/alcc-recipes/cctbx/modules/kokkos/algorithms/src']
+  '''
+
+  if os.getenv('KOKKOS_PATH') is None:
+    os.environ['KOKKOS_PATH'] = libtbx.env.under_dist('simtbx', '../../kokkos')
+  #kokkos_env = env_simtbx.Clone()
+  if os.getenv('CXX') is not None:
+    original_cxx = diffBragg_env['CXX']
+  if 'Cuda' in os.getenv('KOKKOS_DEVICES'):
+    diffBragg_env.Replace(CXX=os.path.join(os.environ['KOKKOS_PATH'], 'bin', 'nvcc_wrapper'))    #os.environ['CXX'])
+    diffBragg_env.Replace(SHCXX=os.path.join(os.environ['KOKKOS_PATH'], 'bin', 'nvcc_wrapper'))  #os.environ['CXX'])
+  elif 'HIP' in os.getenv('KOKKOS_DEVICES'):
+    diffBragg_env.Replace(CXX='hipcc')
+    diffBragg_env.Replace(SHCXX='hipcc') 
+  diffBragg_env.Prepend(CXXFLAGS=['-DCUDAREAL=double'] + kokkos_cxxflags)
+  diffBragg_env.Prepend(CPPFLAGS=['-DCUDAREAL=double'] + kokkos_cxxflags)
+  diffBragg_env.Prepend(CPPPATH=[os.environ['KOKKOS_PATH']])
+  diffBragg_env.Append(LIBS=['kokkoscontainers','kokkoscore'])
+
+  diffBraggKokkos_env = diffBragg_env.Clone()
+  
+  # remove NANOBRAGG_HAVE_CUDA and DIFFBRAGG_HAVE_CUDA compile option
+  if 'CPPDEFINES' in diffBraggKokkos_env:
+    ccflags = diffBraggKokkos_env['CPPDEFINES']
+    for o in ['NANOBRAGG_HAVE_CUDA', 'DIFFBRAGG_HAVE_CUDA', 'DIFFBRAGG_HAVE_KOKKOS']:
+      if o in ccflags:
+        ccflags.remove(o)
+  
+  if 'NVCCFLAGS' in diffBraggKokkos_env:
+    nvccflags = diffBraggKokkos_env['NVCCFLAGS']
+    o = '-DDIFFBRAGG_HAVE_CUDA'
+    if o in nvccflags:
+      nvccflags.remove(o)
+
+  diffBraggKokkos_lib = diffBraggKokkos_env.SharedLibrary(
+    target="#lib/libsimtbx_diffBraggKOKKOS",
+    source=["src/diffBraggKOKKOS.cpp", "src/diffBragg_kokkos_kernel.cpp", "../../kokkostbx/kokkos_utils.cpp"],
+    LIBS=['kokkoscontainers', 'kokkoscore'])
+
+  print("LIBS: ", env_simtbx['LIBS'])
+
+  if 'Cuda' in os.getenv("KOKKOS_DEVICES"):
+    libpath += [os.path.join(os.environ['CUDA_HOME'], 'lib64')]
+    libpath += [os.path.join(os.environ['CUDA_HOME'], 'lib64/stubs')]
+    libpath += [os.path.join(os.environ['CUDA_HOME'], 'compat')]
+    libs += ["cudart", "cuda"]
+  elif 'HIP' in os.getenv('KOKKOS_DEVICES'):
+    libpath += [os.path.join(os.environ['ROCM_PATH'], 'lib')]
+    libs += ["amdhip64", "hsa-runtime64"]
+
 env_simtbx.SharedLibrary(
   target="#lib/simtbx_diffBragg_ext",
-  source=[
-    diffBragg_obj
-  ]
-)
+  source=[diffBragg_obj],
+  LIBPATH=env_simtbx['LIBPATH'] + libpath,
+  LIBS=['simtbx_diffBraggKOKKOS']*env_etc.enable_kokkos + env_simtbx['LIBS'] + libs)
diff --git a/simtbx/diffBragg/device.py b/simtbx/diffBragg/device.py
new file mode 100644
index 0000000000..1dade37c57
--- /dev/null
+++ b/simtbx/diffBragg/device.py
@@ -0,0 +1,32 @@
+from __future__ import division
+import libtbx.load_env  # implicit import
+DIFFBRAGG_HAS_KOKKOS = libtbx.env.build_options.enable_kokkos
+
+if DIFFBRAGG_HAS_KOKKOS:
+    # TODO fix simtbx.kokkos import ...
+    # from simtbx.kokkos import gpu_instance
+    from simtbx.diffBragg import initialize_kokkos, finalize_kokkos
+
+import os
+USE_KOKKOS_GPU = DIFFBRAGG_HAS_KOKKOS and ("DIFFBRAGG_USE_KOKKOS" in os.environ)
+
+
+class DeviceWrapper:
+
+    def __init__(self, gpu_id, force=False):
+        """
+
+        :param gpu_id: gpu device id  (int from 0 to Ngpu-1)
+        :param force: force initialization of kokkos, otherwise, the os.environ will be searched for DIFFBRAGG_USE_KOKKOS
+        """
+        self.gpu_id = gpu_id
+        self.force_kokkos_init = force
+
+    def __enter__(self):
+        if USE_KOKKOS_GPU or self.force_kokkos_init:
+            initialize_kokkos(self.gpu_id)
+        return self
+
+    def __exit__(self, tp, val, tbk):
+        if USE_KOKKOS_GPU or self.force_kokkos_init:
+            finalize_kokkos()
diff --git a/simtbx/diffBragg/hopper_ensemble_utils.py b/simtbx/diffBragg/hopper_ensemble_utils.py
index 061ef28d26..d7c099a331 100644
--- a/simtbx/diffBragg/hopper_ensemble_utils.py
+++ b/simtbx/diffBragg/hopper_ensemble_utils.py
@@ -276,12 +276,8 @@ def _mpi_get_shots_per_rank(self):
 
     def set_device_id(self):
         assert self.SIM is not None
-        if self.params.refiner.randomize_devices:
-            dev = np.random.choice(self.params.refiner.num_devices)
-            MAIN_LOGGER.info("will use randomly chosen device %d on host %s" % (dev, socket.gethostname()))
-        else:
-            dev = COMM.rank % self.params.refiner.num_devices
-            MAIN_LOGGER.info("will use device %d on host %s" % (dev, socket.gethostname()))
+        dev = COMM.rank % self.params.refiner.num_devices
+        MAIN_LOGGER.info("will use device %d on host %s" % (dev, socket.gethostname()))
         self.SIM.D.device_Id = dev
 
     def mpi_set_x_slices(self):
diff --git a/simtbx/diffBragg/phil.py b/simtbx/diffBragg/phil.py
index 3bfa91b2f9..c8f882e0fb 100644
--- a/simtbx/diffBragg/phil.py
+++ b/simtbx/diffBragg/phil.py
@@ -939,9 +939,6 @@
   force_unit_cell = None
     .type = ints(size=6)
     .help = a unit cell tuple to use
-  randomize_devices = True
-    .type = bool
-    .help = randomly select a device Id
   num_devices = 1
     .type = int
     .help = number of cuda devices on current node
diff --git a/simtbx/diffBragg/src/.clang-format b/simtbx/diffBragg/src/.clang-format
new file mode 100644
index 0000000000..7ecc221c5c
--- /dev/null
+++ b/simtbx/diffBragg/src/.clang-format
@@ -0,0 +1,12 @@
+---
+BasedOnStyle: Chromium
+---
+AlignAfterOpenBracket: AlwaysBreak
+BreakStringLiterals: false
+ColumnLimit: 100
+CompactNamespaces: true
+IndentPPDirectives: BeforeHash
+IndentWidth: '4'
+Language: Cpp
+
+...
diff --git a/simtbx/diffBragg/src/diffBragg.cpp b/simtbx/diffBragg/src/diffBragg.cpp
index 1a4bc0d49e..5bb10ccff3 100644
--- a/simtbx/diffBragg/src/diffBragg.cpp
+++ b/simtbx/diffBragg/src/diffBragg.cpp
@@ -1778,10 +1778,12 @@ np::ndarray diffBragg::add_Fhkl_gradients(const af::shared<size_t>& panels_fasts
     np::dtype dt = np::dtype::get_builtin<double>();
     np::ndarray output = np::empty(shape,dt);
 
-    if (errors)
+    if (errors) {
         output = np::from_data(&first_deriv_imgs.Fhkl_hessian[0], dt, shape, stride, boost::python::object());
-    else
+    }
+    else {
         output = np::from_data(&first_deriv_imgs.Fhkl_scale_deriv[0], dt, shape, stride, boost::python::object());
+    }
     return output.copy();
 }
 
@@ -2020,7 +2022,7 @@ void diffBragg::add_diffBragg_spots(const af::shared<size_t>& panels_fasts_slows
 
     //fudge = 1.1013986013; // from manuscript computation
     gettimeofday(&t1,0 );
-    if ((! use_cuda && getenv("DIFFBRAGG_USE_CUDA")==NULL) || force_cpu){
+    if ((! use_cuda && getenv("DIFFBRAGG_USE_CUDA")==NULL && getenv("DIFFBRAGG_USE_KOKKOS")==NULL ) || force_cpu){
         diffBragg_sum_over_steps(
             Npix_to_model, panels_fasts_slows_vec,
             image,
@@ -2033,8 +2035,8 @@ void diffBragg::add_diffBragg_spots(const af::shared<size_t>& panels_fasts_slows
             db_flags);
         last_kernel_on_GPU=false;
         }
-    else { // we are using cuda
-#ifdef NANOBRAGG_HAVE_CUDA
+    else { // we are using cuda or kokkos
+#if defined DIFFBRAGG_HAVE_CUDA || defined DIFFBRAGG_HAVE_KOKKOS
         db_cu_flags.device_Id = device_Id;
         db_cu_flags.update_step_positions = update_step_positions_on_device;
         db_cu_flags.update_panels_fasts_slows = update_panels_fasts_slows_on_device;
@@ -2048,7 +2050,35 @@ void diffBragg::add_diffBragg_spots(const af::shared<size_t>& panels_fasts_slows
         db_cu_flags.update_panel_deriv_vecs = update_panel_deriv_vecs_on_device;
         db_cu_flags.Npix_to_allocate = Npix_to_allocate;
 
+      if (use_cuda || getenv("DIFFBRAGG_USE_CUDA")!=NULL){
+#ifdef DIFFBRAGG_HAVE_CUDA
         diffBragg_sum_over_steps_cuda(
+             Npix_to_model, panels_fasts_slows_vec,
+             image,
+             first_deriv_imgs,
+             second_deriv_imgs,
+             db_steps,
+             db_det,
+             db_beam,
+             db_cryst,
+             db_flags,
+             db_cu_flags,
+             device_pointers,
+             TIMERS);
+        if(verbose)
+           printf("Ran the CUDA kernel\n");
+#else
+        bool DIFFBRAGG_USE_CUDA_flag_unsupported=false;
+        SCITBX_ASSERT(DIFFBRAGG_USE_CUDA_flag_unsupported);
+#endif
+      }
+      else {
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+        if (!diffBragg_runner) {
+          diffBragg_runner = std::make_shared<diffBraggKOKKOS>();
+        }
+
+        diffBragg_runner->diffBragg_sum_over_steps_kokkos(
             Npix_to_model, panels_fasts_slows_vec,
             image,
             first_deriv_imgs,
@@ -2059,12 +2089,18 @@ void diffBragg::add_diffBragg_spots(const af::shared<size_t>& panels_fasts_slows
             db_cryst,
             db_flags,
             db_cu_flags,
-            device_pointers,
             TIMERS);
-        last_kernel_on_GPU=true;
-
+        if (verbose)
+            printf("Ran the Kokkos kernel\n");
+#else
+    bool DIFFBRAGG_USE_KOKKOS_flag_unsupported=false;
+    SCITBX_ASSERT(DIFFBRAGG_USE_KOKKOS_flag_unsupported);
+#endif
+    }
+    last_kernel_on_GPU=true;
 #else
-       // no statement
+    bool DIFFBRAGG_USE_KOKKOS_and_DIFFBRAGG_USE_CUDA_flags_unsupported=false;
+    SCITBX_ASSERT(DIFFBRAGG_USE_KOKKOS_and_DIFFBRAGG_USE_CUDA_flags_unsupported);
 #endif
     }
     gettimeofday(&t2, 0);
@@ -2179,7 +2215,6 @@ void diffBragg::add_diffBragg_spots(const af::shared<size_t>& panels_fasts_slows
     if(verbose) printf("done with pixel loop\n");
 } // END  of add_diffBragg_spots
 
-
 void diffBragg::diffBragg_rot_mats(){
     for (int i_rot=0; i_rot < 3; i_rot++){
         //if (rot_managers[i_rot]->refine_me){
diff --git a/simtbx/diffBragg/src/diffBragg.h b/simtbx/diffBragg/src/diffBragg.h
index 99521e5be4..36ee6c0afb 100644
--- a/simtbx/diffBragg/src/diffBragg.h
+++ b/simtbx/diffBragg/src/diffBragg.h
@@ -1,20 +1,25 @@
-
 #ifndef SIMTBX_DIFFBRAGG_H
 #define SIMTBX_DIFFBRAGG_H
+
 #include <simtbx/nanoBragg/nanoBragg.h>
 #include <simtbx/diffBragg/src/util.h>
 #include <vector>
+#include <memory>
 #include <boost/ptr_container/ptr_vector.hpp>
 #include <Eigen/Dense>
 #include <boost/python.hpp>
 #include <boost/python/numpy.hpp>
-#include<Eigen/StdVector>
+#include <Eigen/StdVector>
 #include <boost/python/numpy.hpp>
 
-#ifdef NANOBRAGG_HAVE_CUDA
+#ifdef DIFFBRAGG_HAVE_CUDA
 #include "diffBraggCUDA.h"
 #endif
 
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+#include "diffBraggKOKKOS.h"
+#endif
+
 //#include <boost/python/numpy.hpp>
 namespace np=boost::python::numpy;
 
@@ -150,13 +155,19 @@ class diffBragg: public nanoBragg{
   cuda_flags db_cu_flags;
   detector db_det;
 
-
-#ifdef NANOBRAGG_HAVE_CUDA
+#ifdef DIFFBRAGG_HAVE_CUDA
     diffBragg_cudaPointers device_pointers;
-    inline void gpu_free(){
+    inline void cuda_free(){
         freedom(device_pointers);
     }
+#endif
 
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+    // diffBragg_cudaPointers cuda_pointers;
+    void kokkos_free() { diffBragg_runner.reset(); }
+    // allocate when needed to avoid problems with kokkos initialization when cuda/kokkos isn't used
+    std::shared_ptr<diffBraggKOKKOS> diffBragg_runner{};
+    // diffBraggKOKKOS diffBragg_runner;
 #endif
 
   // methods
diff --git a/simtbx/diffBragg/src/diffBraggKOKKOS.cpp b/simtbx/diffBragg/src/diffBraggKOKKOS.cpp
new file mode 100644
index 0000000000..7ddf9121b6
--- /dev/null
+++ b/simtbx/diffBragg/src/diffBraggKOKKOS.cpp
@@ -0,0 +1,538 @@
+#include <sys/time.h>
+#include <cstdio>
+
+#include "diffBraggKOKKOS.h"
+#include "diffBragg_kokkos_kernel.h"
+
+void diffBraggKOKKOS::diffBragg_sum_over_steps_kokkos(
+    int Npix_to_model,
+    std::vector<unsigned int>& panels_fasts_slows,
+    image_type& floatimage,
+    images& d_image,
+    images& d2_image,
+    step_arrays& db_steps,
+    detector& db_det,
+    beam& db_beam,
+    crystal& db_cryst,
+    flags& db_flags,
+    cuda_flags& db_cu_flags,
+    // diffBragg_kokkosPointers& kp,
+    timer_variables& TIMERS) {
+    if (db_cryst.phi0 != 0 || db_cryst.phisteps > 1) {
+        printf(
+            "PHI (goniometer position) not supported in GPU code: phi0=%f phisteps=%d, phistep=%d\n",
+            db_cryst.phi0, db_cryst.phisteps, db_cryst.phistep);
+        exit(-1);
+    }
+
+    Kokkos::Tools::pushRegion("diffBragg_sum_over_steps_kokkos");
+    Kokkos::Tools::pushRegion("local detector, beam and crystal");
+    kokkos_detector local_det(db_det);
+    kokkos_beam local_beam(db_beam);
+    // kokkos_crystal local_cryst(db_cryst);
+    Kokkos::Tools::popRegion();
+
+    double time;
+    struct timeval t1, t2;  //, t3 ,t4;
+    gettimeofday(&t1, 0);
+
+    //  determine if we need to allocate pixels, and how many.
+    //  For best usage, one should use the diffBragg property (visible from Python) Npix_to_allocate
+    //  in order to just allocate to the GPU - this is useful for ensemble refinement, where each
+    //  shot can have a variable number of pixels being modeled, and ony only needs to allocate the
+    //  device once (with the largest expected number of pixels for a given shot)
+    // TODO clean up this logic a bit
+    if (m_device_is_allocated && (Npix_to_model > m_npix_allocated)) {
+        printf(
+            "Need to re-allocate pixels, currently have %d allocated, but trying to model %d\n",
+            m_npix_allocated, Npix_to_model);
+        exit(-1);
+    } else if (db_cu_flags.Npix_to_allocate == -1) {
+        db_cu_flags.Npix_to_allocate = Npix_to_model;
+    } else if (Npix_to_model > db_cu_flags.Npix_to_allocate) {
+        printf(
+            "Npix to model=%d is greater than the number of pixel requested for allocation (%d)!\n",
+            Npix_to_model, db_cu_flags.Npix_to_allocate);
+        exit(-1);
+    }
+
+    //  support dynamic allocation for different numbers of sources
+    Kokkos::Tools::pushRegion("resize sources");
+    if (m_previous_nsource != 0 && m_previous_nsource != local_beam.number_of_sources) {
+        // printf("Resizing for %d sources!:\n", local_beam.number_of_sources);
+        resize(m_Fhkl_channels, local_beam.number_of_sources);
+        resize(m_source_X, local_beam.number_of_sources);
+        resize(m_source_Y, local_beam.number_of_sources);
+        resize(m_source_Z, local_beam.number_of_sources);
+        resize(m_source_I, local_beam.number_of_sources);
+        resize(m_source_lambda, local_beam.number_of_sources);
+        m_previous_nsource = local_beam.number_of_sources;
+    }
+    Kokkos::Tools::popRegion();
+
+    if (m_device_is_allocated) {
+        if (db_flags.verbose) {
+            printf(
+                "Will model %d pixels (GPU has %d pre-allocated pix)\n", Npix_to_model,
+                m_npix_allocated);
+        }
+    } else {
+        Kokkos::Tools::pushRegion("Device_allocate");
+        if (db_flags.verbose) {
+            printf(
+                "Will model %d pixels and allocate %d pix\n", Npix_to_model,
+                db_cu_flags.Npix_to_allocate);
+        }
+        // Check the Fhkl geradient arrays
+        if (db_flags.Fhkl_have_scale_factors){
+            resize(m_data_residual, db_cu_flags.Npix_to_allocate);
+            resize(m_data_variance, db_cu_flags.Npix_to_allocate);
+            resize(m_data_freq, db_cu_flags.Npix_to_allocate);
+            resize(m_data_trusted, db_cu_flags.Npix_to_allocate);
+            resize(m_FhklLinear_ASUid, db_cryst.FhklLinear_ASUid.size());
+            resize(m_Fhkl_scale, d_image.Fhkl_scale.size());
+            // alloc Fhkl_scale_deriv to bs same length as Fhkl_scale.size(), as Fhkl_scale_deriv is only set when Fhkl_gradient_mode=True, typpically not first iteration
+            resize(m_Fhkl_scale_deriv, d_image.Fhkl_scale.size());
+            m_Fhkl_grad_arrays_allocated = true;
+        }
+
+        resize(m_Fhkl_channels, local_beam.number_of_sources);
+        resize(m_source_X, local_beam.number_of_sources);
+        resize(m_source_Y, local_beam.number_of_sources);
+        resize(m_source_Z, local_beam.number_of_sources);
+        resize(m_source_I, local_beam.number_of_sources);
+        resize(m_source_lambda, local_beam.number_of_sources);
+        m_previous_nsource = local_beam.number_of_sources;
+
+
+        resize(m_UMATS, db_cryst.UMATS.size());
+        resize(m_UMATS_RXYZ, db_cryst.UMATS_RXYZ.size());
+        resize(m_AMATS, db_cryst.UMATS_RXYZ.size());
+
+        if (db_cryst.UMATS_RXYZ_prime.size() > 0) {
+            resize(m_UMATS_RXYZ_prime, db_cryst.UMATS_RXYZ_prime.size());
+        }
+
+        if (db_cryst.UMATS_RXYZ_dbl_prime.size() > 0) {
+            resize(m_UMATS_RXYZ_dbl_prime, db_cryst.UMATS_RXYZ_dbl_prime.size());
+        }
+
+        resize(m_dB_Mats, db_cryst.dB_Mats.size());
+        resize(m_dB2_Mats, db_cryst.dB2_Mats.size());
+
+        resize(m_RotMats, db_cryst.RotMats.size());
+        resize(m_dRotMats, db_cryst.dRotMats.size());
+        resize(m_d2RotMats, db_cryst.d2RotMats.size());
+
+        resize(m_fdet_vectors, local_det.fdet_vectors.size());
+        resize(m_sdet_vectors, local_det.sdet_vectors.size());
+        resize(m_odet_vectors, local_det.odet_vectors.size());
+        resize(m_pix0_vectors, local_det.pix0_vectors.size());
+        resize(m_close_distances, local_det.close_distances.size());
+
+        if (db_cryst.fpfdp.size() > 0) {
+            resize(m_fpfdp, db_cryst.fpfdp.size());
+            resize(m_atom_data, db_cryst.atom_data.size());
+        }
+        if (db_cryst.fpfdp_derivs.size() > 0) {
+            resize(m_fpfdp_derivs, db_cryst.fpfdp_derivs.size());
+        }
+
+        resize(m_Fhkl, db_cryst.FhklLinear.size());
+
+        if (db_flags.complex_miller) {
+            resize(m_Fhkl2, db_cryst.FhklLinear.size());
+        }
+
+        resize(m_dF_vecs, local_det.dF_vecs.size());
+        resize(m_dS_vecs, local_det.dF_vecs.size());
+
+        resize(m_floatimage, db_cu_flags.Npix_to_allocate);
+
+        if (db_flags.wavelength_img) {
+            resize(m_wavelenimage, db_cu_flags.Npix_to_allocate);
+        }
+        if (db_flags.refine_diffuse) {
+            resize(m_d_diffuse_gamma_images, db_cu_flags.Npix_to_allocate * 3);
+            resize(m_d_diffuse_sigma_images, db_cu_flags.Npix_to_allocate * 3);
+        }
+        if (db_flags.refine_fcell) {
+            resize(m_d_fcell_images, db_cu_flags.Npix_to_allocate);
+            resize(m_d2_fcell_images, db_cu_flags.Npix_to_allocate);
+        }
+        if (db_flags.refine_eta) {
+            resize(m_d_eta_images, db_cu_flags.Npix_to_allocate * 3);
+            resize(m_d2_eta_images, db_cu_flags.Npix_to_allocate * 3);
+        }
+        if (std::count(db_flags.refine_Umat.begin(), db_flags.refine_Umat.end(), true) > 0) {
+            resize(m_d_Umat_images, db_cu_flags.Npix_to_allocate * 3);
+            resize(m_d2_Umat_images, db_cu_flags.Npix_to_allocate * 3);
+        }
+        if (std::count(db_flags.refine_Ncells.begin(), db_flags.refine_Ncells.end(), true) > 0 ||
+            db_flags.refine_Ncells_def) {
+            resize(m_d_Ncells_images, db_cu_flags.Npix_to_allocate * 6);
+            resize(m_d2_Ncells_images, db_cu_flags.Npix_to_allocate * 6);
+        }
+        if (std::count(db_flags.refine_panel_rot.begin(), db_flags.refine_panel_rot.end(), true) >
+            0) {
+            resize(m_d_panel_rot_images, db_cu_flags.Npix_to_allocate * 3);
+        }
+
+        if (std::count(
+                db_flags.refine_panel_origin.begin(), db_flags.refine_panel_origin.end(), true) >
+            0) {
+            resize(m_d_panel_orig_images, db_cu_flags.Npix_to_allocate * 3);
+        }
+
+        if (std::count(db_flags.refine_lambda.begin(), db_flags.refine_lambda.end(), true) > 0) {
+            resize(m_d_lambda_images, db_cu_flags.Npix_to_allocate * 2);
+        }
+        if (std::count(db_flags.refine_Bmat.begin(), db_flags.refine_Bmat.end(), true) > 0) {
+            resize(m_d_Bmat_images, db_cu_flags.Npix_to_allocate * 6);
+            resize(m_d2_Bmat_images, db_cu_flags.Npix_to_allocate * 6);
+        }
+        if (db_flags.refine_fp_fdp) {
+            resize(m_d_fp_fdp_images, db_cu_flags.Npix_to_allocate * 2);
+        }
+
+        if (db_cryst.nominal_hkl.size() > 0) {
+            resize(m_nominal_hkl, db_cu_flags.Npix_to_allocate * 3);
+        }
+
+        resize(m_panels_fasts_slows, db_cu_flags.Npix_to_allocate * 3);
+
+        m_npix_allocated = db_cu_flags.Npix_to_allocate;
+        Kokkos::Tools::popRegion();
+    }  // END of allocation
+
+    bool ALLOC = !m_device_is_allocated;  // shortcut variable
+
+    gettimeofday(&t2, 0);
+    time = (1000000.0 * (t2.tv_sec - t1.tv_sec) + t2.tv_usec - t1.tv_usec) / 1000.0;
+    if (TIMERS.recording)
+        TIMERS.cuda_alloc += time;
+    if (db_flags.verbose > 1)
+        printf("TIME SPENT ALLOCATING (TOTAL):  %3.10f ms \n", time);
+
+    // ALLOC = false;
+    //  BEGIN COPYING DATA
+    gettimeofday(&t1, 0);
+    bool FORCE_COPY = true;
+
+    //  END step position
+    if (db_flags.Fhkl_gradient_mode){
+        transfer(m_data_residual, d_image.residual, Npix_to_model);
+        transfer(m_data_variance, d_image.variance, Npix_to_model);
+        transfer(m_data_trusted, d_image.trusted, Npix_to_model);
+        transfer(m_data_freq, d_image.freq, Npix_to_model);
+    }
+
+    if (db_flags.Fhkl_have_scale_factors && ALLOC){
+        transfer(m_FhklLinear_ASUid, db_cryst.FhklLinear_ASUid);
+    }
+
+    if (db_flags.Fhkl_have_scale_factors){
+        //SCITBX_ASSERT(db_beam.number_of_sources == db_beam.Fhkl_channels.size());
+        transfer(m_Fhkl_channels, db_beam.Fhkl_channels);
+        transfer(m_Fhkl_scale, d_image.Fhkl_scale);
+        ::Kokkos::deep_copy(m_Fhkl_scale_deriv, 0);
+
+        // for (int i=0; i < d_image.Fhkl_scale.size(); i++){
+            // cp.Fhkl_scale[i] = d_image.Fhkl_scale[i];
+            // if (db_flags.Fhkl_gradient_mode){
+            //     cp.Fhkl_scale_deriv[i] = 0;
+            // }
+        // }
+    }
+
+    //  BEGIN sources
+    Kokkos::Tools::pushRegion("BEGIN sources");
+    if (db_cu_flags.update_sources || ALLOC || FORCE_COPY) {
+        int source_count = local_beam.number_of_sources;
+        kokkostbx::transfer_double2kokkos(m_source_X, local_beam.source_X, source_count);
+        kokkostbx::transfer_double2kokkos(m_source_Y, local_beam.source_Y, source_count);
+        kokkostbx::transfer_double2kokkos(m_source_Z, local_beam.source_Z, source_count);
+        kokkostbx::transfer_double2kokkos(m_source_I, local_beam.source_I, source_count);
+        kokkostbx::transfer_double2kokkos(m_source_lambda, local_beam.source_lambda, source_count);
+        auto tmp_src_X = m_source_X;
+        auto tmp_src_Y = m_source_Y;
+        auto tmp_src_Z = m_source_Z;
+        Kokkos::parallel_for(
+            "normalize incident vector", source_count, KOKKOS_LAMBDA(const int& i) {
+                KOKKOS_VEC3 incident{tmp_src_X(i), tmp_src_Y(i), tmp_src_Z(i)};
+                incident.normalize();
+                tmp_src_X(i) = incident.x_val();
+                tmp_src_Y(i) = incident.y_val();
+                tmp_src_Z(i) = incident.z_val();
+            });
+        Kokkos::fence();
+        if (db_flags.verbose > 1)
+            printf("H2D sources\n");
+    }
+
+    Kokkos::Tools::popRegion();
+    //  END sources
+
+    //  UMATS
+    Kokkos::Tools::pushRegion("UMATS");
+    if (db_cu_flags.update_umats || ALLOC || FORCE_COPY) {
+        transfer_KOKKOS_MAT3(m_UMATS, db_cryst.UMATS);
+        transfer_KOKKOS_MAT3(m_UMATS_RXYZ, db_cryst.UMATS_RXYZ);
+        transfer_KOKKOS_MAT3(m_UMATS_RXYZ_prime, db_cryst.UMATS_RXYZ_prime);
+        transfer_KOKKOS_MAT3(m_UMATS_RXYZ_dbl_prime, db_cryst.UMATS_RXYZ_dbl_prime);
+
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying Umats\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END UMATS
+
+    if (db_cu_flags.update_umats || ALLOC || FORCE_COPY) {
+        auto Amat_init = db_cryst.eig_U*db_cryst.eig_B*1e10*(db_cryst.eig_O.transpose());
+        for (int i=0; i<db_cryst.UMATS_RXYZ.size(); ++i) {
+            m_AMATS(i) = to_mat3((db_cryst.UMATS_RXYZ[i]*Amat_init).transpose());
+        }
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying Amats\n");
+    }
+
+    //  BMATS
+    Kokkos::Tools::pushRegion("BMATS");
+    if (db_cu_flags.update_dB_mats || ALLOC || FORCE_COPY) {
+        transfer_KOKKOS_MAT3(m_dB_Mats, db_cryst.dB_Mats);
+        transfer_KOKKOS_MAT3(m_dB2_Mats, db_cryst.dB2_Mats);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying dB_Mats\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END BMATS
+
+    //  ROT MATS
+    Kokkos::Tools::pushRegion("ROT MATS");
+    if (db_cu_flags.update_rotmats || ALLOC || FORCE_COPY) {
+        transfer_KOKKOS_MAT3(m_RotMats, db_cryst.RotMats);
+        transfer_KOKKOS_MAT3(m_dRotMats, db_cryst.dRotMats);
+        transfer_KOKKOS_MAT3(m_d2RotMats, db_cryst.d2RotMats);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying rotmats\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END ROT MATS
+
+    //  DETECTOR VECTORS
+    Kokkos::Tools::pushRegion("DETECTOR VECTORS");
+    if (db_cu_flags.update_detector || ALLOC || FORCE_COPY) {
+        kokkostbx::transfer_vector2kokkos(m_fdet_vectors, local_det.fdet_vectors);
+        kokkostbx::transfer_vector2kokkos(m_sdet_vectors, local_det.sdet_vectors);
+        kokkostbx::transfer_vector2kokkos(m_odet_vectors, local_det.odet_vectors);
+        kokkostbx::transfer_vector2kokkos(m_pix0_vectors, local_det.pix0_vectors);
+        kokkostbx::transfer_vector2kokkos(m_close_distances, local_det.close_distances);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying detector vectors\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END  DETECTOR VECTORS
+
+    if (ALLOC || FORCE_COPY) {
+        transfer(m_nominal_hkl, db_cryst.nominal_hkl);
+        transfer(m_atom_data, db_cryst.atom_data);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying atom data\n");
+
+        transfer(m_fpfdp, db_cryst.fpfdp);
+        transfer(m_fpfdp_derivs, db_cryst.fpfdp_derivs);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying fprime and fdblprime\n");
+    }
+
+    //  BEGIN REFINEMENT FLAGS
+    Kokkos::Tools::pushRegion("BEGIN REFINMENT FLAGS");
+    if (db_cu_flags.update_refine_flags || ALLOC || FORCE_COPY) {
+        kokkostbx::transfer_vector2kokkos(m_refine_Umat, db_flags.refine_Umat);
+        kokkostbx::transfer_vector2kokkos(m_refine_Ncells, db_flags.refine_Ncells);
+        kokkostbx::transfer_vector2kokkos(m_refine_panel_origin, db_flags.refine_panel_origin);
+        kokkostbx::transfer_vector2kokkos(m_refine_panel_rot, db_flags.refine_panel_rot);
+        kokkostbx::transfer_vector2kokkos(m_refine_lambda, db_flags.refine_lambda);
+        kokkostbx::transfer_vector2kokkos(m_refine_Bmat, db_flags.refine_Bmat);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying refinement flags\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END REFINEMENT FLAGS
+
+    //  BEGIN Fhkl
+    Kokkos::Tools::pushRegion("Begin Fhkl");
+    if (db_cu_flags.update_Fhkl || ALLOC || FORCE_COPY) {
+        transfer(m_Fhkl, db_cryst.FhklLinear);
+        if (db_flags.complex_miller) {
+            transfer(m_Fhkl2, db_cryst.Fhkl2Linear);
+        }
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying step Fhkl\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END Fhkl
+
+    //  BEGIN panel derivative vecs
+    Kokkos::Tools::pushRegion("BEGIN panel derivative vecs");
+    if (db_cu_flags.update_panel_deriv_vecs || ALLOC || FORCE_COPY) {
+        kokkostbx::transfer_vector2kokkos(m_dF_vecs, local_det.dF_vecs);
+        kokkostbx::transfer_vector2kokkos(m_dS_vecs, local_det.dS_vecs);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying step panel derivative vectors\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END panel derivative vecs
+
+    //  BEGIN panels fasts slows
+    Kokkos::Tools::pushRegion("BEGIN panels fasts slows");
+    if (db_cu_flags.update_panels_fasts_slows || ALLOC || FORCE_COPY) {
+        kokkostbx::transfer_vector2kokkos(m_panels_fasts_slows, panels_fasts_slows);
+        if (db_flags.verbose > 1)
+            printf("H2D Done copying panels_fasts_slows\n");
+    }
+    Kokkos::Tools::popRegion();
+    //  END panels fasts slows
+
+    gettimeofday(&t2, 0);
+    time = (1000000.0 * (t2.tv_sec - t1.tv_sec) + t2.tv_usec - t1.tv_usec) / 1000.0;
+    if (TIMERS.recording)
+        TIMERS.cuda_copy_to_dev += time;
+    if (db_flags.verbose > 1)
+        printf("TIME SPENT COPYING DATA HOST->DEV:  %3.10f ms \n", time);
+
+    m_device_is_allocated = true;
+    ::Kokkos::fence("after copy to device");
+
+    gettimeofday(&t1, 0);
+
+    int Npanels = local_det.fdet_vectors.size() / 3;
+    int num_atoms = db_cryst.atom_data.size() / 5;
+    // note cannot use atom data if fpfdp is 0, make this cleaner
+    if (db_cryst.fpfdp.size() == 0) {
+        num_atoms = 0;
+    }
+    // int sm_size = number_of_sources*5*sizeof(CUDAREAL);
+    // gpu_sum_over_steps<<<numblocks, blocksize, sm_size >>>(
+    bool aniso_eta = db_cryst.UMATS_RXYZ.size() != db_cryst.UMATS_RXYZ_prime.size();
+    bool use_nominal_hkl = !db_cryst.nominal_hkl.empty();
+    kokkos_sum_over_steps(
+        Npix_to_model, m_panels_fasts_slows, m_floatimage, m_wavelenimage, m_d_Umat_images,
+        m_d2_Umat_images, m_d_Bmat_images, m_d2_Bmat_images, m_d_Ncells_images, m_d2_Ncells_images,
+        m_d_fcell_images, m_d2_fcell_images, m_d_eta_images, m_d2_eta_images, m_d_lambda_images,
+        m_d2_lambda_images, m_d_panel_rot_images, m_d2_panel_rot_images, m_d_panel_orig_images,
+        m_d2_panel_orig_images, m_d_fp_fdp_images, db_steps.Nsteps, db_flags.printout_fpixel,
+        db_flags.printout_spixel, db_flags.printout, db_cryst.default_F, local_det.oversample,
+        db_flags.oversample_omega, local_det.subpixel_size, local_det.pixel_size,
+        local_det.detector_thickstep, local_det.detector_thick, m_close_distances,
+        local_det.detector_attnlen, local_det.detector_thicksteps, local_beam.number_of_sources,
+        db_cryst.phisteps, db_cryst.UMATS.size(), db_flags.use_lambda_coefficients,
+        local_beam.lambda0, local_beam.lambda1, to_mat3(db_cryst.eig_U), to_mat3(db_cryst.eig_O),
+        to_mat3(db_cryst.eig_B), to_mat3(db_cryst.RXYZ), m_dF_vecs, m_dS_vecs, m_UMATS_RXYZ, m_UMATS_RXYZ_prime,
+        m_UMATS_RXYZ_dbl_prime, m_RotMats, m_dRotMats, m_d2RotMats, m_UMATS, m_dB_Mats, m_dB2_Mats,
+        m_AMATS, m_source_X, m_source_Y, m_source_Z, m_source_lambda, m_source_I,
+        local_beam.kahn_factor, db_cryst.Na, db_cryst.Nb, db_cryst.Nc, db_cryst.Nd,
+        db_cryst.Ne, db_cryst.Nf, db_cryst.phi0, db_cryst.phistep,
+        to_vec3(db_cryst.spindle_vec), local_beam.polarization_axis, db_cryst.h_range,
+        db_cryst.k_range, db_cryst.l_range, db_cryst.h_max, db_cryst.h_min,
+        db_cryst.k_max, db_cryst.k_min, db_cryst.l_max, db_cryst.l_min,
+        db_cryst.dmin, db_cryst.fudge, db_flags.complex_miller, db_flags.verbose,
+        db_flags.only_save_omega_kahn, db_flags.isotropic_ncells, db_flags.compute_curvatures,
+        m_Fhkl, m_Fhkl2, m_refine_Bmat, m_refine_Ncells, db_flags.refine_Ncells_def,
+        m_refine_panel_origin, m_refine_panel_rot, db_flags.refine_fcell, m_refine_lambda,
+        db_flags.refine_eta, m_refine_Umat, m_fdet_vectors, m_sdet_vectors, m_odet_vectors,
+        m_pix0_vectors, db_flags.nopolar, db_flags.point_pixel, local_beam.fluence,
+        db_cryst.r_e_sqr, db_cryst.spot_scale, Npanels, aniso_eta, db_flags.no_Nabc_scale,
+        m_fpfdp, m_fpfdp_derivs, m_atom_data, num_atoms, db_flags.refine_fp_fdp, m_nominal_hkl,
+        use_nominal_hkl, to_mat3(db_cryst.anisoU), to_mat3(db_cryst.anisoG), db_flags.use_diffuse,
+        m_d_diffuse_gamma_images, m_d_diffuse_sigma_images, db_flags.refine_diffuse,
+        db_flags.gamma_miller_units, db_flags.refine_Icell, db_flags.wavelength_img,
+        db_cryst.laue_group_num, db_cryst.stencil_size, db_flags.Fhkl_gradient_mode,
+        db_flags.Fhkl_errors_mode, db_flags.using_trusted_mask, db_beam.Fhkl_channels.empty(),
+        db_flags.Fhkl_have_scale_factors, db_cryst.Num_ASU,
+        m_data_residual, m_data_variance,
+        m_data_freq, m_data_trusted,
+        m_FhklLinear_ASUid,
+        m_Fhkl_channels,
+        m_Fhkl_scale, m_Fhkl_scale_deriv
+        );
+
+    ::Kokkos::fence("after kernel call");
+
+    if (db_flags.verbose > 1)
+        printf("KERNEL_COMPLETE gpu_sum_over_steps\n");
+    gettimeofday(&t2, 0);
+    time = (1000000.0 * (t2.tv_sec - t1.tv_sec) + t2.tv_usec - t1.tv_usec) / 1000.0;
+    if (TIMERS.recording)
+        TIMERS.cuda_kernel += time;
+    if (db_flags.verbose > 1)
+        printf("TIME SPENT(KERNEL):  %3.10f ms \n", time);
+
+    gettimeofday(&t1, 0);
+    //  COPY BACK FROM DEVICE
+    Kokkos::Tools::pushRegion("COPY BACK FROM DEVICE");
+    kokkostbx::transfer_kokkos2vector(floatimage, m_floatimage);
+
+    if (db_flags.wavelength_img) {
+        kokkostbx::transfer_kokkos2vector(d_image.wavelength, m_wavelenimage);
+    }
+    if (db_flags.refine_fcell) {
+        kokkostbx::transfer_kokkos2vector(d_image.fcell, m_d_fcell_images);
+        kokkostbx::transfer_kokkos2vector(d2_image.fcell, m_d2_fcell_images);
+    }
+    if (db_flags.Fhkl_gradient_mode){
+        if (db_flags.Fhkl_errors_mode){
+            for (int i=0; i < d_image.Fhkl_hessian.size(); i++)
+                d_image.Fhkl_hessian[i]= m_Fhkl_scale_deriv(i);
+        }
+        else{
+            for (int i=0; i < d_image.Fhkl_scale_deriv.size(); i++)
+                d_image.Fhkl_scale_deriv[i]= m_Fhkl_scale_deriv(i);
+        }
+    }
+    if (std::count(db_flags.refine_Umat.begin(), db_flags.refine_Umat.end(), true) > 0) {
+        kokkostbx::transfer_kokkos2vector(d_image.Umat, m_d_Umat_images);
+        kokkostbx::transfer_kokkos2vector(d2_image.Umat, m_d2_Umat_images);
+    }
+    if (std::count(db_flags.refine_panel_rot.begin(), db_flags.refine_panel_rot.end(), true) > 0) {
+        kokkostbx::transfer_kokkos2vector(d_image.panel_rot, m_d_panel_rot_images);
+    }
+    if (std::count(db_flags.refine_panel_origin.begin(), db_flags.refine_panel_origin.end(), true) >
+        0) {
+        kokkostbx::transfer_kokkos2vector(d_image.panel_orig, m_d_panel_orig_images);
+    }
+    if (db_flags.refine_eta) {
+        kokkostbx::transfer_kokkos2vector(d_image.eta, m_d_eta_images);
+        kokkostbx::transfer_kokkos2vector(d2_image.eta, m_d2_eta_images);
+    }
+    if (std::count(db_flags.refine_Ncells.begin(), db_flags.refine_Ncells.end(), true) > 0 ||
+        db_flags.refine_Ncells_def) {
+        kokkostbx::transfer_kokkos2vector(d_image.Ncells, m_d_Ncells_images);
+        kokkostbx::transfer_kokkos2vector(d2_image.Ncells, m_d2_Ncells_images);
+    }
+    if (db_flags.refine_diffuse) {
+        kokkostbx::transfer_kokkos2vector(d_image.diffuse_gamma, m_d_diffuse_gamma_images);
+        kokkostbx::transfer_kokkos2vector(d_image.diffuse_sigma, m_d_diffuse_sigma_images);
+    }
+    if (std::count(db_flags.refine_Bmat.begin(), db_flags.refine_Bmat.end(), true) > 0) {
+        kokkostbx::transfer_kokkos2vector(d_image.Bmat, m_d_Bmat_images);
+        kokkostbx::transfer_kokkos2vector(d2_image.Bmat, m_d2_Bmat_images);
+    }
+    if (std::count(db_flags.refine_lambda.begin(), db_flags.refine_lambda.end(), true) > 0) {
+        kokkostbx::transfer_kokkos2vector(d_image.lambda, m_d_lambda_images);
+    }
+    if (db_flags.refine_fp_fdp) {
+        kokkostbx::transfer_kokkos2vector(d_image.fp_fdp, m_d_fp_fdp_images);
+    }
+
+    Kokkos::Tools::popRegion();
+    gettimeofday(&t2, 0);
+    time = (1000000.0 * (t2.tv_sec - t1.tv_sec) + t2.tv_usec - t1.tv_usec) / 1000.0;
+    if (TIMERS.recording)
+        TIMERS.cuda_copy_from_dev += time;
+    if (db_flags.verbose > 1)
+        printf("TIME SPENT COPYING BACK :  %3.10f ms \n", time);
+    ::Kokkos::fence("After copy to host");
+
+    Kokkos::Tools::popRegion();
+}
diff --git a/simtbx/diffBragg/src/diffBraggKOKKOS.h b/simtbx/diffBragg/src/diffBraggKOKKOS.h
new file mode 100644
index 0000000000..cc074b938a
--- /dev/null
+++ b/simtbx/diffBragg/src/diffBraggKOKKOS.h
@@ -0,0 +1,145 @@
+#ifndef SIMTBX_DIFFBRAGG_KOKKOS
+#define SIMTBX_DIFFBRAGG_KOKKOS
+
+// #include <cmath>
+#include <iostream>
+#include <vector>
+
+#include "kokkostbx/kokkos_types.h"
+#include "kokkostbx/kokkos_utils.h"
+#include "simtbx/diffBragg/src/util.h"
+#include "simtbx/diffBragg/src/util_kokkos.h"
+
+using vector_vec3_t = view_1d_t<KOKKOS_VEC3>;
+using vector_mat3_t = view_1d_t<KOKKOS_MAT3>;
+
+#define INTEGER_VIEW(varname) vector_int_t varname = vector_int_t(#varname, 0)
+#define CUDAREAL_VIEW(varname) vector_cudareal_t varname = vector_cudareal_t(#varname, 0)
+#define MATRIX3_VIEW(varname) vector_mat3_t varname = vector_mat3_t(#varname, 0)
+
+class diffBraggKOKKOS {
+   private:
+    bool m_device_is_allocated = false;
+    int m_npix_allocated = 0;
+    int m_previous_nsource = 0;
+
+    vector_uint_t m_panels_fasts_slows = vector_uint_t("m_panels_fasts_slows", 0);
+
+    CUDAREAL_VIEW(m_floatimage);
+    CUDAREAL_VIEW(m_wavelenimage);
+    CUDAREAL_VIEW(m_d_diffuse_sigma_images);
+    CUDAREAL_VIEW(m_d_diffuse_gamma_images);
+    CUDAREAL_VIEW(m_d_Umat_images);
+    CUDAREAL_VIEW(m_d_Bmat_images);
+    CUDAREAL_VIEW(m_d_Ncells_images);
+    CUDAREAL_VIEW(m_d_fcell_images);
+    CUDAREAL_VIEW(m_d_eta_images);
+    CUDAREAL_VIEW(m_d2_eta_images);
+    CUDAREAL_VIEW(m_d_lambda_images);
+    CUDAREAL_VIEW(m_d_panel_rot_images);
+    CUDAREAL_VIEW(m_d_panel_orig_images);
+
+    CUDAREAL_VIEW(m_d2_Umat_images);
+    CUDAREAL_VIEW(m_d2_Bmat_images);
+    CUDAREAL_VIEW(m_d2_Ncells_images);
+    CUDAREAL_VIEW(m_d2_fcell_images);
+    CUDAREAL_VIEW(m_d2_lambda_images);
+    CUDAREAL_VIEW(m_d2_panel_rot_images);
+    CUDAREAL_VIEW(m_d2_panel_orig_images);
+
+    CUDAREAL_VIEW(m_d_sausage_XYZ_scale_images);
+    CUDAREAL_VIEW(m_d_fp_fdp_images);
+
+    INTEGER_VIEW(m_subS_pos);
+    INTEGER_VIEW(m_subF_pos);
+    INTEGER_VIEW(m_thick_pos);
+    INTEGER_VIEW(m_source_pos);
+    INTEGER_VIEW(m_mos_pos);
+    INTEGER_VIEW(m_phi_pos);
+    INTEGER_VIEW(m_sausage_pos);
+
+    CUDAREAL_VIEW(m_Fhkl);
+    CUDAREAL_VIEW(m_Fhkl2);
+
+    CUDAREAL_VIEW(m_fdet_vectors);
+    CUDAREAL_VIEW(m_sdet_vectors);
+    CUDAREAL_VIEW(m_odet_vectors);
+    CUDAREAL_VIEW(m_pix0_vectors);
+    CUDAREAL_VIEW(m_close_distances);
+
+    INTEGER_VIEW(m_nominal_hkl);
+    CUDAREAL_VIEW(m_fpfdp);
+    CUDAREAL_VIEW(m_fpfdp_derivs);
+    CUDAREAL_VIEW(m_atom_data);
+
+    CUDAREAL_VIEW(m_source_X);
+    CUDAREAL_VIEW(m_source_Y);
+    CUDAREAL_VIEW(m_source_Z);
+    CUDAREAL_VIEW(m_source_I);
+    CUDAREAL_VIEW(m_source_lambda);
+    int m_sources;
+    bool m_sources_are_allocated = false;
+    bool m_sources_recopy = false;
+
+    MATRIX3_VIEW(m_UMATS);
+    MATRIX3_VIEW(m_dB_Mats);
+    MATRIX3_VIEW(m_dB2_Mats);
+    MATRIX3_VIEW(m_UMATS_RXYZ);
+    MATRIX3_VIEW(m_UMATS_RXYZ_prime);
+    MATRIX3_VIEW(m_UMATS_RXYZ_dbl_prime);
+    MATRIX3_VIEW(m_RotMats);
+    MATRIX3_VIEW(m_dRotMats);
+    MATRIX3_VIEW(m_d2RotMats);
+
+    MATRIX3_VIEW(m_AMATS);
+
+    vector_vec3_t m_dF_vecs = vector_vec3_t("m_dF_vecs", 0);
+    vector_vec3_t m_dS_vecs = vector_vec3_t("m_dS_vecs", 0);
+
+    MATRIX3_VIEW(m_sausages_RXYZ);
+    MATRIX3_VIEW(m_d_sausages_RXYZ);
+    MATRIX3_VIEW(m_sausages_U);
+    CUDAREAL_VIEW(m_sausages_scale);
+
+    vector_bool_t m_refine_Bmat = vector_bool_t("m_refine_Bmat", 6);
+    vector_bool_t m_refine_Umat = vector_bool_t("m_refine_Umat", 3);
+    vector_bool_t m_refine_Ncells = vector_bool_t("m_refine_Ncells", 3);
+    vector_bool_t m_refine_panel_origin = vector_bool_t("m_refine_panel_origin", 3);
+    vector_bool_t m_refine_panel_rot = vector_bool_t("m_refine_panel_rot", 3);
+    vector_bool_t m_refine_lambda = vector_bool_t("m_refine_lambda", 2);
+
+    bool m_Fhkl_gradient_mode;
+    bool m_using_trusted_mask;
+    bool m_Fhkl_channels_empty;
+    bool m_Fhkl_have_scale_factors;
+    // these are copied once at first iteration
+    bool m_Fhkl_grad_arrays_allocated=false;
+    CUDAREAL_VIEW(m_data_residual); // length is number of modeled pixels
+    CUDAREAL_VIEW(m_data_variance); // length is number of modeled pixels
+    INTEGER_VIEW(m_data_freq); // length is number of modeled pixels
+    vector_bool_t m_data_trusted = vector_bool_t("m_data_trusted", 0); // length is number of modeled pixels
+    INTEGER_VIEW(m_FhklLinear_ASUid); // length is number of ASU in FhklLinear
+    CUDAREAL_VIEW(m_Fhkl_channels);
+    // Fhkl_scale is dynamically copied each iteration
+    // Fhkl_scale_deriv is set to 0 each iteration
+    CUDAREAL_VIEW(m_Fhkl_scale);  // length is (number of ASUin FhklLinear) *times* (number of Fhkl channels)
+    CUDAREAL_VIEW(m_Fhkl_scale_deriv); // length is (number of ASUin FhklLinear) *times* (number of Fhkl channels)
+
+   public:
+    void diffBragg_sum_over_steps_kokkos(
+        int Npix_to_model,
+        std::vector<unsigned int>& panels_fasts_slows,
+        image_type& floatimage,
+        images& d_image,
+        images& d2_image,
+        step_arrays& db_steps,
+        detector& db_det,
+        beam& db_beam,
+        crystal& db_cryst,
+        flags& db_flags,
+        cuda_flags& db_cu_flags,
+        // diffBragg_kokkosPointers& kp,
+        timer_variables& TIMERS);
+};
+
+#endif
diff --git a/simtbx/diffBragg/src/diffBragg_ext.cpp b/simtbx/diffBragg/src/diffBragg_ext.cpp
index 109f97329d..993e1eaf34 100644
--- a/simtbx/diffBragg/src/diffBragg_ext.cpp
+++ b/simtbx/diffBragg/src/diffBragg_ext.cpp
@@ -447,6 +447,18 @@ namespace boost_python { namespace {
         return boost::python::make_tuple(diffBragg.pythony_indices,diffBragg.pythony_amplitudes);
   }
 
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+  void finalize_kokkos(){
+    Kokkos::finalize();
+  }
+
+  void initialize_kokkos(int dev){
+    Kokkos::InitArguments kokkos_init;
+    kokkos_init.device_id = dev;
+    Kokkos::initialize(kokkos_init);
+  }
+#endif
+
   void diffBragg_init_module() {
     Py_Initialize();
     boost::python::numpy::initialize();
@@ -455,12 +467,21 @@ namespace boost_python { namespace {
     typedef return_value_policy<return_by_value> rbv;
     typedef default_call_policies dcp;
     typedef return_internal_reference<> rir;
+
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+    def("finalize_kokkos", finalize_kokkos,
+        "calls Kokkos::finalize()");
+
+    def("initialize_kokkos", initialize_kokkos,
+        "the sole argument `dev` (an int from 0 to Ngpu-1) is passed to Kokkos::initialize()");
+#endif
+
     class_<simtbx::nanoBragg::diffBragg, bases<simtbx::nanoBragg::nanoBragg> >
             ("diffBragg", no_init)
       /* constructor that takes a dxtbx detector and beam model */
       .def(init<const dxtbx::model::Detector&,
                 const dxtbx::model::Beam&,
-                int >(
+                int>(
         (arg_("detector"),
          arg_("beam"),
          arg_("verbose")=0),
@@ -589,8 +610,12 @@ namespace boost_python { namespace {
 
       .def("show_heavy_atom_data", &simtbx::nanoBragg::diffBragg::show_heavy_atom_data)
 
-#ifdef NANOBRAGG_HAVE_CUDA
-      .def("gpu_free",&simtbx::nanoBragg::diffBragg::gpu_free)
+#ifdef DIFFBRAGG_HAVE_CUDA
+      .def("gpu_free",&simtbx::nanoBragg::diffBragg::cuda_free)
+#endif
+
+#ifdef DIFFBRAGG_HAVE_KOKKOS
+      .def("kokkos_gpu_free",&simtbx::nanoBragg::diffBragg::kokkos_free)
 #endif
 
       .def("set_mosaic_blocks_prime",
diff --git a/simtbx/diffBragg/src/diffBragg_kokkos_kernel.cpp b/simtbx/diffBragg/src/diffBragg_kokkos_kernel.cpp
new file mode 100644
index 0000000000..7d8cd7b557
--- /dev/null
+++ b/simtbx/diffBragg/src/diffBragg_kokkos_kernel.cpp
@@ -0,0 +1,1268 @@
+#include <cstdio>
+#include "diffBraggKOKKOS.h"
+#include <simtbx/diffBragg/src/diffuse_util_kokkos.h>
+
+// using ::Kokkos::Experimental::exp;
+// using ::Kokkos::Experimental::sin;
+// using ::Kokkos::Experimental::cos;
+// using ::Kokkos::Experimental::sqrt;
+
+void kokkos_sum_over_steps(
+    int Npix_to_model,
+    vector_uint_t panels_fasts_slows,
+    vector_cudareal_t floatimage,
+    vector_cudareal_t wavelenimage,
+    vector_cudareal_t d_Umat_images,
+    vector_cudareal_t d2_Umat_images,
+    vector_cudareal_t d_Bmat_images,
+    vector_cudareal_t d2_Bmat_images,
+    vector_cudareal_t d_Ncells_images,
+    vector_cudareal_t d2_Ncells_images,
+    vector_cudareal_t d_fcell_images,
+    vector_cudareal_t d2_fcell_images,
+    vector_cudareal_t d_eta_images,
+    vector_cudareal_t d2_eta_images,
+    vector_cudareal_t d_lambda_images,
+    vector_cudareal_t d2_lambda_images,
+    vector_cudareal_t d_panel_rot_images,
+    vector_cudareal_t d2_panel_rot_images,
+    vector_cudareal_t d_panel_orig_images,
+    vector_cudareal_t d2_panel_orig_images,
+    vector_cudareal_t d_fp_fdp_images,
+    const int Nsteps,
+    int printout_fpixel,
+    int printout_spixel,
+    bool printout,
+    CUDAREAL default_F,
+    int oversample,
+    bool oversample_omega,
+    CUDAREAL subpixel_size,
+    CUDAREAL pixel_size,
+    CUDAREAL detector_thickstep,
+    CUDAREAL detector_thick,
+    const vector_cudareal_t close_distances,
+    CUDAREAL detector_attnlen,
+    int detector_thicksteps,
+    int sources,
+    int phisteps,
+    int mosaic_domains,
+    bool use_lambda_coefficients,
+    CUDAREAL lambda0,
+    CUDAREAL lambda1,
+    KOKKOS_MAT3 eig_U,
+    KOKKOS_MAT3 eig_O,
+    KOKKOS_MAT3 eig_B,
+    KOKKOS_MAT3 RXYZ,
+    vector_vec3_t dF_vecs,
+    vector_vec3_t dS_vecs,
+    const vector_mat3_t UMATS_RXYZ,
+    vector_mat3_t UMATS_RXYZ_prime,
+    vector_mat3_t UMATS_RXYZ_dbl_prime,
+    vector_mat3_t RotMats,
+    vector_mat3_t dRotMats,
+    vector_mat3_t d2RotMats,
+    vector_mat3_t UMATS,
+    vector_mat3_t dB_mats,
+    vector_mat3_t dB2_mats,
+    vector_mat3_t Amatrices,
+    const vector_cudareal_t source_X,
+    const vector_cudareal_t source_Y,
+    const vector_cudareal_t source_Z,
+    const vector_cudareal_t source_lambda,
+    const vector_cudareal_t source_I,
+    CUDAREAL kahn_factor,
+    CUDAREAL Na,
+    CUDAREAL Nb,
+    CUDAREAL Nc,
+    CUDAREAL Nd,
+    CUDAREAL Ne,
+    CUDAREAL Nf,
+    CUDAREAL phi0,
+    CUDAREAL phistep,
+    KOKKOS_VEC3 spindle_vec,
+    KOKKOS_VEC3 polarization_axis,
+    int h_range,
+    int k_range,
+    int l_range,
+    int h_max,
+    int h_min,
+    int k_max,
+    int k_min,
+    int l_max,
+    int l_min,
+    CUDAREAL dmin,
+    CUDAREAL fudge,
+    bool complex_miller,
+    int verbose,
+    bool only_save_omega_kahn,
+    bool isotropic_ncells,
+    bool compute_curvatures,
+    const vector_cudareal_t FhklLinear,
+    const vector_cudareal_t Fhkl2Linear,
+    vector_bool_t refine_Bmat,
+    vector_bool_t refine_Ncells,
+    bool refine_Ncells_def,
+    vector_bool_t refine_panel_origin,
+    vector_bool_t refine_panel_rot,
+    bool refine_fcell,
+    vector_bool_t refine_lambda,
+    bool refine_eta,
+    vector_bool_t refine_Umat,
+    const vector_cudareal_t fdet_vectors,
+    const vector_cudareal_t sdet_vectors,
+    const vector_cudareal_t odet_vectors,
+    const vector_cudareal_t pix0_vectors,
+    bool nopolar,
+    bool point_pixel,
+    CUDAREAL fluence,
+    CUDAREAL r_e_sqr,
+    CUDAREAL spot_scale,
+    int Npanels,
+    bool aniso_eta,
+    bool no_Nabc_scale,
+    const vector_cudareal_t fpfdp,
+    const vector_cudareal_t fpfdp_derivs,
+    const vector_cudareal_t atom_data,
+    int num_atoms,
+    bool refine_fp_fdp,
+    const vector_int_t nominal_hkl,
+    bool use_nominal_hkl,
+    KOKKOS_MAT3 anisoU,
+    KOKKOS_MAT3 anisoG,
+    bool use_diffuse,
+    vector_cudareal_t d_diffuse_gamma_images,
+    vector_cudareal_t d_diffuse_sigma_images,
+    bool refine_diffuse,
+    bool gamma_miller_units,
+    bool refine_Icell,
+    bool save_wavelenimage,
+    int laue_group_num,
+    int stencil_size,
+    bool Fhkl_gradient_mode,
+    bool Fhkl_errors_mode,
+    bool using_trusted_mask,
+    bool Fhkl_channels_empty,
+    bool Fhkl_have_scale_factors,
+    int Num_ASU,
+    const vector_cudareal_t data_residual,
+    const vector_cudareal_t data_variance,
+    const vector_int_t data_freq,
+    const vector_bool_t data_trusted,
+    const vector_int_t FhklLinear_ASUid,
+    const vector_cudareal_t Fhkl_channels,
+    const vector_cudareal_t Fhkl_scale,
+    vector_cudareal_t Fhkl_scale_deriv) {  // BEGIN GPU kernel
+
+    // int tid = blockIdx.x * blockDim.x + threadIdx.x;
+    // int thread_stride = blockDim.x * gridDim.x;
+    // __shared__ bool s_Fhkl_channels_empty;
+    // __shared__ bool s_Fhkl_have_scale_factors;
+    // __shared__ bool s_Fhkl_gradient_mode;
+    // __shared__ bool s_Fhkl_errors_mode;
+    // __shared__ int s_Num_ASU;
+    // __shared__ bool s_refine_Icell;
+    // __shared__ bool s_use_diffuse;
+    // __shared__ bool s_use_nominal_hkl;
+    // __shared__ bool s_refine_fp_fdp;
+    // __shared__ bool s_complex_miller;
+    // __shared__ int s_num_atoms;
+    // __shared__ bool s_aniso_eta;
+    // __shared__ bool s_no_Nabc_scale;
+    // __shared__ bool s_compute_curvatures;
+    // __shared__ KOKKOS_MAT3 s_Ot;
+    // __shared__ bool s_refine_diffuse;
+    // __shared__ bool s_gamma_miller_units;
+    // __shared__ KOKKOS_MAT3 _NABC;
+    // __shared__ KOKKOS_MAT3 s_dN;
+    // __shared__ CUDAREAL C;
+    // __shared__ CUDAREAL two_C;
+    // __shared__ KOKKOS_MAT3 Bmat_realspace;
+    // __shared__ KOKKOS_MAT3 Amat_init;
+    //__shared__ CUDAREAL s_Na;
+    //__shared__ CUDAREAL s_Nb;
+    //__shared__ CUDAREAL s_Nc;
+    // __shared__ CUDAREAL s_NaNbNc_squared; // unused?
+    // __shared__ int s_h_max, s_k_max, s_l_max, s_h_min, s_k_min, s_l_min, s_k_range, s_l_range;
+    // __shared__ int s_sources, s_mosaic_domains;
+    // __shared__ CUDAREAL s_detector_attnlen, s_lambda0, s_lambda1;
+    // __shared__ bool s_printout;
+    // __shared__ KOKKOS_VEC3 s_polarization_axis;
+
+    // __shared__ bool s_refine_Umat[3];
+    // __shared__ bool s_refine_panel_origin[3];
+    // __shared__ bool s_refine_panel_rot[3];
+    // __shared__ bool s_refine_Ncells[3];
+    // __shared__ bool s_refine_eta;
+    // __shared__ bool s_refine_Ncells_def;
+    // __shared__ bool s_refine_fcell;
+    // __shared__ bool s_refine_Bmat[6];
+    // __shared__ bool s_refine_lambda[2];
+    // __shared__ double s_NABC_det, s_NABC_det_sq;
+    // extern __shared__ CUDAREAL det_vecs[];
+    //__shared__ int det_stride;
+
+    // TODO can we get speed gains by dividing up the following definitions over more threads ?
+    // if (threadIdx.x == 0) {
+    // for (int i = 0; i < 3; i++) {
+    // s_refine_Ncells[i] = refine_Ncells[i];
+    // s_refine_Umat[i] = refine_Umat[i];
+    // s_refine_panel_origin[i] = refine_panel_origin[i];
+    // s_refine_panel_rot[i] = refine_panel_rot[i];
+    // }
+    // s_Fhkl_channels_empty = Fhkl_channels_empty;
+    // s_Fhkl_have_scale_factors = Fhkl_have_scale_factors;
+    // s_Fhkl_gradient_mode = Fhkl_gradient_mode;
+    // s_Fhkl_errors_mode = Fhkl_errors_mode;
+    // s_Num_ASU = Num_ASU;
+    // s_refine_Icell = refine_Icell;
+    // s_use_nominal_hkl = use_nominal_hkl;
+    // s_aniso_eta = aniso_eta;
+    // s_no_Nabc_scale = no_Nabc_scale;
+    // s_complex_miller = complex_miller;
+    // s_refine_lambda[0] = refine_lambda[0];
+    // s_refine_lambda[1] = refine_lambda[1];
+    // for (int i = 0; i < 6; i++) {
+    //     s_refine_Bmat[i] = refine_Bmat[i];
+    // }
+    // s_use_diffuse = use_diffuse;
+    // s_num_atoms = num_atoms;
+    // s_refine_fcell = refine_fcell;
+    // s_refine_eta = refine_eta;
+    // s_refine_Ncells_def = refine_Ncells_def;
+    // s_compute_curvatures = compute_curvatures;
+    // s_refine_fp_fdp = refine_fp_fdp;
+    // s_refine_diffuse = refine_diffuse;
+    // s_gamma_miller_units = gamma_miller_units;
+
+    const KOKKOS_MAT3 Bmat_realspace = eig_B * 1e10;
+    const KOKKOS_MAT3 eig_Otranspose = eig_O.transpose();
+    const KOKKOS_MAT3 Amat_init = eig_U * Bmat_realspace * eig_Otranspose;
+    const KOKKOS_MAT3 Ainv = eig_U*(Bmat_realspace.transpose().inverse())* (eig_O.inverse());
+    const KOKKOS_MAT3 _NABC {Na, Nd, Nf, Nd, Nb, Ne, Nf, Ne, Nc};
+    const double NABC_det = _NABC.determinant();  // TODO is this slow ?
+    const double NABC_det_sq = NABC_det * NABC_det;
+    const CUDAREAL C = 2 / 0.63 * fudge;
+    const CUDAREAL two_C = 2 * C;
+    KOKKOS_MAT3 anisoG_local;
+    KOKKOS_MAT3 anisoU_local;
+    KOKKOS_MAT3 laue_mats[24];
+    KOKKOS_MAT3 dG_dgam[3];
+    int num_laue_mats;
+    int dhh, dkk, dll;
+    KOKKOS_VEC3 Hmin, Hmax, dHH, Hrange;
+    // s_Na = Na;
+    // s_Nb = Nb;
+    // s_Nc = Nc;
+    // s_NaNbNc_squared = (Na * Nb * Nc);
+    // s_NaNbNc_squared *= s_NaNbNc_squared;
+    // s_h_max = h_max;
+    // s_k_max = k_max;
+    // s_l_max = l_max;
+    // s_h_min = h_min;
+    // s_k_min = k_min;
+    // s_l_min = l_min;
+    // s_h_range = h_range;
+    // s_k_range = k_range;
+    // s_l_range = l_range;
+
+    // s_oversample = oversample;
+    // s_detector_thicksteps = detector_thicksteps;
+    // s_sources = sources;
+    // s_mosaic_domains = mosaic_domains;
+    // s_detector_thickstep = detector_thickstep;
+    // s_detector_attnlen = detector_attnlen;
+    // s_subpixel_size = subpixel_size;
+    // s_pixel_size = pixel_size;
+    // s_detector_thick = _detector_thick;
+    // s_lambda0 = lambda0;
+    // s_lambda1 = lambda1;
+    // s_oversample_omega = _oversample_omega;
+    // s_printout = printout;
+    // s_printout_fpixel = printout_fpixel;
+    // s_printout_spixel = printout_spixel;
+    // s_default_F = default_F;
+    // s_verbose = verbose;
+    // s_polarization_axis = polarization_axis;
+    // s_kahn_factor = kahn_factor;
+    // s_nopolar = nopolar;
+    // sX0 = source_X[0];
+    // sY0 = source_Y[0];
+    // sZ0 = source_Z[0];
+    // s_Nsteps = Nsteps;
+
+    // }
+
+    if (use_diffuse){
+        anisoG_local = anisoG;
+        anisoU_local = anisoU;
+        num_laue_mats = gen_laue_mats(laue_group_num, laue_mats);
+        for (int i_gam=0; i_gam<3; i_gam++){
+            dG_dgam[i_gam] << 0,0,0,0,0,0,0,0,0;
+            dG_dgam[i_gam](i_gam, i_gam) = 1;
+        }
+        if (gamma_miller_units){
+            anisoG_local = anisoG_local * Bmat_realspace;
+            for (int i_gam=0; i_gam<3; i_gam++){
+            dG_dgam[i_gam] = dG_dgam[i_gam] * Bmat_realspace;
+            }
+        }
+        dhh = dkk = dll = stencil_size; // Limits of stencil for diffuse calc
+    }
+    Hmin << h_min, k_min, l_min;
+    Hmax << h_max, k_max, l_max;
+    dHH << dhh, dkk, dll;
+    Hrange << h_range, k_range, l_range;
+
+    const CUDAREAL overall_scale = r_e_sqr * spot_scale * fluence / Nsteps;
+
+    Kokkos::parallel_for(
+        "sum_over_steps", Npix_to_model, KOKKOS_LAMBDA(const int& pixIdx) {
+            // __syncthreads();
+
+            // for (int pixIdx=tid; pixIdx < Npix_to_model; pixIdx+= thread_stride){
+            if (using_trusted_mask) {
+                if (!data_trusted(pixIdx))
+                    return;
+            }
+            int _pid = panels_fasts_slows(pixIdx * 3);
+            int _fpixel = panels_fasts_slows(pixIdx * 3 + 1);
+            int _spixel = panels_fasts_slows(pixIdx * 3 + 2);
+
+            CUDAREAL Fhkl_deriv_coef=0;
+            CUDAREAL Fhkl_hessian_coef=0;
+            if (Fhkl_gradient_mode) {
+                CUDAREAL u = data_residual(pixIdx);
+                CUDAREAL one_by_v = 1/data_variance(pixIdx);
+                CUDAREAL Gterm = 1 - 2*u - u*u*one_by_v;
+                Fhkl_deriv_coef = 0.5 * Gterm*one_by_v / data_freq(pixIdx);
+                if (Fhkl_errors_mode) {
+                    Fhkl_hessian_coef = -0.5*one_by_v*(one_by_v*Gterm - 2  - 2*u*one_by_v -u*u*one_by_v*one_by_v)/data_freq(pixIdx);
+                }
+            }
+
+            // int fcell_idx=1;
+            int nom_h, nom_k, nom_l;
+            if (use_nominal_hkl) {
+                nom_h = nominal_hkl(pixIdx * 3);
+                nom_k = nominal_hkl(pixIdx * 3 + 1);
+                nom_l = nominal_hkl(pixIdx * 3 + 2);
+            }
+            CUDAREAL close_distance = close_distances(_pid);
+
+            // reset photon count for this pixel
+            double _I = 0;
+            double Ilambda = 0;
+
+            // reset derivative photon counts for the various parameters
+            double rot_manager_dI[3] = {0, 0, 0};
+            double rot_manager_dI2[3] = {0, 0, 0};
+            double ucell_manager_dI[6] = {0, 0, 0, 0, 0, 0};
+            double ucell_manager_dI2[6] = {0, 0, 0, 0, 0, 0};
+            double Ncells_manager_dI[6] = {0, 0, 0, 0, 0, 0};
+            double Ncells_manager_dI2[6] = {0, 0, 0, 0, 0, 0};
+            double pan_orig_manager_dI[3] = {0, 0, 0};
+            double pan_orig_manager_dI2[3] = {0, 0, 0};
+            double pan_rot_manager_dI[3] = {0, 0, 0};
+            double pan_rot_manager_dI2[3] = {0, 0, 0};
+            double fcell_manager_dI = 0;
+            double fcell_manager_dI2 = 0;
+            double eta_manager_dI[3] = {0, 0, 0};
+            double eta_manager_dI2[3] = {0, 0, 0};
+            double lambda_manager_dI[2] = {0, 0};
+            double lambda_manager_dI2[2] = {0, 0};
+            double fp_fdp_manager_dI[2] = {0, 0};
+            double dI_diffuse[6] = {0, 0, 0, 0, 0, 0};
+
+            for (int _subS = 0; _subS < oversample; ++_subS) {
+                for (int _subF = 0; _subF < oversample; ++_subF) {
+                    // absolute mm position on detector (relative to its origin)
+                    CUDAREAL _Fdet =
+                        subpixel_size * (_fpixel * oversample + _subF) + subpixel_size / 2.0;
+                    CUDAREAL _Sdet =
+                        subpixel_size * (_spixel * oversample + _subS) + subpixel_size / 2.0;
+
+                    // assume "distance" is to the front of the detector sensor layer
+                    int pid_x = _pid * 3;
+                    int pid_y = _pid * 3 + 1;
+                    int pid_z = _pid * 3 + 2;
+
+                    CUDAREAL fx = fdet_vectors(pid_x);
+                    CUDAREAL fy = fdet_vectors(pid_y);
+                    CUDAREAL fz = fdet_vectors(pid_z);
+                    CUDAREAL sx = sdet_vectors(pid_x);
+                    CUDAREAL sy = sdet_vectors(pid_y);
+                    CUDAREAL sz = sdet_vectors(pid_z);
+                    CUDAREAL ox = odet_vectors(pid_x);
+                    CUDAREAL oy = odet_vectors(pid_y);
+                    CUDAREAL oz = odet_vectors(pid_z);
+                    CUDAREAL px = pix0_vectors(pid_x);
+                    CUDAREAL py = pix0_vectors(pid_y);
+                    CUDAREAL pz = pix0_vectors(pid_z);
+
+                    KOKKOS_VEC3 _o_vec(ox, oy, oz);
+
+                    for (int _thick_tic = 0; _thick_tic < detector_thicksteps; ++_thick_tic) {
+                        CUDAREAL _Odet = _thick_tic * detector_thickstep;
+
+                        CUDAREAL pixposX = _Fdet * fx + _Sdet * sx + _Odet * ox + px;
+                        CUDAREAL pixposY = _Fdet * fy + _Sdet * sy + _Odet * oy + py;
+                        CUDAREAL pixposZ = _Fdet * fz + _Sdet * sz + _Odet * oz + pz;
+                        KOKKOS_VEC3 _pixel_pos(pixposX, pixposY, pixposZ);
+
+                        CUDAREAL _airpath = _pixel_pos.length();
+                        KOKKOS_VEC3 _diffracted = _pixel_pos.get_unit_vector();
+
+                        // solid angle subtended by a pixel: (pix/airpath)^2*cos(2theta)
+                        CUDAREAL _omega_pixel = pixel_size * pixel_size / _airpath / _airpath *
+                                                close_distance / _airpath;
+
+                        // option to turn off obliquity effect, inverse-square-law only
+                        if (point_pixel)
+                            _omega_pixel = 1.0 / _airpath / _airpath;
+
+                        // now calculate detector thickness effects
+                        CUDAREAL _capture_fraction = 1;
+
+                        if (detector_thick > 0.0 && detector_attnlen > 0.0) {
+                            // inverse of effective thickness increase
+                            CUDAREAL _parallax = _diffracted.dot(_o_vec);
+                            _capture_fraction = ::Kokkos::Experimental::exp(
+                                                    -_thick_tic * detector_thickstep /
+                                                    detector_attnlen / _parallax) -
+                                                ::Kokkos::Experimental::exp(
+                                                    -(_thick_tic + 1) * detector_thickstep /
+                                                    detector_attnlen / _parallax);
+                        }
+                        CUDAREAL cap_frac_times_omega = _capture_fraction * _omega_pixel;
+
+                        for (int _source = 0; _source < sources; ++_source) {
+                            KOKKOS_VEC3 _incident(
+                                -source_X(_source), -source_Y(_source), -source_Z(_source));
+                            CUDAREAL _lambda = source_lambda(_source);
+                            CUDAREAL sI = source_I(_source);
+                            CUDAREAL lambda_ang = _lambda * 1e10;
+                            if (use_lambda_coefficients) {
+                                lambda_ang = lambda0 + lambda1 * lambda_ang;
+                                _lambda = lambda_ang * 1e-10;
+                            }
+
+                            // polarization
+                            CUDAREAL polar_for_Fhkl_grad=1;
+                            if (!nopolar && Fhkl_gradient_mode){
+                                //polar_for_Fhkl_grad = diffBragg_gpu_kernel_polarization(_incident, _diffracted,
+                                //                    s_polarization_axis, s_kahn_factor);
+                                // component of diffracted unit vector along incident beam unit vector
+                                CUDAREAL cos2theta = _incident.dot(_diffracted);
+                                CUDAREAL cos2theta_sqr = cos2theta*cos2theta;
+                                CUDAREAL sin2theta_sqr = 1-cos2theta_sqr;
+
+                                CUDAREAL _psi=0;
+                                if(kahn_factor != 0.0){
+                                    // cross product to get "vertical" axis that is orthogonal to the cannonical "polarization"
+                                    KOKKOS_VEC3 B_in = polarization_axis.cross(_incident);
+                                    // cross product with incident beam to get E-vector direction
+                                    KOKKOS_VEC3 E_in = _incident.cross(B_in);
+                                    // get components of diffracted ray projected onto the E-B plane
+                                    CUDAREAL _kEi = _diffracted.dot(E_in);
+                                    CUDAREAL _kBi = _diffracted.dot(B_in);
+                                    // compute the angle of the diffracted ray projected onto the incident E-B plane
+                                    _psi = -atan2(_kBi,_kEi);
+                                }
+                                // correction for polarized incident beam
+                                polar_for_Fhkl_grad = 0.5*(1.0 + cos2theta_sqr - kahn_factor*cos(2*_psi)*sin2theta_sqr);
+                            }
+
+                            KOKKOS_VEC3 _scattering = (_diffracted - _incident) / _lambda;
+
+                            KOKKOS_VEC3 q_vec(_scattering[0], _scattering[1], _scattering[2]);
+                            q_vec *= 1e-10;
+
+                            // TODO rename
+                            CUDAREAL texture_scale = 1;
+                            texture_scale *= cap_frac_times_omega;
+                            texture_scale *= sI;
+
+                            for (int _mos_tic = 0; _mos_tic < mosaic_domains; ++_mos_tic) {
+                                int amat_idx = _mos_tic;
+                                KOKKOS_MAT3 UBO = Amatrices(amat_idx);
+
+                                KOKKOS_VEC3 H_vec = UBO * q_vec;
+                                CUDAREAL _h = H_vec[0];
+                                CUDAREAL _k = H_vec[1];
+                                CUDAREAL _l = H_vec[2];
+
+                                int _h0 = ceil(_h - 0.5);
+                                int _k0 = ceil(_k - 0.5);
+                                int _l0 = ceil(_l - 0.5);
+
+                                KOKKOS_VEC3 H0(_h0, _k0, _l0);
+
+                                KOKKOS_VEC3 delta_H = H_vec - H0;
+                                KOKKOS_VEC3 V = _NABC * delta_H;
+                                CUDAREAL _hrad_sqr = V.dot(V);
+                                CUDAREAL exparg = _hrad_sqr * C / 2;
+                                CUDAREAL I0 = 0;
+
+                                if (exparg < 35)
+                                    if (no_Nabc_scale)
+                                        I0 = ::Kokkos::Experimental::exp(-2 * exparg);
+                                    else
+                                        I0 = (NABC_det_sq) *
+                                             ::Kokkos::Experimental::exp(-2 * exparg);
+
+                                // are we doing diffuse scattering
+                                CUDAREAL step_diffuse_param[6] = {0, 0, 0, 0, 0, 0};
+                                if (use_diffuse) {
+                                    calc_diffuse_at_hkl(H_vec,H0,dHH,Hmin,Hmax,Hrange,Ainv,FhklLinear,num_laue_mats,laue_mats,anisoG_local,anisoU_local,dG_dgam,refine_diffuse,&I0,step_diffuse_param);
+                                } // end s_use_diffuse outer
+
+                                CUDAREAL _F_cell = default_F;
+                                CUDAREAL _F_cell2 = 0;
+                                int i_hklasu=0;
+
+                                if ((_h0 <= h_max) && (_h0 >= h_min) && (_k0 <= k_max) &&
+                                    (_k0 >= k_min) && (_l0 <= l_max) && (_l0 >= l_min)) {
+                                    int Fhkl_linear_index = (_h0 - h_min) * k_range * l_range +
+                                                            (_k0 - k_min) * l_range + (_l0 - l_min);
+                                    //_F_cell = __ldg(&FhklLinear[Fhkl_linear_index]);
+                                    _F_cell = FhklLinear(Fhkl_linear_index);
+                                    // if (complex_miller) _F_cell2 =
+                                    // __ldg(&Fhkl2Linear[Fhkl_linear_index]);
+                                    if (complex_miller)
+                                        _F_cell2 = Fhkl2Linear(Fhkl_linear_index);
+                                    if (Fhkl_have_scale_factors)
+                                        i_hklasu = FhklLinear_ASUid(Fhkl_linear_index);
+                                }
+
+                                CUDAREAL c_deriv_Fcell = 0;
+                                CUDAREAL d_deriv_Fcell = 0;
+                                if (complex_miller) {
+                                    CUDAREAL c_deriv_Fcell_real = 0;
+                                    CUDAREAL c_deriv_Fcell_imag = 0;
+                                    CUDAREAL d_deriv_Fcell_real = 0;
+                                    CUDAREAL d_deriv_Fcell_imag = 0;
+                                    if (num_atoms > 0) {
+                                        CUDAREAL S_2 = 1.e-20 * (_scattering[0] * _scattering[0] +
+                                                                 _scattering[1] * _scattering[1] +
+                                                                 _scattering[2] * _scattering[2]);
+
+                                        // fp is always followed by the fdp value
+                                        CUDAREAL val_fp = fpfdp(2 * _source);
+                                        CUDAREAL val_fdp = fpfdp(2 * _source + 1);
+
+                                        CUDAREAL c_deriv_prime = 0;
+                                        CUDAREAL c_deriv_dblprime = 0;
+                                        CUDAREAL d_deriv_prime = 0;
+                                        CUDAREAL d_deriv_dblprime = 0;
+                                        if (refine_fp_fdp) {
+                                            //   currently only supports two parameter model
+                                            int d_idx = 2 * _source;
+                                            c_deriv_prime = fpfdp_derivs(d_idx);
+                                            c_deriv_dblprime = fpfdp_derivs(d_idx + 1);
+                                            d_deriv_prime = fpfdp_derivs(d_idx + 2 * sources);
+                                            d_deriv_dblprime =
+                                                fpfdp_derivs(d_idx + 1 + 2 * sources);
+                                        }
+
+                                        for (int i_atom = 0; i_atom < num_atoms; i_atom++) {
+                                            // fractional atomic coordinates
+                                            CUDAREAL atom_x = atom_data(i_atom * 5);
+                                            CUDAREAL atom_y = atom_data(i_atom * 5 + 1);
+                                            CUDAREAL atom_z = atom_data(i_atom * 5 + 2);
+                                            CUDAREAL B = atom_data(i_atom * 5 + 3);  // B factor
+                                            B = ::Kokkos::Experimental::exp(
+                                                -B * S_2 / 4.0);  // TODO: speed me up?
+                                            CUDAREAL occ = atom_data(i_atom * 5 + 4);  // occupancy
+                                            CUDAREAL r_dot_h =
+                                                _h0 * atom_x + _k0 * atom_y + _l0 * atom_z;
+                                            CUDAREAL phase = 2 * M_PI * r_dot_h;
+                                            CUDAREAL s_rdoth = ::Kokkos::Experimental::sin(phase);
+                                            CUDAREAL c_rdoth = ::Kokkos::Experimental::cos(phase);
+                                            CUDAREAL Bocc = B * occ;
+                                            CUDAREAL BC = B * c_rdoth;
+                                            CUDAREAL BS = B * s_rdoth;
+                                            CUDAREAL real_part = BC * val_fp - BS * val_fdp;
+                                            CUDAREAL imag_part = BS * val_fp + BC * val_fdp;
+                                            _F_cell += real_part;
+                                            _F_cell2 += imag_part;
+                                            if (refine_fp_fdp) {
+                                                c_deriv_Fcell_real +=
+                                                    BC * c_deriv_prime - BS * c_deriv_dblprime;
+                                                c_deriv_Fcell_imag +=
+                                                    BS * c_deriv_prime + BC * c_deriv_dblprime;
+
+                                                d_deriv_Fcell_real +=
+                                                    BC * d_deriv_prime - BS * d_deriv_dblprime;
+                                                d_deriv_Fcell_imag +=
+                                                    BS * d_deriv_prime + BC * d_deriv_dblprime;
+                                            }
+                                        }
+                                    }
+                                    CUDAREAL Freal = _F_cell;
+                                    CUDAREAL Fimag = _F_cell2;
+                                    _F_cell =
+                                        ::Kokkos::Experimental::sqrt(Freal * Freal + Fimag * Fimag);
+                                    if (refine_fp_fdp) {
+                                        c_deriv_Fcell =
+                                            Freal * c_deriv_Fcell_real + Fimag * c_deriv_Fcell_imag;
+                                        d_deriv_Fcell =
+                                            Freal * d_deriv_Fcell_real + Fimag * d_deriv_Fcell_imag;
+                                    }
+                                }
+                                if (!oversample_omega && ! Fhkl_gradient_mode)
+                                    _omega_pixel = 1;
+
+                                CUDAREAL _I_cell = _F_cell;
+                                if (!refine_Icell)
+                                    _I_cell *= _F_cell;
+                                CUDAREAL hkl=1;
+                                int Fhkl_channel=0;
+                                if (! Fhkl_channels_empty)
+                                    Fhkl_channel = Fhkl_channels(_source);
+                                if (Fhkl_have_scale_factors)
+                                    hkl = Fhkl_scale(i_hklasu + Fhkl_channel*Num_ASU);
+                                if (Fhkl_gradient_mode){
+                                    CUDAREAL Fhkl_deriv_scale = overall_scale*polar_for_Fhkl_grad;
+                                    CUDAREAL I_noFcell=texture_scale*I0;
+                                    CUDAREAL dfhkl = I_noFcell*_I_cell * Fhkl_deriv_scale;
+                                    CUDAREAL grad_incr = dfhkl*Fhkl_deriv_coef;
+                                    int fhkl_grad_idx=i_hklasu + Fhkl_channel*Num_ASU;
+
+                                    if (Fhkl_errors_mode){
+                                        // here we hi-kack the Fhkl_scale_deriv array, if computing errors, in order to store the hessian terms
+                                        // if we are getting the hessian terms, we no longer need the  gradients (e.g. by this point we are done refininig)
+                                        CUDAREAL hessian_incr = Fhkl_hessian_coef*dfhkl*dfhkl;
+                                        ::Kokkos::atomic_add(&Fhkl_scale_deriv(fhkl_grad_idx), hessian_incr);
+                                    }
+                                    else{
+                                        ::Kokkos::atomic_add(&Fhkl_scale_deriv(fhkl_grad_idx), grad_incr);
+                                    }
+                                    continue;
+                                }
+
+                                CUDAREAL _I_total = hkl*_I_cell *I0;
+                                CUDAREAL Iincrement = _I_total * texture_scale;
+                                _I += Iincrement;
+                                if (save_wavelenimage)
+                                    Ilambda += Iincrement * lambda_ang;
+
+                                if (refine_diffuse) {
+                                    CUDAREAL step_scale = texture_scale * _F_cell * _F_cell;
+                                    for (int i_diff = 0; i_diff < 6; i_diff++) {
+                                        dI_diffuse[i_diff] +=
+                                            step_scale * step_diffuse_param[i_diff];
+                                    }
+                                }
+
+                                if (refine_fp_fdp) {
+                                    CUDAREAL I_noFcell = texture_scale * I0;
+                                    fp_fdp_manager_dI[0] += 2 * I_noFcell * (c_deriv_Fcell);
+                                    fp_fdp_manager_dI[1] += 2 * I_noFcell * (d_deriv_Fcell);
+                                }
+
+                                if (verbose > 3)
+                                    printf(
+                                        "hkl= %f %f %f  hkl1= %d %d %d  Fcell=%f\n", _h, _k, _l,
+                                        _h0, _k0, _l0, _F_cell);
+
+                                KOKKOS_MAT3 UBOt;
+                                if (refine_Umat(0) || refine_Umat(1) || refine_Umat(2) ||
+                                    refine_eta) {
+                                    UBOt = Amat_init;
+                                }
+                                if (refine_Umat(0)) {
+                                    KOKKOS_MAT3 RyRzUBOt = RotMats(1) * RotMats(2) * UBOt;
+                                    KOKKOS_VEC3 delta_H_prime =
+                                        (UMATS(_mos_tic) * dRotMats(0) * RyRzUBOt)
+                                            .transpose()
+                                            .dot(q_vec);
+                                    CUDAREAL V_dot_dV = V.dot(_NABC.dot(delta_H_prime));
+                                    CUDAREAL value = -two_C * V_dot_dV * Iincrement;
+                                    CUDAREAL value2 = 0;
+                                    if (compute_curvatures) {
+                                        KOKKOS_VEC3 delta_H_dbl_prime =
+                                            (UMATS(_mos_tic).dot(d2RotMats(0).dot(RyRzUBOt)))
+                                                .transpose()
+                                                .dot(q_vec);
+                                        CUDAREAL dV_dot_dV = (_NABC.dot(delta_H_prime))
+                                                                 .dot(_NABC.dot(delta_H_prime));
+                                        CUDAREAL dV2_dot_V =
+                                            (_NABC.dot(delta_H)).dot(_NABC.dot(delta_H_dbl_prime));
+                                        value2 =
+                                            two_C *
+                                            (two_C * V_dot_dV * V_dot_dV - dV2_dot_V - dV_dot_dV) *
+                                            Iincrement;
+                                    }
+                                    rot_manager_dI[0] += value;
+                                    rot_manager_dI2[0] += value2;
+                                }
+                                if (refine_Umat(1)) {
+                                    KOKKOS_MAT3 UmosRx = UMATS(_mos_tic).dot(RotMats(0));
+                                    KOKKOS_MAT3 RzUBOt = RotMats(2).dot(UBOt);
+                                    KOKKOS_VEC3 delta_H_prime = (UmosRx.dot(dRotMats(1).dot(RzUBOt)))
+                                                             .transpose()
+                                                             .dot(q_vec);
+                                    CUDAREAL V_dot_dV = V.dot(_NABC.dot(delta_H_prime));
+                                    CUDAREAL value = -two_C * V_dot_dV * Iincrement;
+
+                                    CUDAREAL value2 = 0;
+                                    if (compute_curvatures) {
+                                        KOKKOS_VEC3 delta_H_dbl_prime =
+                                            (UmosRx.dot(d2RotMats(1).dot(RzUBOt)))
+                                                .transpose()
+                                                .dot(q_vec);
+                                        CUDAREAL dV_dot_dV = (_NABC.dot(delta_H_prime))
+                                                                 .dot(_NABC.dot(delta_H_prime));
+                                        CUDAREAL dV2_dot_V =
+                                            (_NABC.dot(delta_H)).dot(_NABC.dot(delta_H_dbl_prime));
+                                        value2 =
+                                            two_C *
+                                            (two_C * V_dot_dV * V_dot_dV - dV2_dot_V - dV_dot_dV) *
+                                            Iincrement;
+                                    }
+                                    rot_manager_dI[1] += value;
+                                    rot_manager_dI2[1] += value2;
+                                }
+                                if (refine_Umat(2)) {
+                                    KOKKOS_MAT3 UmosRxRy = UMATS(_mos_tic).dot(RotMats(0).dot(RotMats(1)));
+                                    KOKKOS_VEC3 delta_H_prime = (UmosRxRy.dot(dRotMats(2).dot(UBOt)))
+                                                             .transpose()
+                                                             .dot(q_vec);
+                                    CUDAREAL V_dot_dV = V.dot(_NABC.dot(delta_H_prime));
+                                    CUDAREAL value = -two_C * V_dot_dV * Iincrement;
+
+                                    CUDAREAL value2 = 0;
+                                    if (compute_curvatures) {
+                                        KOKKOS_VEC3 delta_H_dbl_prime =
+                                            (UmosRxRy.dot(d2RotMats(2).dot(UBOt)))
+                                                .transpose()
+                                                .dot(q_vec);
+                                        CUDAREAL dV_dot_dV = (_NABC.dot(delta_H_prime))
+                                                                 .dot(_NABC.dot(delta_H_prime));
+                                        CUDAREAL dV2_dot_V =
+                                            (_NABC.dot(delta_H)).dot(_NABC.dot(delta_H_dbl_prime));
+                                        value2 =
+                                            two_C *
+                                            (two_C * V_dot_dV * V_dot_dV - dV2_dot_V - dV_dot_dV) *
+                                            Iincrement;
+                                    }
+                                    rot_manager_dI[2] += value;
+                                    rot_manager_dI2[2] += value2;
+                                }
+                                // Checkpoint for unit cell derivatives
+                                // KOKKOS_MAT3 Ot = eig_O.transpose();
+                                KOKKOS_MAT3 UmosRxRyRzU;
+                                KOKKOS_VEC3 delta_H_prime;
+                                for (int i_uc = 0; i_uc < 6; i_uc++) {
+                                    if (refine_Bmat(i_uc)) {
+                                        UmosRxRyRzU = UMATS_RXYZ(_mos_tic).dot(eig_U);
+                                        delta_H_prime =
+                                            (UmosRxRyRzU.dot(dB_mats(i_uc).dot(eig_Otranspose)))
+                                                .transpose()
+                                                .dot(q_vec);
+                                        CUDAREAL V_dot_dV = V.dot(_NABC.dot(delta_H_prime));
+                                        CUDAREAL value = -two_C * V_dot_dV * Iincrement;
+                                        CUDAREAL value2 = 0;
+                                        if (compute_curvatures) {
+                                            KOKKOS_VEC3 delta_H_dbl_prime =
+                                                (UmosRxRyRzU.dot(
+                                                     dB2_mats(i_uc).dot(eig_Otranspose)))
+                                                    .transpose()
+                                                    .dot(q_vec);
+                                            CUDAREAL dV_dot_dV = (_NABC.dot(delta_H_prime))
+                                                                     .dot(_NABC.dot(delta_H_prime));
+                                            CUDAREAL dV2_dot_V =
+                                                (_NABC.dot(delta_H))
+                                                    .dot(_NABC.dot(delta_H_dbl_prime));
+                                            value2 = two_C *
+                                                     (two_C * V_dot_dV * V_dot_dV - dV2_dot_V -
+                                                      dV_dot_dV) *
+                                                     Iincrement;
+                                        }
+                                        ucell_manager_dI[i_uc] += value;
+                                        ucell_manager_dI2[i_uc] += value2;
+                                    }
+                                }  // end ucell deriv
+
+                                // Checkpoint for Ncells manager
+                                if (refine_Ncells(0)) {
+                                    int num_ncell_deriv = 1;
+                                    if (!isotropic_ncells)
+                                        num_ncell_deriv = 3;
+                                    for (int i_nc = 0; i_nc < num_ncell_deriv; i_nc++) {
+                                        KOKKOS_MAT3 dN;
+                                        dN(i_nc, i_nc) = 1;
+                                        if (num_ncell_deriv == 1) {
+                                            dN(0, 0) = 1;
+                                            dN(1, 1) = 1;
+                                            dN(2, 2) = 1;
+                                        }
+                                        CUDAREAL N_i = _NABC(i_nc, i_nc);
+                                        KOKKOS_VEC3 dV_dN = dN.dot(delta_H);
+                                        // TODO speedops: precompute these, store shared var
+                                        // _NABC.inverse
+                                        CUDAREAL determ_deriv = (_NABC.inverse().dot(dN)).trace();
+                                        CUDAREAL deriv_coef = determ_deriv - C * (dV_dN.dot(V));
+                                        CUDAREAL value = 2 * Iincrement * deriv_coef;
+                                        CUDAREAL value2 = 0;
+                                        if (compute_curvatures) {
+                                            dN(i_nc, i_nc) = 0;  // TODO check maths
+                                            value2 = (-1 / N_i / N_i - C * (dV_dN.dot(dV_dN))) * 2 *
+                                                     Iincrement;
+                                            value2 += deriv_coef * 2 * value;
+                                        }
+                                        Ncells_manager_dI[i_nc] += value;
+                                        Ncells_manager_dI2[i_nc] += value2;
+                                    }
+                                }  // end Ncells manager deriv
+
+                                if (refine_Ncells_def) {
+                                    for (int i_nc = 3; i_nc < 6; i_nc++) {
+                                        KOKKOS_MAT3 dN;
+                                        if (i_nc == 3)
+                                            dN = KOKKOS_MAT3{0, 1, 0, 1, 0, 0, 0, 0, 0};
+                                        else if (i_nc == 4)
+                                            dN = KOKKOS_MAT3{0, 0, 0, 0, 0, 1, 0, 1, 0};
+                                        else
+                                            dN = KOKKOS_MAT3{0, 0, 1, 0, 0, 0, 1, 0, 0};
+                                        KOKKOS_VEC3 dV_dN = dN.dot(delta_H);
+                                        // TODO speedops: precompute these
+                                        CUDAREAL determ_deriv = (_NABC.inverse().dot(dN)).trace();
+                                        CUDAREAL deriv_coef = determ_deriv - C * (dV_dN.dot(V));
+                                        CUDAREAL value = 2 * Iincrement * deriv_coef;
+                                        Ncells_manager_dI[i_nc] += value;
+                                        CUDAREAL value2 = 0;
+                                        if (compute_curvatures) {
+                                            value2 = deriv_coef * value;
+                                            value2 += -2 * C * Iincrement * (dV_dN.dot(dV_dN));
+                                            Ncells_manager_dI2[i_nc] += value2;
+                                        }
+                                    }
+                                }
+
+                                // Checkpoint for Origin manager
+                                for (int i_pan_orig = 0; i_pan_orig < 3; i_pan_orig++) {
+                                    if (refine_panel_origin(i_pan_orig)) {
+                                        CUDAREAL per_k = 1 / _airpath;
+                                        CUDAREAL per_k3 = pow(per_k, 3.);
+                                        CUDAREAL per_k5 = pow(per_k, 5.);
+                                        CUDAREAL lambda_ang = _lambda * 1e10;
+
+                                        KOKKOS_MAT3 M = -two_C * (_NABC.dot(UBO)) / lambda_ang;
+                                        KOKKOS_VEC3 dk;
+                                        if (i_pan_orig == 0)
+                                            dk = KOKKOS_VEC3{0, 0, 1};
+                                        else if (i_pan_orig == 1)
+                                            dk = KOKKOS_VEC3{1, 0, 0};
+                                        else
+                                            dk = KOKKOS_VEC3{0, 1, 0};
+
+                                        CUDAREAL G = dk.dot(_pixel_pos);
+                                        CUDAREAL pix2 = subpixel_size * subpixel_size;
+                                        KOKKOS_VEC3 dk_hat = -per_k3 * G * _pixel_pos + per_k * dk;
+                                        CUDAREAL coef = (M.dot(dk_hat)).dot(V);
+                                        CUDAREAL coef2 =
+                                            -3 * pix2 * per_k5 * G * (_o_vec.dot(_pixel_pos));
+                                        coef2 += pix2 * per_k3 * (_o_vec.dot(dk));
+                                        CUDAREAL value =
+                                            coef * Iincrement + coef2 * Iincrement / _omega_pixel;
+
+                                        pan_orig_manager_dI[i_pan_orig] += value;
+                                        pan_orig_manager_dI2[i_pan_orig] += 0;
+
+                                    }  // end origin manager deriv
+                                }
+
+                                for (int i_pan_rot = 0; i_pan_rot < 3; i_pan_rot++) {
+                                    if (refine_panel_rot(i_pan_rot)) {
+                                        CUDAREAL per_k = 1 / _airpath;
+                                        CUDAREAL per_k3 = pow(per_k, 3.);
+                                        CUDAREAL per_k5 = pow(per_k, 5.);
+                                        CUDAREAL lambda_ang = _lambda * 1e10;
+                                        KOKKOS_MAT3 M = -two_C * (_NABC.dot(UBO)) / lambda_ang;
+                                        KOKKOS_VEC3 dk = _Fdet * (dF_vecs(_pid * 3 + i_pan_rot)) +
+                                                  _Sdet * (dS_vecs(_pid * 3 + i_pan_rot));
+                                        CUDAREAL G = dk.dot(_pixel_pos);
+                                        CUDAREAL pix2 = subpixel_size * subpixel_size;
+                                        KOKKOS_VEC3 dk_hat = -per_k3 * G * _pixel_pos + per_k * dk;
+                                        CUDAREAL coef = (M.dot(dk_hat)).dot(V);
+                                        CUDAREAL coef2 =
+                                            -3 * pix2 * per_k5 * G * (_o_vec.dot(_pixel_pos));
+                                        coef2 += pix2 * per_k3 * (_o_vec.dot(dk));
+                                        CUDAREAL value =
+                                            coef * Iincrement + coef2 * Iincrement / _omega_pixel;
+
+                                        pan_rot_manager_dI[i_pan_rot] += value;
+                                        pan_rot_manager_dI2[i_pan_rot] += 0;
+                                    }
+                                }
+
+                                // checkpoint for Fcell manager
+                                if (refine_fcell) {
+                                    CUDAREAL value;
+                                    if (refine_Icell)
+                                        value = I0 * texture_scale;
+                                    else
+                                        value = 2 * I0 * _F_cell *
+                                                texture_scale;  // Iincrement/_F_cell ;
+                                    CUDAREAL value2 = 0;
+                                    if (compute_curvatures) {
+                                        //    NOTE if _Fcell >0
+                                        value2 = 2 * I0 * texture_scale;
+                                    }
+                                    // if (fcell_idx >=0 && fcell_idx <=2){
+                                    if (use_nominal_hkl) {
+                                        if (_h0 == nom_h && _k0 == nom_k && _l0 == nom_l) {
+                                            fcell_manager_dI += value;
+                                            fcell_manager_dI2 += value2;
+                                        }
+                                    } else {
+                                        fcell_manager_dI += value;
+                                        fcell_manager_dI2 += value2;
+                                    }
+                                }  // end of fcell man deriv
+
+                                // checkpoint for eta manager
+                                if (refine_eta) {
+                                    for (int i_eta = 0; i_eta < 3; i_eta++) {
+                                        if (i_eta > 0 && !aniso_eta)
+                                            continue;
+                                        int mtic2 = _mos_tic + i_eta * mosaic_domains;
+                                        KOKKOS_VEC3 DeltaH_deriv = (UMATS_RXYZ_prime(mtic2).dot(UBOt))
+                                                                .transpose()
+                                                                .dot(q_vec);
+                                        // vector V is _Nabc*Delta_H
+                                        KOKKOS_VEC3 dV = _NABC.dot(DeltaH_deriv);
+                                        CUDAREAL V_dot_dV = V.dot(dV);
+                                        CUDAREAL Iprime = -two_C * (V_dot_dV)*Iincrement;
+                                        eta_manager_dI[i_eta] += Iprime;
+                                        CUDAREAL Idbl_prime = 0;
+                                        if (compute_curvatures) {
+                                            KOKKOS_VEC3 DeltaH_second_deriv =
+                                                (UMATS_RXYZ_dbl_prime(mtic2).dot(UBOt))
+                                                    .transpose()
+                                                    .dot(q_vec);
+                                            KOKKOS_VEC3 dV2 = _NABC.dot(DeltaH_second_deriv);
+                                            Idbl_prime =
+                                                -two_C * (dV.dot(dV) + V.dot(dV2)) * Iincrement;
+                                            Idbl_prime += -two_C * (V_dot_dV)*Iprime;
+                                        }
+                                        eta_manager_dI2[i_eta] += Idbl_prime;
+                                    }
+                                }  // end of eta man deriv
+
+                                // checkpoint for lambda manager
+                                for (int i_lam = 0; i_lam < 2; i_lam++) {
+                                    if (refine_lambda(i_lam)) {
+                                        CUDAREAL lambda_ang = _lambda * 1e10;
+                                        CUDAREAL NH_dot_V = (_NABC.dot(H_vec)).dot(V);
+                                        CUDAREAL dg_dlambda;
+                                        if (i_lam == 0)
+                                            dg_dlambda = 1;
+                                        else  // i_lam==1
+                                            dg_dlambda = lambda_ang;
+                                        CUDAREAL coef =
+                                            NH_dot_V * two_C * (dg_dlambda) / lambda_ang;
+                                        CUDAREAL value = coef * Iincrement;
+                                        CUDAREAL value2 = 0;
+                                        lambda_manager_dI[i_lam] += value;
+                                        lambda_manager_dI2[i_lam] += value2;
+                                    }
+                                }
+                                // end of lambda deriv
+                                if (printout) {
+                                    if (_subS == 0 && _subF == 0 && _thick_tic == 0 &&
+                                        _source == 0 && _mos_tic == 0) {
+                                        if ((_fpixel == printout_fpixel &&
+                                             _spixel == printout_spixel) ||
+                                            printout_fpixel < 0) {
+                                            printf(
+                                                "%4d %4d :  lambda = %g\n", _fpixel, _spixel,
+                                                _lambda);
+                                            printf(
+                                                "at %g %g %g\n", _pixel_pos[0], _pixel_pos[1],
+                                                _pixel_pos[2]);
+                                            printf(
+                                                "Fdet= %g; Sdet= %g ; Odet= %g\n", _Fdet, _Sdet,
+                                                _Odet);
+                                            printf(
+                                                "PIX0: %f %f %f\n", pix0_vectors(pid_x),
+                                                pix0_vectors(pid_y), pix0_vectors(pid_z));
+                                            printf(
+                                                "F: %f %f %f\n", fdet_vectors(pid_x),
+                                                fdet_vectors(pid_y), fdet_vectors(pid_z));
+                                            printf(
+                                                "S: %f %f %f\n", sdet_vectors(pid_x),
+                                                sdet_vectors(pid_y), sdet_vectors(pid_z));
+                                            printf(
+                                                "O: %f %f %f\n", odet_vectors(pid_x),
+                                                odet_vectors(pid_y), odet_vectors(pid_z));
+                                            printf(
+                                                "pid_x=%d, pid_y=%d; pid_z=%d\n", pid_x, pid_y,
+                                                pid_z);
+
+                                            printf(
+                                                "QVECTOR: %f %f %f\n", q_vec[0], q_vec[1],
+                                                q_vec[2]);
+                                            KOKKOS_MAT3 UU = UMATS_RXYZ(_mos_tic);
+                                            printf(
+                                                "UMAT_RXYZ :\n%f  %f  %f\n%f  %f  %f\n%f  %f  %f\n",
+                                                UU(0, 0), UU(0, 1), UU(0, 2), UU(1, 0), UU(1, 1),
+                                                UU(1, 2), UU(2, 0), UU(2, 1), UU(2, 2));
+                                            UU = Bmat_realspace;
+                                            printf(
+                                                "Bmat_realspace :\n%f  %f  %f\n%f  %f  %f\n%f  %f  %f\n",
+                                                UU(0, 0), UU(0, 1), UU(0, 2), UU(1, 0), UU(1, 1),
+                                                UU(1, 2), UU(2, 0), UU(2, 1), UU(2, 2));
+                                            UU = UBO;
+                                            printf(
+                                                "UBO :\n%f  %f  %f\n%f  %f  %f\n%f  %f  %f\n",
+                                                UU(0, 0), UU(0, 1), UU(0, 2), UU(1, 0), UU(1, 1),
+                                                UU(1, 2), UU(2, 0), UU(2, 1), UU(2, 2));
+
+                                            UU = UBOt;
+                                            printf(
+                                                "UBOt :\n%f  %f  %f\n%f  %f  %f\n%f  %f  %f\n",
+                                                UU(0, 0), UU(0, 1), UU(0, 2), UU(1, 0), UU(1, 1),
+                                                UU(1, 2), UU(2, 0), UU(2, 1), UU(2, 2));
+
+                                            UU = UmosRxRyRzU;
+                                            printf(
+                                                "UmosRxRyRzU :\n%f  %f  %f\n%f  %f  %f\n%f  %f  %f\n",
+                                                UU(0, 0), UU(0, 1), UU(0, 2), UU(1, 0), UU(1, 1),
+                                                UU(1, 2), UU(2, 0), UU(2, 1), UU(2, 2));
+                                            KOKKOS_VEC3 AA = delta_H_prime;
+                                            printf(
+                                                "delta_H_prime :\n%f  %f  %f\n", AA[0], AA[1],
+                                                AA[2]);
+                                            printf("Iincrement: %f\n", Iincrement);
+                                            printf(
+                                                "hkl= %f %f %f  hkl0= %d %d %d\n", _h, _k, _l, _h0,
+                                                _k0, _l0);
+                                            printf(
+                                                " F_cell=%g  F_cell2=%g I_latt=%g   I = %g\n",
+                                                _F_cell, _F_cell2, I0, _I);
+                                            printf("I/steps %15.10g\n", _I / Nsteps);
+                                            // printf("Ilatt diffuse %15.10g\n", I_latt_diffuse);
+                                            printf("omega   %15.10g\n", _omega_pixel);
+                                            printf("default_F= %f\n", default_F);
+                                            printf(
+                                                "Incident[0]=%g, Incident[1]=%g, Incident[2]=%g\n",
+                                                _incident[0], _incident[1], _incident[2]);
+                                            if (complex_miller)
+                                                printf("COMPLEX MILLER!\n");
+                                            if (no_Nabc_scale)
+                                                printf("No Nabc scale!\n");
+                                        }
+                                    }
+                                }
+
+                            }  // end of mos_tic loop
+                        }      // end of source loop
+                    }          // end of thick step loop
+                }              // end of fpos loop
+            }                  // end of spos loop
+            if (Fhkl_gradient_mode)
+                return;
+
+            CUDAREAL _Fdet_ave = pixel_size * _fpixel + pixel_size / 2.0;
+            CUDAREAL _Sdet_ave = pixel_size * _spixel + pixel_size / 2.0;
+            CUDAREAL _Odet_ave = 0;  // Odet;
+            // TODO maybe make this more general for thick detectors?
+
+            KOKKOS_VEC3 _pixel_pos_ave(0, 0, 0);
+            int pid_x = _pid * 3;
+            int pid_y = _pid * 3 + 1;
+            int pid_z = _pid * 3 + 2;
+
+            CUDAREAL fx = fdet_vectors(pid_x);
+            CUDAREAL fy = fdet_vectors(pid_y);
+            CUDAREAL fz = fdet_vectors(pid_z);
+
+            CUDAREAL sx = sdet_vectors(pid_x);
+            CUDAREAL sy = sdet_vectors(pid_y);
+            CUDAREAL sz = sdet_vectors(pid_z);
+
+            CUDAREAL ox = odet_vectors(pid_x);
+            CUDAREAL oy = odet_vectors(pid_y);
+            CUDAREAL oz = odet_vectors(pid_z);
+
+            CUDAREAL px = pix0_vectors(pid_x);
+            CUDAREAL py = pix0_vectors(pid_y);
+            CUDAREAL pz = pix0_vectors(pid_z);
+
+            _pixel_pos_ave[0] = _Fdet_ave * fx + _Sdet_ave * sx + _Odet_ave * ox + px;
+            _pixel_pos_ave[1] = _Fdet_ave * fy + _Sdet_ave * sy + _Odet_ave * oy + py;
+            _pixel_pos_ave[2] = _Fdet_ave * fz + _Sdet_ave * sz + _Odet_ave * oz + pz;
+
+            CUDAREAL _airpath_ave = _pixel_pos_ave.length();
+            KOKKOS_VEC3 _diffracted_ave = _pixel_pos_ave.get_unit_vector();
+            CUDAREAL _omega_pixel_ave = pixel_size * pixel_size / _airpath_ave / _airpath_ave *
+                                        close_distance / _airpath_ave;
+
+            CUDAREAL _polar = 1;
+            if (!nopolar) {
+                KOKKOS_VEC3 _incident(-source_X(0), -source_Y(0), -source_Z(0));
+                _incident.normalize();
+                // component of diffracted unit vector along _incident beam unit vector
+                CUDAREAL cos2theta = _incident.dot(_diffracted_ave);
+                CUDAREAL cos2theta_sqr = cos2theta * cos2theta;
+                CUDAREAL sin2theta_sqr = 1 - cos2theta_sqr;
+
+                CUDAREAL _psi = 0;
+                if (kahn_factor != 0.0) {
+                    // cross product to get "vertical" axis that is orthogonal to the cannonical
+                    // "polarization"
+                    KOKKOS_VEC3 B_in = polarization_axis.cross(_incident);
+                    // cross product with _incident beam to get E-vector direction
+                    KOKKOS_VEC3 E_in = _incident.cross(B_in);
+                    // get components of diffracted ray projected onto the E-B plane
+                    CUDAREAL _kEi = _diffracted_ave.dot(E_in);
+                    CUDAREAL _kBi = _diffracted_ave.dot(B_in);
+                    // compute the angle of the diffracted ray projected onto the incident E-B plane
+                    _psi = -atan2(_kBi, _kEi);
+                }
+                // correction for polarized _incident beam
+                _polar =
+                    0.5 * (1.0 + cos2theta_sqr -
+                           kahn_factor * ::Kokkos::Experimental::cos(2 * _psi) * sin2theta_sqr);
+            }
+
+            CUDAREAL _om = 1;
+            if (!oversample_omega)
+                _om = _omega_pixel_ave;
+            // final scale term to being everything to photon number units
+            CUDAREAL _scale_term = _polar * _om * overall_scale;
+            floatimage(pixIdx) = _scale_term * _I;
+            if (save_wavelenimage)
+                wavelenimage(pixIdx) = Ilambda / _I;
+
+            // udpate the rotation derivative images*
+            for (int i_rot = 0; i_rot < 3; i_rot++) {
+                if (refine_Umat(i_rot)) {
+                    CUDAREAL value = _scale_term * rot_manager_dI[i_rot];
+                    CUDAREAL value2 = _scale_term * rot_manager_dI2[i_rot];
+                    int idx = i_rot * Npix_to_model + pixIdx;
+                    d_Umat_images(idx) = value;
+                    d2_Umat_images(idx) = value2;
+                }
+            }  // end rot deriv image increment
+
+            // update the ucell derivative images
+            for (int i_uc = 0; i_uc < 6; i_uc++) {
+                if (refine_Bmat(i_uc)) {
+                    CUDAREAL value = _scale_term * ucell_manager_dI[i_uc];
+                    CUDAREAL value2 = _scale_term * ucell_manager_dI2[i_uc];
+                    int idx = i_uc * Npix_to_model + pixIdx;
+                    d_Bmat_images(idx) = value;
+                    d2_Bmat_images(idx) = value2;
+                }
+            }  // end ucell deriv image increment
+
+            // update the Ncells derivative image
+            if (refine_Ncells(0)) {
+                CUDAREAL value = _scale_term * Ncells_manager_dI[0];
+                CUDAREAL value2 = _scale_term * Ncells_manager_dI2[0];
+                int idx = pixIdx;
+                d_Ncells_images(idx) = value;
+                d2_Ncells_images(idx) = value2;
+
+                if (!isotropic_ncells) {
+                    value = _scale_term * Ncells_manager_dI[1];
+                    value2 = _scale_term * Ncells_manager_dI2[1];
+                    idx = Npix_to_model + pixIdx;
+                    d_Ncells_images(idx) = value;
+                    d2_Ncells_images(idx) = value2;
+
+                    value = _scale_term * Ncells_manager_dI[2];
+                    value2 = _scale_term * Ncells_manager_dI2[2];
+                    idx = Npix_to_model * 2 + pixIdx;
+                    d_Ncells_images(idx) = value;
+                    d2_Ncells_images(idx) = value2;
+                }
+            }  // end Ncells deriv image increment
+            if (refine_Ncells_def) {
+                for (int i_nc = 3; i_nc < 6; i_nc++) {
+                    CUDAREAL value = _scale_term * Ncells_manager_dI[i_nc];
+                    CUDAREAL value2 = _scale_term * Ncells_manager_dI2[i_nc];
+                    int idx = i_nc * Npix_to_model + pixIdx;
+                    d_Ncells_images(idx) = value;
+                    d2_Ncells_images(idx) = value2;
+                }
+            }
+
+            // update Fcell derivative image
+            if (refine_fcell) {
+                CUDAREAL value = _scale_term * fcell_manager_dI;
+                CUDAREAL value2 = _scale_term * fcell_manager_dI2;
+                d_fcell_images(pixIdx) = value;
+                d2_fcell_images(pixIdx) = value2;
+            }  // end Fcell deriv image increment
+
+            if (refine_fp_fdp) {
+                // c derivative
+                CUDAREAL value = _scale_term * fp_fdp_manager_dI[0];
+                d_fp_fdp_images(pixIdx) = value;
+                // d derivative
+                value = _scale_term * fp_fdp_manager_dI[1];
+                d_fp_fdp_images(Npix_to_model + pixIdx) = value;
+            }
+            if (refine_diffuse) {
+                for (int i_gam = 0; i_gam < 3; i_gam++) {
+                    CUDAREAL val = dI_diffuse[i_gam] * _scale_term;
+                    int img_idx = Npix_to_model * i_gam + pixIdx;
+                    d_diffuse_gamma_images(img_idx) = val;
+                }
+                for (int i_sig = 0; i_sig < 3; i_sig++) {
+                    CUDAREAL val = dI_diffuse[i_sig + 3] * _scale_term;
+                    int img_idx = Npix_to_model * i_sig + pixIdx;
+                    d_diffuse_sigma_images(img_idx) = val;
+                }
+            }
+
+            // update eta derivative image
+            if (refine_eta) {
+                for (int i_eta = 0; i_eta < 3; i_eta++) {
+                    if (i_eta > 0 && !aniso_eta)
+                        continue;
+                    int idx = pixIdx + Npix_to_model * i_eta;
+                    CUDAREAL value = _scale_term * eta_manager_dI[i_eta];
+                    CUDAREAL value2 = _scale_term * eta_manager_dI2[i_eta];
+                    d_eta_images(idx) = value;
+                    d2_eta_images(idx) = value2;
+                }
+            }  // end eta deriv image increment
+
+            // update the lambda derivative images
+            for (int i_lam = 0; i_lam < 2; i_lam++) {
+                if (refine_lambda(i_lam)) {
+                    CUDAREAL value = _scale_term * lambda_manager_dI[i_lam];
+                    CUDAREAL value2 = _scale_term * lambda_manager_dI2[i_lam];
+                    int idx = i_lam * Npix_to_model + pixIdx;
+                    d_lambda_images(idx) = value;
+                    // d2_lambda_images(idx) = value2;
+                }
+            }  // end lambda deriv image increment
+
+            for (int i_pan_rot = 0; i_pan_rot < 3; i_pan_rot++) {
+                if (refine_panel_rot(i_pan_rot)) {
+                    CUDAREAL value = _scale_term * pan_rot_manager_dI[i_pan_rot];
+                    CUDAREAL value2 = _scale_term * pan_rot_manager_dI2[i_pan_rot];
+                    int idx = i_pan_rot * Npix_to_model + pixIdx;
+                    d_panel_rot_images(idx) = value;
+                    // d2_panel_rot_images(idx) = value2;
+                }
+            }  // end panel rot deriv image increment
+
+            for (int i_pan_orig = 0; i_pan_orig < 3; i_pan_orig++) {
+                if (refine_panel_origin(i_pan_orig)) {
+                    CUDAREAL value = _scale_term * pan_orig_manager_dI[i_pan_orig];
+                    CUDAREAL value2 = _scale_term * pan_orig_manager_dI2[i_pan_orig];
+                    int idx = i_pan_orig * Npix_to_model + pixIdx;
+                    d_panel_orig_images(idx) = value;
+                    // d2_panel_orig_images(idx) = value2;
+                }
+            }  // end panel orig deriv image increment
+        });    // end pixIdx loop
+
+}  // END of GPU kernel
diff --git a/simtbx/diffBragg/src/diffBragg_kokkos_kernel.h b/simtbx/diffBragg/src/diffBragg_kokkos_kernel.h
new file mode 100644
index 0000000000..33a1c0df6e
--- /dev/null
+++ b/simtbx/diffBragg/src/diffBragg_kokkos_kernel.h
@@ -0,0 +1,154 @@
+#ifndef SIMTBX_DIFFBRAGG_KOKKOSKERNEL
+#define SIMTBX_DIFFBRAGG_KOKKOSKERNEL
+
+#include "diffBraggKOKKOS.h"
+#include "util_kokkos.h"
+
+void kokkos_sum_over_steps(
+    int Npix_to_model,
+    vector_uint_t panels_fasts_slows,
+    vector_cudareal_t floatimage,
+    vector_cudareal_t wavelenimage,
+    vector_cudareal_t d_Umat_images,
+    vector_cudareal_t d2_Umat_images,
+    vector_cudareal_t d_Bmat_images,
+    vector_cudareal_t d2_Bmat_images,
+    vector_cudareal_t d_Ncells_images,
+    vector_cudareal_t d2_Ncells_images,
+    vector_cudareal_t d_fcell_images,
+    vector_cudareal_t d2_fcell_images,
+    vector_cudareal_t d_eta_images,
+    vector_cudareal_t d2_eta_images,
+    vector_cudareal_t d_lambda_images,
+    vector_cudareal_t d2_lambda_images,
+    vector_cudareal_t d_panel_rot_images,
+    vector_cudareal_t d2_panel_rot_images,
+    vector_cudareal_t d_panel_orig_images,
+    vector_cudareal_t d2_panel_orig_images,
+    vector_cudareal_t d_fp_fdp_images,
+    const int Nsteps,
+    int printout_fpixel,
+    int printout_spixel,
+    bool printout,
+    CUDAREAL default_F,
+    int oversample,
+    bool oversample_omega,
+    CUDAREAL subpixel_size,
+    CUDAREAL pixel_size,
+    CUDAREAL detector_thickstep,
+    CUDAREAL detector_thick,
+    const vector_cudareal_t close_distances,
+    CUDAREAL detector_attnlen,
+    int detector_thicksteps,
+    int sources,
+    int phisteps,
+    int mosaic_domains,
+    bool use_lambda_coefficients,
+    CUDAREAL lambda0,
+    CUDAREAL lambda1,
+    KOKKOS_MAT3 eig_U,
+    KOKKOS_MAT3 eig_O,
+    KOKKOS_MAT3 eig_B,
+    KOKKOS_MAT3 RXYZ,
+    vector_vec3_t dF_vecs,
+    vector_vec3_t dS_vecs,
+    const vector_mat3_t UMATS_RXYZ,
+    vector_mat3_t UMATS_RXYZ_prime,
+    vector_mat3_t UMATS_RXYZ_dbl_prime,
+    vector_mat3_t RotMats,
+    vector_mat3_t dRotMats,
+    vector_mat3_t d2RotMats,
+    vector_mat3_t UMATS,
+    vector_mat3_t dB_mats,
+    vector_mat3_t dB2_mats,
+    vector_mat3_t Amatrices,
+    const vector_cudareal_t source_X,
+    const vector_cudareal_t source_Y,
+    const vector_cudareal_t source_Z,
+    const vector_cudareal_t source_lambda,
+    const vector_cudareal_t source_I,
+    CUDAREAL kahn_factor,
+    CUDAREAL Na,
+    CUDAREAL Nb,
+    CUDAREAL Nc,
+    CUDAREAL Nd,
+    CUDAREAL Ne,
+    CUDAREAL Nf,
+    CUDAREAL phi0,
+    CUDAREAL phistep,
+    KOKKOS_VEC3 spindle_vec,
+    KOKKOS_VEC3 polarization_axis,
+    int h_range,
+    int k_range,
+    int l_range,
+    int h_max,
+    int h_min,
+    int k_max,
+    int k_min,
+    int l_max,
+    int l_min,
+    CUDAREAL dmin,
+    CUDAREAL fudge,
+    bool complex_miller,
+    int verbose,
+    bool only_save_omega_kahn,
+    bool isotropic_ncells,
+    bool compute_curvatures,
+    const vector_cudareal_t FhklLinear,
+    const vector_cudareal_t Fhkl2Linear,
+    vector_bool_t refine_Bmat,
+    vector_bool_t refine_Ncells,
+    bool refine_Ncells_def,
+    vector_bool_t refine_panel_origin,
+    vector_bool_t refine_panel_rot,
+    bool refine_fcell,
+    vector_bool_t refine_lambda,
+    bool refine_eta,
+    vector_bool_t refine_Umat,
+    const vector_cudareal_t fdet_vectors,
+    const vector_cudareal_t sdet_vectors,
+    const vector_cudareal_t odet_vectors,
+    const vector_cudareal_t pix0_vectors,
+    bool nopolar,
+    bool point_pixel,
+    CUDAREAL fluence,
+    CUDAREAL r_e_sqr,
+    CUDAREAL spot_scale,
+    int Npanels,
+    bool aniso_eta,
+    bool no_Nabc_scale,
+    const vector_cudareal_t fpfdp,
+    const vector_cudareal_t fpfdp_derivs,
+    const vector_cudareal_t atom_data,
+    int num_atoms,
+    bool refine_fp_fdp,
+    const vector_int_t nominal_hkl,
+    bool use_nominal_hkl,
+    KOKKOS_MAT3 anisoU,
+    KOKKOS_MAT3 anisoG,
+    bool use_diffuse,
+    vector_cudareal_t d_diffuse_gamma_images,
+    vector_cudareal_t d_diffuse_sigma_images,
+    bool refine_diffuse,
+    bool gamma_miller_units,
+    bool refine_Icell,
+    bool save_wavelenimage,
+    int laue_group_num, int stencil_size,
+    bool Fhkl_gradient_mode,
+    bool Fhkl_errors_mode,
+    bool using_trusted_mask,
+    bool Fhkl_channels_empty,
+    bool Fhkl_have_scale_factors,
+    int Num_ASU,
+    const vector_cudareal_t data_residual,
+    const vector_cudareal_t data_variance,
+    const vector_int_t data_freq,
+    const vector_bool_t data_trusted,
+    const vector_int_t FhklLinear_ASUid,
+    const vector_cudareal_t Fhkl_channels,
+    const vector_cudareal_t Fhkl_scale,
+    vector_cudareal_t Fhkl_scale_deriv
+    );
+
+
+#endif
diff --git a/simtbx/diffBragg/src/diffuse_util_kokkos.h b/simtbx/diffBragg/src/diffuse_util_kokkos.h
new file mode 100644
index 0000000000..b5eec4bacb
--- /dev/null
+++ b/simtbx/diffBragg/src/diffuse_util_kokkos.h
@@ -0,0 +1,625 @@
+#ifndef SIMTBX_DIFFBRAGG_DIFFUSE_UTIL_KOKKOS
+#define SIMTBX_DIFFBRAGG_DIFFUSE_UTIL_KOKKOS
+
+#include <simtbx/diffBragg/src/util_kokkos.h>
+
+KOKKOS_FUNCTION
+int gen_laue_mats(int laue_group_num, KOKKOS_MAT3 *lmats) {
+  if ( laue_group_num < 1 or laue_group_num > 14) {
+    return 0;
+  }
+  if ( laue_group_num == 1 ) {
+  // P -1
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+    return 1;
+  }
+  if ( laue_group_num == 2 ) {
+  // P 1 1 2/m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -x,-y,z
+    lmats[1] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+    return 2;
+  }
+  if ( laue_group_num == 3 ) {
+  // P 1 2/m 1
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -x,y,-z
+    lmats[1] << -1, 0, 0,
+                 0, 1, 0,
+                 0, 0,-1;
+
+    return 2;
+  }
+  if ( laue_group_num == 4 ) {
+  // P 2/m 1 1
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // x,-y,-z
+    lmats[1] <<  1, 0, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+    return 2;
+  }
+  if ( laue_group_num == 5 ) {
+  // P m m m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // x,-y,-z
+    lmats[1] <<  1, 0, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+              // -x,y,-z
+    lmats[2] << -1, 0, 0,
+                 0, 1, 0,
+                 0, 0,-1;
+
+              // -x,-y,z
+    lmats[3] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+    return 4;
+  }
+  if ( laue_group_num == 6 ) {
+  // P 4/m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -y,x,z
+    lmats[1] <<  0,-1, 0,
+                 1, 0, 0,
+                 0, 0, 1;
+
+              // y,-x,z
+    lmats[2] <<  0, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // -x,-y,z
+    lmats[3] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+    return 4;
+  }
+  if ( laue_group_num == 7 ) {
+  // P 4/m m m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -y,x,z
+    lmats[1] <<  0,-1, 0,
+                 1, 0, 0,
+                 0, 0, 1;
+
+              // y,-x,z
+    lmats[2] <<  0, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // x,-y,-z
+    lmats[3] <<  1, 0, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+              // -x,y,-z
+    lmats[4] << -1, 0, 0,
+                 0, 1, 0,
+                 0, 0,-1;
+
+              // -x,-y,z
+    lmats[5] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+              // y,x,-z
+    lmats[6] <<  0, 1, 0,
+                 1, 0, 0,
+                 0, 0,-1;
+
+              // -y,-x,-z
+    lmats[7] <<  0,-1, 0,
+                -1, 0, 0,
+                 0, 0,-1;
+
+    return 8;
+  }
+  if ( laue_group_num == 8 ) {
+  // P -3
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -y,x-y,z
+    lmats[1] <<  0,-1, 0,
+                 1,-1, 0,
+                 0, 0, 1;
+
+              // -x+y,-x,z
+    lmats[2] << -1, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+    return 3;
+  }
+  if ( laue_group_num == 9 ) {
+  // P -3 m 1
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -y,x-y,z
+    lmats[1] <<  0,-1, 0,
+                 1,-1, 0,
+                 0, 0, 1;
+
+              // -x+y,-x,z
+    lmats[2] << -1, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // x-y,-y,-z
+    lmats[3] <<  1,-1, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+              // -x,-x+y,-z
+    lmats[4] << -1, 0, 0,
+                -1, 1, 0,
+                 0, 0,-1;
+
+              // y,x,-z
+    lmats[5] <<  0, 1, 0,
+                 1, 0, 0,
+                 0, 0,-1;
+
+    return 6;
+  }
+  if ( laue_group_num == 10 ) {
+  // P -3 1 m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // -y,x-y,z
+    lmats[1] <<  0,-1, 0,
+                 1,-1, 0,
+                 0, 0, 1;
+
+              // -x+y,-x,z
+    lmats[2] << -1, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // -y,-x,-z
+    lmats[3] <<  0,-1, 0,
+                -1, 0, 0,
+                 0, 0,-1;
+
+              // -x+y,y,-z
+    lmats[4] << -1, 1, 0,
+                 0, 1, 0,
+                 0, 0,-1;
+
+              // x,x-y,-z
+    lmats[5] <<  1, 0, 0,
+                 1,-1, 0,
+                 0, 0,-1;
+
+    return 6;
+  }
+  if ( laue_group_num == 11 ) {
+  // P 6/m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // x-y,x,z
+    lmats[1] <<  1,-1, 0,
+                 1, 0, 0,
+                 0, 0, 1;
+
+              // y,-x+y,z
+    lmats[2] <<  0, 1, 0,
+                -1, 1, 0,
+                 0, 0, 1;
+
+              // -y,x-y,z
+    lmats[3] <<  0,-1, 0,
+                 1,-1, 0,
+                 0, 0, 1;
+
+              // -x+y,-x,z
+    lmats[4] << -1, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // -x,-y,z
+    lmats[5] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+    return 6;
+  }
+  if ( laue_group_num == 12 ) {
+  // P 6/m m m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // x-y,x,z
+    lmats[1] <<  1,-1, 0,
+                 1, 0, 0,
+                 0, 0, 1;
+
+              // y,-x+y,z
+    lmats[2] <<  0, 1, 0,
+                -1, 1, 0,
+                 0, 0, 1;
+
+              // -y,x-y,z
+    lmats[3] <<  0,-1, 0,
+                 1,-1, 0,
+                 0, 0, 1;
+
+              // -x+y,-x,z
+    lmats[4] << -1, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // x-y,-y,-z
+    lmats[5] <<  1,-1, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+              // -x,-x+y,-z
+    lmats[6] << -1, 0, 0,
+                -1, 1, 0,
+                 0, 0,-1;
+
+              // -x,-y,z
+    lmats[7] << -1, 0, 0,
+                 0,-1, 0,
+                 0, 0, 1;
+
+              // y,x,-z
+    lmats[8] <<  0, 1, 0,
+                 1, 0, 0,
+                 0, 0,-1;
+
+              // -y,-x,-z
+    lmats[9] <<  0,-1, 0,
+                -1, 0, 0,
+                 0, 0,-1;
+
+              // -x+y,y,-z
+    lmats[10] << -1, 1, 0,
+                  0, 1, 0,
+                  0, 0,-1;
+
+              // x,x-y,-z
+    lmats[11] <<  1, 0, 0,
+                  1,-1, 0,
+                  0, 0,-1;
+
+    return 12;
+  }
+  if ( laue_group_num == 13 ) {
+  // P m -3
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // z,x,y
+    lmats[1] <<  0, 0, 1,
+                 1, 0, 0,
+                 0, 1, 0;
+
+              // y,z,x
+    lmats[2] <<  0, 1, 0,
+                 0, 0, 1,
+                 1, 0, 0;
+
+              // -y,-z,x
+    lmats[3] <<  0,-1, 0,
+                 0, 0,-1,
+                 1, 0, 0;
+
+              // z,-x,-y
+    lmats[4] <<  0, 0, 1,
+                -1, 0, 0,
+                 0,-1, 0;
+
+              // -y,z,-x
+    lmats[5] <<  0,-1, 0,
+                 0, 0, 1,
+                -1, 0, 0;
+
+              // -z,-x,y
+    lmats[6] <<  0, 0,-1,
+                -1, 0, 0,
+                 0, 1, 0;
+
+              // -z,x,-y
+    lmats[7] <<  0, 0,-1,
+                 1, 0, 0,
+                 0,-1, 0;
+
+              // y,-z,-x
+    lmats[8] <<  0, 1, 0,
+                 0, 0,-1,
+                -1, 0, 0;
+
+              // x,-y,-z
+    lmats[9] <<  1, 0, 0,
+                 0,-1, 0,
+                 0, 0,-1;
+
+              // -x,y,-z
+    lmats[10] << -1, 0, 0,
+                  0, 1, 0,
+                  0, 0,-1;
+
+              // -x,-y,z
+    lmats[11] << -1, 0, 0,
+                  0,-1, 0,
+                  0, 0, 1;
+
+    return 12;
+  }
+  if ( laue_group_num == 14 ) {
+  // P m -3 m
+
+              // x,y,z
+    lmats[0] <<  1, 0, 0,
+                 0, 1, 0,
+                 0, 0, 1;
+
+              // x,-z,y
+    lmats[1] <<  1, 0, 0,
+                 0, 0,-1,
+                 0, 1, 0;
+
+              // x,z,-y
+    lmats[2] <<  1, 0, 0,
+                 0, 0, 1,
+                 0,-1, 0;
+
+              // z,y,-x
+    lmats[3] <<  0, 0, 1,
+                 0, 1, 0,
+                -1, 0, 0;
+
+              // -z,y,x
+    lmats[4] <<  0, 0,-1,
+                 0, 1, 0,
+                 1, 0, 0;
+
+              // -y,x,z
+    lmats[5] <<  0,-1, 0,
+                 1, 0, 0,
+                 0, 0, 1;
+
+              // y,-x,z
+    lmats[6] <<  0, 1, 0,
+                -1, 0, 0,
+                 0, 0, 1;
+
+              // z,x,y
+    lmats[7] <<  0, 0, 1,
+                 1, 0, 0,
+                 0, 1, 0;
+
+              // y,z,x
+    lmats[8] <<  0, 1, 0,
+                 0, 0, 1,
+                 1, 0, 0;
+
+              // -y,-z,x
+    lmats[9] <<  0,-1, 0,
+                 0, 0,-1,
+                 1, 0, 0;
+
+              // z,-x,-y
+    lmats[10] <<  0, 0, 1,
+                 -1, 0, 0,
+                  0,-1, 0;
+
+              // -y,z,-x
+    lmats[11] <<  0,-1, 0,
+                  0, 0, 1,
+                 -1, 0, 0;
+
+              // -z,-x,y
+    lmats[12] <<  0, 0,-1,
+                 -1, 0, 0,
+                  0, 1, 0;
+
+              // -z,x,-y
+    lmats[13] <<  0, 0,-1,
+                  1, 0, 0,
+                  0,-1, 0;
+
+              // y,-z,-x
+    lmats[14] <<  0, 1, 0,
+                  0, 0,-1,
+                 -1, 0, 0;
+
+              // x,-y,-z
+    lmats[15] <<  1, 0, 0,
+                  0,-1, 0,
+                  0, 0,-1;
+
+              // -x,y,-z
+    lmats[16] << -1, 0, 0,
+                  0, 1, 0,
+                  0, 0,-1;
+
+              // -x,-y,z
+    lmats[17] << -1, 0, 0,
+                  0,-1, 0,
+                  0, 0, 1;
+
+              // y,x,-z
+    lmats[18] <<  0, 1, 0,
+                  1, 0, 0,
+                  0, 0,-1;
+
+              // -y,-x,-z
+    lmats[19] <<  0,-1, 0,
+                 -1, 0, 0,
+                  0, 0,-1;
+
+              // z,-y,x
+    lmats[20] <<  0, 0, 1,
+                  0,-1, 0,
+                  1, 0, 0;
+
+              // -z,-y,-x
+    lmats[21] <<  0, 0,-1,
+                  0,-1, 0,
+                 -1, 0, 0;
+
+              // -x,z,y
+    lmats[22] << -1, 0, 0,
+                  0, 0, 1,
+                  0, 1, 0;
+
+              // -x,-z,-y
+    lmats[23] << -1, 0, 0,
+                  0, 0,-1,
+                  0,-1, 0;
+
+    return 24;
+  }
+  else {
+    return 0;
+  }
+  return 0;
+};
+
+KOKKOS_FUNCTION
+void calc_diffuse_at_hkl(KOKKOS_VEC3 H_vec, KOKKOS_VEC3 H0, KOKKOS_VEC3 dHH, KOKKOS_VEC3 Hmin, KOKKOS_VEC3 Hmax, KOKKOS_VEC3 Hrange, KOKKOS_MAT3 Ainv, const vector_cudareal_t FhklLinear, int num_laue_mats, const KOKKOS_MAT3 *laue_mats, KOKKOS_MAT3 anisoG_local, KOKKOS_MAT3 anisoU_local, const KOKKOS_MAT3 *dG_dgam, bool refine_diffuse, CUDAREAL *I0, CUDAREAL *step_diffuse_param){
+  CUDAREAL four_mpi_sq = 4.*M_PI*M_PI;
+  // loop over laue matrices
+  bool h_bounded= (H0[0]+dHH[0]<=Hmax[0]) && (H0[0]-dHH[0]>=Hmin[0]) ;
+  bool k_bounded= (H0[1]+dHH[1]<=Hmax[1]) && (H0[1]-dHH[1]>=Hmin[1]) ;
+  bool l_bounded= (H0[2]+dHH[2]<=Hmax[2]) && (H0[2]-dHH[2]>=Hmin[2]) ;
+  if (h_bounded && k_bounded && l_bounded) {
+    int Fhkl_linear_index_0 = (H0[0]-Hmin[0]) * Hrange[1] * Hrange[2]
+      + (H0[1]-Hmin[1]) * Hrange[2] + (H0[2]-Hmin[2]);
+    CUDAREAL _F_cell_0 = FhklLinear(Fhkl_linear_index_0);
+    KOKKOS_MAT3 Ginv = anisoG_local.inverse();
+    CUDAREAL anisoG_determ = anisoG_local.determinant();
+    for (int hh=-dHH[0]; hh <= dHH[0]; hh++){
+      for (int kk=-dHH[1]; kk <= dHH[1]; kk++){
+        for (int ll=-dHH[2]; ll <= dHH[2]; ll++){
+          CUDAREAL ID_this = 0;
+          CUDAREAL step_diffuse_param_this[6]  = {0,0,0,0,0,0};
+          int Fhkl_linear_index_this = (H0[0]+hh-Hmin[0]) * Hrange[1] * Hrange[2]
+            + (H0[1]+kk-Hmin[1]) * Hrange[2] + (H0[2]+ll-Hmin[2]);
+          CUDAREAL _F_cell_this = FhklLinear(Fhkl_linear_index_this);
+          CUDAREAL _this_diffuse_scale;
+          if (_F_cell_0 != 0.0)
+            _this_diffuse_scale = _F_cell_this/_F_cell_0;
+          else
+            _this_diffuse_scale = 1.0;
+
+          _this_diffuse_scale *= _this_diffuse_scale/(CUDAREAL)num_laue_mats;
+          // TODO: Apply discrete transformations to H0 and delta_H_offset
+          //    like the following to reorient G and recover calmodulin diffuse
+          // KOKKOS_MAT3 xform;
+          // xform << 0.70710678,  -0.70710678,  0., 0.70710678,  0.70710678,  0., 0.,  0., 1.;
+
+          for ( int iL = 0; iL < num_laue_mats; iL++ ){
+            KOKKOS_VEC3 Q0 =Ainv*laue_mats[iL]*H0;
+            CUDAREAL exparg = four_mpi_sq*Q0.dot(anisoU_local*Q0);
+            CUDAREAL dwf = exp(-exparg);
+            KOKKOS_VEC3 H0_offset(H0[0]+hh, H0[1]+kk, H0[2]+ll);
+            KOKKOS_VEC3 delta_H_offset = H_vec - H0_offset;
+            KOKKOS_VEC3 delta_Q = Ainv*laue_mats[iL]*delta_H_offset;
+            KOKKOS_VEC3 anisoG_q = anisoG_local*delta_Q;
+
+            CUDAREAL V_dot_V = anisoG_q.dot(anisoG_q);
+            CUDAREAL gamma_portion_denom = (1.+ V_dot_V* four_mpi_sq);
+            gamma_portion_denom *= gamma_portion_denom;
+            CUDAREAL gamma_portion = 8.*M_PI*anisoG_determ /
+              gamma_portion_denom;
+            CUDAREAL this_I_latt_diffuse = dwf*exparg*gamma_portion;
+
+            ID_this += this_I_latt_diffuse;
+            if (refine_diffuse){ // add the contributions to diffuse scattering gradients here
+              for (int i_gam=0; i_gam<3; i_gam++){
+                KOKKOS_VEC3 dV = dG_dgam[i_gam]*delta_Q;
+                CUDAREAL V_dot_dV = anisoG_q.dot(dV);
+                CUDAREAL deriv = (Ginv*dG_dgam[i_gam]).trace() - 4.*four_mpi_sq*V_dot_dV/(1+four_mpi_sq*V_dot_V);
+                step_diffuse_param_this[i_gam] += gamma_portion*deriv*dwf*exparg;
+              }
+              KOKKOS_MAT3 dU_dsigma;
+              // dU_dsigma << 0,0,0,0,0,0,0,0,0;
+              for (int i_sig = 0;i_sig<3; i_sig++){
+                dU_dsigma(i_sig, i_sig) = 2.*sqrt(anisoU_local(i_sig,i_sig));
+                CUDAREAL dexparg = four_mpi_sq*Q0.dot(dU_dsigma*Q0);
+                dU_dsigma(i_sig, i_sig) = 0.;
+                step_diffuse_param_this[i_sig+3] += gamma_portion*dwf*dexparg*(1. - exparg);
+              }
+            }
+          } // end loop over iL (laue group mats)
+          // Update the lattice interference term here to include diffuse scattering (F_latt squared)
+          *I0 += ID_this * _this_diffuse_scale;
+
+          for (int idp=0; idp < 6; idp++)
+            step_diffuse_param[idp] += step_diffuse_param_this[idp]*_this_diffuse_scale;
+        } // end ll loop
+      } // end kk loop
+    } // end hh loop
+  } // end if bounded
+}
+
+#endif
diff --git a/simtbx/diffBragg/src/util.h b/simtbx/diffBragg/src/util.h
index 44c9fa6b7c..eaef5f7a31 100644
--- a/simtbx/diffBragg/src/util.h
+++ b/simtbx/diffBragg/src/util.h
@@ -3,26 +3,30 @@
 #define SIMTBX_DIFFBRAGG_UTIL
 
 #include <Eigen/Dense>
-#include<Eigen/StdVector>
-#include<vector>
-#include<unordered_map>
-#include<unordered_set>
+#include <Eigen/StdVector>
+#include <vector>
+#include <unordered_map>
+#include <unordered_set>
 #include <string>
 
-typedef std::vector<double> image_type;
+#ifndef CUDAREAL
+    #define CUDAREAL double
+#endif
+
+using image_type = std::vector<CUDAREAL>;
 typedef Eigen::Matrix<double,3,1> VEC3;
 typedef Eigen::Matrix<double,3,3> MAT3;
 typedef std::vector<MAT3,Eigen::aligned_allocator<MAT3> > eigMat3_vec;
 typedef std::vector<VEC3,Eigen::aligned_allocator<VEC3> > eigVec3_vec;
 
 struct timer_variables{
-    double add_spots_pre=0; // times the initializations for add spots kernel
-    double add_spots_post=0; // times the copies that occur after add spots kernel
-    double add_spots_kernel_wrapper=0; // times the add spots kernel overall, either CPU or GPU
-    double cuda_alloc=0; // times the allocation of the device
-    double cuda_copy_to_dev=0; // times the copying from host to device
-    double cuda_copy_from_dev=0; // times the copying back from device to host
-    double cuda_kernel=0; // times the GPU kernel
+    CUDAREAL add_spots_pre=0; // times the initializations for add spots kernel
+    CUDAREAL add_spots_post=0; // times the copies that occur after add spots kernel
+    CUDAREAL add_spots_kernel_wrapper=0; // times the add spots kernel overall, either CPU or GPU
+    CUDAREAL cuda_alloc=0; // times the allocation of the device
+    CUDAREAL cuda_copy_to_dev=0; // times the copying from host to device
+    CUDAREAL cuda_copy_from_dev=0; // times the copying back from device to host
+    CUDAREAL cuda_kernel=0; // times the GPU kernel
     int timings=0; // how many times these variables were incremented
     bool recording=true;
   };
@@ -139,28 +143,28 @@ struct crystal{
     std::vector<Eigen::Matrix3d,Eigen::aligned_allocator<Eigen::Matrix3d> > dU_dsigma;
     Eigen::Matrix3d anisoU;
     int mosaic_domains; // number of mosaic domains to model
-    double Na, Nb, Nc, Nd, Ne, Nf; // mosaic domain terms
-    double phi0; // gonio
-    double phistep;
-    double phisteps;
-    double fudge; // factor for Bragg peak exponential falloff adjustment
-    double spot_scale; // factor applied to intensity
+    CUDAREAL Na, Nb, Nc, Nd, Ne, Nf; // mosaic domain terms
+    CUDAREAL phi0; // gonio
+    CUDAREAL phistep;
+    int phisteps;
+    CUDAREAL fudge; // factor for Bragg peak exponential falloff adjustment
+    CUDAREAL spot_scale; // factor applied to intensity
     int h_range, k_range, l_range;
     int h_max, h_min, k_max, k_min, l_max, l_min;
-    double dmin; //res
+    CUDAREAL dmin; //res
     std::vector<double> dspace_bins;
-    std::vector<double> FhklLinear, Fhkl2Linear; // structure factor amps magnitude (or real, image of complex)
+    std::vector<CUDAREAL> FhklLinear, Fhkl2Linear; // structure factor amps magnitude (or real, image of complex)
     std::vector<double> ASU_dspace, ASU_Fcell;
     std::vector<int> FhklLinear_ASUid;
     std::unordered_map<std::string, int> ASUid_map;
     int Num_ASU;
     std::string hall_symbol =" P 4nw 2abw";
-    std::vector<double> fpfdp; // fprim fdblprime
-    std::vector<double> fpfdp_derivs; // fprime fdblprime deriv
-    std::vector<double> atom_data; // heavy atom data
+    std::vector<CUDAREAL> fpfdp; // fprim fdblprime
+    std::vector<CUDAREAL> fpfdp_derivs; // fprime fdblprime deriv
+    std::vector<CUDAREAL> atom_data; // heavy atom data
     std::vector<int> nominal_hkl; // h,k,l of the pixel (expected)
-    double default_F; // place holder amplitude
-    double r_e_sqr; // electron rad
+    CUDAREAL default_F; // place holder amplitude
+    CUDAREAL r_e_sqr; // electron rad
 
     Eigen::Matrix3d eig_U; // Umatrix
     Eigen::Matrix3d eig_O; // O-matrix
@@ -186,21 +190,23 @@ struct crystal{
 struct beam{
     Eigen::Vector3d polarization_axis;
     std::vector<int> Fhkl_channels; // if refining scale factors for wavelength dependent structure factor intensities
-    double fluence; // total fluence
-    double kahn_factor; // polarization factor
-    double *source_X, *source_Y, *source_Z, *source_lambda, *source_I;   // beam vectors, wavelenths, intensities
-    double lambda0,lambda1; // affine correction to spectra
+    CUDAREAL fluence; // total fluence
+    CUDAREAL kahn_factor; // polarization factor
+    CUDAREAL *source_X, *source_Y, *source_Z; // beam vectors
+    CUDAREAL *source_lambda; // wavelengths
+    CUDAREAL *source_I; // intensities
+    CUDAREAL lambda0,lambda1; // affine correction to spectra
     int number_of_sources; // number of beams
 };
 
 struct detector{
     std::vector<Eigen::Vector3d,Eigen::aligned_allocator<Eigen::Vector3d> > dF_vecs; // derivative of the panel fast direction
     std::vector<Eigen::Vector3d,Eigen::aligned_allocator<Eigen::Vector3d> > dS_vecs; // derivative of the panel slow direction
-    double detector_thickstep, detector_thicksteps, detector_thick, detector_attnlen;
-    std::vector<double> close_distances; // offsets to the detector origins (Z direction)
+    CUDAREAL detector_thickstep, detector_thicksteps, detector_thick, detector_attnlen;
+    std::vector<CUDAREAL> close_distances; // offsets to the detector origins (Z direction)
     int oversample; // determines the pixel subsampling rate
-    double subpixel_size, pixel_size;
-    std::vector<double> fdet_vectors, sdet_vectors, odet_vectors, pix0_vectors; // these define the detector (fast, slow, orth, origin)
+    CUDAREAL subpixel_size, pixel_size;
+    std::vector<CUDAREAL> fdet_vectors, sdet_vectors, odet_vectors, pix0_vectors; // these define the detector (fast, slow, orth, origin)
 };
 
 #endif
diff --git a/simtbx/diffBragg/src/util_kokkos.h b/simtbx/diffBragg/src/util_kokkos.h
new file mode 100644
index 0000000000..966167cc8b
--- /dev/null
+++ b/simtbx/diffBragg/src/util_kokkos.h
@@ -0,0 +1,267 @@
+#ifndef SIMTBX_DIFFBRAGG_UTIL_KOKKOS
+#define SIMTBX_DIFFBRAGG_UTIL_KOKKOS
+
+#include <vector>
+#include "kokkostbx/kokkos_matrix3.h"
+// #include "kokkostbx/kokkos_types.h"
+#include "kokkostbx/kokkos_vector3.h"
+#include "simtbx/diffBragg/src/util.h"
+
+#ifndef CUDAREAL
+    #define CUDAREAL double
+#endif
+
+using image_type = std::vector<CUDAREAL>;
+using KOKKOS_VEC3 = kokkostbx::vector3<CUDAREAL>;
+using KOKKOS_MAT3 = kokkostbx::matrix3<CUDAREAL>;
+// using vector_vec3_t = view_1d_t<KOKKOS_VEC3>;
+// using vector_mat3_t = view_1d_t<KOKKOS_MAT3>;
+
+inline KOKKOS_VEC3 to_vec3(const Eigen::Vector3d& v) {
+    return KOKKOS_VEC3(v[0], v[1], v[2]);
+}
+
+inline KOKKOS_MAT3 to_mat3(const Eigen::Matrix3d& m) {
+    // Eigen matrix is column-major!
+    return KOKKOS_MAT3(m(0, 0), m(0, 1), m(0, 2), m(1, 0), m(1, 1), m(1, 2), m(2, 0), m(2, 1), m(2, 2));
+}
+
+template <class T, class U>
+inline void transfer(T& target, U& source, int size=-1) {
+    const int length = size>=0 ? size : source.size();
+    for (int i=0; i<length; ++i) {
+        target(i) = source[i];
+    }
+}
+
+template <class T, class U>
+inline void transfer_KOKKOS_VEC3(T& target, U& source) {
+    for (int i=0; i<source.size(); ++i) {
+        target(i) = to_vec3(source[i]);
+    }
+}
+
+template <class T, class U>
+inline void transfer_KOKKOS_MAT3(T& target, U& source) {
+    for (int i=0; i<source.size(); ++i) {
+        target(i) = to_mat3(source[i]);
+    }
+}
+
+// CONTAINERS
+struct kokkos_crystal {
+    double Friedel_beta = 1e10; // restraint factor for Friedel pairs
+    double Finit_beta = 1e10; // restraint factor for Friedel pairs
+    std::vector<int> pos_inds; // indices of the positive Friedel mate
+    std::vector<int> neg_inds; // indices of the negative Friedel mate
+    double Fhkl_beta=1e10;
+    bool use_geometric_mean=false;
+    std::unordered_set<int> Fhkl_grad_idx_tracker;
+    int num_Fhkl_channels=1;
+    int laue_group_num=1;
+    int stencil_size=0;
+    KOKKOS_MAT3 anisoG;
+    std::vector<KOKKOS_MAT3> dG_dgamma;
+    std::vector<KOKKOS_MAT3> dU_dsigma;
+    KOKKOS_MAT3 anisoU;
+    int mosaic_domains;               // number of mosaic domains to model
+    CUDAREAL Na, Nb, Nc, Nd, Ne, Nf;  // mosaic domain terms
+    CUDAREAL phi0;                    // gonio
+    CUDAREAL phistep;
+    int phisteps;
+    CUDAREAL fudge;       // factor for Bragg peak exponential falloff adjustment
+    CUDAREAL spot_scale;  // factor applied to intensity
+    int h_range, k_range, l_range;
+    int h_max, h_min, k_max, k_min, l_max, l_min;
+    CUDAREAL dmin;  // res
+    std::vector<double> dspace_bins;
+    std::vector<CUDAREAL> FhklLinear,
+        Fhkl2Linear;              // structure factor amps magnitude (or real, image of complex)
+    std::vector<double> ASU_dspace, ASU_Fcell;
+    std::vector<int> FhklLinear_ASUid;
+    std::unordered_map<std::string, int> ASUid_map;
+    int Num_ASU;
+    std::string hall_symbol =" P 4nw 2abw";
+    std::vector<CUDAREAL> fpfdp;  // fprim fdblprime
+    std::vector<CUDAREAL> fpfdp_derivs;  // fprime fdblprime deriv
+    std::vector<CUDAREAL> atom_data;     // heavy atom data
+    std::vector<int> nominal_hkl;        // h,k,l of the pixel (expected)
+    CUDAREAL default_F;                  // place holder amplitude
+    CUDAREAL r_e_sqr;                    // electron rad
+
+    KOKKOS_MAT3 eig_U;        // Umatrix
+    KOKKOS_MAT3 eig_O;        // O-matrix
+    KOKKOS_MAT3 eig_B;        // B matrix
+    KOKKOS_MAT3 RXYZ;         // Rx*Ry*Rz misset perturtbation matrix (this is whats refined)
+    KOKKOS_VEC3 spindle_vec;  // gonio
+
+    std::vector<KOKKOS_MAT3> UMATS_RXYZ;
+    std::vector<KOKKOS_MAT3> UMATS_RXYZ_prime;
+    std::vector<KOKKOS_MAT3> UMATS_RXYZ_dbl_prime;
+
+    std::vector<KOKKOS_MAT3> RotMats;
+    std::vector<KOKKOS_MAT3> dRotMats;
+    std::vector<KOKKOS_MAT3> d2RotMats;
+    std::vector<KOKKOS_MAT3> UMATS;
+    std::vector<KOKKOS_MAT3> UMATS_prime;
+    std::vector<KOKKOS_MAT3> UMATS_dbl_prime;
+    std::vector<KOKKOS_MAT3> dB_Mats;
+    std::vector<KOKKOS_MAT3> dB2_Mats;
+
+    kokkos_crystal(crystal T)
+        : Friedel_beta(T.Friedel_beta),
+          Finit_beta(T.Finit_beta),
+          pos_inds(T.pos_inds),
+          neg_inds(T.neg_inds),
+          Fhkl_beta(T.Fhkl_beta),
+          use_geometric_mean(T.use_geometric_mean),
+          Fhkl_grad_idx_tracker(T.Fhkl_grad_idx_tracker),
+          num_Fhkl_channels(T.num_Fhkl_channels),
+          laue_group_num(T.laue_group_num),
+          stencil_size(T.stencil_size),
+          anisoG(to_mat3(T.anisoG)),
+          anisoU(to_mat3(T.anisoU)),
+          mosaic_domains(T.mosaic_domains),
+          Na(T.Na),
+          Nb(T.Nb),
+          Nc(T.Nc),
+          Nd(T.Nd),
+          Ne(T.Ne),
+          Nf(T.Nf),
+          phi0(T.phi0),
+          phistep(T.phistep),
+          phisteps(T.phisteps),
+          fudge(T.fudge),
+          spot_scale(T.spot_scale),
+          h_range(T.h_range),
+          k_range(T.k_range),
+          l_range(T.l_range),
+          h_max(T.h_max),
+          h_min(T.h_min),
+          k_max(T.k_max),
+          k_min(T.k_min),
+          l_max(T.l_max),
+          l_min(T.l_min),
+          dmin(T.dmin),
+          dspace_bins(T.dspace_bins),
+          FhklLinear(T.FhklLinear),
+          Fhkl2Linear(T.Fhkl2Linear),
+          ASU_dspace(T.ASU_dspace),
+          ASU_Fcell(T.ASU_Fcell),
+          FhklLinear_ASUid(T.FhklLinear_ASUid),
+          ASUid_map(T.ASUid_map),
+          Num_ASU(T.Num_ASU),
+          hall_symbol(T.hall_symbol),
+          fpfdp(T.fpfdp),
+          fpfdp_derivs(T.fpfdp_derivs),
+          atom_data(T.atom_data),
+          nominal_hkl(T.nominal_hkl),
+          default_F(T.default_F),
+          r_e_sqr(T.r_e_sqr),
+          eig_U(to_mat3(T.eig_U)),
+          eig_O(to_mat3(T.eig_O)),
+          eig_B(to_mat3(T.eig_B)),
+          RXYZ(to_mat3(T.RXYZ)),
+          spindle_vec(to_vec3(T.spindle_vec)) {
+        for (auto& mat : T.dG_dgamma) {
+            dG_dgamma.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.dU_dsigma) {
+            dU_dsigma.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS_RXYZ) {
+            UMATS_RXYZ.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS_RXYZ_prime) {
+            UMATS_RXYZ_prime.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS_RXYZ_dbl_prime) {
+            UMATS_RXYZ_dbl_prime.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.RotMats) {
+            RotMats.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.dRotMats) {
+            dRotMats.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.d2RotMats) {
+            d2RotMats.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS) {
+            UMATS.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS_prime) {
+            UMATS_prime.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.UMATS_dbl_prime) {
+            UMATS_dbl_prime.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.dB_Mats) {
+            dB_Mats.push_back(to_mat3(mat));
+        }
+        for (auto& mat : T.dB2_Mats) {
+            dB2_Mats.push_back(to_mat3(mat));
+        }
+    }
+};
+
+struct kokkos_beam {
+    KOKKOS_VEC3 polarization_axis;
+    std::vector<int> Fhkl_channels; // if refining scale factors for wavelength dependent structure factor intensities
+    CUDAREAL fluence;                          // total fluence
+    CUDAREAL kahn_factor;                      // polarization factor
+    CUDAREAL *source_X, *source_Y, *source_Z;  // beam vectors
+    CUDAREAL* source_lambda;                   // wavelengths
+    CUDAREAL* source_I;                        // intensities
+    CUDAREAL lambda0, lambda1;                 // affine correction to spectra
+    int number_of_sources;                     // number of beams
+
+    kokkos_beam(beam T)
+        : Fhkl_channels(T.Fhkl_channels),
+          fluence(T.fluence),
+          kahn_factor(T.kahn_factor),
+          source_X(T.source_X),
+          source_Y(T.source_Y),
+          source_Z(T.source_Z),
+          source_lambda(T.source_lambda),
+          source_I(T.source_I),
+          lambda0(T.lambda0),
+          lambda1(T.lambda1),
+          number_of_sources(T.number_of_sources),
+          polarization_axis(to_vec3(T.polarization_axis)){ };
+};
+
+struct kokkos_detector {
+    std::vector<KOKKOS_VEC3> dF_vecs;  // derivative of the panel fast direction
+    std::vector<KOKKOS_VEC3> dS_vecs;  // derivative of the panel slow direction
+    CUDAREAL detector_thickstep, detector_thicksteps, detector_thick, detector_attnlen;
+    std::vector<CUDAREAL> close_distances;  // offsets to the detector origins (Z direction)
+    int oversample;                         // determines the pixel subsampling rate
+    CUDAREAL subpixel_size, pixel_size;
+    std::vector<CUDAREAL> fdet_vectors, sdet_vectors, odet_vectors,
+        pix0_vectors;  // these define the detector (fast, slow, orth, origin)
+
+    kokkos_detector(detector T)
+        : detector_thickstep(T.detector_thickstep),
+          detector_thicksteps(T.detector_thicksteps),
+          detector_thick(T.detector_thick),
+          detector_attnlen(T.detector_attnlen),
+          close_distances(T.close_distances),
+          oversample(T.oversample),
+          subpixel_size(T.subpixel_size),
+          pixel_size(T.pixel_size),
+          fdet_vectors(T.fdet_vectors),
+          sdet_vectors(T.sdet_vectors),
+          odet_vectors(T.odet_vectors),
+          pix0_vectors(T.pix0_vectors) {
+        for (auto& vec : T.dF_vecs) {
+            dF_vecs.push_back(to_vec3(vec));
+        }
+        for (auto& vec : T.dS_vecs) {
+            dS_vecs.push_back(to_vec3(vec));
+        }
+    }
+
+};
+
+#endif
diff --git a/simtbx/diffBragg/utils.py b/simtbx/diffBragg/utils.py
index a88898a241..57ff2db410 100644
--- a/simtbx/diffBragg/utils.py
+++ b/simtbx/diffBragg/utils.py
@@ -1690,3 +1690,16 @@ def load_Fhkl_model_from_params_and_expt(params, expt):
         miller_data = open_mtz(sf.mtz_name, sf.mtz_column)
 
     return miller_data
+
+
+def find_diffBragg_instances(globe_objs):
+    """find any instances of diffbragg in globals
+        globe_objs is a return value of globals() (dict)
+    """
+    inst_names = []
+    for name,obj in globe_objs.items():
+        if "simtbx.nanoBragg.sim_data.SimData" in str(obj):
+            inst_names.append(name)
+        if "simtbx_diffBragg_ext.diffBragg" in str(obj):
+            inst_names.append(name)
+    return inst_names
diff --git a/simtbx/kokkos/SConscript b/simtbx/kokkos/SConscript
index 254e98c361..2deb4239d6 100644
--- a/simtbx/kokkos/SConscript
+++ b/simtbx/kokkos/SConscript
@@ -31,7 +31,7 @@ os.environ['CXXFLAGS'] = '-O3 -fPIC -DCUDAREAL=double'
 
 library_flags = "-Llib"
 if use_cuda:
-  library_flags += " -L$(CUDA_HOME)/lib64"
+  library_flags += " -L$(CUDA_HOME)/lib64 -L$(CUDA_HOME)/lib64/stubs"
   if os.getenv('CUDA_COMPATIBILITY'):
     library_flags += " -L$(CUDA_HOME)/compat"
 elif use_sycl:
@@ -204,8 +204,12 @@ print('-'*79)
 kokkos_cxxflags = subprocess.check_output(
   ['make', '-f', 'Makefile.kokkos', 'print-cxx-flags'],
   cwd=os.environ['KOKKOS_PATH'])
-kokkos_cxxflags = kokkos_cxxflags.split(b'\n')
-kokkos_cxxflags = kokkos_cxxflags[1].decode('utf8').split()
+#print('5'*79)
+#print(os.environ['KOKKOS_CXXFLAGS'])
+#print('5'*79)
+#kokkos_cxxflags = kokkos_cxxflags.split(b'\n')
+#kokkos_cxxflags = kokkos_cxxflags[1].decode('utf8').split()
+kokkos_cxxflags = os.environ['KOKKOS_CXXFLAGS'].split()
 if kokkos_cxxflags and kokkos_cxxflags[0] == 'echo':
   kokkos_cxxflags = [f.strip('"') for f in kokkos_cxxflags[1:]]
 print('KOKKOS_CXXFLAGS:', kokkos_cxxflags)
@@ -218,13 +222,24 @@ print('='*79)
 # =============================================================================
 Import("env", "env_etc")
 
+# remove -ffast-math compile option
 kokkos_env = env.Clone()
+ccflags = kokkos_env['SHCCFLAGS']
+o = '-ffast-math'
+if o in ccflags:
+  ccflags.remove(o)
+kokkos_env.Replace(SHCCFLAGS=ccflags)
+
 kokkos_env.Replace(CXX=os.environ['CXX'])
 kokkos_env.Replace(SHCXX=os.environ['CXX'])
+kokkos_env.Replace(SHLINK=os.environ['CXX'])
 kokkos_env.Prepend(CXXFLAGS=['-DCUDAREAL=double'] + kokkos_cxxflags)
 kokkos_env.Prepend(CPPFLAGS=['-DCUDAREAL=double'] + kokkos_cxxflags)
 kokkos_env.Prepend(CPPPATH=[os.environ['KOKKOS_PATH']])
 kokkos_env.Append(LIBS=['kokkoscontainers','kokkoscore'])
+if 'HIP' in os.getenv('KOKKOS_DEVICES'):
+  kokkos_env.Replace(LINK=os.environ['CXX'])
+  kokkos_env.Replace(SHLINK=os.environ['CXX'])
 
 if use_sycl:
   kokkos_env.Replace(SHLINK=os.environ['CXX'])
@@ -243,7 +258,7 @@ simtbx_kokkos_lib = kokkos_env.SharedLibrary(
   source=[
     'detector.cpp',
     'kokkos_instance.cpp',
-    'kokkos_utils.cpp',
+    '../../kokkostbx/kokkos_utils.cpp',
     'simulation.cpp',
     'structure_factors.cpp'
   ]
@@ -266,6 +281,7 @@ if not env_etc.no_boost_python:
     paths=env_etc.simtbx_common_includes + [env_etc.python_include])
   kokkos_ext_env.Replace(CXX=os.environ['CXX'])
   kokkos_ext_env.Replace(SHCXX=os.environ['CXX'])
+  kokkos_ext_env.Replace(SHLINK=os.environ['CXX'])
   if use_sycl:
     kokkos_ext_env.Replace(SHLINK=os.environ['CXX'])
     kokkos_ext_env.Prepend(SHCXXFLAGS=['-DCUDAREAL=double'] + kokkos_cxxflags)
@@ -282,6 +298,7 @@ if not env_etc.no_boost_python:
   if 'Cuda' in os.getenv('KOKKOS_DEVICES'):
     kokkos_ext_env.Append(LIBPATH=[os.path.join(os.environ['CUDA_HOME'], 'lib64')])
     kokkos_ext_env.Append(LIBPATH=[os.path.join(os.environ['CUDA_HOME'], 'compat')])
+    kokkos_ext_env.Append(LIBPATH=[os.path.join(os.environ['CUDA_HOME'], 'lib64/stubs')])
     kokkos_ext_env.Append(LIBS=env_etc.libm + default_libs + ["cudart", "cuda"])
   elif 'HIP' in os.getenv('KOKKOS_DEVICES'):
     kokkos_ext_env.Append(LIBPATH=[os.path.join(os.environ['ROCM_PATH'], 'lib')])
diff --git a/simtbx/kokkos/detector.cpp b/simtbx/kokkos/detector.cpp
index 3141cf8d9d..85715e3077 100644
--- a/simtbx/kokkos/detector.cpp
+++ b/simtbx/kokkos/detector.cpp
@@ -1,7 +1,7 @@
 #include "scitbx/array_family/boost_python/flex_fwd.h"
 //#include "cudatbx/cuda_base.cuh"
 #include "simtbx/kokkos/detector.h"
-#include "simtbx/kokkos/kokkos_utils.h"
+#include "kokkostbx/kokkos_utils.h"
 #include "scitbx/vec3.h"
 #include "scitbx/vec2.h"
 
@@ -159,7 +159,7 @@ namespace simtbx { namespace Kokkos {
     // nB.raw_pixels = af::flex_double(af::flex_grid<>(nB.spixels,nB.fpixels));
     // do not reallocate CPU memory for the data write, as it is not needed
 
-    transfer_kokkos2flex(nB.raw_pixels, m_accumulate_floatimage);
+    kokkostbx::transfer_kokkos2flex(nB.raw_pixels, m_accumulate_floatimage);
     // vector_double_t::HostMirror host_floatimage = create_mirror_view(m_accumulate_floatimage);
     // deep_copy(host_floatimage, m_accumulate_floatimage);
 
@@ -175,7 +175,7 @@ namespace simtbx { namespace Kokkos {
   kokkos_detector::get_raw_pixels(){
     //return the data array for the multipanel detector case
     af::flex_double output_array(af::flex_grid<>(m_panel_count,m_slow_dim_size,m_fast_dim_size), af::init_functor_null<double>());
-    transfer_kokkos2flex(output_array, m_accumulate_floatimage);
+    kokkostbx::transfer_kokkos2flex(output_array, m_accumulate_floatimage);
 
     // vector_double_t::HostMirror host_floatimage = create_mirror_view(m_accumulate_floatimage);
     // deep_copy(host_floatimage, m_accumulate_floatimage);
@@ -189,7 +189,7 @@ namespace simtbx { namespace Kokkos {
   void
   kokkos_detector::set_active_pixels_on_GPU(af::shared<std::size_t> active_pixel_list_value) {
     m_active_pixel_size = active_pixel_list_value.size();
-    transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
+    kokkostbx::transfer_shared2kokkos(m_active_pixel_list, active_pixel_list_value);
     active_pixel_list = active_pixel_list_value;
   }
 
@@ -197,7 +197,7 @@ namespace simtbx { namespace Kokkos {
   kokkos_detector::get_whitelist_raw_pixels(af::shared<std::size_t> selection) {
     //return the data array for the multipanel detector case, but only for whitelist pixels
     vector_size_t active_pixel_selection = vector_size_t("active_pixel_selection", selection.size());
-    transfer_shared2kokkos(active_pixel_selection, selection);
+    kokkostbx::transfer_shared2kokkos(active_pixel_selection, selection);
 
     size_t output_pixel_size = selection.size();
     vector_cudareal_t active_pixel_results = vector_cudareal_t("active_pixel_results", output_pixel_size);
@@ -212,7 +212,7 @@ namespace simtbx { namespace Kokkos {
     });
 
     af::shared<double> output_array(output_pixel_size, af::init_functor_null<double>());
-    transfer_kokkos2shared(output_array, active_pixel_results);
+    kokkostbx::transfer_kokkos2shared(output_array, active_pixel_results);
 
     SCITBX_ASSERT(output_array.size() == output_pixel_size);
     return output_array;
@@ -228,13 +228,13 @@ namespace simtbx { namespace Kokkos {
     resize(m_maskimage, m_total_pixel_count);
     resize(m_floatimage, m_total_pixel_count);
 
-    transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
-    transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
-    transfer_shared2kokkos(m_odet_vector, metrology.odet);
-    transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
-    transfer_shared2kokkos(m_distance, metrology.dists);
-    transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
-    transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
+    kokkostbx::transfer_shared2kokkos(m_sdet_vector, metrology.sdet);
+    kokkostbx::transfer_shared2kokkos(m_fdet_vector, metrology.fdet);
+    kokkostbx::transfer_shared2kokkos(m_odet_vector, metrology.odet);
+    kokkostbx::transfer_shared2kokkos(m_pix0_vector, metrology.pix0);
+    kokkostbx::transfer_shared2kokkos(m_distance, metrology.dists);
+    kokkostbx::transfer_shared2kokkos(m_Xbeam, metrology.Xbeam);
+    kokkostbx::transfer_shared2kokkos(m_Ybeam, metrology.Ybeam);
     fence();
 
     // metrology.show();
diff --git a/simtbx/kokkos/detector.h b/simtbx/kokkos/detector.h
index 82ea1793a1..ced5c4b87d 100644
--- a/simtbx/kokkos/detector.h
+++ b/simtbx/kokkos/detector.h
@@ -14,7 +14,7 @@
 #include "dxtbx/model/detector.h"
 #include "dxtbx/model/beam.h"
 #include "simtbx/nanoBragg/nanoBragg.h"
-#include "simtbx/kokkos/kokkos_types.h"
+#include "kokkostbx/kokkos_types.h"
 
 #include <iostream>
 
diff --git a/simtbx/kokkos/kernel_math.h b/simtbx/kokkos/kernel_math.h
index dcae94e369..b140ec91d9 100644
--- a/simtbx/kokkos/kernel_math.h
+++ b/simtbx/kokkos/kernel_math.h
@@ -1,10 +1,14 @@
 #ifndef SIMTBX_KOKKOS_KERNEL_MATH_H
 #define SIMTBX_KOKKOS_KERNEL_MATH_H
 
+#include <kokkostbx/kokkos_vector3.h>
+
 #ifndef CUDAREAL
 #define CUDAREAL float
 #endif
 
+using kokkostbx::vector3;
+
 // cubic spline interpolation functions
 KOKKOS_FUNCTION static void polint(const CUDAREAL *xa, const CUDAREAL *ya, CUDAREAL x, CUDAREAL *y);
 // vector inner product where vector magnitude is 0th element
@@ -14,6 +18,7 @@ KOKKOS_FUNCTION static CUDAREAL dot_product(const CUDAREAL *x, const CUDAREAL *y
 KOKKOS_FUNCTION static CUDAREAL unitize(CUDAREAL * vector, CUDAREAL *new_unit_vector);
 // polarization factor from vectors
 KOKKOS_FUNCTION static CUDAREAL polarization_factor(CUDAREAL kahn_factor, CUDAREAL *incident, CUDAREAL *diffracted, const vector_cudareal_t axis);
+KOKKOS_FUNCTION static CUDAREAL polarization_factor(CUDAREAL kahn_factor, const vector3<CUDAREAL> incident, const vector3<CUDAREAL> diffracted, const vector3<CUDAREAL> axis);
 // vector cross product where vector magnitude is 0th element
 KOKKOS_INLINE_FUNCTION static void cross_product(CUDAREAL *x, CUDAREAL *y, CUDAREAL *z);
 /* rotate a 3-vector about a unit vector axis */
@@ -113,6 +118,41 @@ KOKKOS_FUNCTION CUDAREAL polarization_factor(CUDAREAL kahn_factor, CUDAREAL *inc
         return 0.5 * (1.0 + cos2theta_sqr - kahn_factor * cos(2 * psi) * sin2theta_sqr);
 }
 
+KOKKOS_FUNCTION CUDAREAL polarization_factor(CUDAREAL kahn_factor, const vector3<CUDAREAL> incident, const vector3<CUDAREAL> diffracted, const vector3<CUDAREAL> axis) {
+        // expects incident, diffracted and axis to be unit vectors
+
+        // component of diffracted unit vector along incident beam unit vector
+        CUDAREAL cos2theta = incident.dot(diffracted);
+        CUDAREAL cos2theta_sqr = cos2theta * cos2theta;
+        CUDAREAL sin2theta_sqr = 1 - cos2theta_sqr;
+
+        CUDAREAL psi = 0.0;
+        if (kahn_factor != 0.0) {
+                // tricky bit here is deciding which direction the E-vector lies in for each source
+                // here we assume it is closest to the "axis" defined above
+
+                // cross product to get "vertical" axis that is orthogonal to the cannonical "polarization"
+                vector3<CUDAREAL> B_in = axis.cross(incident);
+                // make it a unit vector
+                B_in.normalize();
+
+                // cross product with incident beam to get E-vector direction
+                vector3<CUDAREAL> E_in = incident.cross(B_in);
+                // make it a unit vector
+                E_in.normalize();
+
+                // get components of diffracted ray projected onto the E-B plane
+                CUDAREAL E_out = diffracted.dot(E_in);
+                CUDAREAL B_out = diffracted.dot(B_in);
+
+                // compute the angle of the diffracted ray projected onto the incident E-B plane
+                psi = -atan2(B_out, E_out);
+        }
+
+        // correction for polarized incident beam
+        return 0.5 * (1.0 + cos2theta_sqr - kahn_factor * cos(2 * psi) * sin2theta_sqr);
+}
+
 // vector cross product where vector magnitude is 0th element
 KOKKOS_INLINE_FUNCTION void cross_product(CUDAREAL *x, CUDAREAL *y, CUDAREAL *z) {
         z[1] = x[2] * y[3] - x[3] * y[2];
diff --git a/simtbx/kokkos/kokkos_instance.cpp b/simtbx/kokkos/kokkos_instance.cpp
index a23981551b..2624459b8e 100644
--- a/simtbx/kokkos/kokkos_instance.cpp
+++ b/simtbx/kokkos/kokkos_instance.cpp
@@ -7,6 +7,9 @@ using Kokkos::finalize;
 namespace simtbx {
 namespace Kokkos {
 
+  bool kokkos_instance::m_isInitialized = false;
+  int kokkos_instance::m_instances = 0;
+
   kokkos_instance::kokkos_instance() {
     printf("NO OPERATION, NO DEVICE NUMBER");
   }
@@ -15,24 +18,30 @@ namespace Kokkos {
     InitArguments kokkos_init;
     kokkos_init.device_id = t_deviceID;
 
-    initialize(kokkos_init);
-    bFinalized = false;
-    deviceID = t_deviceID;
+    if (!m_isInitialized) {
+      initialize(kokkos_init);
+
+      m_isInitialized = true;
+      m_isFinalized = false;
+      m_deviceID = t_deviceID;
+    }
+    ++m_instances;
   }
 
   int
   kokkos_instance::get_deviceID() const {
-    return deviceID;
+    return m_deviceID;
   }
 
   void
   kokkos_instance::finalize_kokkos() {
     finalize();
-    bFinalized = true;
+    m_isFinalized = true;
   }
 
   kokkos_instance::~kokkos_instance() {
-    if (!bFinalized) { finalize(); }
+    --m_instances;
+    if (!m_isFinalized && m_instances<1) { finalize(); }
   }
 
 
diff --git a/simtbx/kokkos/kokkos_instance.h b/simtbx/kokkos/kokkos_instance.h
index e5d3953e77..b2bc3bc3c7 100644
--- a/simtbx/kokkos/kokkos_instance.h
+++ b/simtbx/kokkos/kokkos_instance.h
@@ -1,7 +1,7 @@
 #ifndef SIMTBX_KOKKOS_INSTANCE_H
 #define SIMTBX_KOKKOS_INSTANCE_H
 
-#include "simtbx/kokkos/kokkos_types.h"
+#include "kokkostbx/kokkos_types.h"
 
 namespace simtbx {
 namespace Kokkos {
@@ -17,8 +17,11 @@ class kokkos_instance {
     int get_deviceID() const;
 
   private:
-    bool bFinalized = false;
-    int deviceID = -1;
+    static bool m_isInitialized;
+    static int m_instances;
+    bool m_isFinalized = false;
+    int m_deviceID = -1;
+
 
 };
 } // Kokkos
diff --git a/simtbx/kokkos/kokkos_types.h b/simtbx/kokkos/kokkos_types.h
deleted file mode 100644
index f2b5a526da..0000000000
--- a/simtbx/kokkos/kokkos_types.h
+++ /dev/null
@@ -1,50 +0,0 @@
-#ifndef SIMTBX_KOKKOS_TYPES_H
-#define SIMTBX_KOKKOS_TYPES_H
-#include <Kokkos_Core.hpp>
-
-#ifdef KOKKOS_ENABLE_CUDA
-#define MemSpace Kokkos::CudaSpace
-#endif
-#ifdef KOKKOS_ENABLE_HIP
-#define MemSpace Kokkos::Experimental::HIPSpace
-#endif
-#ifdef KOKKOS_ENABLE_OPENMPTARGET
-#define MemSpace Kokkos::OpenMPTargetSpace
-#endif
-
-#ifndef MemSpace
-#define MemSpace Kokkos::HostSpace
-#endif
-
-using ExecSpace = MemSpace::execution_space;
-using range_policy = Kokkos::RangePolicy<ExecSpace>;
-
-template<typename T> using view_1d_t = Kokkos::View<T*, MemSpace>;
-
-using vector_bool_t = view_1d_t<bool>;
-using vector_double_t = view_1d_t<double>;
-using vector_float_t = view_1d_t<float>;
-using vector_cudareal_t = view_1d_t<CUDAREAL>;
-using vector_int_t = view_1d_t<int>;
-using vector_size_t = view_1d_t<std::size_t>;
-using vector_ushort_t = view_1d_t<unsigned int short>;
-
-template <typename T>
-void print_view(const view_1d_t<T> &arg_view, size_t arg_first, size_t arg_last) {
-  std::string label = arg_view.label();
-  printf("print_view '%s'\n", label.c_str());
-
-  Kokkos::parallel_for("print_view",
-                        range_policy(arg_first, arg_last),
-                        KOKKOS_LAMBDA (const int i) {
-    printf(" P[ %d ] = '%f'\n", i, (double) arg_view(i) );
-  });
-  Kokkos::fence();
-}
-
-template <typename T>
-void print_view(const view_1d_t<T> &arg_view) {
-  print_view(arg_view, 0, arg_view.span());
-}
-
-#endif // SIMTBX_KOKKOS_TYPES_H
diff --git a/simtbx/kokkos/kokkos_utils.cpp b/simtbx/kokkos/kokkos_utils.cpp
deleted file mode 100644
index d9a2b3e7a8..0000000000
--- a/simtbx/kokkos/kokkos_utils.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-#include "simtbx/kokkos/kokkos_utils.h"
-
-namespace simtbx { namespace Kokkos {
-  void
-  transfer_double2kokkos(vector_cudareal_t &dst, const double *src, const size_t length) {
-    if (true) {
-      // printf("== Transfer %s from %p\n", dst.label().c_str(), (void*) dst.data());
-      // printf(" - size src|dst: %d|%d\n", length, dst.span() );
-    }
-    if (dst.span() < length) {
-      resize(dst, length);
-      // printf(" - size changed, new size: %d\n", dst.span() );
-    }
-
-    transfer_X2kokkos(dst, src, length);
-
-    if (length < 20) {
-      // print_view(dst);
-    }
-  }
-
-} // Kokkos
-} // simtbx
diff --git a/simtbx/kokkos/kokkos_utils.h b/simtbx/kokkos/kokkos_utils.h
deleted file mode 100644
index f5dd69cae1..0000000000
--- a/simtbx/kokkos/kokkos_utils.h
+++ /dev/null
@@ -1,113 +0,0 @@
-#ifndef SIMTBX_KOKKOS_UTILS_H
-#define SIMTBX_KOKKOS_UTILS_H
-
-#include "scitbx/array_family/shared.h"
-#include "scitbx/array_family/flex_types.h"
-#include "simtbx/kokkos/kokkos_types.h"
-
-//*******************************************************************
-// Transfer data
-//*******************************************************************
-
-namespace {
-  template <typename T, typename U>
-  void
-  transfer_X2kokkos(view_1d_t<T> &dst, const U &src) {
-    auto host_view = Kokkos::create_mirror_view(dst);
-    auto src_ptr = src.begin();
-    for (int i=0; i<src.size(); ++i) {
-      host_view( i ) = src_ptr[ i ];
-    }
-    Kokkos::deep_copy(dst, host_view);
-  }
-
-  template <typename T, typename U>
-  void
-  transfer_X2kokkos(view_1d_t<T> &dst, const U *src, const size_t length) {
-    auto host_view = Kokkos::create_mirror_view(dst);
-    for (int i=0; i<length; ++i) {
-      host_view( i ) = src[ i ];
-    }
-    Kokkos::deep_copy(dst, host_view);
-  }
-
-  template <typename T, typename U>
-  void
-  transfer_kokkos2X(T &dst, const view_1d_t<U> &src) {
-    auto host_view = Kokkos::create_mirror_view(src);
-    Kokkos::deep_copy(host_view, src);
-    auto dst_ptr = dst.begin();
-    for (int i=0; i<host_view.span(); ++i) {
-      dst_ptr[ i ] = host_view( i );
-    }
-  }
-}
-
-//*******************************************************************
-// math functions
-//*******************************************************************
-
-template <typename T, typename U>
-void
-add_array( view_1d_t<T> lhs, const view_1d_t<U> rhs ) {
-  Kokkos::parallel_for("add_arrays", lhs.span(), KOKKOS_LAMBDA(const int& i) {
-    lhs( i ) = lhs( i ) + (T)rhs( i );
-    rhs( i ) = 0;
-  });
-}
-
-namespace simtbx { namespace Kokkos {
-
-  namespace af = scitbx::af;
-
-  template <typename T>
-  void
-  transfer_shared2kokkos(view_1d_t<T> &dst, const af::shared<T>  &src) {
-    if (true) {
-      // printf("== Transfer %s from %p\n", dst.label().c_str(), (void*) dst.data());
-      // printf(" - size src|dst: %d|%d\n", src.size(), dst.span() );
-    }
-    if (dst.span() < src.size()) {
-      resize(dst, src.size());
-      // printf(" - size changed, new size: %d\n", dst.span() );
-    }
-
-    transfer_X2kokkos(dst, src);
-    if (src.size() < 20) {
-      // print_view(dst);
-    }
-  }
-
-  void transfer_double2kokkos(vector_cudareal_t &dst, const double *src, const size_t length);
-
-  template <typename T>
-  void
-  transfer_kokkos2shared(af::shared<T> &dst, const view_1d_t<T> &src) {
-    if (true) {
-      // printf("== Transfer %s from %p\n", src.label().c_str(), (void*) src.data());
-    }
-    transfer_kokkos2X(dst, src);
-  }
-
-  template <typename T>
-  void
-  transfer_kokkos2flex(af::flex_double &dst, const view_1d_t<T> &src) {
-    if (true) {
-      // printf("== Transfer %s from %p\n", src.label().c_str(), (void*) src.data());
-    }
-    transfer_kokkos2X(dst, src);
-  }
-
-  template <typename T>
-  void
-  transfer_kokkos2flex(af::flex_int &dst, const view_1d_t<T> &src) {
-    if (true) {
-      // printf("== Transfer %s from %p\n", src.label().c_str(), (void*) src.data());
-    }
-    transfer_kokkos2X(dst, src);
-  }
-
-} // Kokkos
-} // simtbx
-
-#endif // SIMTBX_KOKKOS_UTILS_H
diff --git a/simtbx/kokkos/simulation.cpp b/simtbx/kokkos/simulation.cpp
index 9c0573349c..848acff533 100644
--- a/simtbx/kokkos/simulation.cpp
+++ b/simtbx/kokkos/simulation.cpp
@@ -2,7 +2,7 @@
 //#include <cudatbx/cuda_base.cuh>
 #include "simtbx/kokkos/simulation.h"
 #include "simtbx/kokkos/simulation_kernels.h"
-#include "simtbx/kokkos/kokkos_utils.h"
+#include "kokkostbx/kokkos_utils.h"
 #include "scitbx/array_family/flex_types.h"
 
 #define THREADS_PER_BLOCK_X 128
@@ -108,8 +108,8 @@ namespace Kokkos {
     af::shared<double> weighted_sources_I = af::shared<double>(source_count);
     double* wptr = weighted_sources_I.begin();
     for (std::size_t iwt = 0; iwt < source_count; iwt++){wptr[iwt] = weight*(SIM.source_I[iwt]);}
-    transfer_double2kokkos(m_source_I, wptr, source_count);
-    transfer_double2kokkos(m_source_lambda, SIM.source_lambda, source_count);
+    kokkostbx::transfer_double2kokkos(m_source_I, wptr, source_count);
+    kokkostbx::transfer_double2kokkos(m_source_lambda, SIM.source_lambda, source_count);
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(cu_source_I, SIM.source_I, SIM.sources));
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(cu_source_lambda, SIM.source_lambda, SIM.sources));
 
@@ -197,8 +197,8 @@ namespace Kokkos {
     // transfer source_I, source_lambda
     // the int arguments are for sizes of the arrays
     int source_count = SIM.sources;
-    transfer_double2kokkos(m_source_I, SIM.source_I, source_count);
-    transfer_double2kokkos(m_source_lambda, SIM.source_lambda, source_count);
+    kokkostbx::transfer_double2kokkos(m_source_I, SIM.source_I, source_count);
+    kokkostbx::transfer_double2kokkos(m_source_lambda, SIM.source_lambda, source_count);
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(m_source_I, SIM.source_I, SIM.sources));
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(m_source_lambda, SIM.source_lambda, SIM.sources));
 
@@ -284,11 +284,11 @@ namespace Kokkos {
       vector_cudareal_t c_source_I = vector_cudareal_t("c_source_I", 0);
       vector_cudareal_t c_source_lambda = vector_cudareal_t("c_source_lambda", 0);
       double weighted_I = SIM.source_I[ictr] * weight[ictr];
-      transfer_double2kokkos(c_source_X, &(SIM.source_X[ictr]), 1);
-      transfer_double2kokkos(c_source_Y, &(SIM.source_Y[ictr]), 1);
-      transfer_double2kokkos(c_source_Z, &(SIM.source_Z[ictr]), 1);
-      transfer_double2kokkos(c_source_I, &(weighted_I), 1);
-      transfer_double2kokkos(c_source_lambda, &(SIM.source_lambda[ictr]), 1);
+      kokkostbx::transfer_double2kokkos(c_source_X, &(SIM.source_X[ictr]), 1);
+      kokkostbx::transfer_double2kokkos(c_source_Y, &(SIM.source_Y[ictr]), 1);
+      kokkostbx::transfer_double2kokkos(c_source_Z, &(SIM.source_Z[ictr]), 1);
+      kokkostbx::transfer_double2kokkos(c_source_I, &(weighted_I), 1);
+      kokkostbx::transfer_double2kokkos(c_source_lambda, &(SIM.source_lambda[ictr]), 1);
 
       debranch_maskall_Kernel(
       kdt.m_panel_count, kdt.m_slow_dim_size, kdt.m_fast_dim_size, active_pixel_list.size(),
@@ -332,8 +332,8 @@ namespace Kokkos {
         // transfer source_I, source_lambda
         // the int arguments are for sizes of the arrays
         int sources_count = SIM.sources;
-        transfer_double2kokkos(m_source_I, SIM.source_I, sources_count);
-        transfer_double2kokkos(m_source_lambda, SIM.source_lambda, sources_count);
+        kokkostbx::transfer_double2kokkos(m_source_I, SIM.source_I, sources_count);
+        kokkostbx::transfer_double2kokkos(m_source_lambda, SIM.source_lambda, sources_count);
         // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(m_source_I, SIM.source_I, SIM.sources));
         // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(m_source_lambda, SIM.source_lambda, SIM.sources));
 
@@ -433,11 +433,11 @@ namespace Kokkos {
 
     //const int vector_length = 4;
 
-    transfer_double2kokkos(m_beam_vector, SIM.beam_vector, m_vector_length);
-    transfer_double2kokkos(m_spindle_vector, SIM.spindle_vector, m_vector_length);
-    transfer_double2kokkos(m_a0, SIM.a0, m_vector_length);
-    transfer_double2kokkos(m_b0, SIM.b0, m_vector_length);
-    transfer_double2kokkos(m_c0, SIM.c0, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_beam_vector, SIM.beam_vector, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_spindle_vector, SIM.spindle_vector, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_a0, SIM.a0, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_b0, SIM.b0, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_c0, SIM.c0, m_vector_length);
 
     // cudaSafeCall(cudaMalloc((void ** )&cu_beam_vector, sizeof(*cu_beam_vector) * vector_length));
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(cu_beam_vector, SIM.beam_vector, vector_length));
@@ -458,17 +458,17 @@ namespace Kokkos {
     // Optimization do it only once here rather than multiple time per pixel in the GPU.
     double polar_vector_unitized[4];
     cpu_unitize(SIM.polar_vector, polar_vector_unitized);
-    transfer_double2kokkos(m_polar_vector, polar_vector_unitized, m_vector_length);
+    kokkostbx::transfer_double2kokkos(m_polar_vector, polar_vector_unitized, m_vector_length);
 
     int sources_count = SIM.sources;
-    transfer_double2kokkos(m_source_X, SIM.source_X, sources_count);
-    transfer_double2kokkos(m_source_Y, SIM.source_Y, sources_count);
-    transfer_double2kokkos(m_source_Z, SIM.source_Z, sources_count);
-    transfer_double2kokkos(m_source_I, SIM.source_I, sources_count);
-    transfer_double2kokkos(m_source_lambda, SIM.source_lambda, sources_count);
+    kokkostbx::transfer_double2kokkos(m_source_X, SIM.source_X, sources_count);
+    kokkostbx::transfer_double2kokkos(m_source_Y, SIM.source_Y, sources_count);
+    kokkostbx::transfer_double2kokkos(m_source_Z, SIM.source_Z, sources_count);
+    kokkostbx::transfer_double2kokkos(m_source_I, SIM.source_I, sources_count);
+    kokkostbx::transfer_double2kokkos(m_source_lambda, SIM.source_lambda, sources_count);
 
     int mosaic_domains_count = SIM.mosaic_domains;
-    transfer_double2kokkos(m_mosaic_umats, SIM.mosaic_umats, mosaic_domains_count * 9);
+    kokkostbx::transfer_double2kokkos(m_mosaic_umats, SIM.mosaic_umats, mosaic_domains_count * 9);
 
     // cudaSafeCall(cudaMalloc((void ** )&cu_polar_vector, sizeof(*cu_polar_vector) * vector_length));
     // cudaSafeCall(cudaMemcpyVectorDoubleToDevice(cu_polar_vector, polar_vector_unitized, vector_length));
diff --git a/simtbx/kokkos/simulation.h b/simtbx/kokkos/simulation.h
index 3f882e1d35..126f8da7d2 100644
--- a/simtbx/kokkos/simulation.h
+++ b/simtbx/kokkos/simulation.h
@@ -5,7 +5,7 @@
 #include "simtbx/nanoBragg/nanoBragg.h"
 #include "simtbx/kokkos/structure_factors.h"
 #include "simtbx/kokkos/detector.h"
-#include "kokkos_types.h"
+#include "kokkostbx/kokkos_types.h"
 
 namespace simtbx { namespace Kokkos {
 
diff --git a/simtbx/kokkos/simulation_kernels.h b/simtbx/kokkos/simulation_kernels.h
index aa010d7654..84c4152423 100644
--- a/simtbx/kokkos/simulation_kernels.h
+++ b/simtbx/kokkos/simulation_kernels.h
@@ -1,6 +1,13 @@
-#include <simtbx/kokkos/kokkos_types.h>
+#include <kokkostbx/kokkos_types.h>
+#include <kokkostbx/kokkos_vector3.h>
+#include <kokkostbx/kokkos_matrix3.h>
+using vec3 = kokkostbx::vector3<CUDAREAL>;
+using mat3 = kokkostbx::matrix3<CUDAREAL>;
+
 #include <simtbx/nanoBragg/nanotypes.h>
 #include <simtbx/kokkos/kernel_math.h>
+
+
 using simtbx::nanoBragg::shapetype;
 using simtbx::nanoBragg::hklParams;
 using simtbx::nanoBragg::SQUARE;
@@ -9,6 +16,7 @@ using simtbx::nanoBragg::GAUSS;
 using simtbx::nanoBragg::GAUSS_ARGCHK;
 using simtbx::nanoBragg::TOPHAT;
 
+
 void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
     int roi_ymin, int roi_ymax, int oversample, int point_pixel,
     CUDAREAL pixel_size, CUDAREAL subpixel_size, int steps, CUDAREAL detector_thickstep,
@@ -47,8 +55,6 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
         const int s_k_max = s_k_min + s_k_range - 1;
         const int s_l_max = s_l_min + s_l_range - 1;
 
-
-
         const int total_pixels = spixels * fpixels;
 
         // add background from something amorphous
@@ -57,6 +63,20 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
 
        Kokkos::parallel_for("kokkosSpotsKernel", total_pixels, KOKKOS_LAMBDA(const int& pixIdx) {
 
+                vec3 sdet_tmp {sdet_vector(1), sdet_vector(2), sdet_vector(3)};
+                vec3 fdet_tmp {fdet_vector(1), fdet_vector(2), fdet_vector(3)};
+                vec3 odet_tmp {odet_vector(1), odet_vector(2), odet_vector(3)};
+                vec3 pix0_tmp {pix0_vector(1), pix0_vector(2), pix0_vector(3)};
+
+                vec3 beam_vector_tmp {beam_vector(1), beam_vector(2), beam_vector(3)};
+                vec3 spindle_vector_tmp {spindle_vector(1), spindle_vector(2), spindle_vector(3)};
+
+                vec3 a0_tmp {a0(1), a0(2), a0(3)};
+                vec3 b0_tmp {b0(1), b0(2), b0(3)};
+                vec3 c0_tmp {c0(1), c0(2), c0(3)};
+
+                vec3 polar_vector_tmp {polar_vector(1), polar_vector(2), polar_vector(3)};
+
                 const int fpixel = pixIdx % fpixels;
                 const int spixel = pixIdx / fpixels;
                 // allow for just a region of interest (roi) on detector to be rendered
@@ -106,38 +126,25 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
                                         //                      pixel_X = distance;
                                         //                      pixel_Y = Sdet-Ybeam;
                                         //                      pixel_Z = Fdet-Xbeam;
-                                        //CUDAREAL * pixel_pos = tmpVector1;
-                                        CUDAREAL pixel_pos[4];
-                                        pixel_pos[1] = Fdet * fdet_vector(1)
-                                                     + Sdet * sdet_vector(1)
-                                                     + Odet * odet_vector(1)
-                                                            + pix0_vector(1); // X
-                                        pixel_pos[2] = Fdet * fdet_vector(2)
-                                                     + Sdet * sdet_vector(2)
-                                                     + Odet * odet_vector(2)
-                                                            + pix0_vector(2); // Y
-                                        pixel_pos[3] = Fdet * fdet_vector(3)
-                                                     + Sdet * sdet_vector(3)
-                                                     + Odet * odet_vector(3)
-                                                            + pix0_vector(3); // Z
+                                        vec3 pixel_pos;
+                                        pixel_pos += Fdet * fdet_tmp;
+                                        pixel_pos += Sdet * sdet_tmp;
+                                        pixel_pos += Odet * odet_tmp;
+                                        pixel_pos += pix0_tmp;
+
+                                        CUDAREAL pixel_pos_tmp[] = {0, pixel_pos[0], pixel_pos[1], pixel_pos[2]};
 
                                         if (curved_detector) {
                                                 // construct detector pixel that is always "distance" from the sample
-                                                CUDAREAL dbvector[] = { 0.0, 0.0, 0.0, 0.0 };
-                                                dbvector[1] = distance * beam_vector(1);
-                                                dbvector[2] = distance * beam_vector(2);
-                                                dbvector[3] = distance * beam_vector(3);
+                                                vec3 dbvector = distance * vec3{beam_vector(1), beam_vector(2), beam_vector(3)};
                                                 // treat detector pixel coordinates as radians
-                                                CUDAREAL newvector[] = { 0.0, 0.0, 0.0, 0.0 };
-                                                rotate_axis(dbvector, newvector, sdet_vector, pixel_pos[2] / distance);
-                                                rotate_axis(newvector, pixel_pos, fdet_vector, pixel_pos[3] / distance);
-                                                // rotate(vector,pixel_pos,0,pixel_pos[3]/distance,pixel_pos[2]/distance);
+                                                vec3 newvector = dbvector.rotate_around_axis(sdet_tmp, pixel_pos.y_val() / distance );
+                                                pixel_pos = newvector.rotate_around_axis(fdet_tmp, pixel_pos.z_val() / distance );
                                         }
 
                                         // construct the diffracted-beam unit vector to this sub-pixel
-                                        //CUDAREAL * diffracted = tmpVector2;
-                                        CUDAREAL diffracted[4];
-                                        CUDAREAL airpath = unitize(pixel_pos, diffracted);
+                                        CUDAREAL airpath = pixel_pos.length();
+                                        vec3 diffracted = pixel_pos.get_unit_vector();
 
                                         // solid angle subtended by a pixel: (pix/airpath)^2*cos(2theta)
                                         CUDAREAL omega_pixel = pixel_size * pixel_size / airpath / airpath * close_distance / airpath;
@@ -150,11 +157,8 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
                                         CUDAREAL capture_fraction = 1.0;
                                         if (detector_thick > 0.0 && detector_mu> 0.0) {
                                                 // inverse of effective thickness increase
-                                                CUDAREAL odet[4];
-                                                odet[1] = odet_vector(1);
-                                                odet[2] = odet_vector(2);
-                                                odet[3] = odet_vector(3);
-                                                CUDAREAL parallax = dot_product(odet, diffracted);
+                                                vec3 odet{odet_vector(1), odet_vector(2), odet_vector(3)};
+                                                CUDAREAL parallax = odet.dot(diffracted);
                                                 capture_fraction = exp(-thick_tic * detector_thickstep / detector_mu / parallax)
                                                                 - exp(-(thick_tic + 1) * detector_thickstep / detector_mu / parallax);
                                         }
@@ -164,28 +168,17 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
                                         for (source = 0; source < sources; ++source) {
 
                                                 // retrieve stuff from cache
-                                                CUDAREAL incident[4];
-                                                incident[1] = -source_X(source);
-                                                incident[2] = -source_Y(source);
-                                                incident[3] = -source_Z(source);
+                                                vec3 incident = {-source_X(source), -source_Y(source), -source_Z(source)};
                                                 CUDAREAL lambda = source_lambda(source);
                                                 CUDAREAL source_fraction = source_I(source);
 
                                                 // construct the incident beam unit vector while recovering source distance
                                                 // TODO[Giles]: Optimization! We can unitize the source vectors before passing them in.
-                                                unitize(incident, incident);
+                                                incident.normalize();
 
                                                 // construct the scattering vector for this pixel
-                                                CUDAREAL scattering[4];
-                                                scattering[1] = (diffracted[1] - incident[1]) / lambda;
-                                                scattering[2] = (diffracted[2] - incident[2]) / lambda;
-                                                scattering[3] = (diffracted[3] - incident[3]) / lambda;
-
-                                                #ifdef __CUDA_ARCH__
-                                                CUDAREAL stol = 0.5 * norm3d(scattering[1], scattering[2], scattering[3]);
-                                                #else
-                                                CUDAREAL stol = 0.5 * sqrt(scattering[1]*scattering[1] + scattering[2]*scattering[2] + scattering[3]*scattering[3]);
-                                                #endif
+                                                vec3 scattering = (diffracted - incident) / lambda;
+                                                CUDAREAL stol = 0.5 * scattering.length();
 
                                                 // rough cut to speed things up when we aren't using whole detector
                                                 if (dmin > 0.0 && stol > 0.0) {
@@ -197,7 +190,7 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
                                                 // polarization factor
                                                 if (!nopolar) {
                                                         // need to compute polarization factor
-                                                        polar = polarization_factor(polarization, incident, diffracted, polar_vector);
+                                                        polar = polarization_factor(polarization, incident, diffracted, polar_vector_tmp);
                                                 } else {
                                                         polar = 1.0;
                                                 }
@@ -206,54 +199,37 @@ void kokkosSpotsKernel(int spixels, int fpixels, int roi_xmin, int roi_xmax,
                                                 for (int phi_tic = 0; phi_tic < phisteps; ++phi_tic) {
                                                         CUDAREAL phi = phistep * phi_tic + phi0;
 
-                                                        CUDAREAL ap[4];
-                                                        CUDAREAL bp[4];
-                                                        CUDAREAL cp[4];
-
                                                         // rotate about spindle if necessary
-                                                        rotate_axis(a0, ap, spindle_vector, phi);
-                                                        rotate_axis(b0, bp, spindle_vector, phi);
-                                                        rotate_axis(c0, cp, spindle_vector, phi);
+                                                        vec3 ap = a0_tmp.rotate_around_axis(spindle_vector_tmp, phi);
+                                                        vec3 bp = b0_tmp.rotate_around_axis(spindle_vector_tmp, phi);
+                                                        vec3 cp = c0_tmp.rotate_around_axis(spindle_vector_tmp, phi);
 
                                                         // enumerate mosaic domains
                                                         for (int mos_tic = 0; mos_tic < mosaic_domains; ++mos_tic) {
                                                                 // apply mosaic rotation after phi rotation
-                                                                CUDAREAL a[4];
-                                                                CUDAREAL b[4];
-                                                                CUDAREAL c[4];
+                                                                // CUDAREAL a[4];
+                                                                // CUDAREAL b[4];
+                                                                // CUDAREAL c[4];
+                                                                vec3 a, b, c;
 
                                                                 if (mosaic_spread > 0.0) {
-                                                                        CUDAREAL umat[] = {mosaic_umats(mos_tic * 9 + 0),
-                                                                                           mosaic_umats(mos_tic * 9 + 1),
-                                                                                           mosaic_umats(mos_tic * 9 + 2),
-                                                                                           mosaic_umats(mos_tic * 9 + 3),
-                                                                                           mosaic_umats(mos_tic * 9 + 4),
-                                                                                           mosaic_umats(mos_tic * 9 + 5),
-                                                                                           mosaic_umats(mos_tic * 9 + 6),
-                                                                                           mosaic_umats(mos_tic * 9 + 7),
-                                                                                           mosaic_umats(mos_tic * 9 + 8)};
-
-                                                                        rotate_umat(ap, a, umat);
-                                                                        rotate_umat(bp, b, umat);
-                                                                        rotate_umat(cp, c, umat);
+                                                                        mat3 umat;
+                                                                        for (int i=0; i<9; ++i) {
+                                                                                umat[i] = mosaic_umats(mos_tic * 9 + i);
+                                                                        }
+                                                                        a = umat.dot(ap);
+                                                                        b = umat.dot(bp);
+                                                                        c = umat.dot(cp);
                                                                 } else {
-                                                                        a[1] = ap[1];
-                                                                        a[2] = ap[2];
-                                                                        a[3] = ap[3];
-                                                                        b[1] = bp[1];
-                                                                        b[2] = bp[2];
-                                                                        b[3] = bp[3];
-                                                                        c[1] = cp[1];
-                                                                        c[2] = cp[2];
-                                                                        c[3] = cp[3];
+                                                                        a = ap;
+                                                                        b = bp;
+                                                                        c = cp;
                                                                 }
 
-
                                                                 // construct fractional Miller indicies
-
-                                                                CUDAREAL h = dot_product(a, scattering);
-                                                                CUDAREAL k = dot_product(b, scattering);
-                                                                CUDAREAL l = dot_product(c, scattering);
+                                                                CUDAREAL h = a.dot(scattering);
+                                                                CUDAREAL k = b.dot(scattering);
+                                                                CUDAREAL l = c.dot(scattering);
 
                                                                 // round off to nearest whole index
                                                                 int h0 = ceil(h - 0.5);
@@ -628,6 +604,13 @@ void debranch_maskall_Kernel(int npanels, int spixels, int fpixels, int total_pi
 //                 float * max_I_x_reduction,
 //                 float * max_I_y_reduction, bool * rangemap);
 
+template <typename T, typename U>
+void add_array( view_1d_t<T> lhs, const view_1d_t<U> rhs ) {
+  Kokkos::parallel_for("add_arrays", lhs.span(), KOKKOS_LAMBDA(const int& i) {
+    lhs( i ) = lhs( i ) + (T)rhs( i );
+    rhs( i ) = 0;
+  });
+}
 
 void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int override_source,
     CUDAREAL pixel_size, int spixels, int fpixels, int detector_thicksteps,
@@ -670,6 +653,13 @@ void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int ove
     // const int stride = fstride * sstride;
     Kokkos::parallel_for("add_background", total_pixels, KOKKOS_LAMBDA(const int& pixIdx) {
 
+        vec3 sdet_tmp {sdet_vector(1), sdet_vector(2), sdet_vector(3)};
+        vec3 fdet_tmp {fdet_vector(1), fdet_vector(2), fdet_vector(3)};
+        vec3 odet_tmp {odet_vector(1), odet_vector(2), odet_vector(3)};
+        vec3 pix0_tmp {pix0_vector(1), pix0_vector(2), pix0_vector(3)};
+
+        vec3 polar_vector_tmp {polar_vector(1), polar_vector(2), polar_vector(3)};
+
         const int fpixel = pixIdx % fpixels;
         const int spixel = pixIdx / fpixels;
         // reset background photon count for this pixel
@@ -685,17 +675,21 @@ void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int ove
                 for(int thick_tic=0; thick_tic<detector_thicksteps; ++thick_tic) {
                     // assume "distance" is to the front of the detector sensor layer
                     CUDAREAL Odet = thick_tic*detector_thickstep;
-                    CUDAREAL pixel_pos[4];
+                //     CUDAREAL pixel_pos[4];
+
+                //     pixel_pos[0] = 0.0;
+                //     pixel_pos[1] = Fdet * fdet_vector(1) + Sdet * sdet_vector(1) + Odet * odet_vector(1) + pix0_vector(1); // X
+                //     pixel_pos[2] = Fdet * fdet_vector(2) + Sdet * sdet_vector(2) + Odet * odet_vector(2) + pix0_vector(2); // Y
+                //     pixel_pos[3] = Fdet * fdet_vector(3) + Sdet * sdet_vector(3) + Odet * odet_vector(3) + pix0_vector(3); // Z
 
-                    pixel_pos[0] = 0.0;
-                    pixel_pos[1] = Fdet * fdet_vector(1) + Sdet * sdet_vector(1) + Odet * odet_vector(1) + pix0_vector(1); // X
-                    pixel_pos[2] = Fdet * fdet_vector(2) + Sdet * sdet_vector(2) + Odet * odet_vector(2) + pix0_vector(2); // Y
-                    pixel_pos[3] = Fdet * fdet_vector(3) + Sdet * sdet_vector(3) + Odet * odet_vector(3) + pix0_vector(3); // Z
+                    vec3 pixel_pos = Fdet * fdet_tmp + Sdet * sdet_tmp + Odet * odet_tmp + pix0_tmp;
 
                     // no curved detector option (future implementation)
                     // construct the diffracted-beam unit vector to this pixel
-                    CUDAREAL diffracted[4];
-                    CUDAREAL airpath = unitize(pixel_pos, diffracted);
+                //     CUDAREAL diffracted[4];
+                //     CUDAREAL airpath = unitize(pixel_pos, diffracted);
+                    CUDAREAL airpath = pixel_pos.length();
+                    vec3 diffracted = pixel_pos.get_unit_vector();
 
                     // solid angle subtended by a pixel: (pix/airpath)^2*cos(2theta)
                     CUDAREAL omega_pixel = pixel_size*pixel_size/airpath/airpath*close_distance/airpath;
@@ -706,7 +700,8 @@ void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int ove
                     CUDAREAL capture_fraction = 1.0;
                     if(detector_thick > 0.0){
                         // inverse of effective thickness increase
-                        CUDAREAL parallax = diffracted[1] * odet_vector(1) + diffracted[2] * odet_vector(2) + diffracted[3] * odet_vector(3);
+                        // CUDAREAL parallax = diffracted[1] * odet_vector(1) + diffracted[2] * odet_vector(2) + diffracted[3] * odet_vector(3);
+                        CUDAREAL parallax = diffracted.dot(odet_tmp);
                         capture_fraction = exp(-thick_tic*detector_thickstep/detector_attnlen/parallax)
                                             -exp(-(thick_tic+1)*detector_thickstep/detector_attnlen/parallax);
                     }
@@ -715,23 +710,27 @@ void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int ove
                     for(int source=source_start; source<sources; ++source) {
 
                         // retrieve stuff from cache
-                        CUDAREAL incident[4];
-                        incident[1] = -source_X(source);
-                        incident[2] = -source_Y(source);
-                        incident[3] = -source_Z(source);
+                        // CUDAREAL incident[4];
+                        // incident[1] = -source_X(source);
+                        // incident[2] = -source_Y(source);
+                        // incident[3] = -source_Z(source);
+                        vec3 incident {-source_X(source), -source_Y(source), -source_Z(source)};
                         CUDAREAL lambda = source_lambda(source);
                         CUDAREAL source_fraction = full_spectrum * source_I(source) + n_source_scale;
                         // construct the incident beam unit vector while recovering source distance
-                        unitize(incident, incident);
+                        incident.normalize();
+                        // unitize(incident, incident);
 
                         // construct the scattering vector for this pixel
-                        CUDAREAL scattering[4];
-                        scattering[1] = (diffracted[1]-incident[1])/lambda;
-                        scattering[2] = (diffracted[2]-incident[2])/lambda;
-                        scattering[3] = (diffracted[3]-incident[3])/lambda;
-                        magnitude(scattering);
+                        vec3 scattering = (diffracted - incident) / lambda;
+                        // CUDAREAL scattering[4];
+                        // scattering[1] = (diffracted[1]-incident[1])/lambda;
+                        // scattering[2] = (diffracted[2]-incident[2])/lambda;
+                        // scattering[3] = (diffracted[3]-incident[3])/lambda;
+                        // magnitude(scattering);
                         // sin(theta)/lambda is half the scattering vector length
-                        CUDAREAL stol = 0.5*scattering[0];
+                        // CUDAREAL stol = 0.5*scattering[0];
+                        CUDAREAL stol = 0.5*scattering.length();
 
                         // now we need to find the nearest four "stol file" points
                         while(stol > stol_of(nearest) && nearest <= stols){ ++nearest; };
@@ -761,8 +760,9 @@ void add_background_kokkos_kernel(int sources, int nanoBragg_oversample, int ove
                         CUDAREAL polar = 1.0;
                         if(! nopolar){
                             // need to compute polarization factor
-                            CUDAREAL axis[] = {polar_vector(0), polar_vector(1), polar_vector(2), polar_vector(3)};
-                            polar = polarization_factor(polarization, incident, diffracted, polar_vector);
+                        //     CUDAREAL axis[] = {polar_vector(0), polar_vector(1), polar_vector(2), polar_vector(3)};
+                        //     polar = polarization_factor(polarization, incident, diffracted, polar_vector);
+                            polar = polarization_factor(polarization, incident, diffracted, polar_vector_tmp);
                         }
 
                         // accumulate unscaled pixel intensity from this
diff --git a/simtbx/kokkos/structure_factors.cpp b/simtbx/kokkos/structure_factors.cpp
index 31f0f3af06..db8628d94c 100644
--- a/simtbx/kokkos/structure_factors.cpp
+++ b/simtbx/kokkos/structure_factors.cpp
@@ -1,6 +1,6 @@
 //#include "cudatbx/cuda_base.cuh"
 #include "simtbx/kokkos/structure_factors.h"
-#include "simtbx/kokkos/kokkos_utils.h"
+#include "kokkostbx/kokkos_utils.h"
 #include <cstdio>
 
 //using Kokkos::create_mirror_view;
@@ -15,7 +15,7 @@ namespace simtbx { namespace Kokkos {
     double * raw_ptr = linear_amplitudes.begin();
 
     vector_cudareal_t device_Fhkl( "device_Fhkl", linear_amplitudes.size() );
-    transfer_shared2kokkos(device_Fhkl, linear_amplitudes);
+    kokkostbx::transfer_shared2kokkos(device_Fhkl, linear_amplitudes);
     d_channel_Fhkl.push_back(device_Fhkl);
 
     if (d_channel_Fhkl.size()==1) { //first time through
@@ -28,7 +28,7 @@ namespace simtbx { namespace Kokkos {
                               af::shared<double> linear_amplitudes){
     double * raw_ptr = linear_amplitudes.begin();
     vector_cudareal_t device_this_channel = d_channel_Fhkl[ichannel];
-    transfer_shared2kokkos(device_this_channel, linear_amplitudes);
+    kokkostbx::transfer_shared2kokkos(device_this_channel, linear_amplitudes);
   }
 
   void
diff --git a/simtbx/kokkos/structure_factors.h b/simtbx/kokkos/structure_factors.h
index bf1124a0c9..9ae535e6a6 100644
--- a/simtbx/kokkos/structure_factors.h
+++ b/simtbx/kokkos/structure_factors.h
@@ -5,7 +5,7 @@
 #include "scitbx/array_family/shared.h"
 #include "cctbx/miller.h"
 #include "simtbx/nanoBragg/nanotypes.h"
-#include "simtbx/kokkos/kokkos_types.h"
+#include "kokkostbx/kokkos_types.h"
 
 
 using simtbx::nanoBragg::hklParams;
diff --git a/simtbx/nanoBragg/nanoBragg.h b/simtbx/nanoBragg/nanoBragg.h
index 808316e369..e3f5a39e94 100644
--- a/simtbx/nanoBragg/nanoBragg.h
+++ b/simtbx/nanoBragg/nanoBragg.h
@@ -20,7 +20,7 @@
 #include <boost/math/special_functions/erf.hpp>
 #include <boost/math/special_functions/fpclassify.hpp>
 #include <boost_adaptbx/python_streambuf.h>
-#include <omptbx/omp_or_stubs.h>
+// #include <omptbx/omp_or_stubs.h>
 #include <simtbx/nanoBragg/nanotypes.h>
 
 using boost::math::erf;


Test patch:
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_Fcell_deriv.py b/simtbx/diffBragg/tests/tst_diffBragg_Fcell_deriv.py
index a6f0faec1a..a24b0bee4a 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_Fcell_deriv.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_Fcell_deriv.py
@@ -1,151 +1,157 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--curvatures", action='store_true')
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-if args.plot:
-    import pylab as plt
-    plt.figure()
-import numpy as np
-from scipy.spatial.transform import Rotation
-from scipy.stats import linregress
-
-from scitbx.array_family import flex
-from scitbx.matrix import sqr
-from scitbx.matrix import rec
-
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-from cctbx import uctbx
-from simtbx.nanoBragg.sim_data import SimData
-from dxtbx.model.crystal import Crystal
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-# make a nanoBragg crystal to pass to diffBragg
-nbcryst = NBcrystal(init_defaults=True)
-nbcryst.dxtbx_crystal = C
-nbcryst.n_mos_domains = 1
-nbcryst.thick_mm = 0.01
-nbcryst.Ncells_abc = (7, 7, 7)
-
-# make an instance of diffBRagg, use the simData wrapper
-SIM = SimData(use_default_crystal=True)
-# overwrite the default detector with a smaller pixels one
-SIM.detector = SimData.simple_detector(300, 0.1, (700, 700))
-SIM.crystal = nbcryst
-Fcell = 1e6
-SIM.instantiate_diffBragg(oversample=0, verbose=0, interpolate=0, default_F=Fcell)
-# D is an instance of diffBragg with reasonable parameters
-# and our dxtbx crystal created above
-D = SIM.D
-
-D.progress_meter = True
-
-# initialize the derivative manager for Fcell
-fcell = 11  # internal index of fcell manager within diffBragg
-D.refine(fcell)
-D.initialize_managers()
-
-#roi = ((0, 699), (0, 699))
-#rX = slice(roi[0][0], roi[0][1], 1)
-#rY = slice(roi[1][0], roi[1][1], 1)
-#D.region_of_interest = roi
-
-# compute the scattering and its derivative
-print("Adding diffBragg spots")
-# Set all miller indices to have the same Fcell value
-indices, data = D.Fhkl_tuple
-data = flex.double(np.ones(len(indices))*Fcell)
-D.F000 = Fcell
-D.Fhkl_tuple = indices, data.deep_copy(), None
-D.compute_curvatures = True
-
-# NOTE optionally focus on a single Bragg peak
-#x1,x2 = 335, 365
-#y1,y2 = 155, 185
-#D.region_of_interest = (x1, x2), (y1, y2)
-D.add_diffBragg_spots()
-
-print("Done!")
-img = D.raw_pixels_roi.as_numpy_array()
-
-# reset all pixel values
-D.raw_pixels *= 0
-D.raw_pixels_roi *= 0
-first_deriv = D.get_derivative_pixels(fcell).as_numpy_array()
-if args.curvatures:
-    second_deriv = D.get_second_derivative_pixels(fcell).as_numpy_array()
-
-# iterate over the parameters and do a finite difference test for each one
-shifts = [1e-2*(i**2) for i in range(1, 20)]
-#shifts = np.array([.5,1,2,4,8,16,32]) # 1e-3, 1e-2, 1e-1, 1, 10]
-all_error = []
-all_delta_F = []
-
-for i_shift, percent_shift in enumerate(shifts):
-
-    delta_F = Fcell * percent_shift*1e-2
-
-    D.F000 = Fcell + delta_F
-    D.Fhkl_tuple = indices, data.deep_copy()+delta_F, None
-    D.default_F = Fcell+delta_F
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    if args.plot:
+        import pylab as plt
+        plt.figure()
+    import numpy as np
+    from scipy.spatial.transform import Rotation
+    from scipy.stats import linregress
+
+    from scitbx.array_family import flex
+    from scitbx.matrix import sqr
+    from scitbx.matrix import rec
+
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    from cctbx import uctbx
+    from simtbx.nanoBragg.sim_data import SimData
+    from dxtbx.model.crystal import Crystal
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    # make a nanoBragg crystal to pass to diffBragg
+    nbcryst = NBcrystal(init_defaults=True)
+    nbcryst.dxtbx_crystal = C
+    nbcryst.n_mos_domains = 1
+    nbcryst.thick_mm = 0.01
+    nbcryst.Ncells_abc = (7, 7, 7)
+
+    # make an instance of diffBRagg, use the simData wrapper
+    SIM = SimData(use_default_crystal=True)
+    # overwrite the default detector with a smaller pixels one
+    SIM.detector = SimData.simple_detector(300, 0.1, (700, 700))
+    SIM.crystal = nbcryst
+    Fcell = 1e6
+    SIM.instantiate_diffBragg(oversample=0, verbose=0, interpolate=0, default_F=Fcell)
+    # D is an instance of diffBragg with reasonable parameters
+    # and our dxtbx crystal created above
+    D = SIM.D
+
+    D.progress_meter = True
+
+    # initialize the derivative manager for Fcell
+    fcell = 11  # internal index of fcell manager within diffBragg
+    D.refine(fcell)
+    D.initialize_managers()
+
+    #roi = ((0, 699), (0, 699))
+    #rX = slice(roi[0][0], roi[0][1], 1)
+    #rY = slice(roi[1][0], roi[1][1], 1)
+    #D.region_of_interest = roi
+
+    # compute the scattering and its derivative
+    print("Adding diffBragg spots")
+    # Set all miller indices to have the same Fcell value
+    indices, data = D.Fhkl_tuple
+    data = flex.double(np.ones(len(indices))*Fcell)
+    D.F000 = Fcell
+    D.Fhkl_tuple = indices, data.deep_copy(), None
+    D.compute_curvatures = True
+
+    # NOTE optionally focus on a single Bragg peak
+    #x1,x2 = 335, 365
+    #y1,y2 = 155, 185
+    #D.region_of_interest = (x1, x2), (y1, y2)
     D.add_diffBragg_spots()
 
-    img_forward = D.raw_pixels_roi.as_numpy_array()
-    # reset for next computation
-    D.raw_pixels_roi *= 0
+    print("Done!")
+    img = D.raw_pixels_roi.as_numpy_array()
+
+    # reset all pixel values
     D.raw_pixels *= 0
+    D.raw_pixels_roi *= 0
+    first_deriv = D.get_derivative_pixels(fcell).as_numpy_array()
+    if args.curvatures:
+        second_deriv = D.get_second_derivative_pixels(fcell).as_numpy_array()
 
-    finite_deriv = (img_forward-img) / delta_F
-    ave_error = np.abs(finite_deriv - first_deriv).mean()
-    #ave_error = np.abs(finite_deriv[bragg] - first_deriv[bragg]).mean()
-    all_error.append(ave_error)
-    all_delta_F.append(delta_F)
+    # iterate over the parameters and do a finite difference test for each one
+    shifts = [1e-2*(i**2) for i in range(1, 20)]
+    #shifts = np.array([.5,1,2,4,8,16,32]) # 1e-3, 1e-2, 1e-1, 1, 10]
+    all_error = []
+    all_delta_F = []
 
-    if args.curvatures:
-        # estimate the second derivative
-        D.F000 = Fcell - delta_F
-        D.Fhkl_tuple = indices, data.deep_copy() - delta_F, None
-        D.default_F = Fcell - delta_F
+    for i_shift, percent_shift in enumerate(shifts):
+
+        delta_F = Fcell * percent_shift*1e-2
+
+        D.F000 = Fcell + delta_F
+        D.Fhkl_tuple = indices, data.deep_copy()+delta_F, None
+        D.default_F = Fcell+delta_F
         D.add_diffBragg_spots()
-        img_backward = D.raw_pixels_roi.as_numpy_array()
 
+        img_forward = D.raw_pixels_roi.as_numpy_array()
         # reset for next computation
         D.raw_pixels_roi *= 0
         D.raw_pixels *= 0
 
-        finite_second_deriv = (img_forward - 2*img + img_backward) / delta_F / delta_F
-        assert np.allclose(finite_second_deriv, second_deriv)
-        # NOTE: second derivative is a constant, doesnt depend on F_cell, hence these are always super close
-
-    print("error=%2.7g, step=%f (%d/%d)" % (ave_error, delta_F, i_shift + 1, len(shifts)))
-
-if args.plot:
-    plt.close()
-    plt.plot(all_delta_F, all_error, 'o')
-    plt.show()
-
-# one expects errors to scale linearly with parameter shift for finite differences
-l = linregress(all_delta_F, all_error)
-assert l.rvalue > .9999  # this is definitely a line!
-assert l.slope > 0
-assert l.pvalue < 1e-6
-print("Error versus parameter shift fits a line with slope=%2.7g and Correleation Coef=%2.7g" % (l.slope, l.rvalue))
-print("OK!")
+        finite_deriv = (img_forward-img) / delta_F
+        ave_error = np.abs(finite_deriv - first_deriv).mean()
+        #ave_error = np.abs(finite_deriv[bragg] - first_deriv[bragg]).mean()
+        all_error.append(ave_error)
+        all_delta_F.append(delta_F)
+
+        if args.curvatures:
+            # estimate the second derivative
+            D.F000 = Fcell - delta_F
+            D.Fhkl_tuple = indices, data.deep_copy() - delta_F, None
+            D.default_F = Fcell - delta_F
+            D.add_diffBragg_spots()
+            img_backward = D.raw_pixels_roi.as_numpy_array()
+
+            # reset for next computation
+            D.raw_pixels_roi *= 0
+            D.raw_pixels *= 0
+
+            finite_second_deriv = (img_forward - 2*img + img_backward) / delta_F / delta_F
+            assert np.allclose(finite_second_deriv, second_deriv)
+            # NOTE: second derivative is a constant, doesnt depend on F_cell, hence these are always super close
+
+        print("error=%2.7g, step=%f (%d/%d)" % (ave_error, delta_F, i_shift + 1, len(shifts)))
+
+    if args.plot:
+        plt.close()
+        plt.plot(all_delta_F, all_error, 'o')
+        plt.show()
+
+    # one expects errors to scale linearly with parameter shift for finite differences
+    l = linregress(all_delta_F, all_error)
+    assert l.rvalue > .9999  # this is definitely a line!
+    assert l.slope > 0
+    assert l.pvalue < 1e-6
+    print("Error versus parameter shift fits a line with slope=%2.7g and Correleation Coef=%2.7g" % (l.slope, l.rvalue))
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_Fhkl_complex.py b/simtbx/diffBragg/tests/tst_diffBragg_Fhkl_complex.py
index f17376c7a8..6a23872f9d 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_Fhkl_complex.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_Fhkl_complex.py
@@ -1,53 +1,59 @@
 from __future__ import division
+#from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
 from dxtbx.model import Crystal
 from simtbx.diffBragg.utils import  fcalc_from_pdb
 
 import sys
-if "--cuda" in sys.argv:
+if "--kokkos" in sys.argv:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-Fcomplex = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=(79,79,38.90,90,90), symbol="P43212",as_amplitudes=False)
-Famp = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=(79,79,38.90,90,90), symbol="P43212",as_amplitudes=True)
-
-
-def sim_spots(F):
-
-    a_real = (79,0,0)
-    b_real = (0,79,0)
-    c_real = (0,0,38)
-    C = Crystal(a_real, b_real, c_real, 'P43212')
-
-    nbcryst = NBcrystal(init_defaults=True)
-    nbcryst.dxtbx_crystal = C  # simulate ground truth
-    nbcryst.thick_mm = 0.1
-    nbcryst.Ncells_abc = 10,10,10
-
-    nbcryst.miller_array = F
-    print("Ground truth ncells = %f" % (nbcryst.Ncells_abc[0]))
-
-    # ground truth detector
-    from simtbx.nanoBragg.sim_data import SimData
-    DET_gt = SimData.simple_detector(150, 0.177, (600, 600))
-
-    # initialize the simulator
-    SIM = SimData(use_default_crystal=True)
-    SIM.detector = DET_gt
-    SIM.crystal = nbcryst
-    SIM.instantiate_diffBragg(oversample=0)
-    SIM.D.default_F = 0
-    SIM.D.progress_meter = False
-    SIM.D.add_diffBragg_spots()
-    SIM.D.F000 = 0
-    SPOTS = SIM.D.raw_pixels.as_numpy_array()
-    SIM.D.free_all()
-    SIM.D.free_Fhkl2()
-    return SPOTS
-
-A = sim_spots(Fcomplex)
-B = sim_spots(Famp)
-import numpy as np
-assert np.allclose(A,B)
-print("OK!")
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    Fcomplex = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=(79,79,38.90,90,90), symbol="P43212",as_amplitudes=False)
+    Famp = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=(79,79,38.90,90,90), symbol="P43212",as_amplitudes=True)
+
+    def sim_spots(F):
+
+        a_real = (79,0,0)
+        b_real = (0,79,0)
+        c_real = (0,0,38)
+        C = Crystal(a_real, b_real, c_real, 'P43212')
+
+        nbcryst = NBcrystal(init_defaults=True)
+        nbcryst.dxtbx_crystal = C  # simulate ground truth
+        nbcryst.thick_mm = 0.1
+        nbcryst.Ncells_abc = 10,10,10
+
+        nbcryst.miller_array = F
+        print("Ground truth ncells = %f" % (nbcryst.Ncells_abc[0]))
+
+        # ground truth detector
+        from simtbx.nanoBragg.sim_data import SimData
+        DET_gt = SimData.simple_detector(150, 0.177, (600, 600))
+
+        # initialize the simulator
+        SIM = SimData(use_default_crystal=True)
+        SIM.detector = DET_gt
+        SIM.crystal = nbcryst
+        SIM.instantiate_diffBragg(oversample=0)
+        SIM.D.default_F = 0
+        SIM.D.progress_meter = False
+        SIM.D.add_diffBragg_spots()
+        SIM.D.F000 = 0
+        SPOTS = SIM.D.raw_pixels.as_numpy_array()
+        SIM.D.free_all()
+        SIM.D.free_Fhkl2()
+        return SPOTS
+
+    A = sim_spots(Fcomplex)
+    B = sim_spots(Famp)
+    import numpy as np
+    assert np.allclose(A,B)
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_change_of_basis.py b/simtbx/diffBragg/tests/tst_diffBragg_change_of_basis.py
index 9b626230bb..c0e536615a 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_change_of_basis.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_change_of_basis.py
@@ -1,4 +1,6 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from simtbx.nanoBragg import sim_data
 from scitbx.matrix import sqr
@@ -8,41 +10,44 @@
 from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
 import numpy as np
 import sys
-if "--cuda" in sys.argv:
+if "--kokkos" in sys.argv:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-nbr = NBcrystal()
-nbr.dxtbx_crystal = C
-
-S = sim_data.SimData(use_default_crystal=True)
-S.crystal = nbr
-S.instantiate_diffBragg(auto_set_spotscale=True)
-S.D.add_diffBragg_spots()
-img = S.D.raw_pixels.as_numpy_array()
-
-# simulate the primitive cell directly
-to_p1 = C.get_space_group().info().change_of_basis_op_to_primitive_setting()
-Cp1 = C.change_basis(to_p1)
-nbr2 = NBcrystal()
-nbr2.dxtbx_crystal = Cp1
-
-S2 = sim_data.SimData()
-S2.crystal = nbr2
-S2.instantiate_diffBragg(auto_set_spotscale=True)
-S2.D.add_diffBragg_spots()
-img2 = S2.D.raw_pixels.as_numpy_array()
-
-# rescale because currently volume is computed incorrectly
-img2 = img2 * S.D.spot_scale / S2.D.spot_scale
-
-assert S.D.Omatrix == tuple(to_p1.c_inv().r().transpose().as_double())
-assert S2.D.Omatrix == (1, 0, 0, 0, 1, 0, 0, 0, 1)
-assert np.allclose(img, img2)
-
-print("OK")
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    nbr = NBcrystal()
+    nbr.dxtbx_crystal = C
+
+    S = sim_data.SimData(use_default_crystal=True)
+    S.crystal = nbr
+    S.instantiate_diffBragg(auto_set_spotscale=True)
+    S.D.add_diffBragg_spots()
+    img = S.D.raw_pixels.as_numpy_array()
+
+    # simulate the primitive cell directly
+    to_p1 = C.get_space_group().info().change_of_basis_op_to_primitive_setting()
+    Cp1 = C.change_basis(to_p1)
+    nbr2 = NBcrystal()
+    nbr2.dxtbx_crystal = Cp1
+
+    S2 = sim_data.SimData()
+    S2.crystal = nbr2
+    S2.instantiate_diffBragg(auto_set_spotscale=True)
+    S2.D.add_diffBragg_spots()
+    img2 = S2.D.raw_pixels.as_numpy_array()
+
+    # rescale because currently volume is computed incorrectly
+    img2 = img2 * S.D.spot_scale / S2.D.spot_scale
+
+    assert S.D.Omatrix == tuple(to_p1.c_inv().r().transpose().as_double())
+    assert S2.D.Omatrix == (1, 0, 0, 0, 1, 0, 0, 0, 1)
+    assert np.allclose(img, img2)
+
+    print("OK")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_deriv_rois.py b/simtbx/diffBragg/tests/tst_diffBragg_deriv_rois.py
index 3fad6744e6..bd3100bcb3 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_deriv_rois.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_deriv_rois.py
@@ -1,35 +1,41 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 from simtbx.diffBragg.utils import get_diffBragg_instance
 import sys
-if "--cuda" in sys.argv:
+if "--kokkos" in sys.argv:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
 
-import numpy as np
+    import numpy as np
 
-print ("Instantiating diffBragg")
-D = get_diffBragg_instance()
-D.vectorize_umats()
-print ("Declaring a refinement manager")
-rotX = 0
-D.refine(rotX)  # defaults full image as ROI
-D.initialize_managers()
-print ("Running diffBragg on full image")
-D.add_diffBragg_spots()
-img_sh = D.raw_pixels.as_numpy_array().shape
-print ("Extracting derivative pixels")
-full_deriv_image = D.get_derivative_pixels(rotX).as_numpy_array()
-print()
-print ("Re-running but using only the ROIs")
-rois = (20, 50, 10, 80), (10, 20, 80, 90)
-for x1, x2, y1, y2 in rois:
-    print("Declarind ROI %d %d %d %d" % (x1, x2, y1, y2))
-    D.raw_pixels*= 0
-    D.region_of_interest = ((x1, x2), (y1, y2))
-    print("Running diffBRagg on ROI")
+    print ("Instantiating diffBragg")
+    D = get_diffBragg_instance()
+    D.vectorize_umats()
+    print ("Declaring a refinement manager")
+    rotX = 0
+    D.refine(rotX)  # defaults full image as ROI
+    D.initialize_managers()
+    print ("Running diffBragg on full image")
     D.add_diffBragg_spots()
-    print("Extracting derivative pixels")
-    roi_deriv_image = D.get_derivative_pixels(rotX).as_numpy_array()
-    assert np.allclose(roi_deriv_image, full_deriv_image.reshape(img_sh)[y1:y2, x1:x2].ravel())
-    print("LOOP OVER")
-print("DONE!")
+    img_sh = D.raw_pixels.as_numpy_array().shape
+    print ("Extracting derivative pixels")
+    full_deriv_image = D.get_derivative_pixels(rotX).as_numpy_array()
+    print()
+    print ("Re-running but using only the ROIs")
+    rois = (20, 50, 10, 80), (10, 20, 80, 90)
+    for x1, x2, y1, y2 in rois:
+        print("Declarind ROI %d %d %d %d" % (x1, x2, y1, y2))
+        D.raw_pixels*= 0
+        D.region_of_interest = ((x1, x2), (y1, y2))
+        print("Running diffBRagg on ROI")
+        D.add_diffBragg_spots()
+        print("Extracting derivative pixels")
+        roi_deriv_image = D.get_derivative_pixels(rotX).as_numpy_array()
+        assert np.allclose(roi_deriv_image, full_deriv_image.reshape(img_sh)[y1:y2, x1:x2].ravel())
+        print("LOOP OVER")
+    print("DONE!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_detdist_derivatives.py b/simtbx/diffBragg/tests/tst_diffBragg_detdist_derivatives.py
index 281a8cc124..55f30ef84f 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_detdist_derivatives.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_detdist_derivatives.py
@@ -5,381 +5,387 @@
 one to update those models for beam/detector post-instantiation
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plotimages", action='store_true')
 parser.add_argument("--plotlines", action='store_true')
 parser.add_argument("--curvatures",action="store_true")
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-from scipy.stats import linregress
-import numpy as np
-import pylab as plt
-from simtbx.nanoBragg import sim_data
-from dxtbx.model import Panel
-from scitbx.matrix import sqr, col
-
-# part 1:
-# simple k_diffracted derivative
-fs = np.array([1, 0, 0])
-ss = np.array([0, -1, 0])
-
-# FIXME:
-k_incident = np.array([0, 0, -1])
-
-i, j = 10, 10
-pixsize_mm = 0.11
-detdist_mm = 150
-camera_size_mm = 50, 50
-origin_z = -detdist_mm
-origin_mm = np.array([-camera_size_mm[1]/2., camera_size_mm[0]/2., origin_z])
-
-unit = 1e-3
-
-# diffracted beam
-k = origin_mm + fs*i*pixsize_mm + ss*j*pixsize_mm
-k *= unit  # convert to meters like the code
-
-# FIXME mod1
-k = np.array([.01685, .02055, -.15])
-#h_vals = [1e-5*2**i for i in range(20)]
-
-h_vals = [(2*i*(1e-2))*1e-3 for i in range(1, 30, 2)]
-
-all_q_err = []
-all_q_err2 = []
-all_uk_err = []
-for h in h_vals:
-    # derivative of k vector w.r.t. detdist
-    #origin2_mm = np.array([-camera_size_mm[1]/2., camera_size_mm[0]/2., origin_z+h])
-    #k2 = origin2_mm + fs*i*pixsize_mm + ss*j*pixsize_mm
-    #k2 *= unit  # convert to meters
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    from scipy.stats import linregress
+    import numpy as np
+    import pylab as plt
+    from simtbx.nanoBragg import sim_data
+    from dxtbx.model import Panel
+    from scitbx.matrix import sqr, col
+
+    # part 1:
+    # simple k_diffracted derivative
+    fs = np.array([1, 0, 0])
+    ss = np.array([0, -1, 0])
+
+    # FIXME:
+    k_incident = np.array([0, 0, -1])
+
+    i, j = 10, 10
+    pixsize_mm = 0.11
+    detdist_mm = 150
+    camera_size_mm = 50, 50
+    origin_z = -detdist_mm
+    origin_mm = np.array([-camera_size_mm[1]/2., camera_size_mm[0]/2., origin_z])
+
+    unit = 1e-3
+
+    # diffracted beam
+    k = origin_mm + fs*i*pixsize_mm + ss*j*pixsize_mm
+    k *= unit  # convert to meters like the code
 
     # FIXME mod1
-    k2 = np.array([.01685, .02055, -.15+h])
-    k0 = np.array([.01685, .02055, -.15-h])
-    fdiff = (k2-k) / h  # / unit
-    dk = np.array((0, 0, 1))
+    k = np.array([.01685, .02055, -.15])
+    #h_vals = [1e-5*2**i for i in range(20)]
+
+    h_vals = [(2*i*(1e-2))*1e-3 for i in range(1, 30, 2)]
+
+    all_q_err = []
+    all_q_err2 = []
+    all_uk_err = []
+    for h in h_vals:
+        # derivative of k vector w.r.t. detdist
+        #origin2_mm = np.array([-camera_size_mm[1]/2., camera_size_mm[0]/2., origin_z+h])
+        #k2 = origin2_mm + fs*i*pixsize_mm + ss*j*pixsize_mm
+        #k2 *= unit  # convert to meters
+
+        # FIXME mod1
+        k2 = np.array([.01685, .02055, -.15+h])
+        k0 = np.array([.01685, .02055, -.15-h])
+        fdiff = (k2-k) / h  # / unit
+        dk = np.array((0, 0, 1))
+
+
+        #print ("\ndiffracted vector:")
+        #print ("finite deriv", fdiff)
+        #print ("deriv", dk)
+
+        # take derivative of unit diffracted vector
+        air_path = np.linalg.norm(k)
+        unit_k = k / air_path
+
+        air_path2 = np.linalg.norm(k2)
+        unit_k2 = k2 / air_path2
+
+        air_path0 = np.linalg.norm(k0)
+        unit_k0 = k0 / air_path0
+
+        fdiff = (unit_k2 - unit_k) / h  # / unit
+
+        G = np.dot(k, dk)
+        per_k = 1./air_path
+        per_k3 = np.power(per_k, 3)
+        per_k5 = np.power(per_k, 5)
+        deriv = dk*per_k - k*(per_k3*G)
+        d_unitk = deriv
+
+        #print ("\nunit diffracted vector:")
+        #print ("finite deriv", fdiff)
+        #print ("deriv", d_unitk)
+        err = np.abs(d_unitk-fdiff).mean()
+        #print("error = %2.7g, h=%2.7g" % (err, h))
+        all_uk_err.append(err)
+        #print("h=%2.7g" % h)
+        # now take derivative of q_vecor
+
+        wavelen = 1.8
+        q = 1 / wavelen * (unit_k - k_incident)
+        q2 = 1 / wavelen * (unit_k2 - k_incident)
+        q0 = 1 / wavelen * (unit_k0 - k_incident)
+        fdiff = (q2 - q) / h  # / unit
+
+        deriv = 1 / wavelen * d_unitk
+
+        dq = 1/wavelen*d_unitk
+        err = np.abs(dq-fdiff).mean()
+        #print("error = %2.7g, h=%2.7g" % (err, h))
+        all_q_err.append(err)
+
+        # second derivative
+        fdiff2 = (q2+q0 - 2.*q) / h/h  # / unit
+
+        # analytical
+        d_unitk2 = (3*per_k5*G*G)*k - per_k3*(np.dot(dk, dk))*k - (2*per_k3*G)*dk
+        dq2 = 1 / wavelen * d_unitk2
+
+        err2 = np.abs(dq2-fdiff2).mean()
+        #print("\terror2 = %2.7g, h=%2.7g" % (err, h))
+        all_q_err2.append(err2)
+
+        print ("\nmomentum transfer vector:")
+        print ("finite deriv2", fdiff2)
+        print ("deriv2", dq2)
+
+    luk = linregress(h_vals, all_uk_err)
+    lq = linregress(h_vals, all_q_err)
+    lq2 = linregress(np.array(h_vals)**2, all_q_err2)
+
+    print (luk.rvalue, luk.slope)
+    print (lq.rvalue, lq.slope)
+    print (lq2.rvalue, lq2.slope)
+    print ("\nmomentum transfer vector:")
+    print ("finite deriv", fdiff)
+    #print ("deriv", deriv)
+    print ("deriv", dq)
+
+    print ("\nmomentum transfer vector:")
+    print ("finite deriv2", fdiff2)
+    print ("deriv", dq2)
 
+    assert luk.rvalue > 0.99
+    assert lq.rvalue > 0.99
+    assert lq2.rvalue > 0.99
 
-    #print ("\ndiffracted vector:")
-    #print ("finite deriv", fdiff)
-    #print ("deriv", dk)
 
-    # take derivative of unit diffracted vector
+    wavelen = 1.8
+    h = h_vals[0]
+    k2 = np.array([.01685, .02055, -.15 + h])
+    k0 = np.array([.01685, .02055, -.15 - h])
+    dk = np.array((0, 0, 1))
     air_path = np.linalg.norm(k)
     unit_k = k / air_path
-
     air_path2 = np.linalg.norm(k2)
     unit_k2 = k2 / air_path2
-
     air_path0 = np.linalg.norm(k0)
     unit_k0 = k0 / air_path0
 
-    fdiff = (unit_k2 - unit_k) / h  # / unit
+    q = 1/wavelen * (unit_k - k_incident)
+    q2 = 1/wavelen * (unit_k2 - k_incident)
+    q0 = 1/wavelen * (unit_k0 - k_incident)
+    fdiff = (q2-q) / h
 
-    G = np.dot(k, dk)
-    per_k = 1./air_path
+    per_k = 1/air_path
     per_k3 = np.power(per_k, 3)
     per_k5 = np.power(per_k, 5)
-    deriv = dk*per_k - k*(per_k3*G)
-    d_unitk = deriv
-
-    #print ("\nunit diffracted vector:")
-    #print ("finite deriv", fdiff)
-    #print ("deriv", d_unitk)
-    err = np.abs(d_unitk-fdiff).mean()
-    #print("error = %2.7g, h=%2.7g" % (err, h))
-    all_uk_err.append(err)
-    #print("h=%2.7g" % h)
-    # now take derivative of q_vecor
+    G = np.dot(dk, k)
+    d_unitk = dk * per_k - k * (per_k3 * G)
+    dq = 1/wavelen * d_unitk
+
+    print ("\nunit momentum transfer vector:")
+    print ("finite deriv", fdiff)
+    print ("deriv", dq)
+
+
+    # take derivative of V-vector
+    # V = N*(U*B*q- h0)
+    SIM = sim_data.SimData(use_default_crystal=True)
+    C = SIM.crystal.dxtbx_crystal
+    B = sqr(C.get_B()).inverse().transpose()
+    U = sqr(C.get_U())
+    O = SIM.crystal.Omatrix
+    UBO = (U*B*O).transpose()
+
+    nn = SIM.crystal.Ncells_abc[0]
+    N = sqr((nn, 0, 0, 0, nn, 0, 0, 0, nn))
+    h0 = col((4, 3, -1))
+    V = N*(UBO*col(q) - h0)
+    V2 = N*(UBO*col(q2)-h0)
+    V0 = N*(UBO*col(q0)-h0)
+    fdiff = (V2-V)/h #/unit
+    fdiff2 = (V2+V0-2*V)/h/h #/unit
+
+    deriv = N*(UBO*col(dq))
+    dV = deriv
+
+    d_unitk2 = (3 * per_k5 * G * G) * k - per_k3 * (np.dot(dk, dk)) * k - (2 * per_k3 * G) * dk
+    dq2 = 1 / wavelen * d_unitk2
+    dV2 = N*(UBO*col(dq2))
 
-    wavelen = 1.8
-    q = 1 / wavelen * (unit_k - k_incident)
-    q2 = 1 / wavelen * (unit_k2 - k_incident)
-    q0 = 1 / wavelen * (unit_k0 - k_incident)
-    fdiff = (q2 - q) / h  # / unit
+    print("\nV-vector")
+    print("Values", V.elems)
+    print("finite deriv", fdiff.elems)
+    print("deriv", dV.elems)
+    print("finite deriv2", fdiff2.elems)
+    print("deriv2", dV2.elems)
 
-    deriv = 1 / wavelen * d_unitk
+    hrad_sqr = np.dot(V.elems, V.elems)
+    hrad_sqr2 = np.dot(V2.elems, V2.elems)
+    fd_hrad = (hrad_sqr2 - hrad_sqr) / h
 
-    dq = 1/wavelen*d_unitk
-    err = np.abs(dq-fdiff).mean()
-    #print("error = %2.7g, h=%2.7g" % (err, h))
-    all_q_err.append(err)
+    Fl = nn**3*np.exp(-(hrad_sqr / 0.63 * 1))
+    Fl2 = nn**3*np.exp(-(hrad_sqr2 / 0.63 * 1))
+    fdiff_Fl = (Fl2-Fl)/h
+    dH = 2 * np.dot(V.elems, dV.elems)
 
-    # second derivative
-    fdiff2 = (q2+q0 - 2.*q) / h/h  # / unit
+    print ("\nHrad_sqr")
+    print(" finite diff=%f" % fd_hrad)
+    print(" derivative=%f" % dH)
 
-    # analytical
-    d_unitk2 = (3*per_k5*G*G)*k - per_k3*(np.dot(dk, dk))*k - (2*per_k3*G)*dk
-    dq2 = 1 / wavelen * d_unitk2
+    deriv_Fl = -1/.63*Fl * dH
 
-    err2 = np.abs(dq2-fdiff2).mean()
-    #print("\terror2 = %2.7g, h=%2.7g" % (err, h))
-    all_q_err2.append(err2)
+    print("\ndFlatt")
+    print ("finite deriv", fdiff_Fl)
+    print("deriv: %f" % deriv_Fl)
 
-    print ("\nmomentum transfer vector:")
-    print ("finite deriv2", fdiff2)
-    print ("deriv2", dq2)
-
-luk = linregress(h_vals, all_uk_err)
-lq = linregress(h_vals, all_q_err)
-lq2 = linregress(np.array(h_vals)**2, all_q_err2)
-
-print (luk.rvalue, luk.slope)
-print (lq.rvalue, lq.slope)
-print (lq2.rvalue, lq2.slope)
-print ("\nmomentum transfer vector:")
-print ("finite deriv", fdiff)
-#print ("deriv", deriv)
-print ("deriv", dq)
-
-print ("\nmomentum transfer vector:")
-print ("finite deriv2", fdiff2)
-print ("deriv", dq2)
+    #deriv_Fl2 = -2/.63*((dV2*V)*Fl + (dV*dV)*Fl + (dV*V)*deriv_Fl)
+
+    print("\n---------------------------------\n")
+
+    # make a simple detector
+    #det = sim_data.SimData.simple_detector(detector_distance_mm=200, pixelsize_mm=0.025, image_shape=(2400, 2400))
 
-assert luk.rvalue > 0.99
-assert lq.rvalue > 0.99
-assert lq2.rvalue > 0.99
+    ##############################
 
 
-wavelen = 1.8
-h = h_vals[0]
-k2 = np.array([.01685, .02055, -.15 + h])
-k0 = np.array([.01685, .02055, -.15 - h])
-dk = np.array((0, 0, 1))
-air_path = np.linalg.norm(k)
-unit_k = k / air_path
-air_path2 = np.linalg.norm(k2)
-unit_k2 = k2 / air_path2
-air_path0 = np.linalg.norm(k0)
-unit_k0 = k0 / air_path0
+    #   BEGIN FINITE DIFF TEST
 
-q = 1/wavelen * (unit_k - k_incident)
-q2 = 1/wavelen * (unit_k2 - k_incident)
-q0 = 1/wavelen * (unit_k0 - k_incident)
-fdiff = (q2-q) / h
-
-per_k = 1/air_path
-per_k3 = np.power(per_k, 3)
-per_k5 = np.power(per_k, 5)
-G = np.dot(dk, k)
-d_unitk = dk * per_k - k * (per_k3 * G)
-dq = 1/wavelen * d_unitk
-
-print ("\nunit momentum transfer vector:")
-print ("finite deriv", fdiff)
-print ("deriv", dq)
-
-
-# take derivative of V-vector
-# V = N*(U*B*q- h0)
-SIM = sim_data.SimData(use_default_crystal=True)
-C = SIM.crystal.dxtbx_crystal
-B = sqr(C.get_B()).inverse().transpose()
-U = sqr(C.get_U())
-O = SIM.crystal.Omatrix
-UBO = (U*B*O).transpose()
-
-nn = SIM.crystal.Ncells_abc[0]
-N = sqr((nn, 0, 0, 0, nn, 0, 0, 0, nn))
-h0 = col((4, 3, -1))
-V = N*(UBO*col(q) - h0)
-V2 = N*(UBO*col(q2)-h0)
-V0 = N*(UBO*col(q0)-h0)
-fdiff = (V2-V)/h #/unit
-fdiff2 = (V2+V0-2*V)/h/h #/unit
-
-deriv = N*(UBO*col(dq))
-dV = deriv
-
-d_unitk2 = (3 * per_k5 * G * G) * k - per_k3 * (np.dot(dk, dk)) * k - (2 * per_k3 * G) * dk
-dq2 = 1 / wavelen * d_unitk2
-dV2 = N*(UBO*col(dq2))
-
-print("\nV-vector")
-print("Values", V.elems)
-print("finite deriv", fdiff.elems)
-print("deriv", dV.elems)
-print("finite deriv2", fdiff2.elems)
-print("deriv2", dV2.elems)
-
-hrad_sqr = np.dot(V.elems, V.elems)
-hrad_sqr2 = np.dot(V2.elems, V2.elems)
-fd_hrad = (hrad_sqr2 - hrad_sqr) / h
-
-Fl = nn**3*np.exp(-(hrad_sqr / 0.63 * 1))
-Fl2 = nn**3*np.exp(-(hrad_sqr2 / 0.63 * 1))
-fdiff_Fl = (Fl2-Fl)/h
-dH = 2 * np.dot(V.elems, dV.elems)
-
-print ("\nHrad_sqr")
-print(" finite diff=%f" % fd_hrad)
-print(" derivative=%f" % dH)
-
-deriv_Fl = -1/.63*Fl * dH
-
-print("\ndFlatt")
-print ("finite deriv", fdiff_Fl)
-print("deriv: %f" % deriv_Fl)
-
-#deriv_Fl2 = -2/.63*((dV2*V)*Fl + (dV*dV)*Fl + (dV*V)*deriv_Fl)
-
-print("\n---------------------------------\n")
-
-# make a simple detector
-#det = sim_data.SimData.simple_detector(detector_distance_mm=200, pixelsize_mm=0.025, image_shape=(2400, 2400))
-
-##############################
-
-
-#   BEGIN FINITE DIFF TEST
-
-
-##############################
-
-det = sim_data.SimData.simple_detector(detector_distance_mm=150, pixelsize_mm=0.1, image_shape=(512, 512))
-orig_idx = 10  # id of origin coordinate in diffBragg
-
-B = SIM.beam.nanoBragg_constructor_beam
-
-# set the detector
-SIM.detector = det
-SIM.instantiate_diffBragg(auto_set_spotscale=True)
-D = SIM.D
-
-D.oversample_omega = True
-D.nopolar = True
-
-D.refine(orig_idx)
-D.initialize_managers()
-#D.region_of_interest = ((2070, 2130), (70, 150))
-D.add_diffBragg_spots()
 
-# get the simulated image and derivative
-img = D.raw_pixels_roi.as_numpy_array()
-deriv = D.get_derivative_pixels(orig_idx).as_numpy_array()
-deriv2 = D.get_second_derivative_pixels(orig_idx).as_numpy_array()
-D.raw_pixels_roi *= 0
-D.raw_pixels *= 0
+    ##############################
 
-node = det[0]
-node_d = node.to_dict()
-O = node_d["origin"][0], node_d["origin"][1], node_d["origin"][2]
-distance = O[2]
+    det = sim_data.SimData.simple_detector(detector_distance_mm=150, pixelsize_mm=0.1, image_shape=(512, 512))
+    orig_idx = 10  # id of origin coordinate in diffBragg
 
-# update the detector distance
-percs = [np.power(1e-3*i, 2) for i in range(1, 30, 2)]
-all_shifts = []
-all_errors = []
-all_errors2 = []
-shifts_mm = [2*i*(1e-2) for i in range(1,30,2)]
-print (shifts_mm)
+    B = SIM.beam.nanoBragg_constructor_beam
 
-for i_shift, p in enumerate(percs):
-    # update the detector model
-    #delta_shift = abs(p*1e-2*distance)
-    delta_shift = shifts_mm[i_shift]  # + distance
+    # set the detector
+    SIM.detector = det
+    SIM.instantiate_diffBragg(auto_set_spotscale=True)
+    D = SIM.D
 
-    O2 = O[0], O[1], (distance + delta_shift)
-    node_d["origin"] = O2
+    D.oversample_omega = True
+    D.nopolar = True
 
-    det[0] = Panel.from_dict(node_d)
-    D.update_dxtbx_geoms(det, B, 0)
+    D.refine(orig_idx)
+    D.initialize_managers()
+    #D.region_of_interest = ((2070, 2130), (70, 150))
     D.add_diffBragg_spots()
-    img_forward = D.raw_pixels_roi.as_numpy_array()
+
+    # get the simulated image and derivative
+    img = D.raw_pixels_roi.as_numpy_array()
+    deriv = D.get_derivative_pixels(orig_idx).as_numpy_array()
+    deriv2 = D.get_second_derivative_pixels(orig_idx).as_numpy_array()
     D.raw_pixels_roi *= 0
     D.raw_pixels *= 0
-    delta_shift_meters = delta_shift*1e-3
-    fdiff = (img_forward-img)/delta_shift_meters
 
-    if args.curvatures:
-        O2 = O[0], O[1], (distance - delta_shift)
+    node = det[0]
+    node_d = node.to_dict()
+    O = node_d["origin"][0], node_d["origin"][1], node_d["origin"][2]
+    distance = O[2]
+
+    # update the detector distance
+    percs = [np.power(1e-3*i, 2) for i in range(1, 30, 2)]
+    all_shifts = []
+    all_errors = []
+    all_errors2 = []
+    shifts_mm = [2*i*(1e-2) for i in range(1,30,2)]
+    print (shifts_mm)
+
+    for i_shift, p in enumerate(percs):
+        # update the detector model
+        #delta_shift = abs(p*1e-2*distance)
+        delta_shift = shifts_mm[i_shift]  # + distance
+
+        O2 = O[0], O[1], (distance + delta_shift)
         node_d["origin"] = O2
+
         det[0] = Panel.from_dict(node_d)
         D.update_dxtbx_geoms(det, B, 0)
-
         D.add_diffBragg_spots()
-        img_backward = D.raw_pixels_roi.as_numpy_array()
+        img_forward = D.raw_pixels_roi.as_numpy_array()
         D.raw_pixels_roi *= 0
         D.raw_pixels *= 0
+        delta_shift_meters = delta_shift*1e-3
+        fdiff = (img_forward-img)/delta_shift_meters
 
-        fdiff2 = (img_forward - 2*img + img_backward) / delta_shift_meters / delta_shift_meters
+        if args.curvatures:
+            O2 = O[0], O[1], (distance - delta_shift)
+            node_d["origin"] = O2
+            det[0] = Panel.from_dict(node_d)
+            D.update_dxtbx_geoms(det, B, 0)
 
-    bragg = img > 1e-2
+            D.add_diffBragg_spots()
+            img_backward = D.raw_pixels_roi.as_numpy_array()
+            D.raw_pixels_roi *= 0
+            D.raw_pixels *= 0
 
-    error = (np.abs(fdiff[bragg] - deriv[bragg]) ).mean()
-    all_errors.append(error)
+            fdiff2 = (img_forward - 2*img + img_backward) / delta_shift_meters / delta_shift_meters
+
+        bragg = img > 1e-2
+
+        error = (np.abs(fdiff[bragg] - deriv[bragg]) ).mean()
+        all_errors.append(error)
 
-    if args.curvatures:
-        error2 = (np.abs(fdiff2[bragg] - deriv2[bragg]) / 1).mean()
-        all_errors2.append(error2)
-
-    all_shifts.append(delta_shift_meters)
-
-    print("Error=%2.7g shift=%2.7g mm, originZ=%2.7g mm" % (error, delta_shift, D.distance_mm))
-    if args.plotimages:
-        plt.clf()
-        y = slice(40, 65, 1)
-        x = slice(415, 437, 1)
-        plt.subplot(121)
-        plt.imshow(fdiff[y, x])
-        plt.title("finite diff")
-        plt.subplot(122)
-        plt.imshow(deriv[y, x])
-        plt.title("analytical")
-        plt.draw()
-        plt.suptitle("originZ=%f mm, delta=%f mm, Shift %d / %d"
-                     % (distance+delta_shift, delta_shift, i_shift + 1, len(percs)))
-        plt.pause(0.3)
         if args.curvatures:
+            error2 = (np.abs(fdiff2[bragg] - deriv2[bragg]) / 1).mean()
+            all_errors2.append(error2)
+
+        all_shifts.append(delta_shift_meters)
+
+        print("Error=%2.7g shift=%2.7g mm, originZ=%2.7g mm" % (error, delta_shift, D.distance_mm))
+        if args.plotimages:
             plt.clf()
+            y = slice(40, 65, 1)
+            x = slice(415, 437, 1)
             plt.subplot(121)
-            plt.imshow(fdiff2[y, x])
-            plt.title("finite 2nd diff")
+            plt.imshow(fdiff[y, x])
+            plt.title("finite diff")
             plt.subplot(122)
-            plt.imshow(deriv2[y, x])
+            plt.imshow(deriv[y, x])
             plt.title("analytical")
             plt.draw()
             plt.suptitle("originZ=%f mm, delta=%f mm, Shift %d / %d"
-                         % (distance + delta_shift, delta_shift, i_shift + 1, len(percs)))
+                         % (distance+delta_shift, delta_shift, i_shift + 1, len(percs)))
             plt.pause(0.3)
+            if args.curvatures:
+                plt.clf()
+                plt.subplot(121)
+                plt.imshow(fdiff2[y, x])
+                plt.title("finite 2nd diff")
+                plt.subplot(122)
+                plt.imshow(deriv2[y, x])
+                plt.title("analytical")
+                plt.draw()
+                plt.suptitle("originZ=%f mm, delta=%f mm, Shift %d / %d"
+                             % (distance + delta_shift, delta_shift, i_shift + 1, len(percs)))
+                plt.pause(0.3)
+
+    print( "\n\nSecond derivative\nerror          (shift)^2\n-----------------------------")
+    for err2, shift in zip(all_errors2, all_shifts):
+        print ("%10.7g  %10.7g" %(err2, shift**2))
 
-print( "\n\nSecond derivative\nerror          (shift)^2\n-----------------------------")
-for err2, shift in zip(all_errors2, all_shifts):
-    print ("%10.7g  %10.7g" %(err2, shift**2))
-
-if args.plotlines:
-    plt.close()
-
-    plt.plot(all_shifts, all_errors, 'o')
-    plt.suptitle("finite 1st difference error", fontsize=16)
-    plt.xlabel("h", fontsize=16)
-    plt.show()
-
-l = linregress(all_shifts, all_errors)
-print("finite diff l.rvalue=%10.7g" % l.rvalue)
-assert l.rvalue > .99
-assert l.slope > 0
-assert l.pvalue < 1e-6
-
-if args.curvatures:
     if args.plotlines:
         plt.close()
 
-        plt.plot(np.array(all_shifts)**2, all_errors2, 'o')
-        plt.suptitle("finite 2nd difference error", fontsize=16)
-        plt.xlabel("$h^2$", fontsize=16)
+        plt.plot(all_shifts, all_errors, 'o')
+        plt.suptitle("finite 1st difference error", fontsize=16)
+        plt.xlabel("h", fontsize=16)
         plt.show()
 
-    l = linregress(np.array(all_shifts)**2, all_errors2)
-    print("finite 2nd diff l.rvalue=%10.7g" % l.rvalue)
+    l = linregress(all_shifts, all_errors)
+    print("finite diff l.rvalue=%10.7g" % l.rvalue)
     assert l.rvalue > .99
     assert l.slope > 0
     assert l.pvalue < 1e-6
-print("OK!")
+
+    if args.curvatures:
+        if args.plotlines:
+            plt.close()
+
+            plt.plot(np.array(all_shifts)**2, all_errors2, 'o')
+            plt.suptitle("finite 2nd difference error", fontsize=16)
+            plt.xlabel("$h^2$", fontsize=16)
+            plt.show()
+
+        l = linregress(np.array(all_shifts)**2, all_errors2)
+        print("finite 2nd diff l.rvalue=%10.7g" % l.rvalue)
+        assert l.rvalue > .99
+        assert l.slope > 0
+        assert l.pvalue < 1e-6
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_diffuse_properties.py b/simtbx/diffBragg/tests/tst_diffBragg_diffuse_properties.py
index 973a947cfa..700d0a6275 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_diffuse_properties.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_diffuse_properties.py
@@ -2,10 +2,12 @@
 This test checks the setter and getter for Ncells parameter
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--onlyDiffuse", action='store_true')
 parser.add_argument("--sigma", default=[1,1,1], type=float, nargs=3)
@@ -19,97 +21,101 @@
 import os
 
 args = parser.parse_args()
-if args.cuda:
-    os.environ["DIFFBRAGG_USE_CUDA"] = "1"
-
-from simtbx.nanoBragg import sim_data
-from simtbx.diffBragg import utils
-
-S = sim_data.SimData(use_default_crystal=True)
-det_shape = (1024,1024)
-S.detector = sim_data.SimData.simple_detector(180, 0.1, det_shape)
-S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
-#S.D.record_time = True
-S.D.spot_scale = 100000
-S.D.use_diffuse = True
-if args.laue:
-    symbol = S.crystal.space_group_info.type().lookup_symbol()
-    S.D.laue_group_num = utils.get_laue_group_number(symbol)
-    if symbol.replace(" ","")=="P43212":
-        assert S.D.laue_group_num==7  # for default P43212
-    print("Laue group number set to %d" % S.D.laue_group_num)
-S.D.stencil_size = args.stencil
-assert S.D.stencil_size== args.stencil
-print("diffuse scattering stencil size set to %d" % S.D.stencil_size)
-S.D.diffuse_gamma = tuple(args.gamma)
-S.D.diffuse_sigma = tuple(args.sigma)
-
-default_gamma = args.gamma
-default_sigma = args.sigma
-
-import numpy as np
-assert np.allclose(S.D.diffuse_gamma, args.gamma)
-assert np.allclose(S.D.diffuse_sigma, args.sigma)
-
-diffuse_id = 23
-S.D.refine(diffuse_id)
-S.D.add_diffBragg_spots()
-img = S.D.raw_pixels_roi.as_numpy_array()
-bragg = img > 1e-1  # select bragg scattering regions
-
-if args.grad == 'gamma':
-    derivs_abc = list(map(lambda x: x.as_numpy_array(), S.D.get_diffuse_gamma_derivative_pixels()))
-    diff_params = args.gamma
-else:
-    derivs_abc = list(map(lambda x: x.as_numpy_array(), S.D.get_diffuse_sigma_derivative_pixels()))
-    diff_params = args.sigma
-
-S.D.fix(diffuse_id)
-
-from scipy.stats import linregress
-perc = 0.001, 0.01, 0.1, 1, 10
-for ii in range(3):
-    if ii != args.idx:
-        continue
-
-    all_error = []
-    all_error2 = []
-    shifts = []
-    shifts2 = []
-    gi = diff_params[ii]
-    for i_shift, p in enumerate(perc):
-        delta_gi = gi*p*0.01
-        print(i_shift, delta_gi, gi)
-        diff_vals = diff_params*1
-        diff_vals[ii] = gi+delta_gi
-
-        if args.grad == 'gamma':
-            S.D.diffuse_gamma = tuple(diff_vals)
-        else:
-            S.D.diffuse_sigma = tuple(diff_vals)
-
-        shifts.append(delta_gi)
-
-        S.D.raw_pixels_roi *= 0
-        #S.D.printout_pixel_fastslow = 10, 10
-        S.D.add_diffBragg_spots()
-        img2 = S.D.raw_pixels_roi.as_numpy_array()
-
-        fdiff = (img2 - img) / delta_gi
-        deriv = derivs_abc[ii]
-        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-        all_error.append(error)
-
-        print ("error=%f, step=%f" % (error, delta_gi))
-
-    if args.plot:
-        plt.plot(shifts, all_error, 'o')
-        plt.show()
-
-    l = linregress(shifts, all_error)
-    assert l.rvalue > .9999  # this is definitely a line!
-    assert l.slope > 0
-    assert l.pvalue < 1e-6
-
-det_sh = 1024, 1024
-print("OK")
+if args.kokkos:
+    os.environ["DIFFBRAGG_USE_KOKKOS"] = "1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    from simtbx.nanoBragg import sim_data
+    from simtbx.diffBragg import utils
+
+    S = sim_data.SimData(use_default_crystal=True)
+    det_shape = (1024,1024)
+    S.detector = sim_data.SimData.simple_detector(180, 0.1, det_shape)
+    S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
+    #S.D.record_time = True
+    S.D.spot_scale = 100000
+    S.D.use_diffuse = True
+    if args.laue:
+        symbol = S.crystal.space_group_info.type().lookup_symbol()
+        S.D.laue_group_num = utils.get_laue_group_number(symbol)
+        if symbol.replace(" ","")=="P43212":
+            assert S.D.laue_group_num==7  # for default P43212
+        print("Laue group number set to %d" % S.D.laue_group_num)
+    S.D.stencil_size = args.stencil
+    assert S.D.stencil_size== args.stencil
+    print("diffuse scattering stencil size set to %d" % S.D.stencil_size)
+    S.D.diffuse_gamma = tuple(args.gamma)
+    S.D.diffuse_sigma = tuple(args.sigma)
+
+    default_gamma = args.gamma
+    default_sigma = args.sigma
+
+    import numpy as np
+    assert np.allclose(S.D.diffuse_gamma, args.gamma)
+    assert np.allclose(S.D.diffuse_sigma, args.sigma)
+
+    diffuse_id = 23
+    S.D.refine(diffuse_id)
+    S.D.add_diffBragg_spots()
+    img = S.D.raw_pixels_roi.as_numpy_array()
+    bragg = img > 1e-1  # select bragg scattering regions
+
+    if args.grad == 'gamma':
+        derivs_abc = list(map(lambda x: x.as_numpy_array(), S.D.get_diffuse_gamma_derivative_pixels()))
+        diff_params = args.gamma
+    else:
+        derivs_abc = list(map(lambda x: x.as_numpy_array(), S.D.get_diffuse_sigma_derivative_pixels()))
+        diff_params = args.sigma
+
+    S.D.fix(diffuse_id)
+
+    from scipy.stats import linregress
+    perc = 0.001, 0.01, 0.1, 1, 10
+    for ii in range(3):
+        if ii != args.idx:
+            continue
+
+        all_error = []
+        all_error2 = []
+        shifts = []
+        shifts2 = []
+        gi = diff_params[ii]
+        for i_shift, p in enumerate(perc):
+            delta_gi = gi*p*0.01
+            print(i_shift, delta_gi, gi)
+            diff_vals = diff_params*1
+            diff_vals[ii] = gi+delta_gi
+
+            if args.grad == 'gamma':
+                S.D.diffuse_gamma = tuple(diff_vals)
+            else:
+                S.D.diffuse_sigma = tuple(diff_vals)
+
+            shifts.append(delta_gi)
+
+            S.D.raw_pixels_roi *= 0
+            #S.D.printout_pixel_fastslow = 10, 10
+            S.D.add_diffBragg_spots()
+            img2 = S.D.raw_pixels_roi.as_numpy_array()
+
+            fdiff = (img2 - img) / delta_gi
+            deriv = derivs_abc[ii]
+            error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+            all_error.append(error)
+
+            print ("error=%f, step=%f" % (error, delta_gi))
+
+        if args.plot:
+            plt.plot(shifts, all_error, 'o')
+            plt.show()
+
+        l = linregress(shifts, all_error)
+        assert l.rvalue > .9999  # this is definitely a line!
+        assert l.slope > 0
+        assert l.pvalue < 1e-6
+
+    det_sh = 1024, 1024
+    print("OK")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_eta_derivs.py b/simtbx/diffBragg/tests/tst_diffBragg_eta_derivs.py
index b5cd31739e..ab3a3c5dd8 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_eta_derivs.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_eta_derivs.py
@@ -1,198 +1,205 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
+
 from argparse import ArgumentParser
 parser = ArgumentParser()
 parser.add_argument("--plot", action="store_true")
 parser.add_argument("--curvatures", action="store_true")
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--aniso", type=int, choices=[0,1,2], default=None)
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-from dxtbx.model.crystal import Crystal
-from cctbx import uctbx
-from scitbx.matrix import rec, col
-import numpy as np
-from scipy.spatial.transform import Rotation
-from scitbx.matrix import sqr
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-from simtbx.nanoBragg.sim_data import SimData
-from simtbx.diffBragg.utils import fcalc_from_pdb
-import pylab as plt
-
-
-ucell = (79, 79, 38, 90, 90, 90)
-symbol = "P43212"
-N_MOS_DOMAINS = 100
-MOS_SPREAD = 1
-ANISO_MOS_SPREAD = 0.5, 0.75, 1
-eta_diffBragg_id = 19
-
-miller_array_GT = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=ucell, symbol=symbol)
-Ncells_gt = 15, 15, 15
-
-np.random.seed(3142019)
-# generate a random rotation
-rotation = Rotation.random(num=1, random_state=100)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-x = col((-1, 0, 0))
-y = col((0, -1, 0))
-z = col((0, 0, -1))
-rx, ry, rz = np.random.uniform(-180, 180, 3)
-RX = x.axis_and_angle_as_r3_rotation_matrix(rx, deg=True)
-RY = y.axis_and_angle_as_r3_rotation_matrix(ry, deg=True)
-RZ = z.axis_and_angle_as_r3_rotation_matrix(rz, deg=True)
-M = RX * RY * RZ
-a_real = M * col(a_real)
-b_real = M * col(b_real)
-c_real = M * col(c_real)
-C = Crystal(a_real, b_real, c_real, symbol)
-C.rotate_around_origin(rot_axis, rot_ang)
-
-# Setup the simulation and create a realistic image
-# with background and noise
-# <><><><><><><><><><><><><><><><><><><><><><><><><>
-nbcryst = NBcrystal(init_defaults=True)
-nbcryst.dxtbx_crystal = C   # simulate ground truth
-nbcryst.thick_mm = 0.1
-nbcryst.Ncells_abc = Ncells_gt  # ground truth Ncells
-nbcryst.mos_spread_deg = MOS_SPREAD
-if args.aniso is not None:
-  nbcryst.anisotropic_mos_spread_deg = ANISO_MOS_SPREAD
-  assert nbcryst.has_anisotropic_mosaicity
-else:
-  assert not nbcryst.has_anisotropic_mosaicity
-
-nbcryst.n_mos_domains = N_MOS_DOMAINS
-nbcryst.miller_array = miller_array_GT
-print("Ground truth ncells = %f" % (nbcryst.Ncells_abc[0]))
-
-# ground truth detector
-DET_gt = SimData.simple_detector(150, 0.177, (600, 600))
-
-# initialize the simulator
-SIM = SimData()
-if args.aniso is None:
-  SIM.Umats_method = 2
-else:
-  SIM.Umats_method = 3
-SIM.detector = DET_gt
-SIM.crystal = nbcryst
-SIM.instantiate_diffBragg(oversample=1, verbose=0)
-SIM.D.refine(eta_diffBragg_id)
-SIM.D.initialize_managers()
-SIM.D.spot_scale = 100000
-SIM.D.default_F = 0
-SIM.D.progress_meter = False
-SIM.water_path_mm = 0.15
-SIM.air_path_mm = 0.1
-SIM.add_air = True
-SIM.add_water = True
-SIM.include_noise = True
-SIM.D.use_cuda = args.cuda
-SIM.D.compute_curvatures = args.curvatures
-SIM.D.add_diffBragg_spots()
-
-if args.aniso is None:
-  deriv = SIM.D.get_derivative_pixels(eta_diffBragg_id).as_numpy_array()
-else:
-  deriv = SIM.D.get_aniso_eta_deriv_pixels()[args.aniso].as_numpy_array()
-
-if args.curvatures:
-  if args.aniso is None:
-    deriv2 = SIM.D.get_second_derivative_pixels(eta_diffBragg_id).as_numpy_array()
-  else:
-    deriv2 = SIM.D.get_aniso_eta_second_deriv_pixels()[args.aniso].as_numpy_array()
-SPOTS = SIM.D.raw_pixels_roi.as_numpy_array()
-
-SIM.D.readout_noise_adu = 1
-SIM._add_background()
-SIM._add_noise()
-
-# This is the ground truth image:
-img = SIM.D.raw_pixels.as_numpy_array()
-SIM.D.raw_pixels_roi *= 0
-SIM.D.raw_pixels *= 0
-
-all_errors = []
-all_shifts = []
-all_errors2 = []
-all_shifts2 = []
-for finite_diff_step in [1, 2, 4, 8, 16]:
-  # update Umats to do finite difference test
-  delta_eta = 0.001*finite_diff_step
-
-  if args.aniso is not None:
-    eta_update = list(ANISO_MOS_SPREAD)
-    eta_update[args.aniso] = eta_update[args.aniso]+ delta_eta
-    crystal = nbcryst.dxtbx_crystal
-  else:
-    eta_update = MOS_SPREAD + delta_eta
-    crystal = None
-
-  SIM.update_umats_for_refinement(eta_update)
-  SIM.D.add_diffBragg_spots()
-
-  img_forward = SIM.D.raw_pixels_roi.as_numpy_array()
-  SIM.D.raw_pixels_roi *= 0
-  SIM.D.raw_pixels *= 0
-  fdiff = (img_forward - SPOTS) / delta_eta
-  bragg = SPOTS > 1e-2
-  error = (np.abs(fdiff[bragg] - deriv[bragg])).mean()
-  all_errors.append(error)
-  all_shifts.append(delta_eta)
-  if args.curvatures:
-
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    from dxtbx.model.crystal import Crystal
+    from cctbx import uctbx
+    from scitbx.matrix import rec, col
+    import numpy as np
+    from scipy.spatial.transform import Rotation
+    from scitbx.matrix import sqr
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    from simtbx.nanoBragg.sim_data import SimData
+    from simtbx.diffBragg.utils import fcalc_from_pdb
+    import pylab as plt
+
+
+    ucell = (79, 79, 38, 90, 90, 90)
+    symbol = "P43212"
+    N_MOS_DOMAINS = 100
+    MOS_SPREAD = 1
+    ANISO_MOS_SPREAD = 0.5, 0.75, 1
+    eta_diffBragg_id = 19
+
+    miller_array_GT = fcalc_from_pdb(resolution=2, wavelength=1, algorithm='fft', ucell=ucell, symbol=symbol)
+    Ncells_gt = 15, 15, 15
+
+    np.random.seed(3142019)
+    # generate a random rotation
+    rotation = Rotation.random(num=1, random_state=100)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    x = col((-1, 0, 0))
+    y = col((0, -1, 0))
+    z = col((0, 0, -1))
+    rx, ry, rz = np.random.uniform(-180, 180, 3)
+    RX = x.axis_and_angle_as_r3_rotation_matrix(rx, deg=True)
+    RY = y.axis_and_angle_as_r3_rotation_matrix(ry, deg=True)
+    RZ = z.axis_and_angle_as_r3_rotation_matrix(rz, deg=True)
+    M = RX * RY * RZ
+    a_real = M * col(a_real)
+    b_real = M * col(b_real)
+    c_real = M * col(c_real)
+    C = Crystal(a_real, b_real, c_real, symbol)
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    # Setup the simulation and create a realistic image
+    # with background and noise
+    # <><><><><><><><><><><><><><><><><><><><><><><><><>
+    nbcryst = NBcrystal(init_defaults=True)
+    nbcryst.dxtbx_crystal = C   # simulate ground truth
+    nbcryst.thick_mm = 0.1
+    nbcryst.Ncells_abc = Ncells_gt  # ground truth Ncells
+    nbcryst.mos_spread_deg = MOS_SPREAD
     if args.aniso is not None:
-      eta_update = list(ANISO_MOS_SPREAD)
-      eta_update[args.aniso] = eta_update[args.aniso] - delta_eta
-      crystal = nbcryst.dxtbx_crystal
+      nbcryst.anisotropic_mos_spread_deg = ANISO_MOS_SPREAD
+      assert nbcryst.has_anisotropic_mosaicity
     else:
-      eta_update = MOS_SPREAD - delta_eta
-      crystal= None
+      assert not nbcryst.has_anisotropic_mosaicity
+
+    nbcryst.n_mos_domains = N_MOS_DOMAINS
+    nbcryst.miller_array = miller_array_GT
+    print("Ground truth ncells = %f" % (nbcryst.Ncells_abc[0]))
 
-    SIM.update_umats_for_refinement(eta_update)
+    # ground truth detector
+    DET_gt = SimData.simple_detector(150, 0.177, (600, 600))
 
-    all_shifts2.append(delta_eta ** 2)
+    # initialize the simulator
+    SIM = SimData()
+    if args.aniso is None:
+      SIM.Umats_method = 2
+    else:
+      SIM.Umats_method = 3
+    SIM.detector = DET_gt
+    SIM.crystal = nbcryst
+    SIM.instantiate_diffBragg(oversample=1, verbose=0)
+    SIM.D.refine(eta_diffBragg_id)
+    SIM.D.initialize_managers()
+    SIM.D.spot_scale = 100000
+    SIM.D.default_F = 0
+    SIM.D.progress_meter = False
+    SIM.water_path_mm = 0.15
+    SIM.air_path_mm = 0.1
+    SIM.add_air = True
+    SIM.add_water = True
+    SIM.include_noise = True
+    SIM.D.use_cuda = args.kokkos
+    SIM.D.compute_curvatures = args.curvatures
+    SIM.D.add_diffBragg_spots()
 
+    if args.aniso is None:
+      deriv = SIM.D.get_derivative_pixels(eta_diffBragg_id).as_numpy_array()
+    else:
+      deriv = SIM.D.get_aniso_eta_deriv_pixels()[args.aniso].as_numpy_array()
+
+    if args.curvatures:
+      if args.aniso is None:
+        deriv2 = SIM.D.get_second_derivative_pixels(eta_diffBragg_id).as_numpy_array()
+      else:
+        deriv2 = SIM.D.get_aniso_eta_second_deriv_pixels()[args.aniso].as_numpy_array()
+    SPOTS = SIM.D.raw_pixels_roi.as_numpy_array()
+
+    SIM.D.readout_noise_adu = 1
+    SIM._add_background()
+    SIM._add_noise()
+
+    # This is the ground truth image:
+    img = SIM.D.raw_pixels.as_numpy_array()
     SIM.D.raw_pixels_roi *= 0
     SIM.D.raw_pixels *= 0
-    SIM.D.add_diffBragg_spots()
-    img_backward = SIM.D.raw_pixels_roi.as_numpy_array()
-
-    fdiff2 = (img_forward - 2*SPOTS + img_backward) / delta_eta/ delta_eta
-
-    second_deriv = deriv2
-    error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
-    all_errors2.append(error2)
-
-  print("\n\n<><><><><><><><>\n\tError:", error, "shift:", delta_eta)
-  if args.curvatures:
-    print("\terror2=%f, step=%f\n<><><><><><><><>\n\n" % (error2, delta_eta))
-
-from scipy.stats import linregress
-l = linregress(all_shifts, all_errors)
-print("finite diff l.rvalue=%10.7g" % l.rvalue)
-if args.plot:
-  plt.figure()
-  plt.plot(all_shifts, all_errors, 'o')
-  plt.show()
-  if args.curvatures:
-    plt.plot(all_shifts2, all_errors2, 'o')
-    plt.title("second finite diff error")
-    plt.xlabel("delta eta")
-    plt.show()
-
-assert l.rvalue > .99, "%f" % l.rvalue
-assert l.slope > 0, "%f" % l.slope
-assert l.pvalue < 1e-6, "%f" % l.pvalue
-if args.curvatures:
-  l = linregress(all_shifts2, all_errors2)
-  assert l.rvalue > .9999  # this is definitely a line!
-  assert l.slope > 0
-  assert l.pvalue < 1e-6
-print("OK")
+
+    all_errors = []
+    all_shifts = []
+    all_errors2 = []
+    all_shifts2 = []
+    for finite_diff_step in [1, 2, 4, 8, 16]:
+      # update Umats to do finite difference test
+      delta_eta = 0.001*finite_diff_step
+
+      if args.aniso is not None:
+        eta_update = list(ANISO_MOS_SPREAD)
+        eta_update[args.aniso] = eta_update[args.aniso]+ delta_eta
+        crystal = nbcryst.dxtbx_crystal
+      else:
+        eta_update = MOS_SPREAD + delta_eta
+        crystal = None
+
+      SIM.update_umats_for_refinement(eta_update)
+      SIM.D.add_diffBragg_spots()
+
+      img_forward = SIM.D.raw_pixels_roi.as_numpy_array()
+      SIM.D.raw_pixels_roi *= 0
+      SIM.D.raw_pixels *= 0
+      fdiff = (img_forward - SPOTS) / delta_eta
+      bragg = SPOTS > 1e-2
+      error = (np.abs(fdiff[bragg] - deriv[bragg])).mean()
+      all_errors.append(error)
+      all_shifts.append(delta_eta)
+      if args.curvatures:
+
+        if args.aniso is not None:
+          eta_update = list(ANISO_MOS_SPREAD)
+          eta_update[args.aniso] = eta_update[args.aniso] - delta_eta
+          crystal = nbcryst.dxtbx_crystal
+        else:
+          eta_update = MOS_SPREAD - delta_eta
+          crystal= None
+
+        SIM.update_umats_for_refinement(eta_update)
+
+        all_shifts2.append(delta_eta ** 2)
+
+        SIM.D.raw_pixels_roi *= 0
+        SIM.D.raw_pixels *= 0
+        SIM.D.add_diffBragg_spots()
+        img_backward = SIM.D.raw_pixels_roi.as_numpy_array()
+
+        fdiff2 = (img_forward - 2*SPOTS + img_backward) / delta_eta/ delta_eta
+
+        second_deriv = deriv2
+        error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
+        all_errors2.append(error2)
+
+      print("\n\n<><><><><><><><>\n\tError:", error, "shift:", delta_eta)
+      if args.curvatures:
+        print("\terror2=%f, step=%f\n<><><><><><><><>\n\n" % (error2, delta_eta))
+
+    from scipy.stats import linregress
+    l = linregress(all_shifts, all_errors)
+    print("finite diff l.rvalue=%10.7g" % l.rvalue)
+    if args.plot:
+      plt.figure()
+      plt.plot(all_shifts, all_errors, 'o')
+      plt.show()
+      if args.curvatures:
+        plt.plot(all_shifts2, all_errors2, 'o')
+        plt.title("second finite diff error")
+        plt.xlabel("delta eta")
+        plt.show()
+
+    assert l.rvalue > .99, "%f" % l.rvalue
+    assert l.slope > 0, "%f" % l.slope
+    assert l.pvalue < 1e-6, "%f" % l.pvalue
+    if args.curvatures:
+      l = linregress(all_shifts2, all_errors2)
+      assert l.rvalue > .9999  # this is definitely a line!
+      assert l.slope > 0
+      assert l.pvalue < 1e-6
+    print("OK")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine.py b/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine.py
index b70c72bd95..1609175af1 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine.py
@@ -1,7 +1,7 @@
 from __future__ import division
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--curvatures", action='store_true')
 parser.add_argument("--readout", type=float, default=0)
 parser.add_argument("--perturb", choices=["G", "Nabc", "detz_shift", "crystal", "eta", "spec"], type=str, nargs="+", default=["crystal"] )
@@ -14,320 +14,326 @@
 name = "hopper_refine_%s" % "-".join(args.perturb)
 import os
 
-if args.cuda:
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-from dxtbx.model.crystal import Crystal
-from cctbx import uctbx
-from scitbx.matrix import sqr, rec, col
-import numpy as np
-from scipy.spatial.transform import Rotation
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-from simtbx.nanoBragg.sim_data import SimData
-from simtbx.diffBragg import utils
-from scipy.signal import windows
-from dxtbx.model import Experiment
-from simtbx.nanoBragg import make_imageset
-from simtbx.diffBragg.phil import hopper_phil, philz
-from libtbx.phil import parse
-
-phil_scope = parse(hopper_phil+philz)
-
-ucell = (55, 65, 75, 90, 95, 90)
-ucell2 = (55.1, 65.2, 74.9, 90, 94.9, 90)
-symbol = "P121"
-
-# generate a random raotation
-rotation = Rotation.random(num=1, random_state=100)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-
-# generate a small perturbation rotation
-np.random.seed(1)
-perturb_rot_axis = np.random.random(3)
-perturb_rot_axis /= np.linalg.norm(perturb_rot_axis)
-perturb_rot_ang = 0.15  # degree random perturbtation
-
-# make the ground truth crystal:
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-C.rotate_around_origin(rot_axis, rot_ang)
-
-a2_real, b2_real, c2_real = sqr(uctbx.unit_cell(ucell2).orthogonalization_matrix()).transpose().as_list_of_lists()
-C2 = Crystal(a2_real, b2_real, c2_real, symbol)
-C2.rotate_around_origin(rot_axis, rot_ang)
-assert np.allclose(C2.get_U(), C.get_U())
-C2.rotate_around_origin(col(perturb_rot_axis), perturb_rot_ang)
-
-# Setup the simulation and create a realistic image
-# with background and noise
-# <><><><><><><><><><><><><><><><><><><><><><><><><>
-nbcryst = NBcrystal()
-nbcryst.dxtbx_crystal = C   # simulate ground truth
-nbcryst.thick_mm = 0.1
-nbcryst.isotropic_ncells = False
-if "eta" in args.perturb:
-    nbcryst.n_mos_domains = 1000
-    ETA_ABC_GT = args.eta
-    nbcryst.anisotropic_mos_spread_deg = ETA_ABC_GT
-    NCELLS_GT = 12,12,11
-else:
-    NCELLS_GT = 12,12,11
-nbcryst.Ncells_abc = NCELLS_GT
-
-SIM = SimData(use_default_crystal=True)
-if "spec" in args.perturb:
-    # initialize the simulator
-    spec = SIM.beam.spectrum
-    total_flux = spec[0][1]
-    wave = spec[0][0]
-
-    en = utils.ENERGY_CONV / wave
-    delta_en = 1.5
-    ens_gt = np.arange(en - 5, en + 6, delta_en)
-    waves_gt = utils.ENERGY_CONV / ens_gt
-    num_energies = len(ens_gt)
-    fluxes_gt = np.ones(num_energies) * total_flux / num_energies
-    fluxes_gt = fluxes_gt*windows.hann(num_energies)
-    fluxes_gt /= fluxes_gt.sum()
-    fluxes_gt *= total_flux
-
-    spectrum_GT = list(zip(waves_gt, fluxes_gt))
-    gt_lambda0 = waves_gt[0]
-    gt_lambda1 = waves_gt[1] - waves_gt[0]
-    spec_idx = np.arange(num_energies)
-    assert np.allclose(waves_gt, gt_lambda0 + spec_idx*gt_lambda1)
-
-    lam0 = np.random.normal(gt_lambda0, gt_lambda0 * 0.002)
-    lam1 = np.random.normal(gt_lambda1, abs(gt_lambda1) * 0.002)
-    waves_perturbed = lam0 + spec_idx * lam1
-    print("ENERGY TRUTH=%.4f" % (utils.ENERGY_CONV / gt_lambda0))
-    print("ENERGY PERTURBED=%.4f" % (utils.ENERGY_CONV / lam0))
-    perturbed_spec = list(zip(waves_perturbed, fluxes_gt))
-    SIM.beam.spectrum = spectrum_GT
-
-#SIM.detector = SimData.simple_detector(150, 0.1, (513, 512))
-if "eta" in args.perturb:
-    shape = 513*3, 512*3
-    #detdist = 70
-else:
-    shape = 513, 512
-detdist = 150
-SIM.detector = SimData.simple_detector(detdist, 0.1, shape)
-SIM.crystal = nbcryst
-SIM.instantiate_diffBragg(oversample=0, auto_set_spotscale=True)
-SIM.D.default_F = 0
-SIM.D.F000 = 0
-SIM.D.progress_meter = False
-SIM.water_path_mm = 0.005
-SIM.air_path_mm = 0.1
-SIM.add_air = True
-SIM.add_Water = True
-SIM.include_noise = True
-SIM.D.verbose = 2
-SIM.D.add_diffBragg_spots()
-SIM.D.verbose = 0
-spots = SIM.D.raw_pixels.as_numpy_array()
-SIM._add_background()
-SIM.D.readout_noise_adu=args.readout
-SIM._add_noise()
-
-# This is the ground truth image:
-img = SIM.D.raw_pixels.as_numpy_array()
-if args.plot:
-    import pylab as plt
-    plt.imshow(img, vmax=100)
-    plt.title("Ground truth image")
-    plt.show()
-
-cbf_name = name + ".cbf"
-if args.cmdlineHopper:
-    from IPython import embed
-    embed()
-    SIM.D.to_cbf(cbf_name)
-
-SIM.D.raw_pixels *= 0
-
-P = phil_scope.extract()
-E = Experiment()
-
-GT_spot_scale = SIM.D.spot_scale
-if "G" in args.perturb:
-    P.init.G = GT_spot_scale*10
-else:
-    P.init.G = GT_spot_scale
-
-P.types.G = args.typeG
-P.types.Nabc = args.typeNabc
-
-if "crystal" in args.perturb:
-    E.crystal = C2
-else:
-    E.crystal = C
-
-if "Nabc" in args.perturb:
-    P.init.Nabc = 20,20,20
-else:
-    P.init.Nabc = SIM.crystal.Ncells_abc
-
-if "detz_shift" in args.perturb:
-    P.init.detz_shift = 1
-else:
-    P.init.detz_shift = 0
-
-if "eta" in args.perturb:
-    P.init.eta_abc = [0.12, 0.13, 0.14]
-    P.simulator.crystal.num_mosaicity_samples = 250  # in practive, the number of mosaic domains we model should be smaller than whats in the crystal .. .
-    P.simulator.crystal.has_isotropic_mosaicity = False
-    P.fix.eta_abc = False
-
-if "spec" in args.perturb:
-    P.fix.spec = False
-    P.init.spec = [0,1]
-    P.fix.Nabc=True
-    P.fix.G=True
-    P.fix.RotXYZ=True
-    P.fix.ucell = True
-    P.fix.detz_shift = True
-    P.ftol=1e-15
-
-E.detector = SIM.detector
-E.beam = SIM.D.beam
-E.imageset = make_imageset([img], E.beam, E.detector)
-#refls = utils.refls_from_sims([img], E.detector, E.beam, thresh=18)
-refls = utils.refls_from_sims([spots], E.detector, E.beam, thresh=18)
-print("%d REFLS" % len(refls))
-utils.refls_to_q(refls, E.detector, E.beam, update_table=True)
-utils.refls_to_hkl(refls, E.detector, E.beam, E.crystal, update_table=True)
-
-P.roi.shoebox_size = 20
-P.relative_tilt = False
-P.roi.fit_tilt = False
-P.roi.pad_shoebox_for_background_estimation=10
-P.roi.reject_edge_reflections = False
-P.refiner.sigma_r = SIM.D.readout_noise_adu
-P.refiner.adu_per_photon = SIM.D.quantum_gain
-P.simulator.init_scale = 1 #SIM.D.spot_scale
-P.simulator.beam.size_mm = SIM.beam.size_mm
-P.simulator.total_flux = SIM.D.flux
-P.use_restraints = False
-mtz_name = name +".mtz"
-SIM.crystal.miller_array.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_name)
-P.simulator.structure_factors.mtz_name = mtz_name
-P.simulator.structure_factors.mtz_column = "F(+),F(-)"
-P.niter = 0
-P.logging.parameters=True
-P.niter_per_J = 1
-P.method="L-BFGS-B"
-P.ftol = 1e-10
-if "eta" in args.perturb:
-    P.ftol=1e-8
-#P.method="Nelder-Mead"
-#P.fix.G = True
-#P.fix.Nabc =True
-#P.fix.detz_shift=True
-
-import logging
-import sys
-h = logging.StreamHandler(sys.stdout)
-logging.basicConfig(level=logging.DEBUG, handlers=[h])
-
-from simtbx.diffBragg import hopper_utils
-spec = None
-if "spec" in args.perturb:
-    spec = "tst_hopper_refine_spec.lam"
-    wave, wt = map(np.array, zip(*perturbed_spec))
-    utils.save_spectra_file(spec, wave, wt)
-
-if args.cmdlineHopper:
-    from dxtbx.model import ExperimentList
-    el_name = "%s.expt" % name
-    import_cmd = "dials.import %s output.experiments=%s" % (cbf_name, el_name)
-    os.system(import_cmd)
-    # add the crystal to the imported expt
-    El = ExperimentList.from_file(el_name)
-    El[0].crystal = E.crystal
-    refl_name = "%s.refl" % name
-    El.as_file(el_name)
-    refls.as_file(refl_name)
-    hopper_input_lst = "%s.lst" % name
-    with open(hopper_input_lst, "w") as o:
-        o.write("%s %s\n" % (os.path.abspath(el_name), os.path.abspath(refl_name)))
-
-    # save the above modified phil params to a file to be read in by hopper
-    phil_file = name+ ".phil"
-    modified_phil = phil_scope.format(python_object=P)
-    with open(phil_file, 'w') as o:
-        modified_phil.show(o)
-    outdir = name + ".outdir"
-    cmd = "hopper %s exp_ref_spec_file=%s outdir=%s logging.rank0_level=high" % (phil_file, hopper_input_lst, outdir)
-    os.system(cmd)
-    # TODO open the pandas output file and optimized expt in outdir and verify the optimized parameters are similar to ground
-    exit()
-
-Eopt,_, Mod, SIM_used_by_hopper, x = hopper_utils.refine(E, refls, P, spec=spec, return_modeler=True)
-
-G, rotX,rotY, rotZ, Na,Nb,Nc,_,_,_,_,_,_,_,_,_,a,b,c,al,be,ga,detz_shift = hopper_utils.get_param_from_x(x, Mod)
-eta_abc_opt = hopper_utils.get_mosaicity_from_x(x, Mod, SIM_used_by_hopper)
-
-print("Na, Nb, Nc= %f %f %f" % (Na, Nb, Nc))
-print("eta_abc optimized:", eta_abc_opt)
-
-# check crystal
-Copt = Eopt.crystal
-misset, misset_init = utils.compare_with_ground_truth(*C.get_real_space_vectors(), dxcryst_models=[Copt, E.crystal], symbol=symbol)
-print(misset_init, "init misset with ground truth")
-print(misset, "misset with ground truth")
-if "detz_shift" in args.perturb or "spec" in args.perturb:
-    assert misset < 0.007, misset
-else:
-    assert misset < 0.005, misset
-
-# check mosaic domain
-assert all(np.subtract(NCELLS_GT, [Na,Nb,Nc]) < 0.2), "%d, %d, %d" % (Na,Nb,Nb)
-
-# check spot scale
-perc_diff_G = abs(GT_spot_scale - G)/ GT_spot_scale * 100
-print("spot scale gt: %f; spot scale opt: %f; percent diff: %f %%" % (GT_spot_scale, G, perc_diff_G))
-max_Gperc = 1
-if "eta" in args.perturb:
-    max_Gperc = 2
-assert perc_diff_G < max_Gperc, perc_diff_G
-
-# check detz
-print("detdist shift %f (should be 0)" % detz_shift)
-assert detz_shift < 0.2, detz_shift
-
-ucell_diff_init = np.abs(np.subtract(ucell , ucell2))
-ucell_diff = np.abs(np.subtract(ucell , Copt.get_unit_cell().parameters()))
-
-init_dev, init_dev_ang = ucell_diff_init[:3].sum(), ucell_diff_init[-3:].sum()
-dev, dev_ang = ucell_diff[:3].sum(), ucell_diff[-3:].sum()
-print("initial ucell dev: %f Angstrom; %f degree" % (init_dev, init_dev_ang))
-print("optimized ucell dev: %f Angstrom; %f degree" % (dev, dev_ang))
-assert dev_ang < init_dev_ang and dev_ang < 0.025, "init: %f curr: %f" % (init_dev_ang, dev_ang)
-if "detz_shift" not in args.perturb:
-    assert dev < init_dev and dev < 0.025, "init: %f  curr: %f" % (init_dev, dev)
-
-if "eta" in args.perturb:
-    print("eta_abc GT:", ETA_ABC_GT)
-    u = np.array(eta_abc_opt)
-    v = np.array(ETA_ABC_GT)
-    perc_diff = np.abs(u-v) / v * 100.
-    assert np.all(perc_diff < 22)  # this is acceptable for now, as we simulated with 5000 blocks, yet modeled with 600
-print("OK")
-
-if "spec" in args.perturb:
-    p0 = Mod.P["lambda_offset"]
-    p1 = Mod.P["lambda_scale"]
-    coef = p0.get_val(x[p0.xpos]), p1.get_val(x[p1.xpos])
-    waves_refined = coef[0] + coef[1] * waves_perturbed
-    fluxsum = sum(fluxes_gt)
-    en_ref_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_refined) / fluxsum)
-    en_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_gt) / fluxsum)
-    en_init_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_perturbed) / fluxsum)
-
-    print("Before refinement: COM energy=%f" % en_init_com)
-    print("AFTER refinement: COM energy=%f" % en_ref_com)
-    print("Ground truth COM energy = %f" % en_com)
-    assert abs(en_ref_com - en_com) < 1
+if args.kokkos:
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    from dxtbx.model.crystal import Crystal
+    from cctbx import uctbx
+    from scitbx.matrix import sqr, rec, col
+    import numpy as np
+    from scipy.spatial.transform import Rotation
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    from simtbx.nanoBragg.sim_data import SimData
+    from simtbx.diffBragg import utils
+    from scipy.signal import windows
+    from dxtbx.model import Experiment
+    from simtbx.nanoBragg import make_imageset
+    from simtbx.diffBragg.phil import hopper_phil, philz
+    from libtbx.phil import parse
+
+    phil_scope = parse(hopper_phil+philz)
+
+    ucell = (55, 65, 75, 90, 95, 90)
+    ucell2 = (55.1, 65.2, 74.9, 90, 94.9, 90)
+    symbol = "P121"
+
+    # generate a random raotation
+    rotation = Rotation.random(num=1, random_state=100)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+
+    # generate a small perturbation rotation
+    np.random.seed(1)
+    perturb_rot_axis = np.random.random(3)
+    perturb_rot_axis /= np.linalg.norm(perturb_rot_axis)
+    perturb_rot_ang = 0.15  # degree random perturbtation
+
+    # make the ground truth crystal:
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    a2_real, b2_real, c2_real = sqr(uctbx.unit_cell(ucell2).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C2 = Crystal(a2_real, b2_real, c2_real, symbol)
+    C2.rotate_around_origin(rot_axis, rot_ang)
+    assert np.allclose(C2.get_U(), C.get_U())
+    C2.rotate_around_origin(col(perturb_rot_axis), perturb_rot_ang)
+
+    # Setup the simulation and create a realistic image
+    # with background and noise
+    # <><><><><><><><><><><><><><><><><><><><><><><><><>
+    nbcryst = NBcrystal()
+    nbcryst.dxtbx_crystal = C   # simulate ground truth
+    nbcryst.thick_mm = 0.1
+    nbcryst.isotropic_ncells = False
+    if "eta" in args.perturb:
+        nbcryst.n_mos_domains = 1000
+        ETA_ABC_GT = args.eta
+        nbcryst.anisotropic_mos_spread_deg = ETA_ABC_GT
+        NCELLS_GT = 12,12,11
+    else:
+        NCELLS_GT = 12,12,11
+    nbcryst.Ncells_abc = NCELLS_GT
+
+    SIM = SimData(use_default_crystal=True)
+    if "spec" in args.perturb:
+        # initialize the simulator
+        spec = SIM.beam.spectrum
+        total_flux = spec[0][1]
+        wave = spec[0][0]
+
+        en = utils.ENERGY_CONV / wave
+        delta_en = 1.5
+        ens_gt = np.arange(en - 5, en + 6, delta_en)
+        waves_gt = utils.ENERGY_CONV / ens_gt
+        num_energies = len(ens_gt)
+        fluxes_gt = np.ones(num_energies) * total_flux / num_energies
+        fluxes_gt = fluxes_gt*windows.hann(num_energies)
+        fluxes_gt /= fluxes_gt.sum()
+        fluxes_gt *= total_flux
+
+        spectrum_GT = list(zip(waves_gt, fluxes_gt))
+        gt_lambda0 = waves_gt[0]
+        gt_lambda1 = waves_gt[1] - waves_gt[0]
+        spec_idx = np.arange(num_energies)
+        assert np.allclose(waves_gt, gt_lambda0 + spec_idx*gt_lambda1)
+
+        lam0 = np.random.normal(gt_lambda0, gt_lambda0 * 0.002)
+        lam1 = np.random.normal(gt_lambda1, abs(gt_lambda1) * 0.002)
+        waves_perturbed = lam0 + spec_idx * lam1
+        print("ENERGY TRUTH=%.4f" % (utils.ENERGY_CONV / gt_lambda0))
+        print("ENERGY PERTURBED=%.4f" % (utils.ENERGY_CONV / lam0))
+        perturbed_spec = list(zip(waves_perturbed, fluxes_gt))
+        SIM.beam.spectrum = spectrum_GT
+
+    #SIM.detector = SimData.simple_detector(150, 0.1, (513, 512))
+    if "eta" in args.perturb:
+        shape = 513*3, 512*3
+        #detdist = 70
+    else:
+        shape = 513, 512
+    detdist = 150
+    SIM.detector = SimData.simple_detector(detdist, 0.1, shape)
+    SIM.crystal = nbcryst
+    SIM.instantiate_diffBragg(oversample=0, auto_set_spotscale=True)
+    SIM.D.default_F = 0
+    SIM.D.F000 = 0
+    SIM.D.progress_meter = False
+    SIM.water_path_mm = 0.005
+    SIM.air_path_mm = 0.1
+    SIM.add_air = True
+    SIM.add_Water = True
+    SIM.include_noise = True
+    SIM.D.verbose = 2
+    SIM.D.add_diffBragg_spots()
+    SIM.D.verbose = 0
+    spots = SIM.D.raw_pixels.as_numpy_array()
+    SIM._add_background()
+    SIM.D.readout_noise_adu=args.readout
+    SIM._add_noise()
+
+    # This is the ground truth image:
+    img = SIM.D.raw_pixels.as_numpy_array()
+    if args.plot:
+        import pylab as plt
+        plt.imshow(img, vmax=100)
+        plt.title("Ground truth image")
+        plt.show()
+
+    cbf_name = name + ".cbf"
+    if args.cmdlineHopper:
+        from IPython import embed
+        embed()
+        SIM.D.to_cbf(cbf_name)
+
+    SIM.D.raw_pixels *= 0
+
+    P = phil_scope.extract()
+    E = Experiment()
+
+    GT_spot_scale = SIM.D.spot_scale
+    if "G" in args.perturb:
+        P.init.G = GT_spot_scale*10
+    else:
+        P.init.G = GT_spot_scale
+
+    P.types.G = args.typeG
+    P.types.Nabc = args.typeNabc
+
+    if "crystal" in args.perturb:
+        E.crystal = C2
+    else:
+        E.crystal = C
+
+    if "Nabc" in args.perturb:
+        P.init.Nabc = 20,20,20
+    else:
+        P.init.Nabc = SIM.crystal.Ncells_abc
+
+    if "detz_shift" in args.perturb:
+        P.init.detz_shift = 1
+    else:
+        P.init.detz_shift = 0
+
+    if "eta" in args.perturb:
+        P.init.eta_abc = [0.12, 0.13, 0.14]
+        P.simulator.crystal.num_mosaicity_samples = 250  # in practive, the number of mosaic domains we model should be smaller than whats in the crystal .. .
+        P.simulator.crystal.has_isotropic_mosaicity = False
+        P.fix.eta_abc = False
+
+    if "spec" in args.perturb:
+        P.fix.spec = False
+        P.init.spec = [0,1]
+        P.fix.Nabc=True
+        P.fix.G=True
+        P.fix.RotXYZ=True
+        P.fix.ucell = True
+        P.fix.detz_shift = True
+        P.ftol=1e-15
+
+    E.detector = SIM.detector
+    E.beam = SIM.D.beam
+    E.imageset = make_imageset([img], E.beam, E.detector)
+    #refls = utils.refls_from_sims([img], E.detector, E.beam, thresh=18)
+    refls = utils.refls_from_sims([spots], E.detector, E.beam, thresh=18)
+    print("%d REFLS" % len(refls))
+    utils.refls_to_q(refls, E.detector, E.beam, update_table=True)
+    utils.refls_to_hkl(refls, E.detector, E.beam, E.crystal, update_table=True)
+
+    P.roi.shoebox_size = 20
+    P.relative_tilt = False
+    P.roi.fit_tilt = False
+    P.roi.pad_shoebox_for_background_estimation=10
+    P.roi.reject_edge_reflections = False
+    P.refiner.sigma_r = SIM.D.readout_noise_adu
+    P.refiner.adu_per_photon = SIM.D.quantum_gain
+    P.simulator.init_scale = 1 #SIM.D.spot_scale
+    P.simulator.beam.size_mm = SIM.beam.size_mm
+    P.simulator.total_flux = SIM.D.flux
+    P.use_restraints = False
+    mtz_name = name +".mtz"
+    SIM.crystal.miller_array.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_name)
+    P.simulator.structure_factors.mtz_name = mtz_name
+    P.simulator.structure_factors.mtz_column = "F(+),F(-)"
+    P.niter = 0
+    P.logging.parameters=True
+    P.niter_per_J = 1
+    P.method="L-BFGS-B"
+    P.ftol = 1e-10
+    if "eta" in args.perturb:
+        P.ftol=1e-8
+    #P.method="Nelder-Mead"
+    #P.fix.G = True
+    #P.fix.Nabc =True
+    #P.fix.detz_shift=True
+
+    import logging
+    import sys
+    h = logging.StreamHandler(sys.stdout)
+    logging.basicConfig(level=logging.DEBUG, handlers=[h])
+
+    from simtbx.diffBragg import hopper_utils
+    spec = None
+    if "spec" in args.perturb:
+        spec = "tst_hopper_refine_spec.lam"
+        wave, wt = map(np.array, zip(*perturbed_spec))
+        utils.save_spectra_file(spec, wave, wt)
+
+    if args.cmdlineHopper:
+        from dxtbx.model import ExperimentList
+        el_name = "%s.expt" % name
+        import_cmd = "dials.import %s output.experiments=%s" % (cbf_name, el_name)
+        os.system(import_cmd)
+        # add the crystal to the imported expt
+        El = ExperimentList.from_file(el_name)
+        El[0].crystal = E.crystal
+        refl_name = "%s.refl" % name
+        El.as_file(el_name)
+        refls.as_file(refl_name)
+        hopper_input_lst = "%s.lst" % name
+        with open(hopper_input_lst, "w") as o:
+            o.write("%s %s\n" % (os.path.abspath(el_name), os.path.abspath(refl_name)))
+
+        # save the above modified phil params to a file to be read in by hopper
+        phil_file = name+ ".phil"
+        modified_phil = phil_scope.format(python_object=P)
+        with open(phil_file, 'w') as o:
+            modified_phil.show(o)
+        outdir = name + ".outdir"
+        cmd = "hopper %s exp_ref_spec_file=%s outdir=%s logging.rank0_level=high" % (phil_file, hopper_input_lst, outdir)
+        os.system(cmd)
+        # TODO open the pandas output file and optimized expt in outdir and verify the optimized parameters are similar to ground
+        exit()
+
+    Eopt,_, Mod, SIM_used_by_hopper, x = hopper_utils.refine(E, refls, P, spec=spec, return_modeler=True)
+
+    G, rotX,rotY, rotZ, Na,Nb,Nc,_,_,_,_,_,_,_,_,_,a,b,c,al,be,ga,detz_shift = hopper_utils.get_param_from_x(x, Mod)
+    eta_abc_opt = hopper_utils.get_mosaicity_from_x(x, Mod, SIM_used_by_hopper)
+
+    print("Na, Nb, Nc= %f %f %f" % (Na, Nb, Nc))
+    print("eta_abc optimized:", eta_abc_opt)
+
+    # check crystal
+    Copt = Eopt.crystal
+    misset, misset_init = utils.compare_with_ground_truth(*C.get_real_space_vectors(), dxcryst_models=[Copt, E.crystal], symbol=symbol)
+    print(misset_init, "init misset with ground truth")
+    print(misset, "misset with ground truth")
+    if "detz_shift" in args.perturb or "spec" in args.perturb:
+        assert misset < 0.007, misset
+    else:
+        assert misset < 0.005, misset
+
+    # check mosaic domain
+    assert all(np.subtract(NCELLS_GT, [Na,Nb,Nc]) < 0.2), "%d, %d, %d" % (Na,Nb,Nb)
+
+    # check spot scale
+    perc_diff_G = abs(GT_spot_scale - G)/ GT_spot_scale * 100
+    print("spot scale gt: %f; spot scale opt: %f; percent diff: %f %%" % (GT_spot_scale, G, perc_diff_G))
+    max_Gperc = 1
+    if "eta" in args.perturb:
+        max_Gperc = 2
+    assert perc_diff_G < max_Gperc, perc_diff_G
+
+    # check detz
+    print("detdist shift %f (should be 0)" % detz_shift)
+    assert detz_shift < 0.2, detz_shift
+
+    ucell_diff_init = np.abs(np.subtract(ucell , ucell2))
+    ucell_diff = np.abs(np.subtract(ucell , Copt.get_unit_cell().parameters()))
+
+    init_dev, init_dev_ang = ucell_diff_init[:3].sum(), ucell_diff_init[-3:].sum()
+    dev, dev_ang = ucell_diff[:3].sum(), ucell_diff[-3:].sum()
+    print("initial ucell dev: %f Angstrom; %f degree" % (init_dev, init_dev_ang))
+    print("optimized ucell dev: %f Angstrom; %f degree" % (dev, dev_ang))
+    assert dev_ang < init_dev_ang and dev_ang < 0.025, "init: %f curr: %f" % (init_dev_ang, dev_ang)
+    if "detz_shift" not in args.perturb:
+        assert dev < init_dev and dev < 0.025, "init: %f  curr: %f" % (init_dev, dev)
+
+    if "eta" in args.perturb:
+        print("eta_abc GT:", ETA_ABC_GT)
+        u = np.array(eta_abc_opt)
+        v = np.array(ETA_ABC_GT)
+        perc_diff = np.abs(u-v) / v * 100.
+        assert np.all(perc_diff < 22)  # this is acceptable for now, as we simulated with 5000 blocks, yet modeled with 600
+    print("OK")
+
+    if "spec" in args.perturb:
+        p0 = Mod.P["lambda_offset"]
+        p1 = Mod.P["lambda_scale"]
+        coef = p0.get_val(x[p0.xpos]), p1.get_val(x[p1.xpos])
+        waves_refined = coef[0] + coef[1] * waves_perturbed
+        fluxsum = sum(fluxes_gt)
+        en_ref_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_refined) / fluxsum)
+        en_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_gt) / fluxsum)
+        en_init_com = utils.ENERGY_CONV / (sum(fluxes_gt * waves_perturbed) / fluxsum)
+
+        print("Before refinement: COM energy=%f" % en_init_com)
+        print("AFTER refinement: COM energy=%f" % en_ref_com)
+        print("Ground truth COM energy = %f" % en_com)
+        assert abs(en_ref_com - en_com) < 1
+
+    del SIM_used_by_hopper.D
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine_Fhkl.py b/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine_Fhkl.py
index 85052075d0..b593ea25e1 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine_Fhkl.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_hopper_refine_Fhkl.py
@@ -2,7 +2,7 @@
 import glob
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--nolog", action="store_true")
 parser.add_argument("--readout", type=float, default=3)
 parser.add_argument("--scale", type=float, default=1)
@@ -16,334 +16,340 @@
 args = parser.parse_args()
 import os
 
-if args.cuda:
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import logging
-import sys
-import pandas
-import numpy as np
-
-from cctbx import miller
-from dials.array_family import flex
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-from simtbx.nanoBragg.sim_data import SimData
-from simtbx.diffBragg import hopper_utils
-from simtbx.diffBragg import utils
-from simtbx.diffBragg.hopper_ensemble_utils import load_inputs
-from dxtbx.model import Experiment
-from simtbx.nanoBragg import make_imageset
-from simtbx.diffBragg.phil import hopper_phil, philz
-from libtbx.phil import parse
-
-phil_scope = parse(hopper_phil+philz)
-
-
-p65_cryst = {'__id__': 'crystal',
-             'real_space_a': (43.32309880004587, 25.5289818883498, 60.49634260901813),
-             'real_space_b': (34.201635357808115, -38.82573591182249, -59.255697149884924),
-             'real_space_c': (41.42476391176581, 229.70849483520402, -126.60059788183489),
-             'space_group_hall_symbol': ' P 65 2 (x,y,z+1/12)',
-             'ML_half_mosaicity_deg': 0.06671930026192037,
-             'ML_domain_size_ang': 6349.223840307989}
-from dxtbx.model.crystal import CrystalFactory
-p65_C = CrystalFactory.from_dict(p65_cryst)
-ucell = p65_C.get_unit_cell().parameters()
-symbol = p65_C.get_space_group().info().type().lookup_symbol()
-
-# Setup the simulation and create a realistic image
-# with background and noise
-# <><><><><><><><><><><><><><><><><><><><><><><><><>
-nbcryst = NBcrystal()
-nbcryst.dxtbx_crystal = p65_C
-nbcryst.thick_mm = 0.005
-nbcryst.isotropic_ncells = False
-NCELLS_GT = 12,12,11
-nbcryst.Ncells_abc = NCELLS_GT
-nbcryst.space_group = "P6522"
-ma = utils.make_miller_array(symbol, ucell, d_min=1.5)
-np.random.seed(0)
-new_data = ma.d_spacings().data()*10
-
-ma = miller.array(ma.set(), new_data).set_observation_type_xray_amplitude()
-ma_map = {h:v for h,v in zip(ma.indices(), ma.data())}
-nbcryst.miller_array = ma
-assert ma.is_xray_amplitude_array()
-
-SIM = SimData(use_default_crystal=False)
-shape = 1000, 1001
-detdist = 140
-SIM.detector = SimData.simple_detector(detdist, 0.1, shape)
-SIM.crystal = nbcryst
-SIM.instantiate_diffBragg(oversample=1, auto_set_spotscale=True, default_F=0)
-
-# test the code for computing the acerage structure factor intensity with resolution
-# (this is why we set the structure factor data to be the same as the resolution (x10)
-num_dspace_bins = 10
-SIM.set_dspace_binning(num_dspace_bins, verbose=True)
-dspace_bins = SIM.D.dspace_bins
-ave_I_cell = SIM.D.ave_I_cell()[0]
-assert len(ave_I_cell) == num_dspace_bins
-assert len(dspace_bins) == num_dspace_bins + 1
-aves = []
-dspaces = []
-for i, (d1,d2) in enumerate(zip(dspace_bins, dspace_bins[1:])):
-    ave_val = np.sqrt(ave_I_cell[i]) / 10.
-    if not args.geo: assert d1 < ave_val < d2
-    aves.append(ave_val)
-    dspaces.append(.5*(d1+d2))
-
-print("1 0 7: ", ma.value_at_index((1,0,7)))
-SIM.D.default_F = 0
-SIM.D.F000 = 0
-SIM.D.progress_meter = False
-SIM.water_path_mm = 0.005
-SIM.air_path_mm = 0.1
-SIM.add_air = True
-SIM.add_Water = True
-SIM.include_noise = True
-SIM.D.verbose = 2
-SIM.D.add_diffBragg_spots()
-SIM.D.verbose = 0
-spots = SIM.D.raw_pixels.as_numpy_array()
-SIM._add_background()
-SIM.D.readout_noise_adu=args.readout
-SIM._add_noise()
-
-# This is the ground truth image:
-img = SIM.D.raw_pixels.as_numpy_array()
-if args.plot:
-    import pylab as plt
-    plt.imshow(img, vmax=100)
-    plt.title("Ground truth image")
-    plt.figure()
-    plt.plot(dspaces, aves)
-    plt.xlabel("Angstrom")
-    plt.show()
-SIM.D.raw_pixels *= 0
-#pfs = 0,270,175
-#utils.show_diffBragg_state(SIM.D, pfs)
-SIM.D.raw_pixels *= 0
-
-P = phil_scope.extract()
-E = Experiment()
-
-P.init.G = SIM.D.spot_scale
-E.crystal = p65_C
-
-P.init.Nabc = SIM.crystal.Ncells_abc
-P.init.detz_shift = 0
-
-E.detector = SIM.detector
-E.beam = SIM.D.beam
-E.imageset = make_imageset([img], E.beam, E.detector)
-refls = utils.refls_from_sims([spots], E.detector, E.beam, thresh=18)
-print("%d REFLS" % len(refls))
-utils.refls_to_q(refls, E.detector, E.beam, update_table=True)
-utils.refls_to_hkl(refls, E.detector, E.beam, E.crystal, update_table=True)
-
-P.roi.shoebox_size = 10
-P.roi.allow_overlapping_spots = True
-P.relative_tilt = False
-P.roi.fit_tilt = False
-P.roi.pad_shoebox_for_background_estimation=10
-P.roi.reject_edge_reflections = False
-P.refiner.sigma_r = SIM.D.readout_noise_adu
-P.refiner.adu_per_photon = SIM.D.quantum_gain
-P.simulator.init_scale = 1
-P.simulator.beam.size_mm = SIM.beam.size_mm
-P.simulator.oversample = SIM.D.oversample
-P.simulator.total_flux = SIM.D.flux
-P.use_restraints = False
-
-
-mset = ma.set()
-ma_map_keys, ma_map_values = list(ma_map.keys()), np.array(list(ma_map.values()))
-ma_map_values2 = np.random.normal(ma_map_values, scale=args.scale*ma_map_values)
-bad_map = {h:v for h,v in zip(ma_map_keys, ma_map_values2)}
-if args.scale ==0:
-    assert np.allclose(ma_map_values, ma_map_values2)
-
-new_amps = flex.double()
-for h in mset.indices():
-    amp = bad_map[h]
-    new_amps.append(amp)
-
-ma2 = miller.array(mset, new_amps).set_observation_type_xray_amplitude()
-
-ma2_map = {h:v for h,v in zip(ma2.indices(), ma2.data())}
-name = "hopper_refine_Fhkl.mtz"
-print("1 0 7: ", ma2.value_at_index((1,0,7)))
-assert ma2.is_xray_amplitude_array()
-ma2.as_mtz_dataset(column_root_label="F").mtz_object().write(name)
-P.simulator.structure_factors.mtz_name = name
-P.simulator.structure_factors.mtz_column = "F(+),F(-)"
-P.logging.parameters=False
-P.method="L-BFGS-B"
-P.ftol = 1e-10
-P.space_group = symbol
-P.fix.Fhkl = False
-P.betas.Fhkl = args.beta
-P.fix.G = True
-P.types.G = "positive"
-P.centers.G = SIM.D.spot_scale*2
-P.betas.G=1e8
-P.use_restraints = args.beta is not None
-P.sigmas.G = args.sigmaG
-P.sigmas.Fhkl = args.sigmaFhkl
-if args.perturb is not None and "G" in args.perturb:
-    P.fix.G = False
-    P.init.G = SIM.D.spot_scale*10
-    #P.maxs.G = SIM.D.spot_scale*100
-P.use_geometric_mean_Fhkl = args.geo
-P.fix.ucell=True
-P.fix.RotXYZ=True
-P.fix.Nabc=True
-if args.perturb is not None and "Nabc" in args.perturb:
-    P.fix.Nabc = False
-    P.init.Nabc = 20,20,18
-P.fix.detz_shift=True
-
-if not args.nolog:
-    h = logging.StreamHandler(sys.stdout)
-    logging.basicConfig(level=logging.DEBUG, handlers=[h])
-
-P.outdir="_temp_fhkl_refine"
-if args.maxiter is not None:
-    P.lbfgs_maxiter = args.maxiter
-Eopt,_, Mod,SIM, x = hopper_utils.refine(E, refls, P, return_modeler=True, free_mem=False)
-
-logging.disable()
-print("\nResults\n<><><><><><>")
-
-Mod.exper_name = "dummie.expt"
-Mod.refl_name = "dummie.refl"
-Mod.save_up(x, SIM)
-
-# we can track the dominant hkls in each shoebox occuring within the diffBragg model
-#count_stats = utils.track_fhkl(Mod)
-#
-#
-#main_hkls = []
-#for i_roi in count_stats:
-#    stats = count_stats[i_roi]
-#    stats = sorted( list(stats.items()), key=lambda x: x[1])
-#    main_hkl, frac = stats[-1]
-#    print(main_hkl, frac)
-#    main_hkls.append(main_hkl)
-
-# this should agree with what we put into diffBragg in the reflection tables
-main_hkls_from_refls = utils.map_hkl_list(list(Mod.refls["miller_index"]), symbol=P.space_group)
-#assert len(set(main_hkls)) == len(set(main_hkls_from_refls))
-# good.
-
-# Now, this should also agree with the refined fhkl values, stored in the data table
-# the modeler save_up method creates an output file containing the refined fhkl values
-
-fnames = glob.glob("%s/Fhkl_scale/rank*/*.npz" % P.outdir)
-assert len(fnames)==1
-fhkl_f= fnames[0]
-fhkl_dat = np.load(fhkl_f)
-asu = list(map(tuple,fhkl_dat['asu_hkl']))
-asu_corrections = fhkl_dat['scale_fac']
-asu_corrections_var = fhkl_dat['scale_var']
-asu_is_nominal = fhkl_dat["is_nominal_hkl"]
-scale = {h:s for h,s in zip(asu, asu_corrections)}
-scale_var = {h:s for h,s in zip(asu, asu_corrections_var)}
-is_nominal = {h:is_nom for h,is_nom in zip(asu,  asu_is_nominal)}
-
-num_not_nominal = 0
-# TODO: figure out why some main_hkls are missing from is_nominal
-for hkl in main_hkls_from_refls:
-    if hkl not in is_nominal:
-        continue
-    if not is_nominal[hkl]:
-        num_not_nominal += 1
-
-nominal_hkl_corrections = {h:s for h,s in zip(asu, asu_corrections) if is_nominal[h]}
-not_nominal_hkl_corrections = {h:s for h,s in zip(asu, asu_corrections) if not is_nominal[h]}
-
-nominal_hkl_init = {h:1 for h in asu if is_nominal[h]}
-not_nominal_hkl_init = {h:1 for h in asu if not is_nominal[h]}
-
-
-def compute_r_factor_with_gt(corrections):
-    gt_data = flex.double()
-    opt_data = flex.double()
-    flx_hkls = flex.miller_index()
-    for hkl, scale in corrections.items():
-        gt_amp = ma_map[hkl]
-        gt_data.append(gt_amp)
-
-        opt_amp = np.sqrt(scale) * ma2_map[hkl]
-        opt_data.append(opt_amp)
-
-        h,k,l = map(int, hkl)
-        flx_hkls.append((h,k,l))
-    mset = ma.miller_set(flx_hkls, ma.anomalous_flag())
-    gt_arr = miller.array(mset, gt_data).set_observation_type_xray_amplitude()
-    opt_arr = miller.array(mset, opt_data).set_observation_type_xray_amplitude()
-    return gt_arr.r1_factor(opt_arr)
-
-
-r1_nominal_init = compute_r_factor_with_gt(nominal_hkl_init)
-r1_nominal = compute_r_factor_with_gt(nominal_hkl_corrections)
-
-r1_not_nominal_init = compute_r_factor_with_gt(not_nominal_hkl_init)
-r1_not_nominal = compute_r_factor_with_gt(not_nominal_hkl_corrections)
-
-print("\nResults\n<><><><><><>")
-print("For the dominant HKLs within each modeled shoebox (e.g. those with indexed reflections)")
-print("initial R1 factor=%.2f%%" % (r1_nominal_init*100))
-print("optimized R1 factor=%.2f%%" % (r1_nominal*100))
-
-diffs = []
-all_opts = []
-all_gts = []
-dsp_map = {d:val for d,val in zip(ma.d_spacings().indices(), ma.d_spacings().data())}
-ds = []
-for hkl in nominal_hkl_corrections:
-    dsp = dsp_map[hkl]
-    gt_val = ma_map[hkl]
-    opt_val = np.sqrt(nominal_hkl_corrections[hkl]) * ma2_map[hkl]
-    d = abs(gt_val - opt_val) / gt_val * 100
-    diffs.append(d)
-
-    all_opts.append(opt_val)
-    all_gts.append(gt_val)
-    ds.append(dsp)
-
-print("mean percent diff", np.mean(diffs))
-
-assert r1_nominal < r1_nominal_init, "r1_nom=%f, r1_not_nom=%f" %(r1_nominal, r1_nominal_init)
-assert r1_nominal < 0.04
-
-# test hopper_ensemble_refiner using this one shot
-# dump the refinement data to the reflection table format (e.g. the pixel data and background estimates)
-input_refl = os.path.join(P.outdir, "input_data.refl")
-Mod.dump_gathered_to_refl(input_refl)
-df = pandas.read_pickle("%s/pandas/rank0/stage1_dummie_0.pkl"% P.outdir)
-refl_col = "input_refls"
-df[refl_col] = [input_refl]
-P.refiner.load_data_from_refl = True
-P.refiner.check_expt_format = False
-modelers = load_inputs(df, P, exper_key="opt_exp_name", refls_key=refl_col)
-modelers.outdir=P.outdir
-modelers.prep_for_refinement()
-modelers.Minimize(save=True)
-
-from iotbx.reflection_file_reader import any_reflection_file
-opt_F = any_reflection_file("_temp_fhkl_refine/optimized_channel0.mtz").as_miller_arrays()[0]
-opt_map = {h:v for h,v in zip(opt_F.indices(), opt_F.data())}
-hcommon = set(ma_map).intersection(opt_map)
-
-mset_common = ma.miller_set(flex.miller_index(list(hcommon)), ma.anomalous_flag())
-ma_vals = flex.double([ma_map[h] for h in hcommon])
-opt_vals = flex.double([opt_map[h] for h in hcommon])
-
-ma_common = miller.array(mset_common, ma_vals).set_observation_type_xray_amplitude()
-opt_common = miller.array(mset_common, opt_vals).set_observation_type_xray_amplitude()
-r1 = ma_common.r1_factor(opt_common)
-assert r1 < 0.04
-
-print("OK")
+if args.kokkos:
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import logging
+    import sys
+    import pandas
+    import numpy as np
+
+    from cctbx import miller
+    from dials.array_family import flex
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    from simtbx.nanoBragg.sim_data import SimData
+    from simtbx.diffBragg import hopper_utils
+    from simtbx.diffBragg import utils
+    from simtbx.diffBragg.hopper_ensemble_utils import load_inputs
+    from dxtbx.model import Experiment
+    from simtbx.nanoBragg import make_imageset
+    from simtbx.diffBragg.phil import hopper_phil, philz
+    from libtbx.phil import parse
+
+    phil_scope = parse(hopper_phil+philz)
+
+
+    p65_cryst = {'__id__': 'crystal',
+                 'real_space_a': (43.32309880004587, 25.5289818883498, 60.49634260901813),
+                 'real_space_b': (34.201635357808115, -38.82573591182249, -59.255697149884924),
+                 'real_space_c': (41.42476391176581, 229.70849483520402, -126.60059788183489),
+                 'space_group_hall_symbol': ' P 65 2 (x,y,z+1/12)',
+                 'ML_half_mosaicity_deg': 0.06671930026192037,
+                 'ML_domain_size_ang': 6349.223840307989}
+    from dxtbx.model.crystal import CrystalFactory
+    p65_C = CrystalFactory.from_dict(p65_cryst)
+    ucell = p65_C.get_unit_cell().parameters()
+    symbol = p65_C.get_space_group().info().type().lookup_symbol()
+
+    # Setup the simulation and create a realistic image
+    # with background and noise
+    # <><><><><><><><><><><><><><><><><><><><><><><><><>
+    nbcryst = NBcrystal()
+    nbcryst.dxtbx_crystal = p65_C
+    nbcryst.thick_mm = 0.005
+    nbcryst.isotropic_ncells = False
+    NCELLS_GT = 12,12,11
+    nbcryst.Ncells_abc = NCELLS_GT
+    nbcryst.space_group = "P6522"
+    ma = utils.make_miller_array(symbol, ucell, d_min=1.5)
+    np.random.seed(0)
+    new_data = ma.d_spacings().data()*10
+
+    ma = miller.array(ma.set(), new_data).set_observation_type_xray_amplitude()
+    ma_map = {h:v for h,v in zip(ma.indices(), ma.data())}
+    nbcryst.miller_array = ma
+    assert ma.is_xray_amplitude_array()
+
+    SIM = SimData(use_default_crystal=False)
+    shape = 1000, 1001
+    detdist = 140
+    SIM.detector = SimData.simple_detector(detdist, 0.1, shape)
+    SIM.crystal = nbcryst
+    SIM.instantiate_diffBragg(oversample=1, auto_set_spotscale=True, default_F=0)
+
+    # test the code for computing the acerage structure factor intensity with resolution
+    # (this is why we set the structure factor data to be the same as the resolution (x10)
+    num_dspace_bins = 10
+    SIM.set_dspace_binning(num_dspace_bins, verbose=True)
+    dspace_bins = SIM.D.dspace_bins
+    ave_I_cell = SIM.D.ave_I_cell()[0]
+    assert len(ave_I_cell) == num_dspace_bins
+    assert len(dspace_bins) == num_dspace_bins + 1
+    aves = []
+    dspaces = []
+    for i, (d1,d2) in enumerate(zip(dspace_bins, dspace_bins[1:])):
+        ave_val = np.sqrt(ave_I_cell[i]) / 10.
+        if not args.geo: assert d1 < ave_val < d2
+        aves.append(ave_val)
+        dspaces.append(.5*(d1+d2))
+
+    print("1 0 7: ", ma.value_at_index((1,0,7)))
+    SIM.D.default_F = 0
+    SIM.D.F000 = 0
+    SIM.D.progress_meter = False
+    SIM.water_path_mm = 0.005
+    SIM.air_path_mm = 0.1
+    SIM.add_air = True
+    SIM.add_Water = True
+    SIM.include_noise = True
+    SIM.D.verbose = 2
+    SIM.D.add_diffBragg_spots()
+    SIM.D.verbose = 0
+    spots = SIM.D.raw_pixels.as_numpy_array()
+    SIM._add_background()
+    SIM.D.readout_noise_adu=args.readout
+    SIM._add_noise()
+
+    # This is the ground truth image:
+    img = SIM.D.raw_pixels.as_numpy_array()
+    if args.plot:
+        import pylab as plt
+        plt.imshow(img, vmax=100)
+        plt.title("Ground truth image")
+        plt.figure()
+        plt.plot(dspaces, aves)
+        plt.xlabel("Angstrom")
+        plt.show()
+    SIM.D.raw_pixels *= 0
+    #pfs = 0,270,175
+    #utils.show_diffBragg_state(SIM.D, pfs)
+    SIM.D.raw_pixels *= 0
+
+    P = phil_scope.extract()
+    E = Experiment()
+
+    P.init.G = SIM.D.spot_scale
+    E.crystal = p65_C
+
+    P.init.Nabc = SIM.crystal.Ncells_abc
+    P.init.detz_shift = 0
+
+    E.detector = SIM.detector
+    E.beam = SIM.D.beam
+    E.imageset = make_imageset([img], E.beam, E.detector)
+    refls = utils.refls_from_sims([spots], E.detector, E.beam, thresh=18)
+    print("%d REFLS" % len(refls))
+    utils.refls_to_q(refls, E.detector, E.beam, update_table=True)
+    utils.refls_to_hkl(refls, E.detector, E.beam, E.crystal, update_table=True)
+
+    P.roi.shoebox_size = 10
+    P.roi.allow_overlapping_spots = True
+    P.relative_tilt = False
+    P.roi.fit_tilt = False
+    P.roi.pad_shoebox_for_background_estimation=10
+    P.roi.reject_edge_reflections = False
+    P.refiner.sigma_r = SIM.D.readout_noise_adu
+    P.refiner.adu_per_photon = SIM.D.quantum_gain
+    P.simulator.init_scale = 1
+    P.simulator.beam.size_mm = SIM.beam.size_mm
+    P.simulator.oversample = SIM.D.oversample
+    P.simulator.total_flux = SIM.D.flux
+    P.use_restraints = False
+
+
+    mset = ma.set()
+    ma_map_keys, ma_map_values = list(ma_map.keys()), np.array(list(ma_map.values()))
+    ma_map_values2 = np.random.normal(ma_map_values, scale=args.scale*ma_map_values)
+    bad_map = {h:v for h,v in zip(ma_map_keys, ma_map_values2)}
+    if args.scale ==0:
+        assert np.allclose(ma_map_values, ma_map_values2)
+
+    new_amps = flex.double()
+    for h in mset.indices():
+        amp = bad_map[h]
+        new_amps.append(amp)
+
+    ma2 = miller.array(mset, new_amps).set_observation_type_xray_amplitude()
+
+    ma2_map = {h:v for h,v in zip(ma2.indices(), ma2.data())}
+    name = "hopper_refine_Fhkl.mtz"
+    print("1 0 7: ", ma2.value_at_index((1,0,7)))
+    assert ma2.is_xray_amplitude_array()
+    ma2.as_mtz_dataset(column_root_label="F").mtz_object().write(name)
+    P.simulator.structure_factors.mtz_name = name
+    P.simulator.structure_factors.mtz_column = "F(+),F(-)"
+    P.logging.parameters=False
+    P.method="L-BFGS-B"
+    P.ftol = 1e-10
+    P.space_group = symbol
+    P.fix.Fhkl = False
+    P.betas.Fhkl = args.beta
+    P.fix.G = True
+    P.types.G = "positive"
+    P.centers.G = SIM.D.spot_scale*2
+    P.betas.G=1e8
+    P.use_restraints = args.beta is not None
+    P.sigmas.G = args.sigmaG
+    P.sigmas.Fhkl = args.sigmaFhkl
+    if args.perturb is not None and "G" in args.perturb:
+        P.fix.G = False
+        P.init.G = SIM.D.spot_scale*10
+        #P.maxs.G = SIM.D.spot_scale*100
+    P.use_geometric_mean_Fhkl = args.geo
+    P.fix.ucell=True
+    P.fix.RotXYZ=True
+    P.fix.Nabc=True
+    if args.perturb is not None and "Nabc" in args.perturb:
+        P.fix.Nabc = False
+        P.init.Nabc = 20,20,18
+    P.fix.detz_shift=True
+
+    if not args.nolog:
+        h = logging.StreamHandler(sys.stdout)
+        logging.basicConfig(level=logging.DEBUG, handlers=[h])
+    #del SIM.D
+
+    P.outdir="_temp_fhkl_refine"
+    if args.maxiter is not None:
+        P.lbfgs_maxiter = args.maxiter
+    Eopt,_, Mod,SIM_from_hopper, x = hopper_utils.refine(E, refls, P, return_modeler=True, free_mem=False)
+
+    logging.disable()
+    print("\nResults\n<><><><><><>")
+
+    Mod.exper_name = "dummie.expt"
+    Mod.refl_name = "dummie.refl"
+    Mod.save_up(x, SIM_from_hopper)
+
+    # we can track the dominant hkls in each shoebox occuring within the diffBragg model
+    #count_stats = utils.track_fhkl(Mod)
+    #
+    #
+    #main_hkls = []
+    #for i_roi in count_stats:
+    #    stats = count_stats[i_roi]
+    #    stats = sorted( list(stats.items()), key=lambda x: x[1])
+    #    main_hkl, frac = stats[-1]
+    #    print(main_hkl, frac)
+    #    main_hkls.append(main_hkl)
+
+    # this should agree with what we put into diffBragg in the reflection tables
+    main_hkls_from_refls = utils.map_hkl_list(list(Mod.refls["miller_index"]), symbol=P.space_group)
+    #assert len(set(main_hkls)) == len(set(main_hkls_from_refls))
+    # good.
+
+    # Now, this should also agree with the refined fhkl values, stored in the data table
+    # the modeler save_up method creates an output file containing the refined fhkl values
+
+    fnames = glob.glob("%s/Fhkl_scale/rank*/*.npz" % P.outdir)
+    assert len(fnames)==1
+    fhkl_f= fnames[0]
+    fhkl_dat = np.load(fhkl_f)
+    asu = list(map(tuple,fhkl_dat['asu_hkl']))
+    asu_corrections = fhkl_dat['scale_fac']
+    asu_corrections_var = fhkl_dat['scale_var']
+    asu_is_nominal = fhkl_dat["is_nominal_hkl"]
+    scale = {h:s for h,s in zip(asu, asu_corrections)}
+    scale_var = {h:s for h,s in zip(asu, asu_corrections_var)}
+    is_nominal = {h:is_nom for h,is_nom in zip(asu,  asu_is_nominal)}
+
+    num_not_nominal = 0
+    # TODO: figure out why some main_hkls are missing from is_nominal
+    for hkl in main_hkls_from_refls:
+        if hkl not in is_nominal:
+            continue
+        if not is_nominal[hkl]:
+            num_not_nominal += 1
+
+    nominal_hkl_corrections = {h:s for h,s in zip(asu, asu_corrections) if is_nominal[h]}
+    not_nominal_hkl_corrections = {h:s for h,s in zip(asu, asu_corrections) if not is_nominal[h]}
+
+    nominal_hkl_init = {h:1 for h in asu if is_nominal[h]}
+    not_nominal_hkl_init = {h:1 for h in asu if not is_nominal[h]}
+
+
+    def compute_r_factor_with_gt(corrections):
+        gt_data = flex.double()
+        opt_data = flex.double()
+        flx_hkls = flex.miller_index()
+        for hkl, scale in corrections.items():
+            gt_amp = ma_map[hkl]
+            gt_data.append(gt_amp)
+
+            opt_amp = np.sqrt(scale) * ma2_map[hkl]
+            opt_data.append(opt_amp)
+
+            h,k,l = map(int, hkl)
+            flx_hkls.append((h,k,l))
+        mset = ma.miller_set(flx_hkls, ma.anomalous_flag())
+        gt_arr = miller.array(mset, gt_data).set_observation_type_xray_amplitude()
+        opt_arr = miller.array(mset, opt_data).set_observation_type_xray_amplitude()
+        return gt_arr.r1_factor(opt_arr)
+
+
+    r1_nominal_init = compute_r_factor_with_gt(nominal_hkl_init)
+    r1_nominal = compute_r_factor_with_gt(nominal_hkl_corrections)
+
+    r1_not_nominal_init = compute_r_factor_with_gt(not_nominal_hkl_init)
+    r1_not_nominal = compute_r_factor_with_gt(not_nominal_hkl_corrections)
+
+    print("\nResults\n<><><><><><>")
+    print("For the dominant HKLs within each modeled shoebox (e.g. those with indexed reflections)")
+    print("initial R1 factor=%.2f%%" % (r1_nominal_init*100))
+    print("optimized R1 factor=%.2f%%" % (r1_nominal*100))
+
+    diffs = []
+    all_opts = []
+    all_gts = []
+    dsp_map = {d:val for d,val in zip(ma.d_spacings().indices(), ma.d_spacings().data())}
+    ds = []
+    for hkl in nominal_hkl_corrections:
+        dsp = dsp_map[hkl]
+        gt_val = ma_map[hkl]
+        opt_val = np.sqrt(nominal_hkl_corrections[hkl]) * ma2_map[hkl]
+        d = abs(gt_val - opt_val) / gt_val * 100
+        diffs.append(d)
+
+        all_opts.append(opt_val)
+        all_gts.append(gt_val)
+        ds.append(dsp)
+
+    print("mean percent diff", np.mean(diffs))
+
+    assert r1_nominal < r1_nominal_init, "r1_nom=%f, r1_not_nom=%f" %(r1_nominal, r1_nominal_init)
+    assert r1_nominal < 0.04
+
+    # test hopper_ensemble_refiner using this one shot
+    # dump the refinement data to the reflection table format (e.g. the pixel data and background estimates)
+    input_refl = os.path.join(P.outdir, "input_data.refl")
+    Mod.dump_gathered_to_refl(input_refl)
+    df = pandas.read_pickle("%s/pandas/rank0/stage1_dummie_0.pkl"% P.outdir)
+    refl_col = "input_refls"
+    df[refl_col] = [input_refl]
+    P.refiner.load_data_from_refl = True
+    P.refiner.check_expt_format = False
+    modelers = load_inputs(df, P, exper_key="opt_exp_name", refls_key=refl_col)
+    modelers.outdir=P.outdir
+    modelers.prep_for_refinement()
+    modelers.Minimize(save=True)
+
+    from iotbx.reflection_file_reader import any_reflection_file
+    opt_F = any_reflection_file("_temp_fhkl_refine/optimized_channel0.mtz").as_miller_arrays()[0]
+    opt_map = {h:v for h,v in zip(opt_F.indices(), opt_F.data())}
+    hcommon = set(ma_map).intersection(opt_map)
+
+    mset_common = ma.miller_set(flex.miller_index(list(hcommon)), ma.anomalous_flag())
+    ma_vals = flex.double([ma_map[h] for h in hcommon])
+    opt_vals = flex.double([opt_map[h] for h in hcommon])
+
+    ma_common = miller.array(mset_common, ma_vals).set_observation_type_xray_amplitude()
+    opt_common = miller.array(mset_common, opt_vals).set_observation_type_xray_amplitude()
+    r1 = ma_common.r1_factor(opt_common)
+    assert r1 < 0.04
+
+    del SIM.D
+    del modelers.SIM.D
+    del SIM_from_hopper.D
+    print("OK")
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_lambda_coefficients.py b/simtbx/diffBragg/tests/tst_diffBragg_lambda_coefficients.py
index ace4c1079b..386f6cd212 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_lambda_coefficients.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_lambda_coefficients.py
@@ -3,152 +3,158 @@
 This test checks the lambda coefficients property and derivatives
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--idx", type=int, help="coefficient index (0 or 1)", default=0, choices=[0,1])
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import numpy as np
-import pylab as plt
-from scipy.stats import linregress
-from scipy.spatial.transform import Rotation
-from simtbx.nanoBragg import sim_data
-from scitbx.matrix import sqr, rec
-from cctbx import uctbx
-from dxtbx.model import Crystal
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-S = sim_data.SimData(use_default_crystal=True)
-S.crystal.dxtbx_crystal = C
-spectrum = S.beam.spectrum
-wave, flux = spectrum[0]
-Nwave = 5
-waves = np.linspace(wave-wave*0.002, wave+wave*0.002, Nwave)
-fluxes = np.ones(Nwave) * flux / Nwave
-
-lambda0_GT = 0
-lambda1_GT = 1
-
-S.beam.spectrum = list(zip(waves, fluxes))
-S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
-S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
-S.D.lambda_coefficients = lambda0_GT, lambda1_GT
-S.D.spot_scale = 100000
-S.D.Ncells_abc = 12
-
-if args.idx == 0:
-    S.D.refine(12)
-else:
-    S.D.refine(13)
-S.D.initialize_managers()
-S.D.region_of_interest = ((0, 1023), (0, 1023))
-
-S.D.add_diffBragg_spots()
-img = S.D.raw_pixels.as_numpy_array()
-derivs = S.D.get_lambda_derivative_pixels()
-deriv = derivs[0].as_numpy_array().reshape(img.shape)
-
-S.D.raw_pixels *= 0
-S.D.use_lambda_coefficients = False
-S.D.add_diffBragg_spots()
-test_img = S.D.raw_pixels.as_numpy_array()
-assert np.allclose(img, test_img)
-S.D.use_lambda_coefficients = True
-S.D.raw_pixels *= 0
-print("OK")
-
-bragg = img > 1e-1  # select bragg scattering regions
-
-all_error = []
-all_error2 = []
-shifts = []
-shifts2 = []
-
-from scipy import constants
-ENERGY_CONV = 1e10*constants.c*constants.h / constants.electron_volt
-
-energy_shifts = 0.1, .3, .5, 1, 3, 5, 10   # in electron volt
-b_percs = 0.001, 0.002, 0.004, 0.008,  0.016, 0.032, 0.064
-reference_energy = ENERGY_CONV / wave
-
-for i_shift, en_shift in enumerate(energy_shifts):
-
-    wave_shifted = ENERGY_CONV / (reference_energy + en_shift)
-    wave_shift = wave - wave_shifted
-    delta_a = wave_shift
-    delta_b = lambda1_GT*b_percs[i_shift]
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import numpy as np
+    import pylab as plt
+    from scipy.stats import linregress
+    from scipy.spatial.transform import Rotation
+    from simtbx.nanoBragg import sim_data
+    from scitbx.matrix import sqr, rec
+    from cctbx import uctbx
+    from dxtbx.model import Crystal
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    S = sim_data.SimData(use_default_crystal=True)
+    S.crystal.dxtbx_crystal = C
+    spectrum = S.beam.spectrum
+    wave, flux = spectrum[0]
+    Nwave = 5
+    waves = np.linspace(wave-wave*0.002, wave+wave*0.002, Nwave)
+    fluxes = np.ones(Nwave) * flux / Nwave
+
+    lambda0_GT = 0
+    lambda1_GT = 1
+
+    S.beam.spectrum = list(zip(waves, fluxes))
+    S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
+    S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
+    S.D.lambda_coefficients = lambda0_GT, lambda1_GT
+    S.D.spot_scale = 100000
+    S.D.Ncells_abc = 12
 
     if args.idx == 0:
-        shift = b_percs[i_shift]*0.01
-        new_waves = waves*lambda1_GT + lambda0_GT+shift
+        S.D.refine(12)
     else:
-        shift = b_percs[i_shift]*0.01
-        new_waves = waves*(lambda1_GT+shift) + lambda0_GT
-
-    en = np.mean(ENERGY_CONV/new_waves)
+        S.D.refine(13)
+    S.D.initialize_managers()
+    S.D.region_of_interest = ((0, 1023), (0, 1023))
 
-    if args.idx == 0:
-        S.D.lambda_coefficients = lambda0_GT + shift, lambda1_GT
-        shifts.append(shift)
-    else:
-        S.D.lambda_coefficients = lambda0_GT, lambda1_GT + shift
-        shifts.append(shift)
+    S.D.add_diffBragg_spots()
+    img = S.D.raw_pixels.as_numpy_array()
+    derivs = S.D.get_lambda_derivative_pixels()
+    deriv = derivs[0].as_numpy_array().reshape(img.shape)
 
     S.D.raw_pixels *= 0
-    S.D.region_of_interest = ((0, 1023), (0, 1023))
+    S.D.use_lambda_coefficients = False
     S.D.add_diffBragg_spots()
-    img2 = S.D.raw_pixels.as_numpy_array()
-
-    fdiff = (img2 - img) / shift
-
-    if args.idx == 0:
-        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-    else:
-        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-
-    all_error.append(error)
-
-    print ("error=%f, step=%f, energy=%f" % (error, delta_a, en))
-    #if args.plot:
-    #    plt.subplot(121)
-    #    plt.imshow(fdiff)
-    #    plt.title("finite diff")
-    #    plt.subplot(122)
-    #    plt.imshow(deriv)
-    #    plt.title("analytical")
-    #    plt.draw()
-    #    plt.suptitle("Shift %d / %d"
-    #                 % (i_shift + 1, len(perc)))
-    #    plt.pause(0.8)
-
-if args.plot:
-    #plt.close()
-    plt.plot(shifts, all_error, 'o')
-    plt.show()
-    #if args.curvatures:
-    #    plt.plot(shifts2, all_error2, 'o')
-    #    plt.show()
-
-l = linregress(shifts, all_error)
-assert l.rvalue > .9999  # this is definitely a line!
-assert l.slope > 0
-assert l.pvalue < 1e-6
-
-print("OK!")
+    test_img = S.D.raw_pixels.as_numpy_array()
+    assert np.allclose(img, test_img)
+    S.D.use_lambda_coefficients = True
+    S.D.raw_pixels *= 0
+    print("OK")
+
+    bragg = img > 1e-1  # select bragg scattering regions
+
+    all_error = []
+    all_error2 = []
+    shifts = []
+    shifts2 = []
+
+    from scipy import constants
+    ENERGY_CONV = 1e10*constants.c*constants.h / constants.electron_volt
+
+    energy_shifts = 0.1, .3, .5, 1, 3, 5, 10   # in electron volt
+    b_percs = 0.001, 0.002, 0.004, 0.008,  0.016, 0.032, 0.064
+    reference_energy = ENERGY_CONV / wave
+
+    for i_shift, en_shift in enumerate(energy_shifts):
+
+        wave_shifted = ENERGY_CONV / (reference_energy + en_shift)
+        wave_shift = wave - wave_shifted
+        delta_a = wave_shift
+        delta_b = lambda1_GT*b_percs[i_shift]
+
+        if args.idx == 0:
+            shift = b_percs[i_shift]*0.01
+            new_waves = waves*lambda1_GT + lambda0_GT+shift
+        else:
+            shift = b_percs[i_shift]*0.01
+            new_waves = waves*(lambda1_GT+shift) + lambda0_GT
+
+        en = np.mean(ENERGY_CONV/new_waves)
+
+        if args.idx == 0:
+            S.D.lambda_coefficients = lambda0_GT + shift, lambda1_GT
+            shifts.append(shift)
+        else:
+            S.D.lambda_coefficients = lambda0_GT, lambda1_GT + shift
+            shifts.append(shift)
+
+        S.D.raw_pixels *= 0
+        S.D.region_of_interest = ((0, 1023), (0, 1023))
+        S.D.add_diffBragg_spots()
+        img2 = S.D.raw_pixels.as_numpy_array()
+
+        fdiff = (img2 - img) / shift
+
+        if args.idx == 0:
+            error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+        else:
+            error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+
+        all_error.append(error)
+
+        print ("error=%f, step=%f, energy=%f" % (error, delta_a, en))
+        #if args.plot:
+        #    plt.subplot(121)
+        #    plt.imshow(fdiff)
+        #    plt.title("finite diff")
+        #    plt.subplot(122)
+        #    plt.imshow(deriv)
+        #    plt.title("analytical")
+        #    plt.draw()
+        #    plt.suptitle("Shift %d / %d"
+        #                 % (i_shift + 1, len(perc)))
+        #    plt.pause(0.8)
+
+    if args.plot:
+        #plt.close()
+        plt.plot(shifts, all_error, 'o')
+        plt.show()
+        #if args.curvatures:
+        #    plt.plot(shifts2, all_error2, 'o')
+        #    plt.show()
+
+    l = linregress(shifts, all_error)
+    assert l.rvalue > .9999  # this is definitely a line!
+    assert l.slope > 0
+    assert l.pvalue < 1e-6
+
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_nanoBragg_congruency.py b/simtbx/diffBragg/tests/tst_diffBragg_nanoBragg_congruency.py
index 783d55d7ac..194607a9c6 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_nanoBragg_congruency.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_nanoBragg_congruency.py
@@ -1,13 +1,12 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
+from simtbx.diffBragg.utils import find_diffBragg_instances
+import numpy as np
 
 
 def main():
     from simtbx.nanoBragg.sim_data import SimData
-    import sys
-    if "--cuda" in sys.argv:
-        import os
-        os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
 
     S = SimData(use_default_crystal=True)
     S.instantiate_diffBragg()
@@ -25,11 +24,16 @@ def main():
 
     nano_img = S.D.raw_pixels.as_numpy_array()
 
-    import numpy as np
-
     assert np.allclose(diff_img, nano_img, atol=1e-9)
+    for name in find_diffBragg_instances(globals()): del globals()[name]
 
 
 if __name__ == "__main__":
-    main()
+    import sys
+    if "--kokkos" in sys.argv:
+        import os
+        os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+    from simtbx.diffBragg.device import DeviceWrapper
+    with DeviceWrapper(0) as _:
+        main()
     print ("OK!")
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_ncells_offdiag_property.py b/simtbx/diffBragg/tests/tst_diffBragg_ncells_offdiag_property.py
index 184854a0ae..9bf3dfd801 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_ncells_offdiag_property.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_ncells_offdiag_property.py
@@ -2,139 +2,145 @@
 This test checks the setter and getter for Ncells parameter
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--curvatures", action='store_true')
 parser.add_argument("--idx", default=0, help="0=Nd, 1=Ne, 2=Nf", type=int, choices=[0,1,2])
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import numpy as np
-import pylab as plt
-from scipy.stats import linregress
-from scipy.spatial.transform import Rotation
-from simtbx.nanoBragg import sim_data
-from scitbx.matrix import sqr, rec
-from cctbx import uctbx
-from dxtbx.model import Crystal
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-S = sim_data.SimData(use_default_crystal=True)
-S.crystal.dxtbx_crystal = C
-S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
-S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
-S.D.spot_scale = 100000
-#S.D.oversample = 5
-
-Ncells_def_id = 21
-
-S.D.Ncells_abc_aniso = 12, 13, 14
-S.D.Ncells_def = 10,11,12
-Nd, Ne, Nf = S.D.Ncells_def
-
-assert (Nd,Ne,Nf) == (10,11,12)
-
-print("OK")
-S.D.isotropic_ncells = False
-S.D.refine(Ncells_def_id)
-S.D.initialize_managers()
-ncells_def_GT = 10,11,12
-
-S.D.Ncells_abc_aniso = (20, 15, 17)
-S.D.Ncells_def = ncells_def_GT
-S.D.add_diffBragg_spots()
-
-img = S.D.raw_pixels_roi.as_numpy_array()
-derivs_abc = S.D.get_ncells_def_derivative_pixels()
-derivs_abc = [d.as_numpy_array() for d in derivs_abc]
-if args.curvatures:
-    derivs2_abc = S.D.get_ncells_def_second_derivative_pixels()
-    derivs2_abc = [d.as_numpy_array() for d in derivs2_abc]
-
-perc = 0.001, 0.01, 0.1, 1, 10
-bragg = img > 1e-1  # select bragg scattering regions
-
-Nd, Ne, Nf = ncells_def_GT
-for i_nc in range(3):
-    if i_nc != args.idx:
-        continue
-
-    N = ncells_def_GT[i_nc]
-    print("Testing finite diff %d / 3, Ncells_def=%d" % (i_nc +1, N))
-
-    all_error = []
-    all_error2 = []
-    shifts = []
-    shifts2 = []
-    for i_shift, p in enumerate(perc):
-        delta_N = N*p*0.01
-
-        Ncells_def_vals = [Nd, Ne, Nf]
-        Ncells_def_vals[i_nc] = N+delta_N
-
-        S.D.Ncells_def = tuple(Ncells_def_vals)
-        shifts.append(delta_N)
-
-        S.D.raw_pixels_roi *= 0
-        #S.D.printout_pixel_fastslow = 10, 10
-        S.D.add_diffBragg_spots()
-        img2 = S.D.raw_pixels_roi.as_numpy_array()
-
-        fdiff = (img2 - img) / delta_N
-        deriv = derivs_abc[i_nc]
-        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-        all_error.append(error)
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import numpy as np
+    import pylab as plt
+    from scipy.stats import linregress
+    from scipy.spatial.transform import Rotation
+    from simtbx.nanoBragg import sim_data
+    from scitbx.matrix import sqr, rec
+    from cctbx import uctbx
+    from dxtbx.model import Crystal
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    S = sim_data.SimData(use_default_crystal=True)
+    S.crystal.dxtbx_crystal = C
+    S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
+    S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
+    S.D.spot_scale = 100000
+    #S.D.oversample = 5
+
+    Ncells_def_id = 21
+
+    S.D.Ncells_abc_aniso = 12, 13, 14
+    S.D.Ncells_def = 10,11,12
+    Nd, Ne, Nf = S.D.Ncells_def
+
+    assert (Nd,Ne,Nf) == (10,11,12)
+
+    print("OK")
+    S.D.isotropic_ncells = False
+    S.D.refine(Ncells_def_id)
+    S.D.initialize_managers()
+    ncells_def_GT = 10,11,12
+
+    S.D.Ncells_abc_aniso = (20, 15, 17)
+    S.D.Ncells_def = ncells_def_GT
+    S.D.add_diffBragg_spots()
+
+    img = S.D.raw_pixels_roi.as_numpy_array()
+    derivs_abc = S.D.get_ncells_def_derivative_pixels()
+    derivs_abc = [d.as_numpy_array() for d in derivs_abc]
+    if args.curvatures:
+        derivs2_abc = S.D.get_ncells_def_second_derivative_pixels()
+        derivs2_abc = [d.as_numpy_array() for d in derivs2_abc]
 
-        if args.curvatures:
-            Ncells_vals = [Nd, Ne, Nf]
-            Ncells_vals[i_nc] = N - delta_N
-            S.D.Ncells_def = tuple(Ncells_vals)
-            shifts2.append(delta_N**2)
+    perc = 0.001, 0.01, 0.1, 1, 10
+    bragg = img > 1e-1  # select bragg scattering regions
+
+    Nd, Ne, Nf = ncells_def_GT
+    for i_nc in range(3):
+        if i_nc != args.idx:
+            continue
+
+        N = ncells_def_GT[i_nc]
+        print("Testing finite diff %d / 3, Ncells_def=%d" % (i_nc +1, N))
+
+        all_error = []
+        all_error2 = []
+        shifts = []
+        shifts2 = []
+        for i_shift, p in enumerate(perc):
+            delta_N = N*p*0.01
+
+            Ncells_def_vals = [Nd, Ne, Nf]
+            Ncells_def_vals[i_nc] = N+delta_N
+
+            S.D.Ncells_def = tuple(Ncells_def_vals)
+            shifts.append(delta_N)
 
             S.D.raw_pixels_roi *= 0
+            #S.D.printout_pixel_fastslow = 10, 10
             S.D.add_diffBragg_spots()
-            img_backward = S.D.raw_pixels_roi.as_numpy_array()
+            img2 = S.D.raw_pixels_roi.as_numpy_array()
 
-            fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
+            fdiff = (img2 - img) / delta_N
+            deriv = derivs_abc[i_nc]
+            error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+            all_error.append(error)
 
-            second_deriv = derivs2_abc[i_nc]
+            if args.curvatures:
+                Ncells_vals = [Nd, Ne, Nf]
+                Ncells_vals[i_nc] = N - delta_N
+                S.D.Ncells_def = tuple(Ncells_vals)
+                shifts2.append(delta_N**2)
 
-            error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
-            all_error2.append(error2)
+                S.D.raw_pixels_roi *= 0
+                S.D.add_diffBragg_spots()
+                img_backward = S.D.raw_pixels_roi.as_numpy_array()
 
-        print ("error=%f, step=%f" % (error, delta_N))
+                fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
 
-    if args.plot:
-        plt.plot(shifts, all_error, 'o')
-        plt.show()
-        if args.curvatures:
-            plt.plot(shifts2, all_error2, 'o')
+                second_deriv = derivs2_abc[i_nc]
+
+                error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
+                all_error2.append(error2)
+
+            print ("error=%f, step=%f" % (error, delta_N))
+
+        if args.plot:
+            plt.plot(shifts, all_error, 'o')
             plt.show()
+            if args.curvatures:
+                plt.plot(shifts2, all_error2, 'o')
+                plt.show()
 
-    l = linregress(shifts, all_error)
-    assert l.rvalue > .999  # this is definitely a line!
-    assert l.slope > 0
-    assert l.pvalue < 1e-5
-    if args.curvatures:
-        l = linregress(shifts2, all_error2)
+        l = linregress(shifts, all_error)
         assert l.rvalue > .999  # this is definitely a line!
         assert l.slope > 0
         assert l.pvalue < 1e-5
+        if args.curvatures:
+            l = linregress(shifts2, all_error2)
+            assert l.rvalue > .999  # this is definitely a line!
+            assert l.slope > 0
+            assert l.pvalue < 1e-5
 
-    print("OK!")
+        print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_ncells_property.py b/simtbx/diffBragg/tests/tst_diffBragg_ncells_property.py
index 49033c1593..70e3d28eb7 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_ncells_property.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_ncells_property.py
@@ -2,131 +2,137 @@
 This test checks the setter and getter for Ncells parameter
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--curvatures", action='store_true')
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import numpy as np
-import pylab as plt
-from scipy.stats import linregress
-from scipy.spatial.transform import Rotation
-from simtbx.nanoBragg import sim_data
-from scitbx.matrix import sqr, rec
-from cctbx import uctbx
-from dxtbx.model import Crystal
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-S = sim_data.SimData(use_default_crystal=True)
-S.crystal.dxtbx_crystal = C
-S.crystal.isotropic_ncells = True
-S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
-S.instantiate_diffBragg(verbose=0, oversample=0)
-assert S.D.isotropic_ncells
-S.D.spot_scale = 100000
-#S.D.oversample = 5
-
-Ncells_id = 9
-
-S.D.Ncells_abc = 12
-i = S.D.get_value(Ncells_id)
-S.D.set_value(Ncells_id, 14)
-i2 = S.D.get_value(Ncells_id)
-S.D.Ncells_abc = 80
-i3 = S.D.get_value(Ncells_id)
-S.D.set_value(Ncells_id, 10)
-
-assert i == 12
-assert i2 == 14
-assert i3 == 80
-
-print("OK")
-
-S.D.refine(Ncells_id)
-S.D.initialize_managers()
-S.D.Ncells_abc = 20
-#S.D.printout_pixel_fastslow = 10, 10
-S.D.verbose = 1
-assert S.D.isotropic_ncells
-#S.D.isotropic_ncells = True
-S.D.add_diffBragg_spots()
-S.D.verbose = 0
-img = S.D.raw_pixels_roi.as_numpy_array()
-deriv = S.D.get_ncells_derivative_pixels()[0].as_numpy_array()
-if args.curvatures:
-    deriv2 = S.D.get_ncells_second_derivative_pixels()[0].as_numpy_array()
-
-N = S.D.get_value(Ncells_id)
-perc = 0.001, 0.01, 0.1, 1, 10
-
-bragg = img > 1e-1  # select bragg scattering regions
-
-all_error = []
-all_error2 = []
-shifts = []
-shifts2 = []
-for i_shift, p in enumerate(perc):
-    delta_N = N*p*0.001
-    #S.D.set_value(Ncells_id, N-delta_N)
-
-    Nabc = [N+delta_N, N+delta_N, N+delta_N]
-    shifts.append(delta_N)
-    S.D.Ncells_abc_aniso = tuple(Nabc)
-    S.D.raw_pixels_roi *= 0
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import numpy as np
+    import pylab as plt
+    from scipy.stats import linregress
+    from scipy.spatial.transform import Rotation
+    from simtbx.nanoBragg import sim_data
+    from scitbx.matrix import sqr, rec
+    from cctbx import uctbx
+    from dxtbx.model import Crystal
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    S = sim_data.SimData(use_default_crystal=True)
+    S.crystal.dxtbx_crystal = C
+    S.crystal.isotropic_ncells = True
+    S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
+    S.instantiate_diffBragg(verbose=0, oversample=0)
+    assert S.D.isotropic_ncells
+    S.D.spot_scale = 100000
+    #S.D.oversample = 5
+
+    Ncells_id = 9
+
+    S.D.Ncells_abc = 12
+    i = S.D.get_value(Ncells_id)
+    S.D.set_value(Ncells_id, 14)
+    i2 = S.D.get_value(Ncells_id)
+    S.D.Ncells_abc = 80
+    i3 = S.D.get_value(Ncells_id)
+    S.D.set_value(Ncells_id, 10)
+
+    assert i == 12
+    assert i2 == 14
+    assert i3 == 80
+
+    print("OK")
+
+    S.D.refine(Ncells_id)
+    S.D.initialize_managers()
+    S.D.Ncells_abc = 20
+    #S.D.printout_pixel_fastslow = 10, 10
+    S.D.verbose = 1
+    assert S.D.isotropic_ncells
+    #S.D.isotropic_ncells = True
     S.D.add_diffBragg_spots()
-    img2 = S.D.raw_pixels_roi.as_numpy_array()
+    S.D.verbose = 0
+    img = S.D.raw_pixels_roi.as_numpy_array()
+    deriv = S.D.get_ncells_derivative_pixels()[0].as_numpy_array()
+    if args.curvatures:
+        deriv2 = S.D.get_ncells_second_derivative_pixels()[0].as_numpy_array()
 
-    fdiff = (img2-img) / delta_N
-    error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-    all_error.append(error)
+    N = S.D.get_value(Ncells_id)
+    perc = 0.001, 0.01, 0.1, 1, 10
 
-    if args.curvatures:
-        S.D.set_value(Ncells_id, N - delta_N)
-        shifts2.append(delta_N**2)
+    bragg = img > 1e-1  # select bragg scattering regions
 
+    all_error = []
+    all_error2 = []
+    shifts = []
+    shifts2 = []
+    for i_shift, p in enumerate(perc):
+        delta_N = N*p*0.001
+        #S.D.set_value(Ncells_id, N-delta_N)
+
+        Nabc = [N+delta_N, N+delta_N, N+delta_N]
+        shifts.append(delta_N)
+        S.D.Ncells_abc_aniso = tuple(Nabc)
         S.D.raw_pixels_roi *= 0
         S.D.add_diffBragg_spots()
-        img_backward = S.D.raw_pixels.as_numpy_array()
+        img2 = S.D.raw_pixels_roi.as_numpy_array()
 
-        fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
+        fdiff = (img2-img) / delta_N
+        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+        all_error.append(error)
 
-        error2 = (np.abs(fdiff2[bragg] - deriv2[bragg]) / 1).mean()
-        all_error2.append(error2)
+        if args.curvatures:
+            S.D.set_value(Ncells_id, N - delta_N)
+            shifts2.append(delta_N**2)
 
-    print ("error=%f, step=%f" % (error, delta_N))
+            S.D.raw_pixels_roi *= 0
+            S.D.add_diffBragg_spots()
+            img_backward = S.D.raw_pixels.as_numpy_array()
 
-if args.plot:
-    plt.plot(shifts, all_error, 'o')
-    plt.show()
-    if args.curvatures:
-        plt.plot(shifts2, all_error2, 'o')
+            fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
+
+            error2 = (np.abs(fdiff2[bragg] - deriv2[bragg]) / 1).mean()
+            all_error2.append(error2)
+
+        print ("error=%f, step=%f" % (error, delta_N))
+
+    if args.plot:
+        plt.plot(shifts, all_error, 'o')
         plt.show()
+        if args.curvatures:
+            plt.plot(shifts2, all_error2, 'o')
+            plt.show()
 
-l = linregress(shifts, all_error)
-assert l.rvalue > .9999, l  # this is definitely a line!
-assert l.slope > 0
-assert l.pvalue < 1e-6
-if args.curvatures:
-    l = linregress(shifts2, all_error2)
-    assert l.rvalue > .9999  # this is definitely a line!
+    l = linregress(shifts, all_error)
+    assert l.rvalue > .9999, l  # this is definitely a line!
     assert l.slope > 0
     assert l.pvalue < 1e-6
+    if args.curvatures:
+        l = linregress(shifts2, all_error2)
+        assert l.rvalue > .9999  # this is definitely a line!
+        assert l.slope > 0
+        assert l.pvalue < 1e-6
 
-print("OK!")
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_ncells_property_anisotropic.py b/simtbx/diffBragg/tests/tst_diffBragg_ncells_property_anisotropic.py
index b32abc79ba..8c9dee4013 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_ncells_property_anisotropic.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_ncells_property_anisotropic.py
@@ -2,140 +2,146 @@
 This test checks the setter and getter for Ncells parameter
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--curvatures", action='store_true')
 parser.add_argument("--idx", default=0, help="0=Na, 1=Nb, 2=Nc", type=int, choices=[0,1,2])
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import numpy as np
-import pylab as plt
-from scipy.stats import linregress
-from scipy.spatial.transform import Rotation
-from simtbx.nanoBragg import sim_data
-from scitbx.matrix import sqr, rec
-from cctbx import uctbx
-from dxtbx.model import Crystal
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-S = sim_data.SimData(use_default_crystal=True)
-S.crystal.dxtbx_crystal = C
-S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
-S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
-S.D.spot_scale = 100000
-#S.D.oversample = 5
-
-Ncells_id = 9
-
-S.D.Ncells_abc_aniso = 12, 13, 14
-Na, Nb, Nc = S.D.Ncells_abc_aniso
-
-assert (Na,Nb,Nc) == (12, 13, 14)
-
-print("OK")
-S.D.isotropic_ncells = False
-S.D.refine(Ncells_id)
-S.D.initialize_managers()
-ncells_GT = (20, 15, 17)
-
-S.D.Ncells_abc_aniso = ncells_GT
-S.D.Ncells_def = 0,0,0
-S.D.add_diffBragg_spots()
-
-img = S.D.raw_pixels_roi.as_numpy_array()
-derivs_abc = S.D.get_ncells_derivative_pixels()
-derivs_abc = [d.as_numpy_array() for d in derivs_abc]
-if args.curvatures:
-    derivs2_abc = S.D.get_ncells_second_derivative_pixels()
-    derivs2_abc = [d.as_numpy_array() for d in derivs2_abc]
-
-perc = 0.001, 0.01, 0.1, 1, 10
-bragg = img > 1e-1  # select bragg scattering regions
-
-Na, Nb, Nc = ncells_GT
-for i_nc in range(3):
-    if i_nc != args.idx:
-        continue
-
-    N = ncells_GT[i_nc]
-    print("Testing finite diff %d / 3, Ncells=%d" % (i_nc +1, N))
-
-    all_error = []
-    all_error2 = []
-    shifts = []
-    shifts2 = []
-    for i_shift, p in enumerate(perc):
-        delta_N = N*p*0.01
-
-        Ncells_vals = [Na, Nb, Nc]
-        Ncells_vals[i_nc] = N+delta_N
-
-        S.D.Ncells_abc_aniso = tuple(Ncells_vals)
-        shifts.append(delta_N)
-
-        S.D.raw_pixels_roi *= 0
-        #S.D.printout_pixel_fastslow = 10, 10
-        S.D.add_diffBragg_spots()
-        img2 = S.D.raw_pixels_roi.as_numpy_array()
-
-        fdiff = (img2 - img) / delta_N
-        deriv = derivs_abc[i_nc]
-        error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
-        all_error.append(error)
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import numpy as np
+    import pylab as plt
+    from scipy.stats import linregress
+    from scipy.spatial.transform import Rotation
+    from simtbx.nanoBragg import sim_data
+    from scitbx.matrix import sqr, rec
+    from cctbx import uctbx
+    from dxtbx.model import Crystal
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    S = sim_data.SimData(use_default_crystal=True)
+    S.crystal.dxtbx_crystal = C
+    S.detector = sim_data.SimData.simple_detector(180, 0.1, (1024, 1024))
+    S.instantiate_diffBragg(verbose=0, oversample=0, auto_set_spotscale=True)
+    S.D.spot_scale = 100000
+    #S.D.oversample = 5
+
+    Ncells_id = 9
+
+    S.D.Ncells_abc_aniso = 12, 13, 14
+    Na, Nb, Nc = S.D.Ncells_abc_aniso
+
+    assert (Na,Nb,Nc) == (12, 13, 14)
+
+    print("OK")
+    S.D.isotropic_ncells = False
+    S.D.refine(Ncells_id)
+    S.D.initialize_managers()
+    ncells_GT = (20, 15, 17)
+
+    S.D.Ncells_abc_aniso = ncells_GT
+    S.D.Ncells_def = 0,0,0
+    S.D.add_diffBragg_spots()
+
+    img = S.D.raw_pixels_roi.as_numpy_array()
+    derivs_abc = S.D.get_ncells_derivative_pixels()
+    derivs_abc = [d.as_numpy_array() for d in derivs_abc]
+    if args.curvatures:
+        derivs2_abc = S.D.get_ncells_second_derivative_pixels()
+        derivs2_abc = [d.as_numpy_array() for d in derivs2_abc]
+
+    perc = 0.001, 0.01, 0.1, 1, 10
+    bragg = img > 1e-1  # select bragg scattering regions
+
+    Na, Nb, Nc = ncells_GT
+    for i_nc in range(3):
+        if i_nc != args.idx:
+            continue
+
+        N = ncells_GT[i_nc]
+        print("Testing finite diff %d / 3, Ncells=%d" % (i_nc +1, N))
+
+        all_error = []
+        all_error2 = []
+        shifts = []
+        shifts2 = []
+        for i_shift, p in enumerate(perc):
+            delta_N = N*p*0.01
 
-        if args.curvatures:
             Ncells_vals = [Na, Nb, Nc]
-            Ncells_vals[i_nc] = N - delta_N
+            Ncells_vals[i_nc] = N+delta_N
+
             S.D.Ncells_abc_aniso = tuple(Ncells_vals)
-            shifts2.append(delta_N**2)
+            shifts.append(delta_N)
 
             S.D.raw_pixels_roi *= 0
+            #S.D.printout_pixel_fastslow = 10, 10
             S.D.add_diffBragg_spots()
-            img_backward = S.D.raw_pixels_roi.as_numpy_array()
+            img2 = S.D.raw_pixels_roi.as_numpy_array()
 
-            fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
+            fdiff = (img2 - img) / delta_N
+            deriv = derivs_abc[i_nc]
+            error = np.abs(fdiff[bragg] - deriv[bragg]).mean()
+            all_error.append(error)
 
-            second_deriv = derivs2_abc[i_nc]
-            error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
-            all_error2.append(error2)
+            if args.curvatures:
+                Ncells_vals = [Na, Nb, Nc]
+                Ncells_vals[i_nc] = N - delta_N
+                S.D.Ncells_abc_aniso = tuple(Ncells_vals)
+                shifts2.append(delta_N**2)
 
-        print ("error=%f, step=%f" % (error, delta_N))
-        if args.curvatures:
-            print ("error2=%f, step=%f" % (error2, delta_N))
-            print()
+                S.D.raw_pixels_roi *= 0
+                S.D.add_diffBragg_spots()
+                img_backward = S.D.raw_pixels_roi.as_numpy_array()
 
-    if args.plot:
-        plt.plot(shifts, all_error, 'o')
-        plt.show()
-        if args.curvatures:
-            plt.plot(shifts2, all_error2, 'o')
+                fdiff2 = (img2 - 2*img + img_backward) / delta_N / delta_N
+
+                second_deriv = derivs2_abc[i_nc]
+                error2 = (np.abs(fdiff2[bragg] - second_deriv[bragg]) / 1).mean()
+                all_error2.append(error2)
+
+            print ("error=%f, step=%f" % (error, delta_N))
+            if args.curvatures:
+                print ("error2=%f, step=%f" % (error2, delta_N))
+                print()
+
+        if args.plot:
+            plt.plot(shifts, all_error, 'o')
             plt.show()
+            if args.curvatures:
+                plt.plot(shifts2, all_error2, 'o')
+                plt.show()
 
-    l = linregress(shifts, all_error)
-    assert l.rvalue > .9999  # this is definitely a line!
-    assert l.slope > 0
-    assert l.pvalue < 1e-6
-    if args.curvatures:
-        l = linregress(shifts2, all_error2)
+        l = linregress(shifts, all_error)
         assert l.rvalue > .9999  # this is definitely a line!
         assert l.slope > 0
         assert l.pvalue < 1e-6
+        if args.curvatures:
+            l = linregress(shifts2, all_error2)
+            assert l.rvalue > .9999  # this is definitely a line!
+            assert l.slope > 0
+            assert l.pvalue < 1e-6
 
-    print("OK!")
+        print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_panelXY_derivs.py b/simtbx/diffBragg/tests/tst_diffBragg_panelXY_derivs.py
index 0c5abfb020..2578fa7136 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_panelXY_derivs.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_panelXY_derivs.py
@@ -1,120 +1,126 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 from simtbx.nanoBragg import sim_data
 
 from argparse import ArgumentParser
 parser = ArgumentParser()
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 parser.add_argument("--plotimages", action='store_true')
 parser.add_argument("--panel", choices=["x", "y", "z"], default="x", help="which origin coordinate to check")
 parser.add_argument("--plotlines", action='store_true')
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-SIM = sim_data.SimData(use_default_crystal=True)
-
-det = sim_data.SimData.simple_detector(detector_distance_mm=150, pixelsize_mm=0.1, image_shape=(512, 512))
-
-if args.panel == "x":
-  refine_idx = 15  # id of origin coordinate in diffBragg
-elif args.panel == "y":
-  refine_idx = 16  # id of origin coordinate in diffBragg
-else:  # args.panel==z
-  refine_idx = 10
-
-B = SIM.beam.nanoBragg_constructor_beam
-
-# set the detector
-SIM.detector = det
-SIM.instantiate_diffBragg(auto_set_spotscale=True)
-D = SIM.D
-D.oversample_omega = True  #False
-D.nopolar = True
-
-D.refine(refine_idx)
-
-D.initialize_managers()
-D.add_diffBragg_spots()
-
-# get the simulated image and derivative
-img = D.raw_pixels_roi.as_numpy_array()
-deriv = D.get_derivative_pixels(refine_idx).as_numpy_array()
-D.raw_pixels_roi *= 0
-D.raw_pixels *= 0
-
-node = det[0]
-node_d = node.to_dict()
-O = node_d["origin"][0], node_d["origin"][1], node_d["origin"][2]
-OX,OY,OZ = O
-
-# update the detector distance
-import numpy as np
-all_shifts = []
-all_errors = []
-shifts_mm = [2*i*(1e-7) for i in range(1,30,2)]
-print(shifts_mm)
-import pylab as plt
-
-from dxtbx.model import Panel
-
-for i_shift, delta_shift in enumerate(shifts_mm):
-  # update the detector model
-
-  if args.panel=="x":
-    shifted = OX+delta_shift, OY, OZ
-  elif args.panel=="y":
-    shifted = OX, OY+delta_shift, OZ
-  else:
-    delta_shift = delta_shift*10  # larger shift for Z
-    shifted = OX, OY, OZ + delta_shift
-
-  node_d["origin"] = shifted
-  det[0] = Panel.from_dict(node_d)
-
-  D.update_dxtbx_geoms(det, B, 0)
-  D.add_diffBragg_spots()
-  img_forward = D.raw_pixels_roi.as_numpy_array()
-  D.raw_pixels_roi *= 0
-  D.raw_pixels *= 0
-  delta_shift_meters = delta_shift*1e-3
-  fdiff = (img_forward-img)/delta_shift_meters
-
-  bragg = img > 1e-2
-
-  error = (np.abs(fdiff[bragg] - deriv[bragg])).mean()
-  all_errors.append(error)
-
-  all_shifts.append(delta_shift_meters)
-
-  print("Error=%2.7g shift=%2.7g um" % (error, delta_shift*1000))
-  if args.plotimages:
-    plt.clf()
-    y = slice(40, 65, 1)
-    x = slice(415, 437, 1)
-    plt.subplot(121)
-    plt.imshow(fdiff[y, x])
-    plt.title("finite diff")
-    plt.subplot(122)
-    plt.imshow(deriv[y, x])
-    plt.title("analytical")
-    plt.draw()
-    plt.suptitle("delta=%f mm, Shift %d / %d"
-                 % ( delta_shift, i_shift + 1, len(shifts_mm)))
-    plt.pause(0.3)
-
-if args.plotlines:
-  plt.close()
-
-  plt.plot(all_shifts, all_errors, 'o')
-  plt.suptitle("finite 1st difference error", fontsize=16)
-  plt.xlabel("h", fontsize=16)
-  plt.show()
-
-from scipy.stats import linregress
-l = linregress(all_shifts, all_errors)
-print("finite diff l.rvalue=%10.7g" % l.rvalue)
-assert l.rvalue > .99
-assert l.slope > 0
-assert l.pvalue < 1e-6
-
-print("OK!")
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+    SIM = sim_data.SimData(use_default_crystal=True)
+
+    det = sim_data.SimData.simple_detector(detector_distance_mm=150, pixelsize_mm=0.1, image_shape=(512, 512))
+
+    if args.panel == "x":
+      refine_idx = 15  # id of origin coordinate in diffBragg
+    elif args.panel == "y":
+      refine_idx = 16  # id of origin coordinate in diffBragg
+    else:  # args.panel==z
+      refine_idx = 10
+
+    B = SIM.beam.nanoBragg_constructor_beam
+
+    # set the detector
+    SIM.detector = det
+    SIM.instantiate_diffBragg(auto_set_spotscale=True)
+    D = SIM.D
+    D.oversample_omega = True  #False
+    D.nopolar = True
+
+    D.refine(refine_idx)
+
+    D.initialize_managers()
+    D.add_diffBragg_spots()
+
+    # get the simulated image and derivative
+    img = D.raw_pixels_roi.as_numpy_array()
+    deriv = D.get_derivative_pixels(refine_idx).as_numpy_array()
+    D.raw_pixels_roi *= 0
+    D.raw_pixels *= 0
+
+    node = det[0]
+    node_d = node.to_dict()
+    O = node_d["origin"][0], node_d["origin"][1], node_d["origin"][2]
+    OX,OY,OZ = O
+
+    # update the detector distance
+    import numpy as np
+    all_shifts = []
+    all_errors = []
+    shifts_mm = [2*i*(1e-7) for i in range(1,30,2)]
+    print(shifts_mm)
+    import pylab as plt
+
+    from dxtbx.model import Panel
+
+    for i_shift, delta_shift in enumerate(shifts_mm):
+      # update the detector model
+
+      if args.panel=="x":
+        shifted = OX+delta_shift, OY, OZ
+      elif args.panel=="y":
+        shifted = OX, OY+delta_shift, OZ
+      else:
+        delta_shift = delta_shift*10  # larger shift for Z
+        shifted = OX, OY, OZ + delta_shift
+
+      node_d["origin"] = shifted
+      det[0] = Panel.from_dict(node_d)
+
+      D.update_dxtbx_geoms(det, B, 0)
+      D.add_diffBragg_spots()
+      img_forward = D.raw_pixels_roi.as_numpy_array()
+      D.raw_pixels_roi *= 0
+      D.raw_pixels *= 0
+      delta_shift_meters = delta_shift*1e-3
+      fdiff = (img_forward-img)/delta_shift_meters
+
+      bragg = img > 1e-2
+
+      error = (np.abs(fdiff[bragg] - deriv[bragg])).mean()
+      all_errors.append(error)
+
+      all_shifts.append(delta_shift_meters)
+
+      print("Error=%2.7g shift=%2.7g um" % (error, delta_shift*1000))
+      if args.plotimages:
+        plt.clf()
+        y = slice(40, 65, 1)
+        x = slice(415, 437, 1)
+        plt.subplot(121)
+        plt.imshow(fdiff[y, x])
+        plt.title("finite diff")
+        plt.subplot(122)
+        plt.imshow(deriv[y, x])
+        plt.title("analytical")
+        plt.draw()
+        plt.suptitle("delta=%f mm, Shift %d / %d"
+                     % ( delta_shift, i_shift + 1, len(shifts_mm)))
+        plt.pause(0.3)
+
+    if args.plotlines:
+      plt.close()
+
+      plt.plot(all_shifts, all_errors, 'o')
+      plt.suptitle("finite 1st difference error", fontsize=16)
+      plt.xlabel("h", fontsize=16)
+      plt.show()
+
+    from scipy.stats import linregress
+    l = linregress(all_shifts, all_errors)
+    print("finite diff l.rvalue=%10.7g" % l.rvalue)
+    assert l.rvalue > .99
+    assert l.slope > 0
+    assert l.pvalue < 1e-6
+
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_regions_of_interest.py b/simtbx/diffBragg/tests/tst_diffBragg_regions_of_interest.py
index db40c90415..a888b3fe35 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_regions_of_interest.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_regions_of_interest.py
@@ -1,13 +1,12 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 def main():
 
     import numpy as np
     from simtbx.diffBragg.utils import get_diffBragg_instance
-    import sys
-    if "--cuda" in sys.argv:
-        import os
-        os.environ["DIFFBRAGG_USE_CUDA"]="1"
+    from simtbx.diffBragg.utils import find_diffBragg_instances
 
     D = get_diffBragg_instance()
     D.nopolar = False
@@ -31,8 +30,16 @@ def main():
         diff_roi_pixels = D.raw_pixels_roi.as_numpy_array()[:npix_sim].reshape((y2-y1, x2-x1))
 
         assert np.allclose(diff_roi_pixels, nano_roi_pixels, atol=1e-9)
+        for name in find_diffBragg_instances(globals()): del globals()[name]
 
 
 if __name__ == "__main__":
-    main()
+    import sys
+    if "--kokkos" in sys.argv:
+        import os
+        os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+    from simtbx.diffBragg.device import DeviceWrapper
+
+    with DeviceWrapper(0) as _:
+        main()
     print ("OK!")
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ.py b/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ.py
index c06ddb9063..730f8f9db9 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ.py
@@ -1,13 +1,16 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser("diffBragg tests")
 parser.add_argument("--plot", action='store_true')
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
 
 
 def main():
@@ -21,7 +24,7 @@ def main():
     print (angles_XYZ*180 / np.pi)
 
     D = get_diffBragg_instance()
-    D.use_cuda = args.cuda
+    D.use_cuda = args.kokkos
 
     rotX, rotY, rotZ = 0, 1, 2
     D.refine(rotX)  # rotX
@@ -56,7 +59,7 @@ def main():
         D.Umatrix = U.elems
         D.verbose = 1
         D.add_diffBragg_spots()
-        if args.cuda:
+        if args.kokkos:
             D.gpu_free()
         imgA = D.raw_pixels.as_numpy_array()
 
@@ -67,7 +70,7 @@ def main():
         D.set_value(rotZ, thetaZ)
         D.Umatrix = Uorig
         D.add_diffBragg_spots()
-        if args.cuda:
+        if args.kokkos:
             D.gpu_free()
         imgB = D.raw_pixels.as_numpy_array()
         if args.plot:
@@ -83,8 +86,11 @@ def main():
 
         assert(np.allclose(imgA, imgB, atol=1e-4))
         print("OK (%d / %d)" % (i_ang+1, len(angles_XYZ)))
+        for name in find_diffBragg_instances(globals()): del globals()[name]
 
 
 if __name__ == "__main__":
-    main()
+    from simtbx.diffBragg.device import DeviceWrapper
+    with DeviceWrapper(0) as _:
+        main()
     print("OK")
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ_deriv.py b/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ_deriv.py
index 376ca83099..0bd0b89f9f 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ_deriv.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_rotXYZ_deriv.py
@@ -1,162 +1,168 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 from argparse import ArgumentParser
 parser = ArgumentParser("diffBragg tests")
 parser.add_argument("--plot", action='store_true')
-parser.add_argument("--cuda", action='store_true')
+parser.add_argument("--kokkos", action='store_true')
 parser.add_argument("--curvatures", action='store_true')
 parser.add_argument("--plotimages", action="store_true")
 parser.add_argument("--rotidx", type=int, choices=[0,1,2], required=True)
 parser.add_argument("--randomrotate", type=int, default=None, help="seed for random rotation of Umatrix")
 args = parser.parse_args()
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-if args.plot:
-    import pylab as plt
-from simtbx.nanoBragg.sim_data import SimData
-
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-import numpy as np
-from scitbx.matrix import sqr
-from cctbx import uctbx
-from dxtbx.model import Crystal
-from scipy import stats
-
-rot_idx = args.rotidx
-
-ucell = (70, 60, 50, 90.0, 110, 90.0)
-symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-# make a nanoBragg crystal to pass to diffBragg
-nbcryst = NBcrystal()
-nbcryst.dxtbx_crystal = C
-nbcryst.n_mos_domains = 1
-nbcryst.thick_mm = 0.01
-nbcryst.Ncells_abc = (7, 7, 7)
-
-# make an instance of diffBRagg, use the simData wrapper
-SIM = SimData(use_default_crystal=True)
-# overwrite the default detector with a smaller pixels one
-SIM.detector = SimData.simple_detector(220, 0.1, (1000, 1000))
-SIM.crystal = nbcryst
-
-SIM.instantiate_diffBragg(oversample=0, verbose=0, interpolate=0, default_F=1e3,auto_set_spotscale=True)
-# D is an instance of diffBragg with reasonable parameters
-# and our dxtbx crystal created above
-D = SIM.D
-if args.curvatures:
-    D.compute_curvatures = True
-
-# STEP 1: simulate the un-perturbed image:
-D.refine(rot_idx)
-D.initialize_managers()
-D.set_value(rot_idx, 0)
-D.use_cuda = args.cuda
-#D.printout_pixel_fastslow = 786, 567
-D.add_diffBragg_spots()
-img0 = D.raw_pixels_roi.as_numpy_array()
-bragg = img0 > 1  #np.ones_like(img0).astype(bool)
-if args.plotimages:
-    plt.title("Scattering from crystal")
-    plt.imshow(img0)
-    plt.show()
-
-deriv = D.get_derivative_pixels(rot_idx).as_numpy_array()
-if args.curvatures:
-    second_deriv = D.get_second_derivative_pixels(rot_idx).as_numpy_array()
-
-error_vals = []
-error_vals2 = []
-delta_h = []
-delta_h2 = []
-theta_vals = [0.0005 + i*0.0005 for i in range(8)]   # 0.01, 0.02048, 0.04096, 0.08192, 0.16384, 0.32768, 0.65536, 1.31072, 2.62144
-
-for theta_degrees in theta_vals:
-    theta = theta_degrees * np.pi / 180
-    D.set_value(rot_idx, theta)
-
-    # simulate the scattering in the rotated crystal:
-    D.raw_pixels_roi *= 0
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+    if args.plot:
+        import pylab as plt
+    from simtbx.nanoBragg.sim_data import SimData
+
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    import numpy as np
+    from scitbx.matrix import sqr
+    from cctbx import uctbx
+    from dxtbx.model import Crystal
+    from scipy import stats
+
+    rot_idx = args.rotidx
+
+    ucell = (70, 60, 50, 90.0, 110, 90.0)
+    symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+    # make a nanoBragg crystal to pass to diffBragg
+    nbcryst = NBcrystal()
+    nbcryst.dxtbx_crystal = C
+    nbcryst.n_mos_domains = 1
+    nbcryst.thick_mm = 0.01
+    nbcryst.Ncells_abc = (7, 7, 7)
+
+    # make an instance of diffBRagg, use the simData wrapper
+    SIM = SimData(use_default_crystal=True)
+    # overwrite the default detector with a smaller pixels one
+    SIM.detector = SimData.simple_detector(220, 0.1, (1000, 1000))
+    SIM.crystal = nbcryst
+
+    SIM.instantiate_diffBragg(oversample=0, verbose=0, interpolate=0, default_F=1e3,auto_set_spotscale=True)
+    # D is an instance of diffBragg with reasonable parameters
+    # and our dxtbx crystal created above
+    D = SIM.D
+    if args.curvatures:
+        D.compute_curvatures = True
+
+    # STEP 1: simulate the un-perturbed image:
+    D.refine(rot_idx)
+    D.initialize_managers()
+    D.set_value(rot_idx, 0)
+    D.use_cuda = args.kokkos
+    #D.printout_pixel_fastslow = 786, 567
     D.add_diffBragg_spots()
-    img_plus = D.raw_pixels_roi.as_numpy_array()
+    img0 = D.raw_pixels_roi.as_numpy_array()
+    bragg = img0 > 1  #np.ones_like(img0).astype(bool)
+    if args.plotimages:
+        plt.title("Scattering from crystal")
+        plt.imshow(img0)
+        plt.show()
+
+    deriv = D.get_derivative_pixels(rot_idx).as_numpy_array()
     if args.curvatures:
-        D.set_value(rot_idx, -theta)
-        D.raw_pixels_roi *= 0
-        D.add_diffBragg_spots()
-        img_minus = D.raw_pixels_roi.as_numpy_array()
-        finite_second_diff = (img_plus-2*img0 + img_minus) / theta / theta
-        delta_h2.append(theta**2)
+        second_deriv = D.get_second_derivative_pixels(rot_idx).as_numpy_array()
 
-    # STEP3 : compute finite differenceL
-    finite_diff = (img_plus-img0)/theta
-    delta_h.append(theta)
+    error_vals = []
+    error_vals2 = []
+    delta_h = []
+    delta_h2 = []
+    theta_vals = [0.0005 + i*0.0005 for i in range(8)]   # 0.01, 0.02048, 0.04096, 0.08192, 0.16384, 0.32768, 0.65536, 1.31072, 2.62144
 
-    if args.plotimages:
-        plt.subplot(121)
-        plt.imshow(finite_diff)
-        plt.title("finite diff.")
-        plt.subplot(122)
-        plt.imshow(deriv)
-        plt.title("analytical")
-        plt.suptitle("Theta = %f deg. " % theta_degrees)
-        plt.draw()
-        plt.pause(1)
+    for theta_degrees in theta_vals:
+        theta = theta_degrees * np.pi / 180
+        D.set_value(rot_idx, theta)
+
+        # simulate the scattering in the rotated crystal:
+        D.raw_pixels_roi *= 0
+        D.add_diffBragg_spots()
+        img_plus = D.raw_pixels_roi.as_numpy_array()
         if args.curvatures:
-            plt.clf()
+            D.set_value(rot_idx, -theta)
+            D.raw_pixels_roi *= 0
+            D.add_diffBragg_spots()
+            img_minus = D.raw_pixels_roi.as_numpy_array()
+            finite_second_diff = (img_plus-2*img0 + img_minus) / theta / theta
+            delta_h2.append(theta**2)
+
+        # STEP3 : compute finite differenceL
+        finite_diff = (img_plus-img0)/theta
+        delta_h.append(theta)
+
+        if args.plotimages:
             plt.subplot(121)
-            plt.imshow(finite_second_diff)
-            plt.title("finite 2nd diff.")
+            plt.imshow(finite_diff)
+            plt.title("finite diff.")
             plt.subplot(122)
-            plt.imshow(second_deriv)
+            plt.imshow(deriv)
             plt.title("analytical")
             plt.suptitle("Theta = %f deg. " % theta_degrees)
             plt.draw()
-            plt.pause(0.2)
-
-    error_image = abs(deriv[bragg]-finite_diff[bragg])
-    error = error_image.mean()
-    error_vals.append(error)
+            plt.pause(1)
+            if args.curvatures:
+                plt.clf()
+                plt.subplot(121)
+                plt.imshow(finite_second_diff)
+                plt.title("finite 2nd diff.")
+                plt.subplot(122)
+                plt.imshow(second_deriv)
+                plt.title("analytical")
+                plt.suptitle("Theta = %f deg. " % theta_degrees)
+                plt.draw()
+                plt.pause(0.2)
+
+        error_image = abs(deriv[bragg]-finite_diff[bragg])
+        error = error_image.mean()
+        error_vals.append(error)
 
-    if args.curvatures:
-        error_image2 = abs(second_deriv[bragg]-finite_second_diff[bragg])
-        error2 = error_image2.mean()
-        print("Theta = %.4f deg, 1st error = %2.7g, 2nd %2.7g" %
-                (theta_degrees, error, error2))
-        error_vals2.append(error2)
-    else:
-        print("Theta = %.4f deg, 1st error = %2.7g" %
-              (theta_degrees, error))
-
-if args.plot:
-    plt.close()
-    plt.plot(delta_h, error_vals, '.')
-    ax = plt.gca()
-    ax.set_xlabel("theta (rad.)")
-    ax.set_ylabel(r"$\langle |\,$finite_diff - analytical$\,| \rangle$", fontsize=14)
-    plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
-    plt.subplots_adjust(left=0.12)
-    plt.show()
-    if args.curvatures:
-        plt.plot(delta_h2, error_vals2, '.')
+        if args.curvatures:
+            error_image2 = abs(second_deriv[bragg]-finite_second_diff[bragg])
+            error2 = error_image2.mean()
+            print("Theta = %.4f deg, 1st error = %2.7g, 2nd %2.7g" %
+                    (theta_degrees, error, error2))
+            error_vals2.append(error2)
+        else:
+            print("Theta = %.4f deg, 1st error = %2.7g" %
+                  (theta_degrees, error))
+
+    if args.plot:
+        plt.close()
+        plt.plot(delta_h, error_vals, '.')
         ax = plt.gca()
-        ax.set_xlabel("theta^2 (rad.^2)")
-        ax.set_ylabel(r"$\langle |\,$finite_second_diff - analytical$\,| \rangle$", fontsize=14)
-        plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
+        ax.set_xlabel("theta (rad.)")
+        ax.set_ylabel(r"$\langle |\,$finite_diff - analytical$\,| \rangle$", fontsize=14)
+        plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
         plt.subplots_adjust(left=0.12)
         plt.show()
-
-l = stats.linregress(delta_h, error_vals)
-assert l.rvalue > .9999, "%2.7g" % l.rvalue
-assert l.slope > 0, "%2.7g" % l.slope
-assert l.pvalue < 1e-6, "%2.7g" % l.pvalue
-if args.curvatures:
-    l = stats.linregress(delta_h2, error_vals2)
-    assert l.rvalue > .9999, "2nd deriv rvalue %2.7g" % l.rvalue
-    assert l.slope > 0, "2nd deriv slope %2.7g" % l.slope
-    assert l.pvalue < 1e-6, "2nd deriv pvalue %2.7g" % l.pvalue
-if args.cuda:
-    D.gpu_free()
-print("OK!")
+        if args.curvatures:
+            plt.plot(delta_h2, error_vals2, '.')
+            ax = plt.gca()
+            ax.set_xlabel("theta^2 (rad.^2)")
+            ax.set_ylabel(r"$\langle |\,$finite_second_diff - analytical$\,| \rangle$", fontsize=14)
+            plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
+            plt.subplots_adjust(left=0.12)
+            plt.show()
+
+    l = stats.linregress(delta_h, error_vals)
+    assert l.rvalue > .9999, "%2.7g" % l.rvalue
+    assert l.slope > 0, "%2.7g" % l.slope
+    assert l.pvalue < 1e-6, "%2.7g" % l.pvalue
+    if args.curvatures:
+        l = stats.linregress(delta_h2, error_vals2)
+        assert l.rvalue > .9999, "2nd deriv rvalue %2.7g" % l.rvalue
+        assert l.slope > 0, "2nd deriv slope %2.7g" % l.slope
+        assert l.pvalue < 1e-6, "2nd deriv pvalue %2.7g" % l.pvalue
+    if args.kokkos:
+        D.gpu_free()
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_rotation.py b/simtbx/diffBragg/tests/tst_diffBragg_rotation.py
index b5702954a6..339e3ffcfc 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_rotation.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_rotation.py
@@ -1,4 +1,6 @@
 from __future__ import absolute_import, division, print_function
+from simtbx.kokkos import gpu_instance
+kokkos_run = gpu_instance(deviceId = 0)
 from simtbx.diffBragg import utils, hopper_utils
 from simtbx.nanoBragg.tst_nanoBragg_multipanel import beam as dxtbx_beam
 from simtbx.nanoBragg.tst_nanoBragg_multipanel import whole_det as dxtbx_det
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_structure_factors.py b/simtbx/diffBragg/tests/tst_diffBragg_structure_factors.py
index d1e5c18aad..90f2d363cd 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_structure_factors.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_structure_factors.py
@@ -104,3 +104,7 @@
 assert np.all(np.float32(avals)==np.float32(avals2))
 
 print("OK")
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_unitcell_property.py b/simtbx/diffBragg/tests/tst_diffBragg_unitcell_property.py
index 92a84b0174..d138a671db 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_unitcell_property.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_unitcell_property.py
@@ -1,237 +1,244 @@
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
+
 from argparse import ArgumentParser
 parser = ArgumentParser()
 parser.add_argument("--plot", action='store_true')
 parser.add_argument("--crystalsystem", default='tetragonal',
                     choices=["monoclinic", "tetragonal"])
 parser.add_argument("--curvatures", action='store_true')
-parser.add_argument("--cuda", action="store_true")
+parser.add_argument("--kokkos", action="store_true")
 args = parser.parse_args()
 
-if args.cuda:
+if args.kokkos:
     import os
-    os.environ["DIFFBRAGG_USE_CUDA"]="1"
-
-import numpy as np
-from scipy.spatial.transform import Rotation
-from scipy.stats import pearsonr
-import pylab as plt
-from scipy.stats import linregress
-from simtbx.diffBragg.hopper_utils import UCELL_ID_OFFSET
-from scitbx.matrix import sqr
-from scitbx.matrix import rec
-from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
-from cctbx import uctbx
-from simtbx.nanoBragg.sim_data import SimData
-from simtbx.diffBragg import utils
-from dxtbx.model.crystal import Crystal
-
-
-# STEP 1:
-# make a crystal and orient it randomly
-if args.crystalsystem=="tetragonal":
-    ucell = (55, 55, 77, 90, 90, 90)
-    symbol = "P43212"
-else:  # args.crystalsystem == "monoclinic"
-    ucell = (70, 60, 50, 90.0, 110, 90.0)
-    symbol = "C121"
-
-a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
-C = Crystal(a_real, b_real, c_real, symbol)
-
-# random raotation
-rotation = Rotation.random(num=1, random_state=101)[0]
-Q = rec(rotation.as_quat(), n=(4, 1))
-rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
-C.rotate_around_origin(rot_axis, rot_ang)
-
-# STEP3:
-# create the unit cell parameter manager
-UcellMan = utils.manager_from_params(ucell)
-n_ucell_params = len(UcellMan.variables)
-assert np.allclose(UcellMan.B_recipspace, C.get_B())
-
-# STEP4:
-# make a nanoBragg crystal to pass to diffBragg
-nbcryst = NBcrystal(init_defaults=True)
-nbcryst.dxtbx_crystal = C
-nbcryst.n_mos_domains = 1
-nbcryst.thick_mm = 0.01
-nbcryst.Ncells_abc = (7, 7, 7)
-
-# STEP5: make an instance of diffBRagg, use the simData wrapper
-SIM = SimData()
-# overwrite the default detector to use smaller pixels
-img_sh = 700,700
-SIM.detector = SimData.simple_detector(300, 0.1, img_sh)
-SIM.crystal = nbcryst
-SIM.instantiate_diffBragg(oversample=0, verbose=0, auto_set_spotscale=True)
-# D is an instance of diffBragg with sensible parameters
-# and our dxtbx crystal created above
-D = SIM.D
-D.progress_meter = True
-
-# STEP6:
-# initialize the derivative managers for the unit cell parameters
-for i_param in range(n_ucell_params):
-    D.refine(UCELL_ID_OFFSET+i_param)
-for i in range(n_ucell_params):
-    D.set_ucell_derivative_matrix(UCELL_ID_OFFSET+i, UcellMan.derivative_matrices[i])
-    if args.curvatures:
-        D.set_ucell_second_derivative_matrix(UCELL_ID_OFFSET + i, UcellMan.second_derivative_matrices[i])
-D.initialize_managers()
-
-
-# STEP7:
-# compute the scattering and its derivative
-print("Adding diffBragg spots")
-#D.printout_pixel_fastslow =150, 351
-D.add_diffBragg_spots()
-print("Done!")
-img = D.raw_pixels_roi.as_numpy_array()
-# reset all pixel values
-D.raw_pixels *= 0
-D.raw_pixels_roi *= 0
-
-derivs = []
-second_derivs = []
-for i_param in range(n_ucell_params):
-    analy_deriv = D.get_derivative_pixels(UCELL_ID_OFFSET+i_param).as_numpy_array()
-    derivs.append(analy_deriv)
-    if args.curvatures:
-        second_derivs.append(D.get_second_derivative_pixels(UCELL_ID_OFFSET+i_param).as_numpy_array())
-
-# STEP8
-# iterate over the parameters and do a finite difference test for each one
-# parameter shifts:
-shifts = [1e-4*(2*i) for i in range(1, 12, 2)]
-
-import copy
-starting_var = copy.copy(UcellMan.variables)
-for i_param in range(n_ucell_params):
-    analy_deriv = derivs[i_param]
-    cc_vals = []
-    cc_vals2 = []
-    error = []
-    error2 = []
-    h_vals = []
-    for i_shift, percent_shift in enumerate(shifts):
-
-        var = copy.copy(starting_var)
-
-        param_shift = var[i_param] * percent_shift
-
-        var[i_param] += param_shift
-        UcellMan.variables = var
-
-        D.Bmatrix = UcellMan.B_recipspace
-        D.add_diffBragg_spots()
-
-        img_forward = D.raw_pixels_roi.as_numpy_array()
-
-        # reset for next computation
-        D.raw_pixels_roi *= 0
-        D.raw_pixels *= 0
-
+    os.environ["DIFFBRAGG_USE_KOKKOS"]="1"
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+
+    import numpy as np
+    from scipy.spatial.transform import Rotation
+    from scipy.stats import pearsonr
+    import pylab as plt
+    from scipy.stats import linregress
+    from simtbx.diffBragg.hopper_utils import UCELL_ID_OFFSET
+    from scitbx.matrix import sqr
+    from scitbx.matrix import rec
+    from simtbx.nanoBragg.nanoBragg_crystal import NBcrystal
+    from cctbx import uctbx
+    from simtbx.nanoBragg.sim_data import SimData
+    from simtbx.diffBragg import utils
+    from dxtbx.model.crystal import Crystal
+
+
+    # STEP 1:
+    # make a crystal and orient it randomly
+    if args.crystalsystem=="tetragonal":
+        ucell = (55, 55, 77, 90, 90, 90)
+        symbol = "P43212"
+    else:  # args.crystalsystem == "monoclinic"
+        ucell = (70, 60, 50, 90.0, 110, 90.0)
+        symbol = "C121"
+
+    a_real, b_real, c_real = sqr(uctbx.unit_cell(ucell).orthogonalization_matrix()).transpose().as_list_of_lists()
+    C = Crystal(a_real, b_real, c_real, symbol)
+
+    # random raotation
+    rotation = Rotation.random(num=1, random_state=101)[0]
+    Q = rec(rotation.as_quat(), n=(4, 1))
+    rot_ang, rot_axis = Q.unit_quaternion_as_axis_and_angle()
+    C.rotate_around_origin(rot_axis, rot_ang)
+
+    # STEP3:
+    # create the unit cell parameter manager
+    UcellMan = utils.manager_from_params(ucell)
+    n_ucell_params = len(UcellMan.variables)
+    assert np.allclose(UcellMan.B_recipspace, C.get_B())
+
+    # STEP4:
+    # make a nanoBragg crystal to pass to diffBragg
+    nbcryst = NBcrystal(init_defaults=True)
+    nbcryst.dxtbx_crystal = C
+    nbcryst.n_mos_domains = 1
+    nbcryst.thick_mm = 0.01
+    nbcryst.Ncells_abc = (7, 7, 7)
+
+    # STEP5: make an instance of diffBRagg, use the simData wrapper
+    SIM = SimData()
+    # overwrite the default detector to use smaller pixels
+    img_sh = 700,700
+    SIM.detector = SimData.simple_detector(300, 0.1, img_sh)
+    SIM.crystal = nbcryst
+    SIM.instantiate_diffBragg(oversample=0, verbose=0, auto_set_spotscale=True)
+    # D is an instance of diffBragg with sensible parameters
+    # and our dxtbx crystal created above
+    D = SIM.D
+    D.progress_meter = True
+
+    # STEP6:
+    # initialize the derivative managers for the unit cell parameters
+    for i_param in range(n_ucell_params):
+        D.refine(UCELL_ID_OFFSET+i_param)
+    for i in range(n_ucell_params):
+        D.set_ucell_derivative_matrix(UCELL_ID_OFFSET+i, UcellMan.derivative_matrices[i])
+        if args.curvatures:
+            D.set_ucell_second_derivative_matrix(UCELL_ID_OFFSET + i, UcellMan.second_derivative_matrices[i])
+    D.initialize_managers()
+
+
+    # STEP7:
+    # compute the scattering and its derivative
+    print("Adding diffBragg spots")
+    #D.printout_pixel_fastslow =150, 351
+    D.add_diffBragg_spots()
+    print("Done!")
+    img = D.raw_pixels_roi.as_numpy_array()
+    # reset all pixel values
+    D.raw_pixels *= 0
+    D.raw_pixels_roi *= 0
+
+    derivs = []
+    second_derivs = []
+    for i_param in range(n_ucell_params):
+        analy_deriv = D.get_derivative_pixels(UCELL_ID_OFFSET+i_param).as_numpy_array()
+        derivs.append(analy_deriv)
         if args.curvatures:
-            # estimate the second derivative
-            var[i_param] = var[i_param] - 2*param_shift  # do the backwards finite deriv
+            second_derivs.append(D.get_second_derivative_pixels(UCELL_ID_OFFSET+i_param).as_numpy_array())
+
+    # STEP8
+    # iterate over the parameters and do a finite difference test for each one
+    # parameter shifts:
+    shifts = [1e-4*(2*i) for i in range(1, 12, 2)]
+
+    import copy
+    starting_var = copy.copy(UcellMan.variables)
+    for i_param in range(n_ucell_params):
+        analy_deriv = derivs[i_param]
+        cc_vals = []
+        cc_vals2 = []
+        error = []
+        error2 = []
+        h_vals = []
+        for i_shift, percent_shift in enumerate(shifts):
+
+            var = copy.copy(starting_var)
+
+            param_shift = var[i_param] * percent_shift
+
+            var[i_param] += param_shift
             UcellMan.variables = var
 
             D.Bmatrix = UcellMan.B_recipspace
             D.add_diffBragg_spots()
 
-            img_backward = D.raw_pixels_roi.as_numpy_array()
+            img_forward = D.raw_pixels_roi.as_numpy_array()
 
             # reset for next computation
             D.raw_pixels_roi *= 0
             D.raw_pixels *= 0
 
-        finite_deriv = (img_forward-img) / param_shift
+            if args.curvatures:
+                # estimate the second derivative
+                var[i_param] = var[i_param] - 2*param_shift  # do the backwards finite deriv
+                UcellMan.variables = var
 
-        if second_derivs:
-            finite_second_deriv = (img_forward - 2*img + img_backward) / param_shift / param_shift
+                D.Bmatrix = UcellMan.B_recipspace
+                D.add_diffBragg_spots()
 
-        bragg = img > 0.5
+                img_backward = D.raw_pixels_roi.as_numpy_array()
 
-        ave_error = np.abs(finite_deriv[bragg] - analy_deriv[bragg]).mean()
+                # reset for next computation
+                D.raw_pixels_roi *= 0
+                D.raw_pixels *= 0
 
-        r = pearsonr(analy_deriv[bragg].ravel(), finite_deriv[bragg].ravel())[0]
-        cc_vals.append(r)
+            finite_deriv = (img_forward-img) / param_shift
 
-        error.append(ave_error)
-        h_vals.append( param_shift)
-        print ("\tAverage error=%f; parameter shift h=%f" % (ave_error, abs(param_shift)))
-        if args.curvatures:
-            ave_error2 = np.abs(finite_second_deriv[bragg] - second_derivs[i_param][bragg]).mean()
-            print("\tsecond derivative Average error=%f; parameter shift squared h^2=%f"
-                  % (ave_error2, abs(param_shift)**2))
-            r2 = pearsonr(second_derivs[i_param][bragg].ravel(), finite_second_deriv[bragg].ravel())[0]
+            if second_derivs:
+                finite_second_deriv = (img_forward - 2*img + img_backward) / param_shift / param_shift
 
-            error2.append(ave_error2)
-            cc_vals2.append(r2)
-        if args.plot:
-            plt.subplot(121)
-            plt.imshow(finite_deriv)
-            plt.title("finite diff")
-            plt.subplot(122)
-            plt.imshow(analy_deriv)
-            plt.title("analytical")
-            plt.draw()
-            plt.suptitle("Shift %d / %d"
-                         % (i_shift+1, len(shifts)))
-            plt.pause(0.8)
+            bragg = img > 0.5
+
+            ave_error = np.abs(finite_deriv[bragg] - analy_deriv[bragg]).mean()
+
+            r = pearsonr(analy_deriv[bragg].ravel(), finite_deriv[bragg].ravel())[0]
+            cc_vals.append(r)
+
+            error.append(ave_error)
+            h_vals.append( param_shift)
+            print ("\tAverage error=%f; parameter shift h=%f" % (ave_error, abs(param_shift)))
             if args.curvatures:
+                ave_error2 = np.abs(finite_second_deriv[bragg] - second_derivs[i_param][bragg]).mean()
+                print("\tsecond derivative Average error=%f; parameter shift squared h^2=%f"
+                      % (ave_error2, abs(param_shift)**2))
+                r2 = pearsonr(second_derivs[i_param][bragg].ravel(), finite_second_deriv[bragg].ravel())[0]
+
+                error2.append(ave_error2)
+                cc_vals2.append(r2)
+            if args.plot:
                 plt.subplot(121)
-                plt.imshow(finite_second_deriv)
-                plt.title("finite second diff")
+                plt.imshow(finite_deriv)
+                plt.title("finite diff")
                 plt.subplot(122)
-                plt.imshow(second_derivs[i_param])
+                plt.imshow(analy_deriv)
                 plt.title("analytical")
                 plt.draw()
                 plt.suptitle("Shift %d / %d"
-                             % (i_shift + 1, len(shifts)))
+                             % (i_shift+1, len(shifts)))
                 plt.pause(0.8)
+                if args.curvatures:
+                    plt.subplot(121)
+                    plt.imshow(finite_second_deriv)
+                    plt.title("finite second diff")
+                    plt.subplot(122)
+                    plt.imshow(second_derivs[i_param])
+                    plt.title("analytical")
+                    plt.draw()
+                    plt.suptitle("Shift %d / %d"
+                                 % (i_shift + 1, len(shifts)))
+                    plt.pause(0.8)
+
+        l = linregress(h_vals, error)
+
+        print ("finite diff l.rvalue=%10.7g" % l.rvalue)
+        assert l.rvalue > .99
+        assert l.slope > 0
+        assert l.pvalue < 1e-6
 
-    l = linregress(h_vals, error)
-
-    print ("finite diff l.rvalue=%10.7g" % l.rvalue)
-    assert l.rvalue > .99
-    assert l.slope > 0
-    assert l.pvalue < 1e-6
-
-    if args.curvatures:
-        l2 = linregress(np.array(h_vals)**2, error2)
-        print ("finite 2nd diff l.rvalue=%10.7g" % l2.rvalue)
-        assert l2.rvalue > .99
-        assert l2.slope > 0
-        assert l2.pvalue < 1e-6
-
-    if args.plot:
-        plt.close()
-        plt.plot(shifts, cc_vals, 'o')
-        title = "Unit cell parameter %d / %d" % (i_param+1, n_ucell_params)
-        plt.title(title + "\nPearson corr between finite deriv and analytical")
-        plt.xlabel("unit cell shifts")
-        plt.ylabel("Pearson corr")
-        plt.show()
         if args.curvatures:
+            l2 = linregress(np.array(h_vals)**2, error2)
+            print ("finite 2nd diff l.rvalue=%10.7g" % l2.rvalue)
+            assert l2.rvalue > .99
+            assert l2.slope > 0
+            assert l2.pvalue < 1e-6
+
+        if args.plot:
             plt.close()
-            plt.plot(np.array(shifts)**2, cc_vals2, 'o')
-            title = "Unit cell parameter %d / %d" % (i_param + 1, n_ucell_params)
-            plt.title(title + "\nPearson corr between finite second deriv and analytical")
+            plt.plot(shifts, cc_vals, 'o')
+            title = "Unit cell parameter %d / %d" % (i_param+1, n_ucell_params)
+            plt.title(title + "\nPearson corr between finite deriv and analytical")
             plt.xlabel("unit cell shifts")
             plt.ylabel("Pearson corr")
             plt.show()
-
-    # verify a high correlation for the smallest parameter shift
-    print("Check high pearson R between analytical and finite diff")
-    print("Pearson correlection at smallest parameter shift=%f" % cc_vals[0])
-    assert(cc_vals[0] > .98), "%f" % cc_vals[0]
-    # check monotonic decrease
-    print("Fit polynomial and check monotonic decrease")
-    trend = np.polyval(np.polyfit(shifts, cc_vals, 2), shifts)
-    assert np.all(np.diff(list(zip(trend[:-1], trend[1:])), axis=1) <= 0)
-    if args.curvatures:
-        assert (cc_vals2[0] > .99), "%f" % cc_vals2[0]
-print("OK!")
+            if args.curvatures:
+                plt.close()
+                plt.plot(np.array(shifts)**2, cc_vals2, 'o')
+                title = "Unit cell parameter %d / %d" % (i_param + 1, n_ucell_params)
+                plt.title(title + "\nPearson corr between finite second deriv and analytical")
+                plt.xlabel("unit cell shifts")
+                plt.ylabel("Pearson corr")
+                plt.show()
+
+        # verify a high correlation for the smallest parameter shift
+        print("Check high pearson R between analytical and finite diff")
+        print("Pearson correlection at smallest parameter shift=%f" % cc_vals[0])
+        assert(cc_vals[0] > .98), "%f" % cc_vals[0]
+        # check monotonic decrease
+        print("Fit polynomial and check monotonic decrease")
+        trend = np.polyval(np.polyfit(shifts, cc_vals, 2), shifts)
+        assert np.all(np.diff(list(zip(trend[:-1], trend[1:])), axis=1) <= 0)
+        if args.curvatures:
+            assert (cc_vals2[0] > .99), "%f" % cc_vals2[0]
+    print("OK!")
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_update_dxtbx_geoms.py b/simtbx/diffBragg/tests/tst_diffBragg_update_dxtbx_geoms.py
index 3152caa931..f9154a8638 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_update_dxtbx_geoms.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_update_dxtbx_geoms.py
@@ -5,6 +5,8 @@
 one to update those models for beam/detector post-instantiation
 """
 from __future__ import division
+##from simtbx.kokkos import gpu_instance
+#kokkos_run = gpu_instance(deviceId = 0)
 
 import numpy as np
 from simtbx.nanoBragg import sim_data
@@ -75,3 +77,7 @@
 assert np.allclose(det[0].get_beam_centre(B.get_s0()), SIM.D.beam_center_mm)
 
 print("OK!")
+from simtbx.diffBragg.utils import find_diffBragg_instances
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(0) as _:
+    for name in find_diffBragg_instances(globals()): del globals()[name]
diff --git a/simtbx/diffBragg/tests/tst_diffBragg_utils.py b/simtbx/diffBragg/tests/tst_diffBragg_utils.py
index 412732e0e4..04cdbed417 100644
--- a/simtbx/diffBragg/tests/tst_diffBragg_utils.py
+++ b/simtbx/diffBragg/tests/tst_diffBragg_utils.py
@@ -33,5 +33,4 @@ def test_regions():
 
 
 if __name__ == "__main__":
-    test_regions()
     print("OK")
diff --git a/simtbx/diffBragg/tests/tst_hopper_usecase.py b/simtbx/diffBragg/tests/tst_hopper_usecase.py
new file mode 100644
index 0000000000..c555b0336b
--- /dev/null
+++ b/simtbx/diffBragg/tests/tst_hopper_usecase.py
@@ -0,0 +1,40 @@
+from __future__ import division
+from libtbx.mpi4py import MPI
+COMM = MPI.COMM_WORLD
+from dxtbx.model import DetectorFactory, BeamFactory
+from simtbx.nanoBragg.tst_nanoBragg_multipanel import det_descr, beam_descr
+from simtbx.diffBragg import diffBragg
+
+from argparse import ArgumentParser
+parser = ArgumentParser()
+parser.add_argument("--ngpu", type=int, default=1)
+parser.add_argument("--kokkos", action="store_true")
+args = parser.parse_args()
+NGPU_PER_NODE = args.ngpu
+if args.kokkos:
+    import os
+    os.environ["DIFFBRAGG_USE_KOKKOS"] = "1"
+
+GPU_ID = COMM.rank % NGPU_PER_NODE
+beam = BeamFactory.from_dict(beam_descr)
+det = DetectorFactory.from_dict(det_descr)
+
+from simtbx.diffBragg.device import DeviceWrapper
+with DeviceWrapper(GPU_ID) as wrapper:
+    for i in range(COMM.size*3):
+        if i % COMM.size != COMM.rank:
+            continue
+        D = diffBragg(det, beam)
+        D.verbose = 1
+        D.vectorize_umats()
+        for _ in range(5):
+            D.add_diffBragg_spots()
+
+        D.free_all()
+        D.free_Fhkl2()
+        print("RANK %d, done with Iter %d" % (COMM.rank, i+1))
+
+    # TODO: the following deletion of the diffBragg instance is necessary to prevent the deallocation error
+    # is there someway around this ????
+    del D
+    print("OK!")
diff --git a/simtbx/run_tests.py b/simtbx/run_tests.py
index ec5a27317d..70be561b2b 100644
--- a/simtbx/run_tests.py
+++ b/simtbx/run_tests.py
@@ -14,11 +14,12 @@
 
 db_tst_list_nonCuda = ["$D/diffBragg/tests/tst_diffBragg_utils.py",
                        "$D/diffBragg/tests/tst_diffBragg_structure_factors.py"]
-db_tst_list_onlyCuda =[["$D/diffBragg/tests/tst_diffBragg_hopper_refine.py", "--perturb eta --cuda"]]
+db_tst_list_onlyCuda =[["$D/diffBragg/tests/tst_diffBragg_hopper_refine.py", "--perturb eta --kokkos"]]
 
 db_tst_list = [
     ["$D/diffBragg/tests/tst_diffBragg_hopper_refine_Fhkl.py", "--scale .2"],
     "$D/diffBragg/tests/tst_diffBragg_Fhkl_complex.py",
+    "$D/diffBragg/tests/tst_hopper_usecase.py",
     "$D/diffBragg/tests/tst_diffBragg_change_of_basis.py",
     "$D/diffBragg/tests/tst_diffBragg_update_dxtbx_geoms.py",
     "$D/diffBragg/tests/tst_diffBragg_deriv_rois.py",
@@ -91,13 +92,13 @@
      ["$D/tests/tst_unified.py","context=kokkos_gpu"],# GPU, exaFEL full API
      ["$D/gpu/tst_shoeboxes.py","context=kokkos_gpu"],# GPU, test whitelist API
    ]
-if OPT.enable_cuda:
+if OPT.enable_kokkos:
   if OPT.enable_cxx11 and sys.platform != 'win32':
       for tst in db_tst_list:
           if isinstance(tst, str):
-              par_tst = [tst, "--cuda"]
+              par_tst = [tst, "--kokkos"]
           else:
-              par_tst = tst + ["--cuda"]
+              par_tst = tst + ["--kokkos"]
           tst_list_parallel.append(par_tst)
 
       tst_list_parallel += db_tst_list_onlyCuda