Repository: PolusAI/nyxus (language: python)

Issue:
Arrow writers refactor
- Refactor process dataset functions to be overloaded to separate csv/buffer functionality from Arrow functionality
- Remove passing status to get_arrow_table and check for nullptr instead
- Remove low level functionality from nyxus main to process functions and the environment


Discussion / hints:
## Apply [Sweep Rules](https://docs.sweep.dev/usage/config#rules) to your PR?
- [ ] **Apply:**  Leftover TODOs in the code should be handled.
- [ ] **Apply:**  All new business logic should have corresponding unit tests in the tests/ directory.
- [ ] **Apply:**  Any clearly inefficient or repeated code should be optimized or refactored.

Fix patch:
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6a785ccd..fba19162 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -412,7 +412,7 @@ endif()
 
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 	SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
-	list(APPEND Nyxus_LIBRARIES stdc++fs)
+	list(APPEND Nyxus_LIBRARIES stdc++fs -static-libgcc -static-libstdc++)
 endif()
 
 if(BUILD_LIB)
diff --git a/src/nyx/arrow_output_stream.cpp b/src/nyx/arrow_output_stream.cpp
index 0d27c508..ce13ffd6 100644
--- a/src/nyx/arrow_output_stream.cpp
+++ b/src/nyx/arrow_output_stream.cpp
@@ -1,34 +1,45 @@
 #include "arrow_output_stream.h"
-
 #ifdef USE_ARROW
 
-std::shared_ptr<ApacheArrowWriter> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header) {
 
-    if(arrow_file_path != "" && !fs::is_directory(arrow_file_path) && !(Nyxus::ends_with_substr(arrow_file_path, ".arrow") || Nyxus::ends_with_substr(arrow_file_path, ".feather") || Nyxus::ends_with_substr(arrow_file_path, ".parquet"))) {
-        throw std::invalid_argument("The arrow file path must end in \".arrow\"");
-    }
 
-    if (arrow_file_type != Nyxus::SaveOption::saveArrowIPC && arrow_file_type != Nyxus::SaveOption::saveParquet) {
-        throw std::invalid_argument("The valid save options are Nyxus::SaveOption::saveArrowIPC or Nyxus::SaveOption::saveParquet.");
-    }
 
-    std::string extension = (arrow_file_type == Nyxus::SaveOption::saveParquet) ? ".parquet" : ".arrow";
+    auto valid_extension = [&arrow_file_path](){
+        if(	arrow_file_path != "" && 
+            !fs::is_directory(arrow_file_path)){
+                if( auto ext = fs::path(arrow_file_path).extension(); 
+                    ext == ".arrow" || ext == ".feather" || ext == ".arrow"){
+                        return true;
+                } else {
+                    return false;
+                } 
+            }
+        return false;
+    }(); 
 
-    if (arrow_file_path == "") {
-        arrow_file_path_ = "NyxusFeatures" + extension;
-    } else {
+    if (valid_extension) {
         arrow_file_path_ = arrow_file_path;
+    } else {
+        std::string extension = (arrow_file_type == Nyxus::SaveOption::saveParquet) ? "parquet" : "arrow";
+        if (arrow_file_path == "") {
+            arrow_file_path_ = "NyxusFeatures." + extension;
+        } else if (fs::is_directory(arrow_file_path)) {
+            arrow_file_path_ = arrow_file_path + "/NyxusFeatures." + extension;
+        } else {
+            arrow_file_path_ = fs::path(arrow_file_path).replace_extension(extension).string();
+        }
     }
 
-    if (fs::is_directory(arrow_file_path)) {
-        arrow_file_path_ += "/NyxusFeatures" + extension;
+    std::optional<std::string> error_msg;
+    std::tie(writer_, error_msg) = WriterFactory::create_writer(arrow_file_path_, header);
+    if (writer_) {
+        return {true, std::nullopt};
+    } else {
+        return {false, error_msg};
     }
-
-    writer_ = WriterFactory::create_writer(arrow_file_path_, header);
-
-    return writer_;
 }
 
 
@@ -45,30 +56,50 @@ std::string ArrowOutputStream::get_arrow_path() {
     return arrow_file_path_;
 }
 
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features){
+    if (writer_){
+        auto status = writer_->write(features);
+        if (status.ok()) {
+            return {true, std::nullopt};
+        }
+        else {
+            return {false, status.ToString()};
+        }
+    }
+    return {false, "Arrow Writer is not initialized."};
+}
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::close_arrow_file (){
+    if (writer_){
+        auto status = writer_->close();
+        if (status.ok()) {
+            return {true, std::nullopt};
+        }
+        else {
+            return {false, status.ToString()};
+        }
+    }
+    return {false, "Arrow Writer is not initialized."};
+}
+
 #else 
 
-std::shared_ptr<ApacheArrowWriter> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header) {
     
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
 
-    return nullptr;
+    return {false, "Apache Arrow functionality is not available."};
 }
 
-
-std::shared_ptr<arrow::Table> ArrowOutputStream::get_arrow_table(const std::string& file_path) {
-
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features){
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
-
-    return nullptr;
+    return {false, "Apache Arrow functionality is not available."};
 }
-
-std::string ArrowOutputStream::get_arrow_path() {
-
+std::tuple<bool, std::optional<std::string>> ArrowOutputStream::close_arrow_file (){
     std::cerr << "Apache Arrow functionality is not available. Please install Nyxus with Arrow enabled to use this functionality." << std::endl;
-    
-    return "";
+    return {false, "Apache Arrow functionality is not available."};
 }
 
+
 #endif
\ No newline at end of file
diff --git a/src/nyx/arrow_output_stream.h b/src/nyx/arrow_output_stream.h
index 33ec7014..e720ef89 100644
--- a/src/nyx/arrow_output_stream.h
+++ b/src/nyx/arrow_output_stream.h
@@ -1,7 +1,9 @@
 #pragma once
 
+#include <optional>
 #include <string>
 #include <memory>
+#include <tuple>
 
 #include "output_writers.h"
 #include "helpers/helpers.h"
@@ -31,25 +33,22 @@ class ArrowOutputStream {
 private:
 
   std::string arrow_file_path_ = "";
-	std::shared_ptr<ApacheArrowWriter> writer_ = nullptr;
+	std::unique_ptr<ApacheArrowWriter> writer_ = nullptr;
 	std::string arrow_output_type_ = "";
   std::shared_ptr<arrow::Table> arrow_table_ = nullptr;
 
 public:
-    std::shared_ptr<ApacheArrowWriter> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+    std::tuple<bool, std::optional<std::string>> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header);
     std::shared_ptr<arrow::Table> get_arrow_table(const std::string& file_path);
     std::string get_arrow_path();
+    std::tuple<bool, std::optional<std::string>> write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features);
+    std::tuple<bool, std::optional<std::string>> close_arrow_file ();
 };
 
 #else 
 
-// Replace arrow::Table with a dummy variable
-namespace arrow {
-  using Table = bool;
-};
-
 /**
  * @brief Class to write to Apache Arrow formats
  * 
@@ -58,19 +57,12 @@ namespace arrow {
  */
 class ArrowOutputStream {
 
-private:
-
-  std::string arrow_file_path_ = "";
-	std::shared_ptr<ApacheArrowWriter> writer_ = nullptr;
-	std::string arrow_output_type_ = "";
-  std::shared_ptr<arrow::Table> arrow_table_ = nullptr;
-
 public:
-    std::shared_ptr<ApacheArrowWriter> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
+    std::tuple<bool, std::optional<std::string>> create_arrow_file(const Nyxus::SaveOption& arrow_file_type,
                                                          const std::string& arrow_file_path,
                                                          const std::vector<std::string>& header);
-    std::shared_ptr<arrow::Table> get_arrow_table(const std::string& file_path);
-    std::string get_arrow_path();
+    std::tuple<bool, std::optional<std::string>> write_arrow_file (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features);
+    std::tuple<bool, std::optional<std::string>> close_arrow_file ();
 };
 
 
diff --git a/src/nyx/environment.cpp b/src/nyx/environment.cpp
index 3323db7a..df7ca61f 100644
--- a/src/nyx/environment.cpp
+++ b/src/nyx/environment.cpp
@@ -838,16 +838,21 @@ bool Environment::parse_cmdline(int argc, char **argv)
 	}
 
 	//==== Output type
-#ifdef USE_ARROW 
 	VERBOSLVL1(std::cout << "\n*-*-*-*- Using Apache output -*-*-*-*\n");
 
 	auto rawOutpTypeUC = Nyxus::toupper(rawOutpType);
-	if (rawOutpTypeUC != Nyxus::toupper(OT_SINGLECSV) && 
-	    rawOutpTypeUC != Nyxus::toupper(OT_SEPCSV) && 
-		rawOutpTypeUC != Nyxus::toupper(OT_ARROWIPC) &&
-		rawOutpTypeUC != Nyxus::toupper(OT_PARQUET))
+	if (!((rawOutpTypeUC == Nyxus::toupper(OT_SINGLECSV))  || 
+	      (rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV)) || 
+		  (rawOutpTypeUC == Nyxus::toupper(OT_ARROWIPC)) ||
+		  (rawOutpTypeUC == Nyxus::toupper(OT_PARQUET))
+		))	
 	{
-		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " << OT_SINGLECSV << ", or" << OT_PARQUET << "."  "\n";
+		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " 
+					<< OT_SINGLECSV <<", "
+					#ifdef USE_ARROW
+					<< OT_ARROWIPC <<", or" << OT_PARQUET <<
+					#endif 
+					"."  "\n";
 		return false;
 	}
 
@@ -861,31 +866,16 @@ bool Environment::parse_cmdline(int argc, char **argv)
 		} 
 	}();
 
-	if (saveOption == SaveOption::saveCSV) {
-		separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
-	}
-
-#else // no Apache support
-	auto rawOutpTypeUC = Nyxus::toupper(rawOutpType);
-	if (rawOutpTypeUC != Nyxus::toupper(OT_SINGLECSV) &&
-		rawOutpTypeUC != Nyxus::toupper(OT_SEPCSV))
-	{
-		std::cout << "Error: valid values of " << OUTPUTTYPE << " are " << OT_SEPCSV << ", " << OT_SINGLECSV << "\n";
-
-		// Intercept an attempt of running Nyxus with Apache options
-		if (rawOutpTypeUC != Nyxus::toupper(OT_ARROWIPC) ||
-			rawOutpTypeUC != Nyxus::toupper(OT_PARQUET)) 
-			std::cout << "Error: Nyxus must be built with Apache Arrow enabled to use Arrow output types. Please rebuild with the flag USEARROW=ON." << std::endl;
-
+#ifndef USE_ARROW // no Apache support
+	if (saveOption == SaveOption::saveArrowIPC || saveOption == SaveOption::saveParquet) {
+		std::cout << "Error: Nyxus must be built with Apache Arrow enabled to use Arrow output types. Please rebuild with the flag USEARROW=ON." << std::endl;
 		return false;
 	}
+#endif
 
-	separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
-
-	if (rawOutpTypeUC == Nyxus::toupper(OT_SINGLECSV) || rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV)) {
-		saveOption = Nyxus::SaveOption::saveCSV;
+	if (saveOption == SaveOption::saveCSV) {
+		separateCsv = rawOutpTypeUC == Nyxus::toupper(OT_SEPCSV);
 	}
-#endif
 
 	//==== Check numeric parameters
 	if (!loader_threads.empty())
diff --git a/src/nyx/environment.h b/src/nyx/environment.h
index 024aa98f..969d5b46 100644
--- a/src/nyx/environment.h
+++ b/src/nyx/environment.h
@@ -8,8 +8,6 @@
 #include "cli_gabor_options.h"
 #include "cli_nested_roi_options.h"
 #include "save_option.h"
-
-#include "output_writers.h"
 #include "arrow_output_stream.h"
 
 
@@ -119,7 +117,6 @@ class Environment: public BasicEnvironment
 
 
 	ArrowOutputStream  arrow_stream;
-	std::shared_ptr<ApacheArrowWriter> arrow_writer = nullptr;
 
 	std::string embedded_pixel_size = "";
 
diff --git a/src/nyx/globals.h b/src/nyx/globals.h
index 63074ab3..b620d229 100644
--- a/src/nyx/globals.h
+++ b/src/nyx/globals.h
@@ -40,7 +40,7 @@ namespace Nyxus
 
 	bool scanFilePairParallel(const std::string& intens_fpath, const std::string& label_fpath, int num_fastloader_threads, int num_sensemaker_threads, int filepair_index, int tot_num_filepairs);
 	std::string getPureFname(const std::string& fpath);
-	int processDataset(const std::vector<std::string>& intensFiles, const std::vector<std::string>& labelFiles, int numFastloaderThreads, int numSensemakerThreads, int numReduceThreads, int min_online_roi_size, SaveOption saveOption, const std::string& outputDir);
+	int processDataset(const std::vector<std::string>& intensFiles, const std::vector<std::string>& labelFiles, int numFastloaderThreads, int numSensemakerThreads, int numReduceThreads, int min_online_roi_size, const SaveOption saveOption, const std::string& outputPath);
 	bool gatherRoisMetrics(const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads);
 	bool processTrivialRois (const std::vector<int>& trivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads, size_t memory_limit);
 	bool processNontrivialRois (const std::vector<int>& nontrivRoiLabels, const std::string& intens_fpath, const std::string& label_fpath, int num_FL_threads);
@@ -54,7 +54,7 @@ namespace Nyxus
 	bool gatherRoisMetricsInMemory (const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_image, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_image, int start_idx);
 	bool processIntSegImagePairInMemory (const std::string& intens_fpath, const std::string& label_fpath, int filepair_index, const std::string& intens_name, const std::string& seg_name);
 	int processMontage(const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intensFiles, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& labelFiles, int numReduceThreads, const std::vector<std::string>& intensity_names,
-		const std::vector<std::string>& seg_names, std::string& error_message, SaveOption saveOption,  const std::string& outputDir="");
+		const std::vector<std::string>& seg_names, std::string& error_message, const SaveOption saveOption,  const std::string& outputPath="");
 	bool scanTrivialRois (const std::vector<int>& batch_labels, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_images, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_images, int start_idx);
 	bool processTrivialRoisInMemory (const std::vector<int>& trivRoiLabels, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& intens_fpath, const py::array_t<unsigned int, py::array::c_style | py::array::forcecast>& label_fpath, int start_idx, size_t memory_limit);
 #endif
diff --git a/src/nyx/output_writers.cpp b/src/nyx/output_writers.cpp
index 6a850cd9..343ecea6 100644
--- a/src/nyx/output_writers.cpp
+++ b/src/nyx/output_writers.cpp
@@ -386,44 +386,30 @@ arrow::Status ArrowIPCWriter::close () {
 }
 
 
-std::shared_ptr<ApacheArrowWriter> WriterFactory::create_writer(const std::string &output_file, const std::vector<std::string> &header) {
+std::tuple<std::unique_ptr<ApacheArrowWriter>, std::optional<std::string>> WriterFactory::create_writer(const std::string &output_file, const std::vector<std::string> &header) {
     
     if (Nyxus::ends_with_substr(output_file, ".parquet")) {
         
-        return std::make_shared<ParquetWriter>(output_file, header);
+        return {std::make_unique<ParquetWriter>(output_file, header), std::nullopt};
 
     } else if (Nyxus::ends_with_substr(output_file, ".arrow") || Nyxus::ends_with_substr(output_file, ".feather")) {
         
-        return std::make_shared<ArrowIPCWriter>(output_file, header);
+        return {std::make_unique<ArrowIPCWriter>(output_file, header), std::nullopt};
 
     } else {
 
         std::filesystem::path path(output_file);
 
-        if (path.has_extension()) {
-            std::string file_extension = path.extension().string();
-            
-            throw std::invalid_argument("No writer option for extension \"" + file_extension + "\". Valid options are \".parquet\" or \".arrow\".");
-
-        } else {
-
-            throw std::invalid_argument("No extension type was provided in the path. ");
+        auto error_msg = [&path](){        
+            if (path.has_extension()) 
+            {
+                return "No writer option for extension \"" + path.extension().string() + "\". Valid options are \".parquet\" or \".arrow\".";
+            } else {
+                return std::string{"No extension type was provided in the path."};
+            }
+        }; 
 
-        }
+        return {nullptr, error_msg()};
     }
 }
-#else
-
-    std::shared_ptr<arrow::Table> ApacheArrowWriter::get_arrow_table(const std::string& file_path) {
-        return nullptr;
-    }
-
-    arrow::Status ApacheArrowWriter::write (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features) {
-        return arrow::Status();
-    }
-
-    arrow::Status ApacheArrowWriter::close () {
-        return arrow::Status();
-    }
-
 #endif
\ No newline at end of file
diff --git a/src/nyx/output_writers.h b/src/nyx/output_writers.h
index 04fcdee2..ac31d1f5 100644
--- a/src/nyx/output_writers.h
+++ b/src/nyx/output_writers.h
@@ -4,6 +4,8 @@
 #include <vector>
 #include <string>
 #include <memory>
+#include <tuple>
+#include <optional>
 
 #ifdef USE_ARROW
 #include <arrow/api.h>
@@ -86,6 +88,8 @@ class ApacheArrowWriter
 
     virtual arrow::Status close () = 0;
 
+    virtual ~ApacheArrowWriter() = default;
+
 };
 
 /**
@@ -161,65 +165,8 @@ class WriterFactory {
          * @brief Create an ApacheArrowWriter based on the type of file passed.
          * 
          * @param output_file Path to output file (.arrow or .parquet)
-         * @return std::shared_ptr<ApacheArrowWriter> 
+         * @return std::unique_ptr<ApacheArrowWriter> 
          */
-        static std::shared_ptr<ApacheArrowWriter> create_writer(const std::string &output_file, const std::vector<std::string> &header);
-};
-
-#else 
-
-
-namespace arrow {
-
-    using Table = bool;
-
-    class Status {
-
-    public:
-
-        bool ok() {return false;}
-
-        std::string ToString() {return "Apache Arrow support is not enabled. Please reinstall Nyxus with Arrow support enabled.";}
-
-    };
-
+        static std::tuple<std::unique_ptr<ApacheArrowWriter>, std::optional<std::string>> create_writer(const std::string &output_file, const std::vector<std::string> &header);
 };
-
-/**
- * @brief Base class for creating Apache Arrow output writers
- * 
- * This class provides methods for the Arrow table used for writing to Arrow formats and
- * provides virtual functions to overridden for writing to different formats
- * 
- */
-class ApacheArrowWriter
-{
-
-private: 
-    std::shared_ptr<arrow::Table> table_ = nullptr;
-
-public:
-
-    /**
-     * @brief Get the arrow table object
-     * 
-     * @return std::shared_ptr<arrow::Table> 
-     */
-    std::shared_ptr<arrow::Table> get_arrow_table(const std::string& file_path);
-
-    /**
-     * @brief Write Nyxus data to Arrow file
-     * 
-     * @param header Header data
-     * @param string_columns String data
-     * @param numeric_columns Numeric data
-     * @param number_of_rows Number of rows
-     * @return arrow::Status 
-     */
-    virtual arrow::Status write (const std::vector<std::tuple<std::vector<std::string>, int, std::vector<double>>>& features);
-
-    virtual arrow::Status close ();
-
-};
-
 #endif
diff --git a/src/nyx/scan_fastloader_way.cpp b/src/nyx/scan_fastloader_way.cpp
index 8fdcc9a3..2d68b2a0 100644
--- a/src/nyx/scan_fastloader_way.cpp
+++ b/src/nyx/scan_fastloader_way.cpp
@@ -26,11 +26,6 @@ namespace py = pybind11;
 #include "globals.h"
 #include "helpers/timing.h"
 
-#ifdef USE_ARROW
-#include "arrow_output_stream.h"
-#include "output_writers.h"
-#endif
-
 // Sanity
 #ifdef _WIN32
 #include<windows.h>
@@ -209,8 +204,8 @@ namespace Nyxus
 		int numSensemakerThreads,
 		int numReduceThreads,
 		int min_online_roi_size,
-		SaveOption saveOption,
-		const std::string& outputDir)
+		const SaveOption saveOption,
+		const std::string& outputPath)
 	{
 
 		#ifdef CHECKTIMING
@@ -227,11 +222,10 @@ namespace Nyxus
 		if (write_apache) {
 
 			theEnvironment.arrow_stream = ArrowOutputStream();
-
-			try {
-				theEnvironment.arrow_writer = theEnvironment.arrow_stream.create_arrow_file(saveOption, outputDir,   Nyxus::get_header(theFeatureSet.getEnabledFeatures()));
-			} catch (const std::exception &err) {
-				std::cout << "Error creating Arrow file: " << err.what() << std::endl;
+			auto [status, msg] = theEnvironment.arrow_stream.create_arrow_file(saveOption, outputPath, Nyxus::get_header(theFeatureSet.getEnabledFeatures()));
+			
+			if (!status) {
+				std::cout << "Error creating Arrow file: " << msg.value() << std::endl;
 				return 1;
 			}
 		}
@@ -278,15 +272,14 @@ namespace Nyxus
 
 			if (write_apache) {
 
-				auto status = theEnvironment.arrow_writer->write(Nyxus::get_feature_values());
+				auto [status, msg] = theEnvironment.arrow_stream.write_arrow_file(Nyxus::get_feature_values());
 				
-				if (!status.ok()) {
-                    // Handle read error
-                    std::cout << "Error writing Arrow file: " << status.ToString() << std::endl;
+				if (!status) {
+					std::cout << "Error writing Arrow file: " << msg.value() << std::endl;
 					return 2;
 				}
 			} else if (saveOption == SaveOption::saveCSV) {
-				ok = save_features_2_csv(ifp, lfp, outputDir);
+				ok = save_features_2_csv(ifp, lfp, outputPath);
 
 				if (ok == false)
 				{
@@ -349,12 +342,10 @@ namespace Nyxus
 
 		if (write_apache) {
 			// close arrow file after use
-			auto status = theEnvironment.arrow_writer->close();
-			
-			if (!status.ok()) {
-				// Handle read error
-				std::cout << "Error closing Arrow file: " << status.ToString() << std::endl;
-				return 2;
+			auto [status, msg] = theEnvironment.arrow_stream.close_arrow_file();
+			if (!status) {
+				std::cout << "Error closing Arrow file: " << msg.value() << std::endl;
+					return 2;
 			}
 		}
 	
@@ -370,19 +361,17 @@ namespace Nyxus
 		const std::vector<std::string>& intensity_names,
 		const std::vector<std::string>& seg_names,
 		std::string& error_message,
-		SaveOption saveOption,
-		const std::string& outputDir)
+		const SaveOption saveOption,
+		const std::string& outputPath)
 	{	
 		bool write_apache = (saveOption == SaveOption::saveArrowIPC || saveOption == SaveOption::saveParquet);
 
 		if (write_apache) {
 
 			theEnvironment.arrow_stream = ArrowOutputStream();
-
-			try {
-				theEnvironment.arrow_writer = theEnvironment.arrow_stream.create_arrow_file(saveOption, outputDir,  Nyxus::get_header(theFeatureSet.getEnabledFeatures()));
-			} catch (const std::exception &err) {
-				error_message = err.what();
+			auto [status, msg] = theEnvironment.arrow_stream.create_arrow_file(saveOption, outputPath,   Nyxus::get_header(theFeatureSet.getEnabledFeatures()));
+			if (!status) {
+				std::cout << "Error creating Arrow file: " << msg.value() << std::endl;
 				return 1;
 			}
 		}
@@ -413,14 +402,11 @@ namespace Nyxus
 
 			if (write_apache) {
 			
-				auto status = theEnvironment.arrow_writer->write(Nyxus::get_feature_values());
-				
-				if (!status.ok()) {
-                    // Handle read error
-                    error_message = "Error writing Arrow file: " + status.ToString();
+				auto [status, msg] = theEnvironment.arrow_stream.write_arrow_file(Nyxus::get_feature_values());
+				if (!status) {
+					std::cout << "Error writing Arrow file: " << msg.value() << std::endl;
 					return 2;
 				}
-
 			} else {
 
 				ok = save_features_2_buffer(theResultsCache);
@@ -453,15 +439,12 @@ namespace Nyxus
 
 		if (write_apache) {
 			// close arrow file after use
-			auto status = theEnvironment.arrow_writer->close();
-			
-			if (!status.ok()) {
-				// Handle read error
-				error_message = "Error closing Arrow file: " + status.ToString();
+			auto [status, msg] = theEnvironment.arrow_stream.close_arrow_file();
+			if (!status) {
+				std::cout << "Error closing Arrow file: " << msg.value() << std::endl;
 				return 2;
 			}
 		}
-		
 		return 0; // success
 	}
 #endif


Test patch:
diff --git a/.github/workflows/build_and_test_mac.yml b/.github/workflows/build_and_test_mac.yml
index cbc24b69..7d16ea2d 100644
--- a/.github/workflows/build_and_test_mac.yml
+++ b/.github/workflows/build_and_test_mac.yml
@@ -77,4 +77,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
\ No newline at end of file
+      run: python -m pytest tests/python/ -vv
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_ubuntu.yml b/.github/workflows/build_and_test_ubuntu.yml
index 49bcd88c..8b5d5c5a 100644
--- a/.github/workflows/build_and_test_ubuntu.yml
+++ b/.github/workflows/build_and_test_ubuntu.yml
@@ -76,4 +76,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
\ No newline at end of file
+      run: python -m pytest tests/python/  -vv
\ No newline at end of file
diff --git a/.github/workflows/build_and_test_windows.yml b/.github/workflows/build_and_test_windows.yml
index de769fcc..8de643a2 100644
--- a/.github/workflows/build_and_test_windows.yml
+++ b/.github/workflows/build_and_test_windows.yml
@@ -80,4 +80,4 @@ jobs:
 
     - name: Run PyTest
       working-directory: ${{github.workspace}}
-      run: python -m pytest tests/python/
\ No newline at end of file
+      run: python -m pytest tests/python/  -vv
\ No newline at end of file