Repository: planetlabs/gpq
Pull request #22 (2023-05-22 20:28:37)

Allow limited converter options
This makes it so the defaults are used for minFeatures, maxFeatures, and compression when converting to GeoParquet from GeoJSON.


Patch:
diff --git a/internal/geojson/geojson.go b/internal/geojson/geojson.go
index 83ddbb8..3e07294 100644
--- a/internal/geojson/geojson.go
+++ b/internal/geojson/geojson.go
@@ -587,14 +587,30 @@ func ToParquet(input io.Reader, output io.Writer, convertOptions *ConvertOptions
 	if convertOptions == nil {
 		convertOptions = defaultOptions
 	}
-	converter, converterErr := reader.Converter(convertOptions.MinFeatures, convertOptions.MaxFeatures)
+
+	minFeatures := convertOptions.MinFeatures
+	if minFeatures == 0 {
+		minFeatures = defaultOptions.MinFeatures
+	}
+
+	maxFeatures := convertOptions.MaxFeatures
+	if maxFeatures == 0 {
+		maxFeatures = defaultOptions.MaxFeatures
+	}
+
+	converter, converterErr := reader.Converter(minFeatures, maxFeatures)
 	if converterErr != nil {
 		return converterErr
 	}
 
 	schema := parquet.SchemaOf(reflect.New(converter.Type).Elem().Interface())
 
-	codec, codecErr := getCodec(convertOptions.Compression)
+	compression := convertOptions.Compression
+	if compression == "" {
+		compression = defaultOptions.Compression
+	}
+
+	codec, codecErr := getCodec(compression)
 	if codecErr != nil {
 		return codecErr
 	}


Test patch:
diff --git a/internal/geojson/geojson_test.go b/internal/geojson/geojson_test.go
index 7074e3e..5a7f90b 100644
--- a/internal/geojson/geojson_test.go
+++ b/internal/geojson/geojson_test.go
@@ -775,7 +775,7 @@ func TestCodecUncompressed(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "uncompressed"}
+	convertOptions := &geojson.ConvertOptions{Compression: "uncompressed"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -791,7 +791,7 @@ func TestCodecSnappy(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "snappy"}
+	convertOptions := &geojson.ConvertOptions{Compression: "snappy"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -807,7 +807,7 @@ func TestCodecGzip(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "gzip"}
+	convertOptions := &geojson.ConvertOptions{Compression: "gzip"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -823,7 +823,7 @@ func TestCodecBrotli(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "brotli"}
+	convertOptions := &geojson.ConvertOptions{Compression: "brotli"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -839,7 +839,7 @@ func TestCodecZstd(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "zstd"}
+	convertOptions := &geojson.ConvertOptions{Compression: "zstd"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -855,7 +855,7 @@ func TestCodecLz4raw(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "lz4raw"}
+	convertOptions := &geojson.ConvertOptions{Compression: "lz4raw"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.NoError(t, toParquetErr)
 
@@ -871,7 +871,7 @@ func TestCodecInvalid(t *testing.T) {
 	require.NoError(t, openErr)
 
 	parquetBuffer := &bytes.Buffer{}
-	convertOptions := &geojson.ConvertOptions{MinFeatures: 10, MaxFeatures: 100, Compression: "invalid"}
+	convertOptions := &geojson.ConvertOptions{Compression: "invalid"}
 	toParquetErr := geojson.ToParquet(geojsonFile, parquetBuffer, convertOptions)
 	assert.EqualError(t, toParquetErr, "invalid compression codec invalid")
 }