Repository: brimsec/zq
Pull request #2418 (2021-03-24 21:04:26)

change map and array refs with missing items to return missing


Patch:
diff --git a/Makefile b/Makefile
index baf4876fc..0b65f7bae 100644
--- a/Makefile
+++ b/Makefile
@@ -91,28 +91,28 @@ test-unit:
 	@go test -short ./...
 
 test-system: build bin/minio bin/$(ZEEKPATH) bin/$(SURICATAPATH)
-	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin:$(CURDIR)/bin/$(ZEEKPATH):$(CURDIR)/bin/$(SURICATAPATH)" go test -v .
+	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin:$(CURDIR)/bin/$(ZEEKPATH):$(CURDIR)/bin/$(SURICATAPATH)" go test .
 
 test-run: build bin/minio bin/$(ZEEKPATH) bin/$(SURICATAPATH)
-	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin:$(CURDIR)/bin/$(ZEEKPATH):$(CURDIR)/bin/$(SURICATAPATH)" go test -v . -run $(TEST)
+	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin:$(CURDIR)/bin/$(ZEEKPATH):$(CURDIR)/bin/$(SURICATAPATH)" go test . -run $(TEST)
 
 test-heavy: build $(SAMPLEDATA)
-	@go test -v -tags=heavy ./tests
+	@go test -tags=heavy ./tests
 
 test-pcapingest: bin/$(ZEEKPATH)
-	@ZEEK="$(CURDIR)/bin/$(ZEEKPATH)/zeekrunner" go test -v -run=PcapPost -tags=pcapingest ./ppl/zqd
+	@ZEEK="$(CURDIR)/bin/$(ZEEKPATH)/zeekrunner" go test -run=PcapPost -tags=pcapingest ./ppl/zqd
 
 .PHONY: test-services
 test-services: build bin/tctl
 	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" \
 		ZTEST_TAG=services \
-		go test -v -run TestZq/ppl/zqd/db/postgresdb/ztests .
+		go test -run TestZq/ppl/zqd/db/postgresdb/ztests .
 	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" \
 		ZTEST_TAG=services \
-		go test -v -run TestZq/ppl/zqd/ztests/redis .
+		go test -run TestZq/ppl/zqd/ztests/redis .
 	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" \
 		ZTEST_TAG=services \
-		go test -v -run TestZq/ppl/zqd/temporal/ztests .
+		go test -run TestZq/ppl/zqd/temporal/ztests .
 
 .PHONY: test-services-docker
 test-services-docker: export TEMPORAL_VERSION := $(TEMPORAL_VERSION)
@@ -131,14 +131,14 @@ test-cluster: build install
 	time zapi -s files postpath s3://brim-sampledata/wrccdc/zeek-logs/files.log.gz
 	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" \
 		ZTEST_TAG=cluster \
-		go test -v -run TestZq/ppl/zqd/ztests/cluster .
+		go test -run TestZq/ppl/zqd/ztests/cluster .
 
 # test-temporal target assumes zqd-temporal endpoint is available at port 9868
 .PHONY: test-temporal
 test-temporal: build install
 	@ZTEST_PATH="$(CURDIR)/dist:$(CURDIR)/bin" \
 		ZTEST_TAG=temporal \
-		go test -v -run TestZq/ppl/zqd/temporal/ztests .
+		go test -run TestZq/ppl/zqd/temporal/ztests .
 
 perf-compare: build $(SAMPLEDATA)
 	scripts/comparison-test.sh
diff --git a/expr/eval.go b/expr/eval.go
index cb36fc5a1..f9b27739a 100644
--- a/expr/eval.go
+++ b/expr/eval.go
@@ -555,7 +555,7 @@ func getNthFromContainer(container zcode.Bytes, idx uint) (zcode.Bytes, error) {
 			return zv, nil
 		}
 	}
-	return nil, ErrIndexOutOfBounds
+	return nil, zng.ErrMissing
 }
 
 func lookupKey(mapBytes, target zcode.Bytes) (zcode.Bytes, bool) {
@@ -618,7 +618,7 @@ func indexArray(typ *zng.TypeArray, array zcode.Bytes, index zng.Value) (zng.Val
 	if zng.IsSigned(id) {
 		v, _ := zng.DecodeInt(index.Bytes)
 		if idx < 0 {
-			return zng.NewErrorf("array index out of bounds"), nil
+			return zng.Value{}, zng.ErrMissing
 		}
 		idx = uint(v)
 	} else {
@@ -653,7 +653,7 @@ func indexMap(typ *zng.TypeMap, mapBytes zcode.Bytes, key zng.Value) (zng.Value,
 	if valBytes, ok := lookupKey(mapBytes, key.Bytes); ok {
 		return zng.Value{typ.ValType, valBytes}, nil
 	}
-	return zng.NewErrorf("key not found in map: %s", key), nil
+	return zng.Value{}, zng.ErrMissing
 }
 
 type Conditional struct {


Test patch:
diff --git a/expr/expr_test.go b/expr/expr_test.go
index e66890e0d..b382ff2a4 100644
--- a/expr/expr_test.go
+++ b/expr/expr_test.go
@@ -571,9 +571,6 @@ func TestArrayIndex(t *testing.T) {
 	testSuccessful(t, "x[i]", record, zint64(2))
 	testSuccessful(t, "i+1", record, zint64(2))
 	testSuccessful(t, "x[i+1]", record, zint64(3))
-
-	testError(t, "x[-1]", record, expr.ErrIndexOutOfBounds, "negative array index")
-	testError(t, "x[3]", record, expr.ErrIndexOutOfBounds, "array index too large")
 }
 
 func TestFieldReference(t *testing.T) {
diff --git a/expr/ztests/array-bounds-2.yaml b/expr/ztests/array-bounds-2.yaml
new file mode 100644
index 000000000..ef482520f
--- /dev/null
+++ b/expr/ztests/array-bounds-2.yaml
@@ -0,0 +1,7 @@
+zql: cut m1=missing(a[-1]),m2=missing(a[3])
+
+input: |
+  {x:[1,2,3],i:1 (uint16)} (=0)
+
+output: |
+  {m1:true,m2:true}
diff --git a/expr/ztests/array-bounds.yaml b/expr/ztests/array-bounds.yaml
new file mode 100644
index 000000000..443bbb316
--- /dev/null
+++ b/expr/ztests/array-bounds.yaml
@@ -0,0 +1,9 @@
+zql: put b=missing(a[2])
+
+input: |
+  {a:[1,2,3]}
+  {a:[1,2]}
+
+output: |
+  {a:[1,2,3],b:false}
+  {a:[1,2],b:true}
diff --git a/expr/ztests/type-map.yaml b/expr/ztests/type-map.yaml
new file mode 100644
index 000000000..c64674c4b
--- /dev/null
+++ b/expr/ztests/type-map.yaml
@@ -0,0 +1,25 @@
+zql: |
+  type conn = {_path:string, id:{src:ip,dst:ip}, etc:string}
+  type dns = {_path:string, id:{src:ip,dst:ip}, query:string, query_size:uint64}
+  const schemas = |{
+    "conn": conn,
+    "dns": dns
+  }|
+  split (
+    => cut schema=schemas[_path]
+    => missing(schemas[_path]) | put _UNCLASSIFIED=true
+  ) | sort .
+
+input: |
+  {_path:"conn", id:{src:"192.168.1.1",dst:"192.168.1.2"}, etc:"foo"}
+  {_path:"dns", id:{src:"192.168.1.1",dst:"192.168.1.2"}, query:"www.google.com", query_size:23}
+  {_path:"conn", id:{src:"192.168.1.1",dst:"192.168.1.3"}, etc:"foo"}
+  {_path:"unknown", x:"has unknown _path"}
+  {y:"has no _path"}
+
+output: |
+  {_path:"unknown",x:"has unknown _path",_UNCLASSIFIED:true}
+  {schema:(conn=({_path:string,id:{src:ip,dst:ip},etc:string}))}
+  {schema:(conn=({_path:string,id:{src:ip,dst:ip},etc:string}))}
+  {schema:(dns=({_path:string,id:{src:ip,dst:ip},query:string,query_size:uint64}))}
+  {y:"has no _path",_UNCLASSIFIED:true}